[CalendarServer-changes] [2625] CalendarServer/branches/users/cdaboo/sqlpropstore-2623
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jun 25 20:39:32 PDT 2008
Revision: 2625
http://trac.macosforge.org/projects/calendarserver/changeset/2625
Author: cdaboo at apple.com
Date: 2008-06-25 20:39:32 -0700 (Wed, 25 Jun 2008)
Log Message:
-----------
Merged forward from trunk.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.element.base.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.delete.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.static.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/__init__.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/config.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/extensions.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/propfind.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/put_common.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/root.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sql.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/static.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.noneprops.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.resource.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sqlprops.py
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.element.base.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.element.base.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.element.base.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -115,7 +115,7 @@
+ PrintXML(document, stream=output)
+ else:
+ output.write("<?xml version='1.0' encoding='UTF-8'?>\r\n")
-+ self.writeToStream(output, "", 0, True)
++ self.writeToStream(output, None, 0, True)
+ output.write("\r\n")
+
+ def writeToStream(self, output, ns, level, pretty):
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -53,16 +53,25 @@
yield destparent
destparent = destparent.getResult()
-@@ -144,7 +155,19 @@
+@@ -144,10 +155,29 @@
log.err(msg)
raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, msg))
- x = waitForDeferred(move(self.fp, request.uri, destination.fp, destination_uri, depth))
+- yield x
+- yield x.getResult()
+ # Lets optimise a move within the same directory to a new resource as a simple move
+ # rather than using the full transaction based storeResource api. This allows simple
+ # "rename" operations to work quickly.
+ if (not destination.exists()) and destparent == parent:
+ x = waitForDeferred(move(self.fp, request.uri, destination.fp, destination_uri, depth))
++ yield x
++ result = x.getResult()
+
++ # Explicitly move properties - no one else does it
++ destination.deadProperties().move(self.deadProperties())
++
++ yield result
+ else:
+ x = waitForDeferred(put_common.storeResource(request,
+ source=self,
@@ -71,6 +80,9 @@
+ destination_uri=destination_uri,
+ deletesource=True,
+ depth=depth))
- yield x
- yield x.getResult()
++ yield x
++ yield x.getResult()
++
+ http_MOVE = deferredGenerator(http_MOVE)
+ def prepareForCopy(self, request):
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.delete.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.delete.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.delete.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -2,7 +2,7 @@
===================================================================
--- twisted/web2/dav/method/delete.py (revision 19773)
+++ twisted/web2/dav/method/delete.py (working copy)
-@@ -58,8 +58,28 @@
+@@ -58,8 +58,31 @@
yield x
x.getResult()
@@ -23,6 +23,9 @@
- yield x.getResult()
+ result = x.getResult()
++ # Explicitly delete properties - no one else does it
++ self.deadProperties().deleteAll()
++
+ # Adjust quota
+ if myquota is not None:
+ d = waitForDeferred(self.quotaSizeAdjust(request, -old_size))
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -2,8 +2,28 @@
===================================================================
--- twisted/web2/dav/method/propfind.py (revision 19773)
+++ twisted/web2/dav/method/propfind.py (working copy)
-@@ -200,7 +200,7 @@
+@@ -100,6 +100,10 @@
+ request_uri = request.uri
+ depth = request.headers.getHeader("depth", "infinity")
++ # For depth one cache all the child resource properties in one go if possible
++ if depth == "1":
++ self.deadProperties().cacheAllChildProperties()
++
+ xml_responses = []
+
+ # FIXME: take advantage of the new generative properties of findChildren
+@@ -108,7 +112,7 @@
+ if self.isCollection() and not my_url.endswith("/"):
+ my_url += "/"
+
+- # Do some optimisation of access control calculation by determining any inherited ACLs outside of
++ # Do some optimization of access control calculation by determining any inherited ACLs outside of
+ # the child resource loop and supply those to the checkPrivileges on each child.
+ filtered_aces = waitForDeferred(self.inheritedACEsforChildren(request))
+ yield filtered_aces
+@@ -200,7 +204,7 @@
+
def propertyName(name):
property_namespace, property_name = name
- class PropertyName (davxml.WebDAVEmptyElement):
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -2,7 +2,7 @@
===================================================================
--- twisted/web2/dav/method/put_common.py (revision 0)
+++ twisted/web2/dav/method/put_common.py (revision 0)
-@@ -0,0 +1,266 @@
+@@ -0,0 +1,273 @@
+##
+# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+#
@@ -211,6 +211,10 @@
+ yield response
+ response = response.getResult()
+
++ # Copy dead properties first, before adding overridden values
++ if source is not None:
++ destination.deadProperties().copy(source.deadProperties())
++
+ # Update the MD5 value on the resource
+ if source is not None:
+ # Copy MD5 value from source to destination
@@ -256,6 +260,9 @@
+ delete(source_uri, source.fp, depth)
+ rollback.source_deleted = True
+
++ # Explicitly delete properties - no one else does it
++ source.deadProperties().deleteAll()
++
+ # Can now commit changes and forget the rollback details
+ rollback.Commit()
+
Copied: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.noneprops.patch (from rev 2624, CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.noneprops.patch)
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.noneprops.patch (rev 0)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.noneprops.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -0,0 +1,34 @@
+Index: twisted/web2/dav/noneprops.py
+===================================================================
+--- twisted/web2/dav/noneprops.py (revision 19773)
++++ twisted/web2/dav/noneprops.py (working copy)
+@@ -49,6 +49,9 @@
+ def __init__(self, resource):
+ pass
+
++ def cacheAllChildProperties(self):
++ pass
++
+ def get(self, qname):
+ raise HTTPError(StatusResponse(responsecode.NOT_FOUND, "No such property: {%s}%s" % qname))
+
+@@ -60,8 +63,19 @@
+ # non-existing property is not an error.
+ pass
+
++ def deleteall(self, qname):
++ # RFC 2518 Section 12.13.1 says that removal of
++ # non-existing property is not an error.
++ pass
++
+ def contains(self, qname):
+ return False
+
+ def list(self):
+ return ()
++
++ def copy(self, props):
++ pass
++
++ def move(self, props):
++ pass
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -2,7 +2,7 @@
===================================================================
--- twisted/web2/dav/xattrprops.py (revision 19773)
+++ twisted/web2/dav/xattrprops.py (working copy)
-@@ -33,15 +33,24 @@
+@@ -33,12 +33,19 @@
import urllib
import sys
@@ -11,7 +11,7 @@
+from random import random
+from errno import EAGAIN
+from zlib import compress, decompress
-+from cPickle import loads as unpickle, PicklingError, UnpicklingError
++from cPickle import loads as unpickle, UnpicklingError
import xattr
@@ -19,18 +19,14 @@
raise ImportError("wrong xattr package imported")
+from twisted.python import log
-+from twisted.python.failure import Failure
from twisted.web2 import responsecode
from twisted.web2.http import HTTPError, StatusResponse
from twisted.web2.dav import davxml
-+from twisted.web2.dav.http import statusForFailure
-
- class xattrPropertyStore (object):
- """
-@@ -66,16 +75,8 @@
+@@ -65,21 +72,13 @@
+ if sys.platform == "linux2":
deadPropertyXattrPrefix = "user."
- def _encode(clazz, name):
+- def _encode(clazz, name):
- #
- # FIXME: The xattr API in Mac OS 10.4.2 breaks if you have "/" in an
- # attribute name (radar://4202440). We'll quote the strings to get rid
@@ -41,13 +37,25 @@
- c = result[i]
- if c in "%/": result[i] = "%%%02X" % (ord(c),)
- r = clazz.deadPropertyXattrPrefix + ''.join(result)
++ def _encode(cls, name):
+ result = urllib.quote("{%s}%s" % name, safe='{}:')
-+ r = clazz.deadPropertyXattrPrefix + result
++ r = cls.deadPropertyXattrPrefix + result
return r
- def _decode(clazz, name):
-@@ -97,19 +98,86 @@
+- def _decode(clazz, name):
+- name = urllib.unquote(name[len(clazz.deadPropertyXattrPrefix):])
++ def _decode(cls, name):
++ name = urllib.unquote(name[len(cls.deadPropertyXattrPrefix):])
+ index = name.find("}")
+
+@@ -95,21 +94,79 @@
+ self.resource = resource
+ self.attrs = xattr.xattr(self.resource.fp.path)
+
++ def cacheAllChildProperties(self):
++ pass
++
def get(self, qname):
try:
- value = self.attrs[self._encode(qname)]
@@ -57,13 +65,6 @@
responsecode.NOT_FOUND,
"No such property: {%s}%s" % qname
))
-+ except Exception, e:
-+ log.error("Unable to read property {%s}%s: %s"
-+ % (qname[0], qname[1], e))
-+ raise HTTPError(StatusResponse(
-+ statusForFailure(Failure()),
-+ "Unable to read property: {%s}%s" % qname
-+ ))
- doc = davxml.WebDAVDocument.fromString(value)
+ #
@@ -123,53 +124,39 @@
+ data = compress(property.toxml())
+ try:
+ self.attrs[self._encode(property.qname())] = data
-+ except Exception, e:
-+ if e.errno == EAGAIN and n < 19:
-+ sleep(random() / 10) # OMG Brutal Hax
-+ else:
-+ log.error("Unable to write property %s: %s" % (property.sname(), e))
-+ raise HTTPError(StatusResponse(
-+ statusForFailure(Failure()),
-+ "Unable to write property: %s" % (property.sname(),)
-+ ))
++ except IOError, error:
++ if error.errno != EAGAIN:
++ raise
++ sleep(random() / 10) # OMG Brutal Hax
+ else:
+ break
# Update the resource because we've modified it
self.resource.fp.restat()
-@@ -121,15 +189,34 @@
- # RFC 2518 Section 12.13.1 says that removal of
+@@ -122,6 +179,10 @@
# non-existing property is not an error.
pass
-+ except Exception, e:
-+ log.error("Unable to delete property {%s}%s: %s" % (qname[0], qname[1], e))
-+ raise HTTPError(StatusResponse(
-+ statusForFailure(Failure()),
-+ "Unable to delete property: {%s}%s" % qname
-+ ))
++ def deleteAll(self):
++ # xattrs deleted when the underlying file is deleted.
++ pass
++
def contains(self, qname):
try:
return self._encode(qname) in self.attrs
- except TypeError:
+@@ -129,7 +190,12 @@
return False
-+ except Exception, e:
-+ log.error("Unable to read properties: %s" % (e,))
-+ raise HTTPError(StatusResponse(
-+ statusForFailure(Failure()),
-+ "Unable to read properties"
-+ ))
def list(self):
- prefix = self.deadPropertyXattrPrefix
- prefix_len = len(prefix)
+- prefix = self.deadPropertyXattrPrefix
+- prefix_len = len(prefix)
++ return [ self._decode(name) for name in self.attrs if name.startswith(self.deadPropertyXattrPrefix) ]
- return [ self._decode(name) for name in self.attrs if name.startswith(prefix) ]
-+ try:
-+ return [ self._decode(name) for name in self.attrs if name.startswith(prefix) ]
-+ except Exception, e:
-+ log.error("Unable to list properties: %s" % (e,))
-+ raise HTTPError(StatusResponse(
-+ statusForFailure(Failure()),
-+ "Unable to list properties"
-+ ))
++ def copy(self, props):
++ for item in props.list():
++ self.set(props.get(item))
++
++ def move(self, props):
++ # xattrs move with the underlying file
++ pass
Copied: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.resource.patch (from rev 2624, CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.resource.patch)
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.resource.patch (rev 0)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.resource.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -0,0 +1,12 @@
+Index: twisted/web2/resource.py
+===================================================================
+--- twisted/web2/resource.py (revision 19773)
++++ twisted/web2/resource.py (working copy)
+@@ -196,6 +196,7 @@
+ @param child: an object adaptable to L{iweb.IResource}.
+ """
+ setattr(self, 'child_%s' % (path, ), child)
++ child.parent_resource = self
+
+ def http_GET(self, request):
+ if self.addSlash and request.prepath[-1] != '':
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.static.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.static.patch 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/lib-patches/Twisted/twisted.web2.static.patch 2008-06-26 03:39:32 UTC (rev 2625)
@@ -2,7 +2,7 @@
===================================================================
--- twisted/web2/static.py (revision 19773)
+++ twisted/web2/static.py (working copy)
-@@ -200,7 +200,10 @@
+@@ -200,7 +200,11 @@
super(File, self).__init__()
self.putChildren = {}
@@ -11,6 +11,15 @@
+ self.fp = path
+ else:
+ self.fp = filepath.FilePath(path)
++
# Remove the dots from the path to split
self.defaultType = defaultType
self.ignoredExts = list(ignoredExts)
+@@ -304,6 +308,7 @@
+ @param child: the child to register
+ """
+ self.putChildren[name] = child
++ child.parent_resource = self
+
+ def getChild(self, name):
+ """
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/__init__.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/__init__.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/__init__.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -47,6 +47,7 @@
"root",
"schedule",
"sql",
+ "sqlprops",
"static",
]
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/config.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/config.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -237,6 +237,11 @@
self.setDefaults(defaults)
self._data = copy.deepcopy(self._defaults)
self._configFile = None
+
+ from twistedcaldav.sqlprops import sqlPropertyStore
+ self.PropertyStoreClass = sqlPropertyStore
+ #from twistedcaldav.extensions import CachingXattrPropertyStore
+ #self.PropertyStoreClass = CachingXattrPropertyStore
def __str__(self):
return str(self._data)
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/extensions.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/extensions.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/extensions.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -49,10 +49,10 @@
from twisted.web2.dav.util import joinURL
from twisted.web2.dav.xattrprops import xattrPropertyStore
+from twistedcaldav.directory.directory import DirectoryService
+from twistedcaldav.directory.sudo import SudoDirectoryService
from twistedcaldav.log import Logger, LoggingMixIn
from twistedcaldav.util import submodule, Alternator, printTracebacks
-from twistedcaldav.directory.sudo import SudoDirectoryService
-from twistedcaldav.directory.directory import DirectoryService
log = Logger()
@@ -448,6 +448,13 @@
"""
Extended L{twisted.web2.dav.static.DAVFile} implementation.
"""
+
+ def deadProperties(self):
+ if not hasattr(self, "_dead_properties"):
+ from twistedcaldav.config import config
+ self._dead_properties = config.PropertyStoreClass(self)
+ return self._dead_properties
+
def readProperty(self, property, request):
if type(property) is tuple:
qname = property
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/propfind.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/propfind.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/propfind.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -106,6 +106,10 @@
request_uri = request.uri
depth = request.headers.getHeader("depth", "infinity")
+ # For depth one cache all the child resource properties in one go if possible
+ if depth == "1":
+ self.deadProperties().cacheAllChildProperties()
+
xml_responses = []
# FIXME: take advantage of the new generative properties of findChildren
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/put_common.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/method/put_common.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
+from twisted.web2.dav.fileop import copy
"""
PUT/COPY/MOVE common behavior.
@@ -515,7 +516,7 @@
# Do put or copy based on whether source exists
if source is not None:
- response = maybeDeferred(copyWithXAttrs, source.fp, destination.fp, destination_uri)
+ response = maybeDeferred(copy, source.fp, destination.fp, destination_uri, "0")
else:
md5 = MD5StreamWrapper(MemoryStream(calendardata))
response = maybeDeferred(putWithXAttrs, md5, destination.fp)
@@ -523,6 +524,10 @@
yield response
response = response.getResult()
+ # Copy dead properties first, before adding overridden values
+ if source is not None:
+ destination.deadProperties().copy(source.deadProperties())
+
# Update the MD5 value on the resource
if source is not None:
# Copy MD5 value from source to destination
@@ -598,6 +603,9 @@
rollback.source_deleted = True
log.debug("Source removed %s" % (source.fp.path,))
+ # Explicitly delete properties - no one else does it
+ source.deadProperties().deleteAll()
+
def doSourceIndexRecover():
"""
Do source resource indexing. This only gets called when restoring
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/root.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/root.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/root.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -28,7 +28,7 @@
from twisted.web2.http import HTTPError
from twisted.web2.auth.wrapper import UnauthorizedResponse
-from twistedcaldav.extensions import DAVFile, CachingXattrPropertyStore
+from twistedcaldav.extensions import DAVFile
from twistedcaldav.config import config
from twistedcaldav.cache import _CachedResponseResource
from twistedcaldav.cache import MemcacheResponseCache, MemcacheChangeNotifier
@@ -36,6 +36,7 @@
from twistedcaldav.log import Logger
from twistedcaldav.static import CalendarHomeFile
from twistedcaldav.directory.principal import DirectoryPrincipalResource
+from twistedcaldav.static import CalDAVFile
log = Logger()
@@ -93,7 +94,7 @@
def deadProperties(self):
if not hasattr(self, '_dead_properties'):
- self._dead_properties = CachingXattrPropertyStore(self)
+ self._dead_properties = config.PropertyStoreClass(self)
return self._dead_properties
@@ -206,6 +207,14 @@
return super(RootResource, self).locateChild(request, segments)
+ def createSimilarFile(self, path):
+ if path == self.fp.path:
+ return self
+ else:
+ child = CalDAVFile(path, principalCollections=self.principalCollections())
+ child.parent_resource = self
+ return child
+
def http_COPY (self, request): return responsecode.FORBIDDEN
def http_MOVE (self, request): return responsecode.FORBIDDEN
def http_DELETE (self, request): return responsecode.FORBIDDEN
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sql.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sql.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -40,7 +40,7 @@
A generic SQL database.
"""
- def __init__(self, dbpath, persistent, autocommit=False):
+ def __init__(self, dbpath, persistent, autocommit=False, utf8=False):
"""
@param dbpath: the path where the db file is stored.
@@ -50,10 +50,14 @@
@type persistent: bool
@param autocommit: C{True} if auto-commit mode is desired, C{False} otherwise
@type autocommit: bool
+ @param utf8: C{True} if utf8 encoded C{str} should be returned for SQl TEXT data,
+ C{False} if C{unicode} should be returned.
+ @type utf8: bool
"""
self.dbpath = dbpath
self.persistent = persistent
self.autocommit = autocommit
+ self.utf8 = utf8
def __repr__(self):
return "<%s %r>" % (self.__class__.__name__, self.dbpath)
@@ -85,6 +89,8 @@
except:
log.err("Unable to open database: %s" % (self,))
raise
+ if self.utf8:
+ self._db_connection.text_factory = str
#
# Set up the schema
Copied: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sqlprops.py (from rev 2624, CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/sqlprops.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sqlprops.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/sqlprops.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -0,0 +1,520 @@
+##
+# Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+##
+from uuid import uuid4
+
+"""
+DAV Property store using an sqlite database.
+"""
+
+__all__ = ["sqlPropertyStore"]
+
+import os
+
+from twisted.web2 import responsecode
+from twisted.web2.dav import davxml
+from twisted.web2.http import HTTPError, StatusResponse
+
+from twistedcaldav.log import Logger
+from twistedcaldav.sql import AbstractSQLDatabase
+
+log = Logger()
+
+class sqlPropertyStore (object):
+ """
+ A dead property store that uses an SQLite database back end.
+ """
+
+ def __init__(self, resource, use_cache=True):
+ self.resource = resource
+ if os.path.exists(os.path.dirname(resource.fp.path)):
+ if resource.isCollection():
+ self.childindex = SQLPropertiesDatabase(resource.fp.path, use_cache)
+ else:
+ self.childindex = None
+
+ from twistedcaldav.root import RootResource
+ if resource.isCollection() and isinstance(resource, RootResource):
+ self.rname = ""
+ self.index = self.childindex
+ else:
+ self.rname = os.path.basename(resource.fp.path)
+ if hasattr(self.resource, "parent_resource"):
+ self.index = self.resource.parent_resource.deadProperties().childindex
+ else:
+ self.index = SQLPropertiesDatabase(os.path.dirname(resource.fp.path), use_cache)
+ else:
+ log.err("No sqlPropertyStore file for %s" % (os.path.dirname(resource.fp.path),))
+ self.index = None
+ self.childindex = None
+
+ def cacheAllChildProperties(self):
+ """
+ Cache all properties for all child resources
+ """
+
+ if self.childindex:
+ self.childindex.cacheAllChildProperties()
+
+ def get(self, qname):
+ """
+ Read property from index.
+
+ @param qname: C{tuple} of property namespace and name.
+ """
+
+ if self.index:
+ value = self.index.getOnePropertyForResource(self.rname, qname)
+ if not value:
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: {%s}%s" % qname
+ ))
+
+ return value
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: {%s}%s" % qname
+ ))
+
+ def _getAll(self, hidden=False):
+ """
+ Read all properties from index.
+
+ @param hidden: C{True} to return hidden properties, C{False} otherwise.
+ @return: a C{dict} containing property name/value.
+ """
+
+ if self.index:
+ return self.index.getAllPropertiesForResource(self.rname, hidden)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property"
+ ))
+
+ def set(self, property):
+ """
+ Write property into index.
+
+ @param property: C{Property} to write
+ """
+
+ if self.index:
+ self.index.setOnePropertyForResource(self.rname, property)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
+
+ def _setSeveral(self, properties):
+ """
+ Write specific properties into index.
+
+ @param properties: C{list} of properties to write
+ """
+
+ if self.index:
+ self.index.setSeveralPropertiesForResource(self.rname, properties)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
+
+ def _setAll(self, properties):
+ """
+ Write all properties into index.
+
+ @param properties: C{list} of properties to write
+ """
+
+ self.index.removeAllPropertiesForResource(self.rname)
+ self.index.setSeveralPropertiesForResource(self.rname, properties)
+
+ def delete(self, qname):
+ """
+ Delete property from index.
+
+ DELETE from PROPERTIES where NAME=<<rname>> and PROPNAME=<<pname>>
+
+ @param qname:
+ """
+
+ if self.index:
+ self.index.removeOnePropertyForResource(self.rname, qname)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
+
+ def deleteAll(self):
+ """
+ Delete all properties from index.
+
+ DELETE from PROPERTIES where NAME=<<rname>>
+ """
+
+ if self.index:
+ self.index.removeAllPropertiesForResource(self.rname)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
+
+ def contains(self, qname):
+
+ if self.index:
+ return self.index.getOnePropertyForResource(self.rname, qname) is not None
+ else:
+ return False
+
+ def list(self):
+ """
+ List all property names for this resource.
+
+ SELECT PROPNAME from PROPERTIES where NAME=<<rname>>
+
+ """
+
+ if self.index:
+ return self.index.listPropertiesForResource(self.rname)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
+
+ def copy(self, props):
+ """
+ Copy properties from another property store into this one, replacing everything
+ currently present.
+
+ @param props: a property store to copy from
+ @type props: C{sqlPropertyStore}
+ """
+
+ oldprops = props._getAll(hidden=True)
+ self._setAll(oldprops.itervalues())
+
+ def move(self, props):
+ """
+ Move properties from another property store into this one, replacing everything
+ currently present.
+
+ @param props: a property store to copy from
+ @type props: C{sqlPropertyStore}
+ """
+
+ oldprops = props._getAll(hidden=True)
+ self._setAll(oldprops.itervalues())
+ props.deleteAll()
+
+class SQLPropertiesDatabase(AbstractSQLDatabase):
+ """
+ A database to store properties on resources in a collection.
+ It maintains a cache of values read from the database.
+
+ SCHEMA:
+
+ Properties Database:
+
+ ROW: RESOURCENAME, PROPERTYNAME, PROPERTYXML
+
+ """
+
+ dbType = "SQLPROPERTIES"
+ dbFilename = ".db.sqlproperties"
+ dbFormatVersion = "1"
+
+ @classmethod
+ def _encodePropertyName(cls, name):
+ return "{%s}%s" % name
+
+ @classmethod
+ def _decodePropertyName(cls, name):
+ index = name.find("}")
+
+ if (index is -1 or not len(name) > index or not name[0] == "{"):
+ raise ValueError("Invalid encoded name: %r" % (name,))
+
+ return (name[1:index], name[index+1:])
+
+ @classmethod
+ def _decodePropertyValue(cls, pname, pxml):
+ """
+ @param pname: name of the property
+ @type pname: str
+ @param pxml: a property encoded as XML
+ @type pxml: str
+ @return: the davxml Element of the property, or C{None} if pxml is empty or C{None}
+ """
+
+ result = None
+ if pxml:
+ try:
+ doc = davxml.WebDAVDocument.fromString(pxml)
+ result = doc.root_element
+ except ValueError:
+ msg = ("Invalid property value stored on server: {%s}%s %s"
+ % (pname[0], pname[1], pxml))
+ log.err(msg)
+ raise HTTPError(StatusResponse(responsecode.INTERNAL_SERVER_ERROR, msg))
+
+ return result
+
+ def __init__(self, path, use_cache=True):
+ path = os.path.join(path, SQLPropertiesDatabase.dbFilename)
+ super(SQLPropertiesDatabase, self).__init__(path, True, utf8=True)
+
+ self.use_cache = use_cache
+ self.cache = {}
+ self.instance = uuid4()
+
+ def cacheAllChildProperties(self):
+
+ self.cache = {}
+
+ log.debug("[%s]: Caching all child properties" % (self.instance,))
+ for row in self._db_execute("select RESOURCENAME, PROPERTYNAME, PROPERTYXML from PROPERTIES"):
+ rname = row[0]
+ pname = self._decodePropertyName(row[1])
+ pvalue = self._decodePropertyValue(pname, row[2])
+ self.cache.setdefault(rname, {})[pname] = pvalue
+
+ def getOnePropertyForResource(self, rname, pname):
+ """
+ Get a property.
+
+ @param rname: a C{str} containing the resource name.
+ @param pname: a C{str} containing the name of the property to get.
+ @return: an object representing the property.
+ """
+
+ # Check cache first
+ if self.use_cache:
+ # Make sure we have a cache entry
+ if not self.cache.has_key(rname):
+ log.debug("[%s]: Caching properties for %s, triggered by {%s}%s" % (self.instance, rname, pname[0], pname[1],))
+ self.cache[rname] = {}
+ for row in self._db_execute("select PROPERTYNAME, PROPERTYXML from PROPERTIES where RESOURCENAME = :1", rname):
+ sqlpname = self._decodePropertyName(row[0])
+ sqlpvalue = self._decodePropertyValue(sqlpname, row[1])
+ self.cache[rname][sqlpname] = sqlpvalue
+
+ # Get the property and do negative caching if not present
+ return self.cache[rname].setdefault(pname, None)
+ else:
+ log.debug("[%s] Getting property {%s}%s for %s" % (self.instance, pname[0], pname[1], rname,))
+ pxml = self._db_value_for_sql("select PROPERTYXML from PROPERTIES where RESOURCENAME = :1 and PROPERTYNAME = :2", rname, self._encodePropertyName(pname))
+ pvalue = self._decodePropertyValue(pname, pxml)
+ return pvalue
+
+ def getAllPropertiesForResource(self, rname, hidden):
+ """
+ Get specified property values from specific resource.
+
+ @param rname: a C{str} containing the resource name.
+ @param hidden: C{True} to return hidden properties, C{False} otherwise.
+ @return: a C{dict} containing property name/value.
+ """
+
+ log.debug("[%s] Caching all properties for %s" % (self.instance, rname,))
+ properties = {}
+ statement = "select PROPERTYNAME, PROPERTYXML from PROPERTIES where RESOURCENAME = :1"
+ if not hidden:
+ statement += " and HIDDEN = 'F'"
+ for row in self._db_execute(statement, rname):
+ pname = self._decodePropertyName(row[0])
+ pvalue = self._decodePropertyValue(pname, row[1])
+ properties[pname] = pvalue
+
+ return properties
+
+ def setOnePropertyForResource(self, rname, property):
+ """
+ Add a property.
+
+ @param rname: a C{str} containing the resource name.
+ @param pname: a C{str} containing the name of the property to set.
+ @param pvalue: the property to set.
+ @param hidden: C{True} for a hidden property, C{False} otherwise.
+ """
+
+ pname = property.qname()
+ log.debug("[%s] Setting property {%s}%s for %s" % (self.instance, pname[0], pname[1], rname,))
+ self._add_to_db(rname, self._encodePropertyName(pname), property.toxml(), property.hidden)
+ self._db_commit()
+
+ if self.use_cache and self.cache.has_key(rname):
+ self.cache[rname][pname] = property
+
+ def setSeveralPropertiesForResource(self, rname, properties):
+ """
+ Add a set of properties.
+
+ @param rname: a C{str} containing the resource name.
+ @param properties: a C{dict} containing the name of the properties to set.
+ """
+
+ # Add properties.
+ log.debug("[%s] Setting properties for %s" % (self.instance, rname,))
+ for property in properties:
+ self._add_to_db(rname, self._encodePropertyName(property.qname()), property.toxml(), property.hidden)
+ self._db_commit()
+
+ if self.use_cache and self.cache.has_key(rname):
+ for property in properties:
+ self.cache[rname][property.qname()] = property
+
+ def removeOnePropertyForResource(self, rname, pname):
+ """
+ Remove a property.
+
+ @param rname: a C{str} containing the resource name.
+ @param pname: a C{str} containing the name of the property to remove.
+ """
+
+ log.debug("[%s] Removing property {%s}%s from %s" % (self.instance, pname[0], pname[1], rname,))
+ self._delete_from_db(rname, self._encodePropertyName(pname))
+ self._db_commit()
+
+ if self.use_cache and self.cache.has_key(rname):
+ self.cache[rname][pname] = None
+
+ def removeAllPropertiesForResource(self, rname):
+ """
+ Remove all properties for resource.
+
+ @param rname: a C{str} containing the resource name.
+ """
+
+ log.debug("[%s] Removing all properties from %s" % (self.instance, rname,))
+ self._delete_all_from_db(rname)
+ self._db_commit()
+
+ if self.use_cache:
+ try:
+ del self.cache[rname]
+ except KeyError:
+ pass
+
+ def listPropertiesForResource(self, rname):
+ """
+ List all properties in resource.
+
+ @param rname: a C{str} containing the resource name.
+ @return: a C{set} containing the property names.
+ """
+
+ members = set()
+ if self.use_cache and self.cache.has_key(rname):
+ log.debug("[%s] Listing all properties for %s from cache" % (self.instance, rname,))
+ members.update([k for k, v in self.cache[rname].iteritems() if v is not None])
+ else:
+ log.debug("[%s] Listing all properties for %s via query" % (self.instance, rname,))
+ for row in self._db_execute("select PROPERTYNAME from PROPERTIES where RESOURCENAME = :1", rname):
+ members.add(self._decodePropertyName(row[0]))
+ return members
+
+ def _add_to_db(self, rname, pname, pxml, hidden):
+ """
+ Add a property.
+
+ @param rname: a C{str} containing the resource name.
+ @param pname: a C{str} containing the name of the property to set.
+ @param pxml: a C{str} containing the XML representation of the property object.
+ @param hidden: a C{bool} indicating whether the property is hidden.
+ """
+
+ self._db_execute(
+ """
+ insert or replace into PROPERTIES (RESOURCENAME, PROPERTYNAME, PROPERTYXML, HIDDEN)
+ values (:1, :2, :3, :4)
+ """, rname, pname, pxml, "T" if hidden else "F"
+ )
+
+ def _delete_from_db(self, rname, pname):
+ """
+ Remove a property.
+
+ @param rname: a C{str} containing the resource name.
+ @param pname: a C{str} containing the name of the property to get.
+ """
+
+ self._db_execute("delete from PROPERTIES where RESOURCENAME = :1 and PROPERTYNAME = :2", rname, pname)
+
+ def _delete_all_from_db(self, rname):
+ """
+ Remove a property.
+
+ @param rname: a C{str} containing the resource name.
+ @param pname: a C{str} containing the name of the property to get.
+ """
+
+ self._db_execute("delete from PROPERTIES where RESOURCENAME = :1", rname)
+
+ def _db_type(self):
+ """
+ @return: the collection type assigned to this index.
+ """
+ return SQLPropertiesDatabase.dbType
+
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return SQLPropertiesDatabase.dbFormatVersion
+
+ def _db_init_data_tables(self, q):
+ """
+ Initialize the underlying database tables.
+ @param q: a database cursor to use.
+ """
+
+ #
+ # PROPERTIES table
+ #
+ # (RESOURCENAME, PROPERTYNAME) is unique and we default to replacing whatever is
+ # already present for those pairs. Using UNIQUE on those creates an index for us.
+ #
+ q.execute(
+ """
+ create table PROPERTIES (
+ RESOURCENAME text,
+ PROPERTYNAME text,
+ PROPERTYXML text,
+ HIDDEN text(1),
+ UNIQUE (RESOURCENAME, PROPERTYNAME) ON CONFLICT REPLACE
+ )
+ """
+ )
+
+ # When deleting a resource as a whole we search on the RESOURCENAME so index that column.
+ q.execute(
+ """
+ create index RESOURCE on PROPERTIES (RESOURCENAME)
+ """
+ )
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/static.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/static.py 2008-06-26 00:27:39 UTC (rev 2624)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2623/twistedcaldav/static.py 2008-06-26 03:39:32 UTC (rev 2625)
@@ -55,7 +55,6 @@
from twistedcaldav.caldavxml import caldav_namespace
from twistedcaldav.config import config
from twistedcaldav.extensions import DAVFile
-from twistedcaldav.extensions import CachingXattrPropertyStore
from twistedcaldav.ical import Component as iComponent
from twistedcaldav.ical import Property as iProperty
from twistedcaldav.index import Index, IndexSchedule
@@ -87,7 +86,7 @@
def deadProperties(self):
if not hasattr(self, "_dead_properties"):
- self._dead_properties = CachingXattrPropertyStore(self)
+ self._dead_properties = config.PropertyStoreClass(self)
return self._dead_properties
@@ -457,10 +456,12 @@
def provisionChild(self, name):
if name == uidsResourceName:
- return CalendarHomeUIDProvisioningFile(self.fp.child(name).path, self)
+ child = CalendarHomeUIDProvisioningFile(self.fp.child(name).path, self)
+ else:
+ child = CalendarHomeTypeProvisioningFile(self.fp.child(name).path, self, name)
+ child.parent_resource = self
+ return child
- return CalendarHomeTypeProvisioningFile(self.fp.child(name).path, self, name)
-
def createSimilarFile(self, path):
raise HTTPError(responsecode.NOT_FOUND)
@@ -501,6 +502,7 @@
childPath = self.fp.child(name[0:2]).child(name[2:4]).child(name)
child = self.homeResourceClass(childPath.path, self, record)
+ child.parent_resource = self
if not child.exists():
self.provision()
@@ -581,6 +583,7 @@
if cls is not None:
child = cls(self.fp.child(name).path, self)
+ child.parent_resource = self
child.cacheNotifier = self.cacheNotifier
return child
@@ -591,6 +594,7 @@
return self
else:
similar = CalDAVFile(path, principalCollections=self.principalCollections())
+ similar.parent_resource = self
similar.cacheNotifier = self.cacheNotifier
return similar
@@ -613,7 +617,9 @@
if path == self.fp.path:
return self
else:
- return CalDAVFile(path, principalCollections=self.principalCollections())
+ child = CalDAVFile(path, principalCollections=self.principalCollections())
+ child.parent_resource = self
+ return child
def index(self):
"""
@@ -690,7 +696,9 @@
if path == self.fp.path:
return self
else:
- return DropBoxCollectionFile(path, self)
+ child = DropBoxCollectionFile(path, self)
+ child.parent_resource = self
+ return child
def __repr__(self):
return "<%s (dropbox home collection): %s>" % (self.__class__.__name__, self.fp.path)
@@ -704,7 +712,9 @@
if path == self.fp.path:
return self
else:
- return DropBoxChildFile(path, self)
+ child = DropBoxChildFile(path, self)
+ child.parent_resource = self
+ return child
def __repr__(self):
return "<%s (dropbox collection): %s>" % (self.__class__.__name__, self.fp.path)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080625/8efbaa92/attachment-0001.htm
More information about the calendarserver-changes
mailing list