[CalendarServer-changes] [2604] CalendarServer/branches/users/cdaboo/sqlpropstore-2563
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jun 19 10:47:44 PDT 2008
Revision: 2604
http://trac.macosforge.org/projects/calendarserver/changeset/2604
Author: cdaboo at apple.com
Date: 2008-06-19 10:47:39 -0700 (Thu, 19 Jun 2008)
Log Message:
-----------
Make sure we always use a single instance of the sqlite DB when querying a bunch of resources in
the same collection. Also optimize the Depth:1 case by pre-caching all dead properties of all children.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.static.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/method/propfind.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/root.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/sqlprops.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/static.py
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/test/test_sqlprops.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.noneprops.patch
CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.resource.patch
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.method.propfind.patch 2008-06-19 17:47:39 UTC (rev 2604)
@@ -2,8 +2,28 @@
===================================================================
--- twisted/web2/dav/method/propfind.py (revision 19773)
+++ twisted/web2/dav/method/propfind.py (working copy)
-@@ -200,7 +200,7 @@
+@@ -100,6 +100,10 @@
+ request_uri = request.uri
+ depth = request.headers.getHeader("depth", "infinity")
++ # For depth one cache all the child resource properties in one go if possible
++ if depth == "1":
++ self.deadProperties().cacheAllChildProperties()
++
+ xml_responses = []
+
+ # FIXME: take advantage of the new generative properties of findChildren
+@@ -108,7 +112,7 @@
+ if self.isCollection() and not my_url.endswith("/"):
+ my_url += "/"
+
+- # Do some optimisation of access control calculation by determining any inherited ACLs outside of
++ # Do some optimization of access control calculation by determining any inherited ACLs outside of
+ # the child resource loop and supply those to the checkPrivileges on each child.
+ filtered_aces = waitForDeferred(self.inheritedACEsforChildren(request))
+ yield filtered_aces
+@@ -200,7 +204,7 @@
+
def propertyName(name):
property_namespace, property_name = name
- class PropertyName (davxml.WebDAVEmptyElement):
Added: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.noneprops.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.noneprops.patch (rev 0)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.noneprops.patch 2008-06-19 17:47:39 UTC (rev 2604)
@@ -0,0 +1,34 @@
+Index: twisted/web2/dav/noneprops.py
+===================================================================
+--- twisted/web2/dav/noneprops.py (revision 19773)
++++ twisted/web2/dav/noneprops.py (working copy)
+@@ -49,6 +49,9 @@
+ def __init__(self, resource):
+ pass
+
++ def cacheAllChildProperties(self):
++ pass
++
+ def get(self, qname):
+ raise HTTPError(StatusResponse(responsecode.NOT_FOUND, "No such property: {%s}%s" % qname))
+
+@@ -60,8 +63,19 @@
+ # non-existing property is not an error.
+ pass
+
++ def deleteall(self, qname):
++ # RFC 2518 Section 12.13.1 says that removal of
++ # non-existing property is not an error.
++ pass
++
+ def contains(self, qname):
+ return False
+
+ def list(self):
+ return ()
++
++ def copy(self, props):
++ pass
++
++ def move(self, props):
++ pass
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.dav.xattrprops.patch 2008-06-19 17:47:39 UTC (rev 2604)
@@ -49,8 +49,13 @@
index = name.find("}")
-@@ -97,19 +96,74 @@
+@@ -95,21 +94,79 @@
+ self.resource = resource
+ self.attrs = xattr.xattr(self.resource.fp.path)
++ def cacheAllChildProperties(self):
++ pass
++
def get(self, qname):
try:
- value = self.attrs[self._encode(qname)]
@@ -128,7 +133,7 @@
# Update the resource because we've modified it
self.resource.fp.restat()
-@@ -122,6 +176,10 @@
+@@ -122,6 +179,10 @@
# non-existing property is not an error.
pass
@@ -139,7 +144,7 @@
def contains(self, qname):
try:
return self._encode(qname) in self.attrs
-@@ -129,7 +187,12 @@
+@@ -129,7 +190,12 @@
return False
def list(self):
Added: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.resource.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.resource.patch (rev 0)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.resource.patch 2008-06-19 17:47:39 UTC (rev 2604)
@@ -0,0 +1,12 @@
+Index: twisted/web2/resource.py
+===================================================================
+--- twisted/web2/resource.py (revision 19773)
++++ twisted/web2/resource.py (working copy)
+@@ -196,6 +196,7 @@
+ @param child: an object adaptable to L{iweb.IResource}.
+ """
+ setattr(self, 'child_%s' % (path, ), child)
++ child.parent_resource = self
+
+ def http_GET(self, request):
+ if self.addSlash and request.prepath[-1] != '':
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.static.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.static.patch 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/lib-patches/Twisted/twisted.web2.static.patch 2008-06-19 17:47:39 UTC (rev 2604)
@@ -15,3 +15,11 @@
# Remove the dots from the path to split
self.defaultType = defaultType
self.ignoredExts = list(ignoredExts)
+@@ -304,6 +308,7 @@
+ @param child: the child to register
+ """
+ self.putChildren[name] = child
++ child.parent_resource = self
+
+ def getChild(self, name):
+ """
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/method/propfind.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/method/propfind.py 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/method/propfind.py 2008-06-19 17:47:39 UTC (rev 2604)
@@ -106,6 +106,10 @@
request_uri = request.uri
depth = request.headers.getHeader("depth", "infinity")
+ # For depth one cache all the child resource properties in one go if possible
+ if depth == "1":
+ self.deadProperties().cacheAllChildProperties()
+
xml_responses = []
# FIXME: take advantage of the new generative properties of findChildren
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/root.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/root.py 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/root.py 2008-06-19 17:47:39 UTC (rev 2604)
@@ -187,7 +187,9 @@
if path == self.fp.path:
return self
else:
- return CalDAVFile(path, principalCollections=self.principalCollections())
+ child = CalDAVFile(path, principalCollections=self.principalCollections())
+ child.parent_resource = self
+ return child
def http_COPY (self, request): return responsecode.FORBIDDEN
def http_MOVE (self, request): return responsecode.FORBIDDEN
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/sqlprops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/sqlprops.py 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/sqlprops.py 2008-06-19 17:47:39 UTC (rev 2604)
@@ -19,6 +19,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
##
+from uuid import uuid4
"""
DAV Property store using an sqlite database.
@@ -45,43 +46,55 @@
def __init__(self, resource, use_cache=True):
self.resource = resource
if os.path.exists(os.path.dirname(resource.fp.path)):
+ if resource.isCollection():
+ self.childindex = SQLPropertiesDatabase(resource.fp.path, use_cache)
+ else:
+ self.childindex = None
+
from twistedcaldav.root import RootResource
if resource.isCollection() and isinstance(resource, RootResource):
self.rname = ""
- indexpath = resource.fp.path
+ self.index = self.childindex
else:
self.rname = os.path.basename(resource.fp.path)
- indexpath = os.path.dirname(resource.fp.path)
- self.index = SQLPropertiesDatabase(indexpath, use_cache)
- if resource.isCollection():
- self.childindex = SQLPropertiesDatabase(resource.fp.path, use_cache)
- else:
- self.childindex = None
+ if hasattr(self.resource, "parent_resource"):
+ self.index = self.resource.parent_resource.deadProperties().childindex
+ else:
+ self.index = SQLPropertiesDatabase(os.path.dirname(resource.fp.path), use_cache)
else:
log.err("No sqlPropertyStore file for %s" % (os.path.dirname(resource.fp.path),))
self.index = None
self.childindex = None
+ def cacheAllChildProperties(self):
+ """
+ Cache all properties for all child resources
+ """
+
+ if self.childindex:
+ self.childindex.cacheAllChildProperties()
+
def get(self, qname):
"""
Read property from index.
@param qname: C{tuple} of property namespace and name.
"""
- if not self.index:
+
+ if self.index:
+ value = self.index.getOnePropertyForResource(self.rname, qname)
+ if not value:
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: {%s}%s" % qname
+ ))
+
+ return value
+ else:
raise HTTPError(StatusResponse(
responsecode.NOT_FOUND,
"No such property: {%s}%s" % qname
))
-
- value = self.index.getOnePropertyForResource(self.rname, qname)
- if not value:
- raise HTTPError(StatusResponse(
- responsecode.NOT_FOUND,
- "No such property: {%s}%s" % qname
- ))
-
- return value
def _getAll(self, hidden=False):
"""
@@ -90,13 +103,14 @@
@param hidden: C{True} to return hidden properties, C{False} otherwise.
@return: a C{dict} containing property name/value.
"""
- if not self.index:
+
+ if self.index:
+ return self.index.getAllPropertiesForResource(self.rname, hidden)
+ else:
raise HTTPError(StatusResponse(
responsecode.NOT_FOUND,
- "No properties"
+ "No such property"
))
-
- return self.index.getAllPropertiesForResource(self.rname, hidden)
def set(self, property):
"""
@@ -107,6 +121,11 @@
if self.index:
self.index.setOnePropertyForResource(self.rname, property)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
def _setSeveral(self, properties):
"""
@@ -117,6 +136,11 @@
if self.index:
self.index.setSeveralPropertiesForResource(self.rname, properties)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
def _setAll(self, properties):
"""
@@ -125,9 +149,8 @@
@param properties: C{list} of properties to write
"""
- if self.index:
- self.index.removeAllPropertiesForResource(self.rname)
- self.index.setSeveralPropertiesForResource(self.rname, properties)
+ self.index.removeAllPropertiesForResource(self.rname)
+ self.index.setSeveralPropertiesForResource(self.rname, properties)
def delete(self, qname):
"""
@@ -140,6 +163,11 @@
if self.index:
self.index.removeOnePropertyForResource(self.rname, qname)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
def deleteAll(self):
"""
@@ -150,11 +178,16 @@
if self.index:
self.index.removeAllPropertiesForResource(self.rname)
+ else:
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
def contains(self, qname):
+
if self.index:
- value = self.index.getOnePropertyForResource(self.rname, qname)
- return value is not None
+ return self.index.getOnePropertyForResource(self.rname, qname) is not None
else:
return False
@@ -169,7 +202,10 @@
if self.index:
return self.index.listPropertiesForResource(self.rname)
else:
- return ()
+ raise HTTPError(StatusResponse(
+ responsecode.INTERNAL_SERVER_ERROR,
+ "Property store does not exist"
+ ))
def copy(self, props):
"""
@@ -255,7 +291,19 @@
self.use_cache = use_cache
self.cache = {}
+ self.instance = uuid4()
+ def cacheAllChildProperties(self):
+
+ self.cache = {}
+
+ log.debug("[%s]: Caching all child properties" % (self.instance,))
+ for row in self._db_execute("select RESOURCENAME, PROPERTYNAME, PROPERTYXML from PROPERTIES"):
+ rname = row[0]
+ pname = self._decodePropertyName(row[1])
+ pvalue = self._decodePropertyValue(pname, row[2])
+ self.cache.setdefault(rname, {})[pname] = pvalue
+
def getOnePropertyForResource(self, rname, pname):
"""
Get a property.
@@ -269,6 +317,7 @@
if self.use_cache:
# Make sure we have a cache entry
if not self.cache.has_key(rname):
+ log.debug("[%s]: Caching properties for %s, triggered by {%s}%s" % (self.instance, rname, pname[0], pname[1],))
self.cache[rname] = {}
for row in self._db_execute("select PROPERTYNAME, PROPERTYXML from PROPERTIES where RESOURCENAME = :1", rname):
sqlpname = self._decodePropertyName(row[0])
@@ -278,6 +327,7 @@
# Get the property and do negative caching if not present
return self.cache[rname].setdefault(pname, None)
else:
+ log.debug("[%s] Getting property {%s}%s for %s" % (self.instance, pname[0], pname[1], rname,))
pxml = self._db_value_for_sql("select PROPERTYXML from PROPERTIES where RESOURCENAME = :1 and PROPERTYNAME = :2", rname, self._encodePropertyName(pname))
pvalue = self._decodePropertyValue(pname, pxml)
return pvalue
@@ -291,6 +341,7 @@
@return: a C{dict} containing property name/value.
"""
+ log.debug("[%s] Caching all properties for %s" % (self.instance, rname,))
properties = {}
statement = "select PROPERTYNAME, PROPERTYXML from PROPERTIES where RESOURCENAME = :1"
if not hidden:
@@ -313,6 +364,7 @@
"""
pname = property.qname()
+ log.debug("[%s] Setting property {%s}%s for %s" % (self.instance, pname[0], pname[1], rname,))
self._add_to_db(rname, self._encodePropertyName(pname), property.toxml(), property.hidden)
self._db_commit()
@@ -327,7 +379,8 @@
@param properties: a C{dict} containing the name of the properties to set.
"""
- # Remove what is there, then add it back.
+ # Add properties.
+ log.debug("[%s] Setting properties for %s" % (self.instance, rname,))
for property in properties:
self._add_to_db(rname, self._encodePropertyName(property.qname()), property.toxml(), property.hidden)
self._db_commit()
@@ -344,6 +397,7 @@
@param pname: a C{str} containing the name of the property to remove.
"""
+ log.debug("[%s] Removing property {%s}%s from %s" % (self.instance, pname[0], pname[1], rname,))
self._delete_from_db(rname, self._encodePropertyName(pname))
self._db_commit()
@@ -357,6 +411,7 @@
@param rname: a C{str} containing the resource name.
"""
+ log.debug("[%s] Removing all properties from %s" % (self.instance, rname,))
self._delete_all_from_db(rname)
self._db_commit()
@@ -376,8 +431,10 @@
members = set()
if self.use_cache and self.cache.has_key(rname):
+ log.debug("[%s] Listing all properties for %s from cache" % (self.instance, rname,))
members.update([k for k, v in self.cache[rname].iteritems() if v is not None])
else:
+ log.debug("[%s] Listing all properties for %s via query" % (self.instance, rname,))
for row in self._db_execute("select PROPERTYNAME from PROPERTIES where RESOURCENAME = :1", rname):
members.add(self._decodePropertyName(row[0]))
return members
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/static.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/static.py 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/static.py 2008-06-19 17:47:39 UTC (rev 2604)
@@ -456,10 +456,12 @@
def provisionChild(self, name):
if name == uidsResourceName:
- return CalendarHomeUIDProvisioningFile(self.fp.child(name).path, self)
+ child = CalendarHomeUIDProvisioningFile(self.fp.child(name).path, self)
+ else:
+ child = CalendarHomeTypeProvisioningFile(self.fp.child(name).path, self, name)
+ child.parent_resource = self
+ return child
- return CalendarHomeTypeProvisioningFile(self.fp.child(name).path, self, name)
-
def createSimilarFile(self, path):
raise HTTPError(responsecode.NOT_FOUND)
@@ -500,6 +502,7 @@
childPath = self.fp.child(name[0:2]).child(name[2:4]).child(name)
child = self.homeResourceClass(childPath.path, self, record)
+ child.parent_resource = self
if not child.exists():
self.provision()
@@ -580,6 +583,7 @@
if cls is not None:
child = cls(self.fp.child(name).path, self)
+ child.parent_resource = self
child.cacheNotifier = self.cacheNotifier
return child
@@ -590,6 +594,7 @@
return self
else:
similar = CalDAVFile(path, principalCollections=self.principalCollections())
+ similar.parent_resource = self
similar.cacheNotifier = self.cacheNotifier
return similar
@@ -612,7 +617,9 @@
if path == self.fp.path:
return self
else:
- return CalDAVFile(path, principalCollections=self.principalCollections())
+ child = CalDAVFile(path, principalCollections=self.principalCollections())
+ child.parent_resource = self
+ return child
def index(self):
"""
@@ -689,7 +696,9 @@
if path == self.fp.path:
return self
else:
- return DropBoxCollectionFile(path, self)
+ child = DropBoxCollectionFile(path, self)
+ child.parent_resource = self
+ return child
def __repr__(self):
return "<%s (dropbox home collection): %s>" % (self.__class__.__name__, self.fp.path)
@@ -703,7 +712,9 @@
if path == self.fp.path:
return self
else:
- return DropBoxChildFile(path, self)
+ child = DropBoxChildFile(path, self)
+ child.parent_resource = self
+ return child
def __repr__(self):
return "<%s (dropbox collection): %s>" % (self.__class__.__name__, self.fp.path)
Modified: CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/test/test_sqlprops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/test/test_sqlprops.py 2008-06-19 17:42:20 UTC (rev 2603)
+++ CalendarServer/branches/users/cdaboo/sqlpropstore-2563/twistedcaldav/test/test_sqlprops.py 2008-06-19 17:47:39 UTC (rev 2604)
@@ -345,3 +345,28 @@
# Make a calendar
request = SimpleRequest(self.site, "MKCALENDAR", "/calendar1/")
return self.send(request, doneMake1)
+
+ def test_cacheallproperties(self):
+
+ collection_name, _ignore = self.mkdtemp("sql")
+
+ file1 = CalDAVFile(os.path.join(collection_name, "file1.ics"))
+ file1_index = sqlPropertyStore(file1)
+ for prop in SQLProps.props:
+ self._setProperty(file1_index, prop)
+
+ file2 = CalDAVFile(os.path.join(collection_name, "file2.ics"))
+ file2_index = sqlPropertyStore(file2)
+ for prop in SQLProps.props:
+ self._setProperty(file2_index, prop)
+
+ file1 = None
+ file1_index = None
+ file2 = None
+ file2_index = None
+
+ collection = CalDAVFile(collection_name)
+ collection_index = sqlPropertyStore(collection)
+ collection_index.cacheAllChildProperties()
+
+
\ No newline at end of file
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080619/3fc70710/attachment-0001.htm
More information about the calendarserver-changes
mailing list