[CalendarServer-changes] [9015] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Tue Apr 10 13:41:50 PDT 2012


Revision: 9015
          http://trac.macosforge.org/projects/calendarserver/changeset/9015
Author:   sagen at apple.com
Date:     2012-04-10 13:41:50 -0700 (Tue, 10 Apr 2012)
Log Message:
-----------
SQL query cacher, specifically for the object-with-name query, but more will come soon.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/conf/caldavd-test.plist
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/txdav/base/datastore/util.py
    CalendarServer/trunk/txdav/common/datastore/sql.py

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2012-04-10 20:27:08 UTC (rev 9014)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2012-04-10 20:41:50 UTC (rev 9015)
@@ -236,6 +236,9 @@
             logSQL=config.LogDatabase.SQLStatements,
             logTransactionWaits=config.LogDatabase.TransactionWaitSeconds,
             timeoutTransactions=config.TransactionTimeoutSeconds,
+            cacheQueries=config.QueryCaching.Enabled,
+            cachePool=config.QueryCaching.MemcachedPool,
+            cacheExpireSeconds=config.QueryCaching.ExpireSeconds
         )
     else:
         return CommonFileDataStore(

Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist	2012-04-10 20:27:08 UTC (rev 9014)
+++ CalendarServer/trunk/conf/caldavd-test.plist	2012-04-10 20:41:50 UTC (rev 9015)
@@ -946,6 +946,17 @@
       </array>
     </dict>
 
+    <!-- SQL Query Caching -->
+    <key>QueryCaching</key>
+    <dict>
+      <key>Enabled</key>
+      <true/>
+      <key>MemcachedPool</key>
+      <string>Default</string>
+      <key>ExpireSeconds</key>
+      <integer>3600</integer>
+    </dict>
+
     <!-- Group Membership Caching -->
     <key>GroupCaching</key>
     <dict>

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2012-04-10 20:27:08 UTC (rev 9014)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2012-04-10 20:41:50 UTC (rev 9015)
@@ -882,6 +882,12 @@
         ],
     },
 
+    "QueryCaching" : {
+        "Enabled" : True,
+        "MemcachedPool" : "Default",
+        "ExpireSeconds" : 3600,
+    },
+
     "GroupCaching" : {
         "Enabled": True,
         "MemcachedPool" : "Default",

Modified: CalendarServer/trunk/txdav/base/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/util.py	2012-04-10 20:27:08 UTC (rev 9014)
+++ CalendarServer/trunk/txdav/base/datastore/util.py	2012-04-10 20:41:50 UTC (rev 9015)
@@ -19,6 +19,8 @@
 Common utility functions for a datastores.
 """
 
+from twistedcaldav.memcacher import Memcacher
+
 _unset = object()
 
 class cached(object):
@@ -45,3 +47,34 @@
             else:
                 return cached
         return inner
+
+
+class QueryCacher(Memcacher):
+    """
+    A Memcacher for the object-with-name query (more to come)
+    """
+
+    def __init__(self, cachePool="Default", cacheExpireSeconds=3600):
+        super(QueryCacher, self).__init__(cachePool, pickle=True)
+        self.cacheExpireSeconds = cacheExpireSeconds
+
+    def set(self, key, value):
+        super(QueryCacher, self).set(key, value, expireTime=self.cacheExpireSeconds)
+
+    def keyForObjectWithName(self, homeResourceID, name):
+        return "objectWithName:%s:%s" % (homeResourceID, name)
+
+    def getObjectWithName(self, homeResourceID, name):
+        key = self.keyForObjectWithName(homeResourceID, name)
+        return self.get(key)
+
+    def setObjectWithName(self, transaction, homeResourceID, name, value):
+        key = self.keyForObjectWithName(homeResourceID, name)
+        transaction.postCommit(lambda:self.set(key, value))
+
+    def invalidateObjectWithName(self, transaction, homeResourceID, name):
+        key = self.keyForObjectWithName(homeResourceID, name)
+        # Invalidate immediately and post-commit in case a calendar was created and deleted
+        # within the same transaction
+        self.delete(key)
+        transaction.postCommit(lambda:self.delete(key))

Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py	2012-04-10 20:27:08 UTC (rev 9014)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py	2012-04-10 20:41:50 UTC (rev 9015)
@@ -43,6 +43,8 @@
 
 from twisted.application.service import Service
 
+from txdav.base.datastore.util import QueryCacher
+
 from twext.internet.decorate import memoizedKey
 
 from txdav.common.datastore.sql_legacy import PostgresLegacyNotificationsEmulator
@@ -136,7 +138,9 @@
                  enableCalendars=True, enableAddressBooks=True,
                  label="unlabeled", quota=(2 ** 20),
                  logLabels=False, logStats=False, logSQL=False,
-                 logTransactionWaits=0, timeoutTransactions=0):
+                 logTransactionWaits=0, timeoutTransactions=0,
+                 cacheQueries=True, cachePool="Default",
+                 cacheExpireSeconds=3600):
         assert enableCalendars or enableAddressBooks
 
         self.sqlTxnFactory = sqlTxnFactory
@@ -154,7 +158,13 @@
         self._migrating = False
         self._enableNotifications = True
 
+        if cacheQueries:
+            self.queryCacher = QueryCacher(cachePool=cachePool,
+                cacheExpireSeconds=cacheExpireSeconds)
+        else:
+            self.queryCacher = None
 
+
     def eachCalendarHome(self):
         """
         @see L{ICalendarStore.eachCalendarHome}
@@ -2018,12 +2028,24 @@
         @return: an L{CommonHomeChild} or C{None} if no such child
             exists.
         """
-        if owned:
-            query = cls._resourceIDOwnedByHomeByName
-        else:
-            query = cls._resourceIDSharedToHomeByName
-        data = yield query.on(home._txn,
-                              objectName=name, homeID=home._resourceID)
+        data = None
+        queryCacher = home._txn.store().queryCacher
+        # Only caching non-shared objects so that we don't need to invalidate
+        # in sql_legacy
+        if owned and queryCacher:
+            data = yield queryCacher.getObjectWithName(home._resourceID, name)
+
+        if data is None:
+            if owned:
+                query = cls._resourceIDOwnedByHomeByName
+            else:
+                query = cls._resourceIDSharedToHomeByName
+            data = yield query.on(home._txn,
+                                  objectName=name, homeID=home._resourceID)
+            if owned and data and queryCacher:
+                queryCacher.setObjectWithName(home._txn, home._resourceID,
+                    name, data)
+
         if not data:
             returnValue(None)
         resourceID = data[0][0]
@@ -2219,6 +2241,12 @@
         @return: a L{Deferred} which fires when the modification is complete.
         """
         oldName = self._name
+
+        queryCacher = self._home._txn.store().queryCacher
+        if queryCacher:
+            queryCacher.invalidateObjectWithName(self._home._txn,
+                self._home._resourceID, oldName)
+
         yield self._renameQuery.on(self._txn, name=name,
                                    resourceID=self._resourceID,
                                    homeID=self._home._resourceID)
@@ -2246,6 +2274,12 @@
 
     @inlineCallbacks
     def remove(self):
+
+        queryCacher = self._home._txn.store().queryCacher
+        if queryCacher:
+            queryCacher.invalidateObjectWithName(self._home._txn,
+                self._home._resourceID, self._name)
+
         yield self._deletedSyncToken()
         yield self._deleteQuery.on(self._txn, NoSuchHomeChildError,
                                    resourceID=self._resourceID)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120410/bdb34eac/attachment-0001.html>


More information about the calendarserver-changes mailing list