[CalendarServer-changes] [14229] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Wed Dec 17 12:15:04 PST 2014


Revision: 14229
          http://trac.calendarserver.org//changeset/14229
Author:   sagen at apple.com
Date:     2014-12-17 12:15:04 -0800 (Wed, 17 Dec 2014)
Log Message:
-----------
Wrap each leaf directory service with its own cache so that when a leaf service needs to do further record lookups, those are also cached

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/calendarserver/tools/dashboard.py
    CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
    CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
    CalendarServer/trunk/requirements-stable.txt
    CalendarServer/trunk/txdav/common/datastore/test/util.py
    CalendarServer/trunk/txdav/who/augment.py
    CalendarServer/trunk/txdav/who/cache.py
    CalendarServer/trunk/txdav/who/test/test_util.py
    CalendarServer/trunk/txdav/who/util.py

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -234,11 +234,11 @@
 def storeFromConfigWithoutDPS(config, txnFactory):
     store = storeFromConfig(config, txnFactory, None)
     directory = directoryFromConfig(config, store)
-    if config.DirectoryProxy.InProcessCachingSeconds:
-        directory = CachingDirectoryService(
-            directory,
-            expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
-        )
+    # if config.DirectoryProxy.InProcessCachingSeconds:
+    #     directory = CachingDirectoryService(
+    #         directory,
+    #         expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
+    #     )
     store.setDirectoryService(directory)
     return store
 
@@ -262,11 +262,11 @@
 def storeFromConfigWithDPSServer(config, txnFactory):
     store = storeFromConfig(config, txnFactory, None)
     directory = directoryFromConfig(config, store)
-    if config.DirectoryProxy.InSidecarCachingSeconds:
-        directory = CachingDirectoryService(
-            directory,
-            expireSeconds=config.DirectoryProxy.InSidecarCachingSeconds
-        )
+    # if config.DirectoryProxy.InSidecarCachingSeconds:
+    #     directory = CachingDirectoryService(
+    #         directory,
+    #         expireSeconds=config.DirectoryProxy.InSidecarCachingSeconds
+    #     )
     store.setDirectoryService(directory)
     return store
 

Modified: CalendarServer/trunk/calendarserver/tools/dashboard.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/dashboard.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tools/dashboard.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -1055,7 +1055,7 @@
             if methodName.endswith("-hit"):
                 overallCountRatio += count
                 overallCountCached += count
-            if "-" not in methodName:
+            if methodName.endswith("-miss") or methodName.endswith("-expired"):
                 overallCountRatio += count
                 overallCountUncached += count
             overallTimeSpent += timeSpent

Modified: CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist	2014-12-17 20:15:04 UTC (rev 14229)
@@ -763,6 +763,14 @@
     <key>SharedConnectionPool</key>
     <true/>
 
+    <key>DirectoryProxy</key>
+    <dict>
+      <key>InProcessCachingSeconds</key>
+      <integer>0</integer>
+      <key>InSidecarCachingSeconds</key>
+      <integer>0</integer>
+    </dict>
+
     <!--
         Twisted
       -->

Modified: CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_gateway.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tools/test/test_gateway.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -182,8 +182,8 @@
 
     def _flush(self):
         # Flush both XML directories
-        self.directory._directory.services[0].flush()
         self.directory._directory.services[1].flush()
+        self.directory._directory.services[2].flush()
 
 
     @inlineCallbacks

Modified: CalendarServer/trunk/requirements-stable.txt
===================================================================
--- CalendarServer/trunk/requirements-stable.txt	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/requirements-stable.txt	2014-12-17 20:15:04 UTC (rev 14229)
@@ -5,7 +5,7 @@
 # For CalendarServer development, don't try to get these projects from PyPI; use svn.
 
 -e .
--e svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14226#egg=twextpy
+-e svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14228#egg=twextpy
 -e svn+http://svn.calendarserver.org/repository/calendarserver/PyKerberos/trunk@13420#egg=kerberos
 -e svn+http://svn.calendarserver.org/repository/calendarserver/PyCalendar/trunk@14224#egg=pycalendar
 

Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -623,7 +623,7 @@
 
 def buildTestDirectory(
     store, dataRoot, accounts=None, resources=None, augments=None, proxies=None,
-    serversDB=None
+    serversDB=None, cacheSeconds=0
 ):
     """
     @param store: the store for the directory to use
@@ -711,7 +711,7 @@
     )
     directory = buildDirectory(
         store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo,
-        serversDB
+        serversDB, cacheSeconds
     )
 
     store.setDirectoryService(directory)
@@ -733,7 +733,7 @@
     @inlineCallbacks
     def buildStoreAndDirectory(
         self, accounts=None, resources=None, augments=None, proxies=None,
-        extraUids=None, serversDB=None
+        extraUids=None, serversDB=None, cacheSeconds=0
     ):
 
         self.serverRoot = self.mktemp()
@@ -752,7 +752,7 @@
             self.store, config.DataRoot,
             accounts=accounts, resources=resources,
             augments=augments, proxies=proxies,
-            serversDB=serversDB
+            serversDB=serversDB, cacheSeconds=cacheSeconds
         )
         if extraUids:
             for uid in extraUids:

Modified: CalendarServer/trunk/txdav/who/augment.py
===================================================================
--- CalendarServer/trunk/txdav/who/augment.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/augment.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -110,12 +110,9 @@
         self._store = store
         self._augmentDB = augmentDB
 
-        # Look for an LDAP DS with extra info to expose to the dashboard
+        # An LDAP DS has extra info to expose via the dashboard
+        # This is assigned in buildDirectory()
         self._ldapDS = None
-        for ds in self._directory._services:
-            if hasattr(ds, "poolStats"):
-                self._ldapDS = ds
-                break
 
 
     @classmethod
@@ -457,10 +454,12 @@
         # print("Augmented fields", fields)
 
         # Clone to a new record with the augmented fields
-        returnValue(AugmentedDirectoryRecord(self, record, fields))
+        augmentedRecord = AugmentedDirectoryRecord(self, record, fields)
 
+        returnValue(augmentedRecord)
 
 
+
 class AugmentedDirectoryRecord(DirectoryRecord, CalendarDirectoryRecordMixin):
     """
     Augmented directory record.

Modified: CalendarServer/trunk/txdav/who/cache.py
===================================================================
--- CalendarServer/trunk/txdav/who/cache.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/cache.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -82,21 +82,40 @@
     def __init__(self, directory, expireSeconds=30):
         BaseDirectoryService.__init__(self, directory.realmName)
         self._directory = directory
-        self.serversDB = directory.serversDB
-        self._directoryTiming = hasattr(self._directory, "_addTiming")
+
+        # Patch the wrapped directory service's recordWithXXX to instead
+        # use this cache
+
+        directory._wrapped_recordWithUID = directory.recordWithUID
+        directory.recordWithUID = self.recordWithUID
+
+        directory._wrapped_recordWithGUID = directory.recordWithGUID
+        directory.recordWithGUID = self.recordWithGUID
+
+        directory._wrapped_recordWithShortName = directory.recordWithShortName
+        directory.recordWithShortName = self.recordWithShortName
+
+        directory._wrapped_recordsWithEmailAddress = directory.recordsWithEmailAddress
+        directory.recordsWithEmailAddress = self.recordsWithEmailAddress
+
         self._expireSeconds = expireSeconds
         self.resetCache()
 
-#        from txdav.who.augment import AugmentedDirectoryService
-#        if isinstance(self._directory, AugmentedDirectoryService):
-#            for ds in self._directory._directory.services:
-#                if hasattr(ds, "setMasterDirectory"):
-#                    ds.setMasterDirectory(self)
 
 
+    def setTimingMethod(self, f):
+        """
+        Replace the default no-op timing method
+        """
+        self._addTiming = f
+
+
     def _addTiming(self, key, duration):
-        if self._directoryTiming:
-            self._directory._addTiming(key, duration)
+        """
+        Timing won't get recorded by default -- you must call setTimingMethod
+        with a callable that takes a key such as a method name, and a duration.
+        """
+        pass
 
 
     def resetCache(self):
@@ -160,6 +179,40 @@
                 pass
 
 
+    def purgeRecord(self, record):
+        """
+        Remove a record from all indices in the cache
+
+        @param record: the directory record
+        """
+
+        if record.uid in self._cache[IndexType.uid]:
+            del self._cache[IndexType.uid][record.uid]
+
+        try:
+            if record.guid in self._cache[IndexType.guid]:
+                del self._cache[IndexType.guid][record.guid]
+        except AttributeError:
+            pass
+
+        try:
+            typeName = record.recordType.name
+            for name in record.shortNames:
+                key = (typeName, name)
+                if key in self._cache[IndexType.shortName]:
+                    del self._cache[IndexType.shortName][key]
+        except AttributeError:
+            pass
+
+        try:
+            for emailAddress in record.emailAddresses:
+                if emailAddress in self._cache[IndexType.emailAddress]:
+                    del self._cache[IndexType.emailAddress][emailAddress]
+        except AttributeError:
+            pass
+
+
+
     def purgeExpiredRecords(self):
         """
         Scans the cache for expired records and deletes them
@@ -175,6 +228,7 @@
                     del self._cache[indexType][key]
 
 
+
     def lookupRecord(self, indexType, key, name):
         """
         Looks for a record in the specified index, under the specified key.
@@ -213,7 +267,7 @@
                     key=key
                 )
                 # This record has expired
-                del self._cache[indexType][key]
+                self.purgeRecord(record)
                 self._addTiming("{}-expired".format(name), 0)
                 return None
 
@@ -244,7 +298,7 @@
         # First check our cache
         record = self.lookupRecord(IndexType.uid, uid, "recordWithUID")
         if record is None:
-            record = yield self._directory.recordWithUID(
+            record = yield self._directory._wrapped_recordWithUID(
                 uid, timeoutSeconds=timeoutSeconds
             )
             if record is not None:
@@ -263,7 +317,7 @@
         # First check our cache
         record = self.lookupRecord(IndexType.guid, guid, "recordWithGUID")
         if record is None:
-            record = yield self._directory.recordWithGUID(
+            record = yield self._directory._wrapped_recordWithGUID(
                 guid, timeoutSeconds=timeoutSeconds
             )
             if record is not None:
@@ -286,7 +340,7 @@
             "recordWithShortName"
         )
         if record is None:
-            record = yield self._directory.recordWithShortName(
+            record = yield self._directory._wrapped_recordWithShortName(
                 recordType, shortName, timeoutSeconds=timeoutSeconds
             )
             if record is not None:
@@ -311,7 +365,7 @@
             "recordsWithEmailAddress"
         )
         if record is None:
-            records = yield self._directory.recordsWithEmailAddress(
+            records = yield self._directory._wrapped_recordsWithEmailAddress(
                 emailAddress,
                 limitResults=limitResults, timeoutSeconds=timeoutSeconds
             )
@@ -352,12 +406,12 @@
 
 
     def recordsFromExpression(
-        self, expression, recordTypes=None,
+        self, expression, recordTypes=None, records=None,
         limitResults=None, timeoutSeconds=None
     ):
         # Defer to the directory service we're caching
         return self._directory.recordsFromExpression(
-            expression, recordTypes=recordTypes,
+            expression, recordTypes=recordTypes, records=records,
             limitResults=limitResults, timeoutSeconds=timeoutSeconds
         )
 

Modified: CalendarServer/trunk/txdav/who/test/test_util.py
===================================================================
--- CalendarServer/trunk/txdav/who/test/test_util.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/test/test_util.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -30,6 +30,7 @@
 from twistedcaldav.config import ConfigDict
 from twistedcaldav.stdconfig import config as stdconfig
 from txdav.who.augment import AugmentedDirectoryService
+from txdav.who.cache import CachingDirectoryService
 from txdav.who.delegates import (
     DirectoryService as DelegateDirectoryService,
     RecordType as DelegateRecordType
@@ -44,6 +45,7 @@
 
 hasattr(stdconfig, "Servers")   # Quell pyflakes
 
+
 class StubStore(object):
     pass
 
@@ -108,6 +110,11 @@
                 "Servers": {
                     "Enabled": False,
                 },
+                "DirectoryProxy": {
+                    "SocketPath": "directory-proxy.sock",
+                    "InProcessCachingSeconds": 60,
+                    "InSidecarCachingSeconds": 120,
+                },
             }
         )
 
@@ -125,17 +132,34 @@
         self.assertTrue(isinstance(service._directory, AggregateDirectoryService))
         self.assertEquals(len(service._directory.services), 4)
         self.assertTrue(
-            isinstance(service._directory.services[0], XMLDirectoryService)
+            isinstance(service._directory.services[0], DelegateDirectoryService)
         )
         self.assertEquals(
             set(service._directory.services[0].recordTypes()),
+            set(
+                [
+                    DelegateRecordType.readDelegateGroup,
+                    DelegateRecordType.writeDelegateGroup,
+                    DelegateRecordType.readDelegatorGroup,
+                    DelegateRecordType.writeDelegatorGroup,
+                ]
+            )
+        )
+        self.assertTrue(
+            isinstance(service._directory.services[1], CachingDirectoryService)
+        )
+        self.assertTrue(
+            isinstance(service._directory.services[1]._directory, XMLDirectoryService)
+        )
+        self.assertEquals(
+            set(service._directory.services[1]._directory.recordTypes()),
             set([RecordType.user, RecordType.group])
         )
         self.assertTrue(
-            isinstance(service._directory.services[1], XMLDirectoryService)
+            isinstance(service._directory.services[2]._directory, XMLDirectoryService)
         )
         self.assertEquals(
-            set(service._directory.services[1].recordTypes()),
+            set(service._directory.services[2]._directory.recordTypes()),
             set(
                 [
                     CalRecordType.location,
@@ -145,20 +169,6 @@
             )
         )
         self.assertTrue(
-            isinstance(service._directory.services[2], DelegateDirectoryService)
-        )
-        self.assertEquals(
-            set(service._directory.services[2].recordTypes()),
-            set(
-                [
-                    DelegateRecordType.readDelegateGroup,
-                    DelegateRecordType.writeDelegateGroup,
-                    DelegateRecordType.readDelegatorGroup,
-                    DelegateRecordType.writeDelegatorGroup,
-                ]
-            )
-        )
-        self.assertTrue(
             isinstance(service._directory.services[3], WikiDirectoryService)
         )
         self.assertEquals(

Modified: CalendarServer/trunk/txdav/who/util.py
===================================================================
--- CalendarServer/trunk/txdav/who/util.py	2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/util.py	2014-12-17 20:15:04 UTC (rev 14229)
@@ -32,6 +32,7 @@
 from twisted.python.reflect import namedClass
 from twistedcaldav.config import fullServerPath
 from txdav.who.augment import AugmentedDirectoryService
+from txdav.who.cache import CachingDirectoryService
 from txdav.who.delegates import DirectoryService as DelegateDirectoryService
 from txdav.who.idirectory import (
     RecordType as CalRecordType,
@@ -72,14 +73,15 @@
         [config.DirectoryService, config.ResourceService],
         config.AugmentService,
         config.Authentication.Wiki,
-        serversDB=serversDB
+        serversDB=serversDB,
+        cachingSeconds=config.DirectoryProxy.InSidecarCachingSeconds
     )
 
 
 
 def buildDirectory(
     store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo,
-    serversDB=None
+    serversDB=None, cachingSeconds=0
 ):
     """
     Return a directory without using a config object; suitable for tests
@@ -97,6 +99,8 @@
     """
 
     aggregatedServices = []
+    cachingServices = []
+    ldapService = None  # LDAP DS has extra stats (see augment.py)
 
     for serviceValue in servicesInfo:
 
@@ -106,6 +110,7 @@
         directoryType = serviceValue.type.lower()
         params = serviceValue.params
 
+
         if "xml" in directoryType:
             xmlFile = params.xmlFile
             xmlFile = fullServerPath(dataRoot, xmlFile)
@@ -179,6 +184,7 @@
                     CalRecordType.address: extraFilters.get("addresses", ""),
                 }
             )
+            ldapService = directory
 
         elif "inmemory" in directoryType:
             from txdav.who.test.support import CalendarInMemoryDirectoryService
@@ -215,6 +221,14 @@
             (directory.fieldName, CalFieldName)
         )
         fieldNames.append(directory.fieldName)
+
+        if cachingSeconds:
+            directory = CachingDirectoryService(
+                directory,
+                expireSeconds=cachingSeconds
+            )
+            cachingServices.append(directory)
+
         aggregatedServices.append(directory)
 
     #
@@ -254,7 +268,9 @@
         userDirectory.realmName,
         store
     )
-    aggregatedServices.append(delegateDirectory)
+    # (put at front of list so we don't try to ask the actual DS services
+    # about the delegate-related principals, for performance)
+    aggregatedServices.insert(0, delegateDirectory)
 
     # Wiki service
     if wikiServiceInfo.Enabled:
@@ -284,6 +300,14 @@
         # FIXME: is there a better pattern to use here?
         delegateDirectory.setMasterDirectory(augmented)
 
+        # Tell each caching service what method to use when reporting
+        # times and cache stats
+        for cachingService in cachingServices:
+            cachingService.setTimingMethod(augmented._addTiming)
+
+        # LDAP has additional stats to report
+        augmented._ldapDS = ldapService
+
     except Exception as e:
         log.error("Could not create directory service", error=e)
         raise
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20141217/cf5a5cd9/attachment-0001.html>


More information about the calendarserver-changes mailing list