[CalendarServer-changes] [14229] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Dec 17 12:15:04 PST 2014
Revision: 14229
http://trac.calendarserver.org//changeset/14229
Author: sagen at apple.com
Date: 2014-12-17 12:15:04 -0800 (Wed, 17 Dec 2014)
Log Message:
-----------
Wrap each leaf directory service with its own cache so that when a leaf service needs to do further record lookups, those are also cached
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/calendarserver/tools/dashboard.py
CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
CalendarServer/trunk/requirements-stable.txt
CalendarServer/trunk/txdav/common/datastore/test/util.py
CalendarServer/trunk/txdav/who/augment.py
CalendarServer/trunk/txdav/who/cache.py
CalendarServer/trunk/txdav/who/test/test_util.py
CalendarServer/trunk/txdav/who/util.py
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -234,11 +234,11 @@
def storeFromConfigWithoutDPS(config, txnFactory):
store = storeFromConfig(config, txnFactory, None)
directory = directoryFromConfig(config, store)
- if config.DirectoryProxy.InProcessCachingSeconds:
- directory = CachingDirectoryService(
- directory,
- expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
- )
+ # if config.DirectoryProxy.InProcessCachingSeconds:
+ # directory = CachingDirectoryService(
+ # directory,
+ # expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
+ # )
store.setDirectoryService(directory)
return store
@@ -262,11 +262,11 @@
def storeFromConfigWithDPSServer(config, txnFactory):
store = storeFromConfig(config, txnFactory, None)
directory = directoryFromConfig(config, store)
- if config.DirectoryProxy.InSidecarCachingSeconds:
- directory = CachingDirectoryService(
- directory,
- expireSeconds=config.DirectoryProxy.InSidecarCachingSeconds
- )
+ # if config.DirectoryProxy.InSidecarCachingSeconds:
+ # directory = CachingDirectoryService(
+ # directory,
+ # expireSeconds=config.DirectoryProxy.InSidecarCachingSeconds
+ # )
store.setDirectoryService(directory)
return store
Modified: CalendarServer/trunk/calendarserver/tools/dashboard.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/dashboard.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tools/dashboard.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -1055,7 +1055,7 @@
if methodName.endswith("-hit"):
overallCountRatio += count
overallCountCached += count
- if "-" not in methodName:
+ if methodName.endswith("-miss") or methodName.endswith("-expired"):
overallCountRatio += count
overallCountUncached += count
overallTimeSpent += timeSpent
Modified: CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist 2014-12-17 20:15:04 UTC (rev 14229)
@@ -763,6 +763,14 @@
<key>SharedConnectionPool</key>
<true/>
+ <key>DirectoryProxy</key>
+ <dict>
+ <key>InProcessCachingSeconds</key>
+ <integer>0</integer>
+ <key>InSidecarCachingSeconds</key>
+ <integer>0</integer>
+ </dict>
+
<!--
Twisted
-->
Modified: CalendarServer/trunk/calendarserver/tools/test/test_gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_gateway.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/calendarserver/tools/test/test_gateway.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -182,8 +182,8 @@
def _flush(self):
# Flush both XML directories
- self.directory._directory.services[0].flush()
self.directory._directory.services[1].flush()
+ self.directory._directory.services[2].flush()
@inlineCallbacks
Modified: CalendarServer/trunk/requirements-stable.txt
===================================================================
--- CalendarServer/trunk/requirements-stable.txt 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/requirements-stable.txt 2014-12-17 20:15:04 UTC (rev 14229)
@@ -5,7 +5,7 @@
# For CalendarServer development, don't try to get these projects from PyPI; use svn.
-e .
--e svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14226#egg=twextpy
+-e svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14228#egg=twextpy
-e svn+http://svn.calendarserver.org/repository/calendarserver/PyKerberos/trunk@13420#egg=kerberos
-e svn+http://svn.calendarserver.org/repository/calendarserver/PyCalendar/trunk@14224#egg=pycalendar
Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -623,7 +623,7 @@
def buildTestDirectory(
store, dataRoot, accounts=None, resources=None, augments=None, proxies=None,
- serversDB=None
+ serversDB=None, cacheSeconds=0
):
"""
@param store: the store for the directory to use
@@ -711,7 +711,7 @@
)
directory = buildDirectory(
store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo,
- serversDB
+ serversDB, cacheSeconds
)
store.setDirectoryService(directory)
@@ -733,7 +733,7 @@
@inlineCallbacks
def buildStoreAndDirectory(
self, accounts=None, resources=None, augments=None, proxies=None,
- extraUids=None, serversDB=None
+ extraUids=None, serversDB=None, cacheSeconds=0
):
self.serverRoot = self.mktemp()
@@ -752,7 +752,7 @@
self.store, config.DataRoot,
accounts=accounts, resources=resources,
augments=augments, proxies=proxies,
- serversDB=serversDB
+ serversDB=serversDB, cacheSeconds=cacheSeconds
)
if extraUids:
for uid in extraUids:
Modified: CalendarServer/trunk/txdav/who/augment.py
===================================================================
--- CalendarServer/trunk/txdav/who/augment.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/augment.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -110,12 +110,9 @@
self._store = store
self._augmentDB = augmentDB
- # Look for an LDAP DS with extra info to expose to the dashboard
+ # An LDAP DS has extra info to expose via the dashboard
+ # This is assigned in buildDirectory()
self._ldapDS = None
- for ds in self._directory._services:
- if hasattr(ds, "poolStats"):
- self._ldapDS = ds
- break
@classmethod
@@ -457,10 +454,12 @@
# print("Augmented fields", fields)
# Clone to a new record with the augmented fields
- returnValue(AugmentedDirectoryRecord(self, record, fields))
+ augmentedRecord = AugmentedDirectoryRecord(self, record, fields)
+ returnValue(augmentedRecord)
+
class AugmentedDirectoryRecord(DirectoryRecord, CalendarDirectoryRecordMixin):
"""
Augmented directory record.
Modified: CalendarServer/trunk/txdav/who/cache.py
===================================================================
--- CalendarServer/trunk/txdav/who/cache.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/cache.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -82,21 +82,40 @@
def __init__(self, directory, expireSeconds=30):
BaseDirectoryService.__init__(self, directory.realmName)
self._directory = directory
- self.serversDB = directory.serversDB
- self._directoryTiming = hasattr(self._directory, "_addTiming")
+
+ # Patch the wrapped directory service's recordWithXXX to instead
+ # use this cache
+
+ directory._wrapped_recordWithUID = directory.recordWithUID
+ directory.recordWithUID = self.recordWithUID
+
+ directory._wrapped_recordWithGUID = directory.recordWithGUID
+ directory.recordWithGUID = self.recordWithGUID
+
+ directory._wrapped_recordWithShortName = directory.recordWithShortName
+ directory.recordWithShortName = self.recordWithShortName
+
+ directory._wrapped_recordsWithEmailAddress = directory.recordsWithEmailAddress
+ directory.recordsWithEmailAddress = self.recordsWithEmailAddress
+
self._expireSeconds = expireSeconds
self.resetCache()
-# from txdav.who.augment import AugmentedDirectoryService
-# if isinstance(self._directory, AugmentedDirectoryService):
-# for ds in self._directory._directory.services:
-# if hasattr(ds, "setMasterDirectory"):
-# ds.setMasterDirectory(self)
+ def setTimingMethod(self, f):
+ """
+ Replace the default no-op timing method
+ """
+ self._addTiming = f
+
+
def _addTiming(self, key, duration):
- if self._directoryTiming:
- self._directory._addTiming(key, duration)
+ """
+ Timing won't get recorded by default -- you must call setTimingMethod
+ with a callable that takes a key such as a method name, and a duration.
+ """
+ pass
def resetCache(self):
@@ -160,6 +179,40 @@
pass
+ def purgeRecord(self, record):
+ """
+ Remove a record from all indices in the cache
+
+ @param record: the directory record
+ """
+
+ if record.uid in self._cache[IndexType.uid]:
+ del self._cache[IndexType.uid][record.uid]
+
+ try:
+ if record.guid in self._cache[IndexType.guid]:
+ del self._cache[IndexType.guid][record.guid]
+ except AttributeError:
+ pass
+
+ try:
+ typeName = record.recordType.name
+ for name in record.shortNames:
+ key = (typeName, name)
+ if key in self._cache[IndexType.shortName]:
+ del self._cache[IndexType.shortName][key]
+ except AttributeError:
+ pass
+
+ try:
+ for emailAddress in record.emailAddresses:
+ if emailAddress in self._cache[IndexType.emailAddress]:
+ del self._cache[IndexType.emailAddress][emailAddress]
+ except AttributeError:
+ pass
+
+
+
def purgeExpiredRecords(self):
"""
Scans the cache for expired records and deletes them
@@ -175,6 +228,7 @@
del self._cache[indexType][key]
+
def lookupRecord(self, indexType, key, name):
"""
Looks for a record in the specified index, under the specified key.
@@ -213,7 +267,7 @@
key=key
)
# This record has expired
- del self._cache[indexType][key]
+ self.purgeRecord(record)
self._addTiming("{}-expired".format(name), 0)
return None
@@ -244,7 +298,7 @@
# First check our cache
record = self.lookupRecord(IndexType.uid, uid, "recordWithUID")
if record is None:
- record = yield self._directory.recordWithUID(
+ record = yield self._directory._wrapped_recordWithUID(
uid, timeoutSeconds=timeoutSeconds
)
if record is not None:
@@ -263,7 +317,7 @@
# First check our cache
record = self.lookupRecord(IndexType.guid, guid, "recordWithGUID")
if record is None:
- record = yield self._directory.recordWithGUID(
+ record = yield self._directory._wrapped_recordWithGUID(
guid, timeoutSeconds=timeoutSeconds
)
if record is not None:
@@ -286,7 +340,7 @@
"recordWithShortName"
)
if record is None:
- record = yield self._directory.recordWithShortName(
+ record = yield self._directory._wrapped_recordWithShortName(
recordType, shortName, timeoutSeconds=timeoutSeconds
)
if record is not None:
@@ -311,7 +365,7 @@
"recordsWithEmailAddress"
)
if record is None:
- records = yield self._directory.recordsWithEmailAddress(
+ records = yield self._directory._wrapped_recordsWithEmailAddress(
emailAddress,
limitResults=limitResults, timeoutSeconds=timeoutSeconds
)
@@ -352,12 +406,12 @@
def recordsFromExpression(
- self, expression, recordTypes=None,
+ self, expression, recordTypes=None, records=None,
limitResults=None, timeoutSeconds=None
):
# Defer to the directory service we're caching
return self._directory.recordsFromExpression(
- expression, recordTypes=recordTypes,
+ expression, recordTypes=recordTypes, records=records,
limitResults=limitResults, timeoutSeconds=timeoutSeconds
)
Modified: CalendarServer/trunk/txdav/who/test/test_util.py
===================================================================
--- CalendarServer/trunk/txdav/who/test/test_util.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/test/test_util.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -30,6 +30,7 @@
from twistedcaldav.config import ConfigDict
from twistedcaldav.stdconfig import config as stdconfig
from txdav.who.augment import AugmentedDirectoryService
+from txdav.who.cache import CachingDirectoryService
from txdav.who.delegates import (
DirectoryService as DelegateDirectoryService,
RecordType as DelegateRecordType
@@ -44,6 +45,7 @@
hasattr(stdconfig, "Servers") # Quell pyflakes
+
class StubStore(object):
pass
@@ -108,6 +110,11 @@
"Servers": {
"Enabled": False,
},
+ "DirectoryProxy": {
+ "SocketPath": "directory-proxy.sock",
+ "InProcessCachingSeconds": 60,
+ "InSidecarCachingSeconds": 120,
+ },
}
)
@@ -125,17 +132,34 @@
self.assertTrue(isinstance(service._directory, AggregateDirectoryService))
self.assertEquals(len(service._directory.services), 4)
self.assertTrue(
- isinstance(service._directory.services[0], XMLDirectoryService)
+ isinstance(service._directory.services[0], DelegateDirectoryService)
)
self.assertEquals(
set(service._directory.services[0].recordTypes()),
+ set(
+ [
+ DelegateRecordType.readDelegateGroup,
+ DelegateRecordType.writeDelegateGroup,
+ DelegateRecordType.readDelegatorGroup,
+ DelegateRecordType.writeDelegatorGroup,
+ ]
+ )
+ )
+ self.assertTrue(
+ isinstance(service._directory.services[1], CachingDirectoryService)
+ )
+ self.assertTrue(
+ isinstance(service._directory.services[1]._directory, XMLDirectoryService)
+ )
+ self.assertEquals(
+ set(service._directory.services[1]._directory.recordTypes()),
set([RecordType.user, RecordType.group])
)
self.assertTrue(
- isinstance(service._directory.services[1], XMLDirectoryService)
+ isinstance(service._directory.services[2]._directory, XMLDirectoryService)
)
self.assertEquals(
- set(service._directory.services[1].recordTypes()),
+ set(service._directory.services[2]._directory.recordTypes()),
set(
[
CalRecordType.location,
@@ -145,20 +169,6 @@
)
)
self.assertTrue(
- isinstance(service._directory.services[2], DelegateDirectoryService)
- )
- self.assertEquals(
- set(service._directory.services[2].recordTypes()),
- set(
- [
- DelegateRecordType.readDelegateGroup,
- DelegateRecordType.writeDelegateGroup,
- DelegateRecordType.readDelegatorGroup,
- DelegateRecordType.writeDelegatorGroup,
- ]
- )
- )
- self.assertTrue(
isinstance(service._directory.services[3], WikiDirectoryService)
)
self.assertEquals(
Modified: CalendarServer/trunk/txdav/who/util.py
===================================================================
--- CalendarServer/trunk/txdav/who/util.py 2014-12-17 20:06:03 UTC (rev 14228)
+++ CalendarServer/trunk/txdav/who/util.py 2014-12-17 20:15:04 UTC (rev 14229)
@@ -32,6 +32,7 @@
from twisted.python.reflect import namedClass
from twistedcaldav.config import fullServerPath
from txdav.who.augment import AugmentedDirectoryService
+from txdav.who.cache import CachingDirectoryService
from txdav.who.delegates import DirectoryService as DelegateDirectoryService
from txdav.who.idirectory import (
RecordType as CalRecordType,
@@ -72,14 +73,15 @@
[config.DirectoryService, config.ResourceService],
config.AugmentService,
config.Authentication.Wiki,
- serversDB=serversDB
+ serversDB=serversDB,
+ cachingSeconds=config.DirectoryProxy.InSidecarCachingSeconds
)
def buildDirectory(
store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo,
- serversDB=None
+ serversDB=None, cachingSeconds=0
):
"""
Return a directory without using a config object; suitable for tests
@@ -97,6 +99,8 @@
"""
aggregatedServices = []
+ cachingServices = []
+ ldapService = None # LDAP DS has extra stats (see augment.py)
for serviceValue in servicesInfo:
@@ -106,6 +110,7 @@
directoryType = serviceValue.type.lower()
params = serviceValue.params
+
if "xml" in directoryType:
xmlFile = params.xmlFile
xmlFile = fullServerPath(dataRoot, xmlFile)
@@ -179,6 +184,7 @@
CalRecordType.address: extraFilters.get("addresses", ""),
}
)
+ ldapService = directory
elif "inmemory" in directoryType:
from txdav.who.test.support import CalendarInMemoryDirectoryService
@@ -215,6 +221,14 @@
(directory.fieldName, CalFieldName)
)
fieldNames.append(directory.fieldName)
+
+ if cachingSeconds:
+ directory = CachingDirectoryService(
+ directory,
+ expireSeconds=cachingSeconds
+ )
+ cachingServices.append(directory)
+
aggregatedServices.append(directory)
#
@@ -254,7 +268,9 @@
userDirectory.realmName,
store
)
- aggregatedServices.append(delegateDirectory)
+ # (put at front of list so we don't try to ask the actual DS services
+ # about the delegate-related principals, for performance)
+ aggregatedServices.insert(0, delegateDirectory)
# Wiki service
if wikiServiceInfo.Enabled:
@@ -284,6 +300,14 @@
# FIXME: is there a better pattern to use here?
delegateDirectory.setMasterDirectory(augmented)
+ # Tell each caching service what method to use when reporting
+ # times and cache stats
+ for cachingService in cachingServices:
+ cachingService.setTimingMethod(augmented._addTiming)
+
+ # LDAP has additional stats to report
+ augmented._ldapDS = ldapService
+
except Exception as e:
log.error("Could not create directory service", error=e)
raise
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20141217/cf5a5cd9/attachment-0001.html>
More information about the calendarserver-changes
mailing list