<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[13822] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/13822">13822</a></dd>
<dt>Author</dt> <dd>sagen@apple.com</dd>
<dt>Date</dt> <dd>2014-08-01 17:49:24 -0700 (Fri, 01 Aug 2014)</dd>
</dl>
<h3>Log Message</h3>
<pre>Adds directory caching, both in-process and in the directory proxy sidecar. The number of seconds records are cached for in each place are configurable.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkcalendarservertapcaldavpy">CalendarServer/trunk/calendarserver/tap/caldav.py</a></li>
<li><a href="#CalendarServertrunkconfcaldavdtestplist">CalendarServer/trunk/conf/caldavd-test.plist</a></li>
<li><a href="#CalendarServertrunktwistedcaldavdirectorytesttest_principalpy">CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py</a></li>
<li><a href="#CalendarServertrunktwistedcaldavstdconfigpy">CalendarServer/trunk/twistedcaldav/stdconfig.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoretestaccountsaccountsxml">CalendarServer/trunk/txdav/common/datastore/test/accounts/accounts.xml</a></li>
<li><a href="#CalendarServertrunktxdavdpsserverpy">CalendarServer/trunk/txdav/dps/server.py</a></li>
<li><a href="#CalendarServertrunktxdavdpstesttest_clientpy">CalendarServer/trunk/txdav/dps/test/test_client.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServertrunktxdavwhocachepy">CalendarServer/trunk/txdav/who/cache.py</a></li>
<li><a href="#CalendarServertrunktxdavwhotesttest_cachepy">CalendarServer/trunk/txdav/who/test/test_cache.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkcalendarservertapcaldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tap/caldav.py (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tap/caldav.py 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -94,9 +94,10 @@
</span><span class="cx"> )
</span><span class="cx"> from txdav.common.datastore.work.inbox_cleanup import InboxCleanupWork
</span><span class="cx"> from txdav.common.datastore.work.revision_cleanup import FindMinValidRevisionWork
</span><del>-from txdav.who.util import directoryFromConfig
</del><span class="cx"> from txdav.dps.client import DirectoryService as DirectoryProxyClientService
</span><ins>+from txdav.who.cache import CachingDirectoryService
</ins><span class="cx"> from txdav.who.groups import GroupCacher
</span><ins>+from txdav.who.util import directoryFromConfig
</ins><span class="cx">
</span><span class="cx"> from twistedcaldav import memcachepool
</span><span class="cx"> from twistedcaldav.config import ConfigurationError
</span><span class="lines">@@ -526,25 +527,24 @@
</span><span class="cx">
</span><span class="cx"> # Add the directory proxy sidecar first so it at least get spawned
</span><span class="cx"> # prior to the caldavd worker processes:
</span><del>- if config.DirectoryProxy.Enabled:
- log.info("Adding directory proxy service")
</del><ins>+ log.info("Adding directory proxy service")
</ins><span class="cx">
</span><del>- dpsArgv = [
- sys.executable,
- sys.argv[0],
- ]
- if config.UserName:
- dpsArgv.extend(("-u", config.UserName))
- if config.GroupName:
- dpsArgv.extend(("-g", config.GroupName))
- dpsArgv.extend((
- "--reactor={}".format(config.Twisted.reactor),
- "-n", "caldav_directoryproxy",
- "-f", self.configPath,
- ))
- self.monitor.addProcess(
- "directoryproxy", dpsArgv, env=PARENT_ENVIRONMENT
- )
</del><ins>+ dpsArgv = [
+ sys.executable,
+ sys.argv[0],
+ ]
+ if config.UserName:
+ dpsArgv.extend(("-u", config.UserName))
+ if config.GroupName:
+ dpsArgv.extend(("-g", config.GroupName))
+ dpsArgv.extend((
+ "--reactor={}".format(config.Twisted.reactor),
+ "-n", "caldav_directoryproxy",
+ "-f", self.configPath,
+ ))
+ self.monitor.addProcess(
+ "directoryproxy", dpsArgv, env=PARENT_ENVIRONMENT
+ )
</ins><span class="cx">
</span><span class="cx"> for slaveNumber in xrange(0, config.MultiProcess.ProcessCount):
</span><span class="cx"> if config.UseMetaFD:
</span><span class="lines">@@ -854,6 +854,11 @@
</span><span class="cx"> directory = DirectoryProxyClientService(config.DirectoryRealmName)
</span><span class="cx"> if config.Servers.Enabled:
</span><span class="cx"> directory.setServersDB(buildServersDB(config.Servers.MaxClients))
</span><ins>+ if config.DirectoryProxy.InProcessCachingSeconds:
+ directory = CachingDirectoryService(
+ directory,
+ expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
+ )
</ins><span class="cx"> store = storeFromConfig(config, txnFactory, directory)
</span><span class="cx"> logObserver = AMPCommonAccessLoggingObserver()
</span><span class="cx"> result = self.requestProcessingService(options, store, logObserver)
</span><span class="lines">@@ -1491,18 +1496,19 @@
</span><span class="cx"> if directory is None:
</span><span class="cx"> # Create a Directory Proxy "Server" service and hand it to
</span><span class="cx"> # the store.
</span><del>- # FIXME: right now the store passed *to* the directory is the
- # calendar/contacts data store, but for a multi-server deployment
- # it will need its own separate store.
</del><span class="cx"> if config.Servers.Enabled:
</span><span class="cx"> serversDB = buildServersDB(config.Servers.MaxClients)
</span><span class="cx"> else:
</span><span class="cx"> serversDB = None
</span><del>- store.setDirectoryService(
- directoryFromConfig(
- config, store=store, serversDB=serversDB
</del><ins>+ directorySvc = directoryFromConfig(
+ config, store=store, serversDB=serversDB
+ )
+ if config.DirectoryProxy.InProcessCachingSeconds:
+ directorySvc = CachingDirectoryService(
+ directorySvc,
+ expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
</ins><span class="cx"> )
</span><del>- )
</del><ins>+ store.setDirectoryService(directorySvc)
</ins><span class="cx">
</span><span class="cx"> pps = PreProcessingService(
</span><span class="cx"> createMainService, cp, store, logObserver, storageService
</span></span></pre></div>
<a id="CalendarServertrunkconfcaldavdtestplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/conf/caldavd-test.plist (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/conf/caldavd-test.plist 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -175,6 +175,14 @@
</span><span class="cx"> <key>MaxAllowedInstances</key>
</span><span class="cx"> <integer>3000</integer>
</span><span class="cx">
</span><ins>+ <key>DirectoryProxy</key>
+ <dict>
+ <key>InProcessCachingSeconds</key>
+ <integer>10</integer>
+ <key>InSidecarCachingSeconds</key>
+ <integer>30</integer>
+ </dict>
+
</ins><span class="cx"> <!--
</span><span class="cx"> Directory service
</span><span class="cx">
</span><span class="lines">@@ -788,7 +796,7 @@
</span><span class="cx"> <dict>
</span><span class="cx"> <key>Enabled</key>
</span><span class="cx"> <true/>
</span><del>-
</del><ins>+
</ins><span class="cx"> <!-- Make these short for testing -->
</span><span class="cx"> <key>RequestDelaySeconds</key>
</span><span class="cx"> <real>0.1</real>
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavdirectorytesttest_principalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_principal.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -297,6 +297,12 @@
</span><span class="cx"> test_items += (principalURL, alternateURL)
</span><span class="cx">
</span><span class="cx"> for address in test_items:
</span><ins>+
+ # For txdav.who.test.test_cache I added two accounts which
+ # share the same email address, but that messes up this test.
+ if "mailto:cache-user" in address:
+ continue
+
</ins><span class="cx"> principal = (
</span><span class="cx"> yield provisioningResource
</span><span class="cx"> .principalForCalendarUserAddress(address)
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavstdconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -898,8 +898,9 @@
</span><span class="cx"> },
</span><span class="cx">
</span><span class="cx"> "DirectoryProxy": {
</span><del>- "Enabled": True,
</del><span class="cx"> "SocketPath": "directory-proxy.sock",
</span><ins>+ "InProcessCachingSeconds": 10,
+ "InSidecarCachingSeconds": 30,
</ins><span class="cx"> },
</span><span class="cx">
</span><span class="cx"> #
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoretestaccountsaccountsxml"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/test/accounts/accounts.xml (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/test/accounts/accounts.xml 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/txdav/common/datastore/test/accounts/accounts.xml 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -393,6 +393,35 @@
</span><span class="cx"> <full-name>Empty Group</full-name>
</span><span class="cx"> </record>
</span><span class="cx">
</span><ins>+ <!-- Caching Directory test records -->
+
+ <record type="user">
+ <uid>cache-uid-1</uid>
+ <guid>8166C681-2D08-4846-90F7-97023A6EDDC5</guid>
+ <short-name>cache-name-1</short-name>
+ <short-name>cache-alt-name-1</short-name>
+ <password>cache-password-1</password>
+ <full-name>Cache User 1</full-name>
+ <email>cache-user-1@example.com</email>
+ </record>
+
+ <record type="user">
+ <uid>cache-uid-duplicate-1</uid>
+ <short-name>cache-name-duplicate-1</short-name>
+ <short-name>cache-alt-name-duplicate-1</short-name>
+ <password>cache-password-duplicate-1</password>
+ <full-name>Cache User Duplicate 1</full-name>
+ <email>cache-user-1@example.com</email>
+ </record>
+
+ <record type="user">
+ <uid>cache-uid-2</uid>
+ <short-name>cache-name-2</short-name>
+ <password>cache-password-2</password>
+ <full-name>Cache User 2</full-name>
+ <email>cache-user-2@example.com</email>
+ </record>
+
</ins><span class="cx"> <!-- Calverify test records -->
</span><span class="cx">
</span><span class="cx"> <record type="user">
</span></span></pre></div>
<a id="CalendarServertrunktxdavdpsserverpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/dps/server.py (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/dps/server.py 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/txdav/dps/server.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -18,6 +18,7 @@
</span><span class="cx"> import datetime
</span><span class="cx"> import uuid
</span><span class="cx">
</span><ins>+from calendarserver.tap.util import getDBPool, storeFromConfig
</ins><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twext.who.expression import MatchType, MatchFlags, Operand
</span><span class="cx"> from twisted.application import service
</span><span class="lines">@@ -39,6 +40,7 @@
</span><span class="cx"> WikiAccessForUIDCommand, ContinuationCommand
</span><span class="cx"> # UpdateRecordsCommand, RemoveRecordsCommand
</span><span class="cx"> )
</span><ins>+from txdav.who.cache import CachingDirectoryService
</ins><span class="cx"> from txdav.who.util import directoryFromConfig
</span><span class="cx"> from txdav.who.wiki import WikiAccessLevel
</span><span class="cx"> from zope.interface import implementer
</span><span class="lines">@@ -571,11 +573,20 @@
</span><span class="cx"> setproctitle("CalendarServer Directory Proxy Service")
</span><span class="cx">
</span><span class="cx"> try:
</span><ins>+ pool, txnFactory = getDBPool(config)
+ store = storeFromConfig(config, txnFactory, None)
</ins><span class="cx"> directory = directoryFromConfig(config)
</span><ins>+ if config.DirectoryProxy.InSidecarCachingSeconds:
+ directory = CachingDirectoryService(
+ directory,
+ expireSeconds=config.DirectoryProxy.InSidecarCachingSeconds
+ )
+ store.setDirectoryService(directory)
</ins><span class="cx"> except Exception as e:
</span><span class="cx"> log.error("Failed to create directory service", error=e)
</span><span class="cx"> raise
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> log.info("Created directory service")
</span><span class="cx">
</span><span class="cx"> return strPortsService(
</span></span></pre></div>
<a id="CalendarServertrunktxdavdpstesttest_clientpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/dps/test/test_client.py (13821 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/dps/test/test_client.py 2014-08-01 21:06:17 UTC (rev 13821)
+++ CalendarServer/trunk/txdav/dps/test/test_client.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -414,7 +414,7 @@
</span><span class="cx"> records = (yield self.client.recordsWithRecordType(
</span><span class="cx"> RecordType.user
</span><span class="cx"> ))
</span><del>- self.assertEquals(len(records), 240)
</del><ins>+ self.assertEquals(len(records), 243)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span></span></pre></div>
<a id="CalendarServertrunktxdavwhocachepy"></a>
<div class="addfile"><h4>Added: CalendarServer/trunk/txdav/who/cache.py (0 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/who/cache.py (rev 0)
+++ CalendarServer/trunk/txdav/who/cache.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -0,0 +1,334 @@
</span><ins>+# -*- test-case-name: txdav.who.test.test_cache -*-
+##
+# Copyright (c) 2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Caching Directory Service
+"""
+
+__all__ = [
+ "CachingDirectoryService",
+]
+
+import time
+
+from zope.interface import implementer
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twext.python.log import Logger
+from twext.who.directory import DirectoryService as BaseDirectoryService
+from twext.who.idirectory import (
+ IDirectoryService, FieldName as BaseFieldName
+)
+from twext.who.util import ConstantsContainer
+
+from txdav.common.idirectoryservice import IStoreDirectoryService
+from txdav.who.directory import (
+ CalendarDirectoryServiceMixin,
+)
+from txdav.who.idirectory import (
+ FieldName
+)
+from twisted.python.constants import Values, ValueConstant
+
+log = Logger()
+
+
+class IndexType(Values):
+ """
+ Constants to use for identifying indexes
+ """
+ uid = ValueConstant("uid")
+ guid = ValueConstant("guid")
+ shortName = ValueConstant("shortName")
+ emailAddress = ValueConstant("emailAddress")
+
+
+@implementer(IDirectoryService, IStoreDirectoryService)
+class CachingDirectoryService(
+ BaseDirectoryService, CalendarDirectoryServiceMixin
+):
+ """
+ Caching directory service.
+
+ This is a directory service that wraps an L{IDirectoryService} and caches
+ directory records.
+ """
+
+ fieldName = ConstantsContainer((
+ BaseFieldName,
+ FieldName,
+ ))
+
+
+
+ def __init__(self, directory, expireSeconds=30):
+ BaseDirectoryService.__init__(self, directory.realmName)
+ self._directory = directory
+ self._expireSeconds = expireSeconds
+ self.resetCache()
+
+
+ def resetCache(self):
+ """
+ Clear the cache
+ """
+ self._cache = {
+ IndexType.uid: {},
+ IndexType.guid: {},
+ IndexType.shortName: {}, # key is (recordType.name, shortName)
+ IndexType.emailAddress: {},
+ }
+ self._hitCount = 0
+ self._requestCount = 0
+
+
+ def setTestTime(self, timestamp):
+ self._test_time = timestamp
+
+
+ def cacheRecord(self, record, indexTypes):
+ """
+ Store a record in the cache, within the specified indexes
+
+ @param record: the directory record
+ @param indexTypes: an iterable of L{IndexType}
+ """
+
+ if hasattr(self, "_test_time"):
+ timestamp = self._test_time
+ else:
+ timestamp = time.time()
+
+ if IndexType.uid in indexTypes:
+ self._cache[IndexType.uid][record.uid] = (timestamp, record)
+
+ if IndexType.guid in indexTypes:
+ try:
+ self._cache[IndexType.guid][record.guid] = (timestamp, record)
+ except AttributeError:
+ pass
+ if IndexType.shortName in indexTypes:
+ try:
+ typeName = record.recordType.name
+ for name in record.shortNames:
+ self._cache[IndexType.shortName][(typeName, name)] = (timestamp, record)
+ except AttributeError:
+ pass
+ if IndexType.emailAddress in indexTypes:
+ try:
+ for emailAddress in record.emailAddresses:
+ self._cache[IndexType.emailAddress][emailAddress] = (timestamp, record)
+ except AttributeError:
+ pass
+
+
+ def lookupRecord(self, indexType, key):
+ """
+ Looks for a record in the specified index, under the specified key
+
+ @param index: an index type
+ @type indexType: L{IndexType}
+
+ @param key: the key to look up in the specified index
+ @type key: any valid type that can be used as a dictionary key
+
+ @return: the cached directory record, or None
+ @rtype: L{DirectoryRecord}
+ """
+
+ self._requestCount += 1
+ if key in self._cache[indexType]:
+
+ if hasattr(self, "_test_time"):
+ now = self._test_time
+ else:
+ now = time.time()
+
+ cachedTime, record = self._cache[indexType].get(key, (0.0, None))
+ if now - self._expireSeconds > cachedTime:
+ log.debug(
+ "Directory cache miss (expired): {index} {key}",
+ index=indexType.value,
+ key=key
+ )
+ # This record has expired
+ del self._cache[indexType][key]
+ return None
+
+ log.debug(
+ "Directory cache hit: {index} {key}",
+ index=indexType.value,
+ key=key
+ )
+ self._hitCount += 1
+ return record
+ else:
+ log.debug(
+ "Directory cache miss: {index} {key}",
+ index=indexType.value,
+ key=key
+ )
+ return None
+
+
+ # Cached methods:
+
+ @inlineCallbacks
+ def recordWithUID(self, uid):
+
+ # First check our cache
+ record = self.lookupRecord(IndexType.uid, uid)
+ if record is None:
+ record = yield self._directory.recordWithUID(uid)
+ if record is not None:
+ # Note we do not index on email address; see below.
+ self.cacheRecord(
+ record,
+ (IndexType.uid, IndexType.guid, IndexType.shortName)
+ )
+
+ returnValue(record)
+
+
+ @inlineCallbacks
+ def recordWithGUID(self, guid):
+
+ # First check our cache
+ record = self.lookupRecord(IndexType.guid, guid)
+ if record is None:
+ record = yield self._directory.recordWithGUID(guid)
+ if record is not None:
+ # Note we do not index on email address; see below.
+ self.cacheRecord(
+ record,
+ (IndexType.uid, IndexType.guid, IndexType.shortName)
+ )
+
+ returnValue(record)
+
+
+ @inlineCallbacks
+ def recordWithShortName(self, recordType, shortName):
+
+ # First check our cache
+ record = self.lookupRecord(
+ IndexType.shortName,
+ (recordType.name, shortName)
+ )
+ if record is None:
+ record = yield self._directory.recordWithShortName(
+ recordType, shortName
+ )
+ if record is not None:
+ # Note we do not index on email address; see below.
+ self.cacheRecord(
+ record,
+ (IndexType.uid, IndexType.guid, IndexType.shortName)
+ )
+
+ returnValue(record)
+
+
+ @inlineCallbacks
+ def recordsWithEmailAddress(self, emailAddress):
+
+ # First check our cache
+ record = self.lookupRecord(IndexType.emailAddress, emailAddress)
+ if record is None:
+ records = yield self._directory.recordsWithEmailAddress(emailAddress)
+ if len(records) == 1:
+ # Only cache if there was a single match (which is the most
+ # common scenario). Caching multiple records for the exact
+ # same key/value complicates the data structures.
+ # Also, this is the only situation where we do index a cached
+ # record on email address. Otherwise, say we had faulted in
+ # on "uid" and then indexed that record on its email address,
+ # the next lookup by email address would only get that record,
+ # but there might be others in the directory service with that
+ # same email address.
+ self.cacheRecord(
+ records[0],
+ (
+ IndexType.uid, IndexType.guid,
+ IndexType.shortName, IndexType.emailAddress
+ )
+ )
+ else:
+ records = [record]
+
+ returnValue(records)
+
+
+ # Uncached methods:
+
+ @property
+ def recordType(self):
+ # Defer to the directory service we're caching
+ return self._directory.recordType
+
+
+ def recordTypes(self):
+ # Defer to the directory service we're caching
+ return self._directory.recordTypes()
+
+
+ def recordsFromExpression(self, expression, recordTypes=None):
+ # Defer to the directory service we're caching
+ return self._directory.recordsFromExpression(
+ expression, recordTypes=recordTypes
+ )
+
+
+ def recordsWithFieldValue(self, fieldName, value):
+ # Defer to the directory service we're caching
+ return self._directory.recordsWithFieldValue(
+ fieldName, value
+ )
+
+
+ def updateRecords(self, records, create=False):
+ # Defer to the directory service we're caching
+ return self._directory.updateRecords(records, create=create)
+
+
+ def removeRecords(self, uids):
+ # Defer to the directory service we're caching
+ return self._directory.removeRecords(uids)
+
+
+ def recordsWithRecordType(self, recordType):
+ # Defer to the directory service we're caching
+ return self._directory.recordsWithRecordType(recordType)
+
+
+ def recordsMatchingTokens(self, *args, **kwds):
+ return CalendarDirectoryServiceMixin.recordsMatchingTokens(
+ self, *args, **kwds
+ )
+
+
+ def recordsMatchingFields(self, *args, **kwds):
+ return CalendarDirectoryServiceMixin.recordsMatchingFields(
+ self, *args, **kwds
+ )
+
+
+ def recordWithCalendarUserAddress(self, *args, **kwds):
+ # This will get cached by the underlying recordWith... call
+ return CalendarDirectoryServiceMixin.recordWithCalendarUserAddress(
+ self, *args, **kwds
+ )
</ins></span></pre></div>
<a id="CalendarServertrunktxdavwhotesttest_cachepy"></a>
<div class="addfile"><h4>Added: CalendarServer/trunk/txdav/who/test/test_cache.py (0 => 13822)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/who/test/test_cache.py (rev 0)
+++ CalendarServer/trunk/txdav/who/test/test_cache.py 2014-08-02 00:49:24 UTC (rev 13822)
</span><span class="lines">@@ -0,0 +1,200 @@
</span><ins>+##
+# Copyright (c) 2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Caching service tests
+"""
+
+from twisted.internet.defer import inlineCallbacks
+from twistedcaldav.test.util import StoreTestCase
+from txdav.who.cache import CachingDirectoryService
+from twext.who.idirectory import (
+ RecordType
+)
+from txdav.who.idirectory import (
+ RecordType as CalRecordType
+)
+import uuid
+
+
+class CacheTest(StoreTestCase):
+
+ @inlineCallbacks
+ def setUp(self):
+ yield super(CacheTest, self).setUp()
+
+ self.cachingDirectory = CachingDirectoryService(
+ self.directory,
+ expireSeconds=10
+ )
+ self.storeUnderTest().setDirectoryService(self.cachingDirectory)
+
+
+
+ @inlineCallbacks
+ def test_cachingPassThrough(self):
+ """
+ Verify the CachingDirectoryService can pass through method calls to
+ the underlying service.
+ """
+
+ dir = self.cachingDirectory
+
+ self.assertTrue(RecordType.user in dir.recordTypes())
+ self.assertTrue(RecordType.group in dir.recordTypes())
+ self.assertTrue(CalRecordType.location in dir.recordTypes())
+ self.assertTrue(CalRecordType.resource in dir.recordTypes())
+
+ records = yield dir.recordsWithRecordType(RecordType.user)
+ self.assertEquals(len(records), 243)
+
+ record = yield dir.recordWithGUID(uuid.UUID("8166C681-2D08-4846-90F7-97023A6EDDC5"))
+ self.assertEquals(record.uid, u"cache-uid-1")
+
+ record = yield dir.recordWithShortName(RecordType.user, u"cache-name-2")
+ self.assertEquals(record.uid, u"cache-uid-2")
+
+ records = yield dir.recordsWithEmailAddress(u"cache-user-1@example.com")
+ self.assertEquals(len(records), 2)
+
+ record = yield dir.recordWithCalendarUserAddress(u"mailto:cache-user-2@example.com")
+ self.assertEquals(record.uid, u"cache-uid-2")
+
+
+
+ @inlineCallbacks
+ def test_cachingHitsAndMisses(self):
+ """
+ Verify faulted in records are indexed appropriately and can be retrieved
+ from the cache even by other attributes.
+ """
+
+ dir = self.cachingDirectory
+
+ # Caching on UID
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 0)
+ record = yield dir.recordWithUID(u"cache-uid-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 1)
+
+ # Repeat the same lookup
+ record = yield dir.recordWithUID(u"cache-uid-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 1)
+ self.assertEquals(dir._requestCount, 2)
+
+ # Lookup the same record, but by GUID
+ record = yield dir.recordWithGUID(uuid.UUID("8166C681-2D08-4846-90F7-97023A6EDDC5"))
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 2)
+ self.assertEquals(dir._requestCount, 3)
+
+ # Lookup by the shortName for that same record, and it should be a hit
+ record = yield dir.recordWithShortName(RecordType.user, u"cache-name-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 3)
+ self.assertEquals(dir._requestCount, 4)
+
+ # Now lookup by a different shortName for that same record, and it
+ # should also be a hit
+ record = yield dir.recordWithShortName(RecordType.user, u"cache-alt-name-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 4)
+ self.assertEquals(dir._requestCount, 5)
+
+
+ dir.resetCache()
+
+ # Look up another record which has a unique email address, first by uid
+ # and then by email address and verify this is a cache miss because we
+ # intentionally don't index on email address when faulting in by another
+ # attribute
+ record = yield dir.recordWithUID(u"cache-uid-2")
+ self.assertEquals(record.uid, u"cache-uid-2")
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 1)
+
+ records = yield dir.recordsWithEmailAddress(u"cache-user-2@example.com")
+ self.assertEquals(len(records), 1)
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 2)
+
+ records = yield dir.recordsWithEmailAddress(u"cache-user-2@example.com")
+ self.assertEquals(len(records), 1)
+ self.assertEquals(dir._hitCount, 1)
+ self.assertEquals(dir._requestCount, 3)
+
+ dir.resetCache()
+
+ # Look up a record which has the same email address as another record.
+ record = yield dir.recordWithUID(u"cache-uid-2")
+ self.assertEquals(record.uid, u"cache-uid-2")
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 1)
+
+ # Now lookup by the email address for that record, and it should
+ # be a miss; Note, because there are two records with this email
+ # address, when we repeat this call it will still be a miss because
+ # for simplicity we're only going to cache records when there is a
+ # single result.
+ records = yield dir.recordsWithEmailAddress(u"cache-user-1@example.com")
+ self.assertEquals(len(records), 2)
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 2)
+
+ records = yield dir.recordsWithEmailAddress(u"cache-user-1@example.com")
+ self.assertEquals(len(records), 2)
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 3)
+
+
+ @inlineCallbacks
+ def test_cachingExpiration(self):
+ """
+ Verify records expire at the expected time; in these tests, 10 seconds
+ """
+
+ dir = self.cachingDirectory
+
+ dir.setTestTime(1.0)
+
+ record = yield dir.recordWithUID(u"cache-uid-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 0)
+ self.assertEquals(dir._requestCount, 1)
+
+ # 1 second later, the record is still cached
+ dir.setTestTime(2.0)
+ record = yield dir.recordWithUID(u"cache-uid-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 1)
+ self.assertEquals(dir._requestCount, 2)
+
+ # 10 seconds later, the record is no longer cached
+ dir.setTestTime(12.0)
+ record = yield dir.recordWithUID(u"cache-uid-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 1)
+ self.assertEquals(dir._requestCount, 3)
+
+ # Wait another 11 seconds, verify it's not cached by other attributes
+ dir.setTestTime(23.0)
+ record = yield dir.recordWithShortName(RecordType.user, u"cache-alt-name-1")
+ self.assertEquals(record.uid, u"cache-uid-1")
+ self.assertEquals(dir._hitCount, 1)
+ self.assertEquals(dir._requestCount, 4)
</ins></span></pre>
</div>
</div>
</body>
</html>