[CalendarServer-changes] [13855] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Thu Aug 7 14:30:06 PDT 2014
Revision: 13855
http://trac.calendarserver.org//changeset/13855
Author: cdaboo at apple.com
Date: 2014-08-07 14:30:05 -0700 (Thu, 07 Aug 2014)
Log Message:
-----------
Make sure the dashboard can get directory stats from the DPS server.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/dashboard_service.py
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tools/dashboard.py
CalendarServer/trunk/txdav/dps/client.py
CalendarServer/trunk/txdav/dps/commands.py
CalendarServer/trunk/txdav/dps/server.py
CalendarServer/trunk/txdav/who/augment.py
CalendarServer/trunk/txdav/who/cache.py
Modified: CalendarServer/trunk/calendarserver/dashboard_service.py
===================================================================
--- CalendarServer/trunk/calendarserver/dashboard_service.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/calendarserver/dashboard_service.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -14,12 +14,19 @@
# limitations under the License.
##
+
from twext.enterprise.jobqueue import JobItem
from twisted.internet.defer import inlineCallbacks, succeed, returnValue
+from twisted.internet.error import ConnectError
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
+from twistedcaldav.config import config
+
+from txdav.dps.client import DirectoryService as DirectoryProxyClientService
+from txdav.who.cache import CachingDirectoryService
+
import json
"""
@@ -162,8 +169,8 @@
@rtype: L{str}
"""
try:
- return succeed(self.factory.store.directoryService().stats())
- except AttributeError:
+ return self.factory.directory.stats()
+ except (AttributeError, ConnectError):
return succeed({})
@@ -176,3 +183,13 @@
self.logger = logObserver
self.limiter = limiter
self.store = None
+ self.directory = None
+
+
+ def makeDirectoryProxyClient(self):
+ self.directory = DirectoryProxyClientService(config.DirectoryRealmName)
+ if config.DirectoryProxy.InProcessCachingSeconds:
+ self.directory = CachingDirectoryService(
+ self.directory,
+ expireSeconds=config.DirectoryProxy.InProcessCachingSeconds
+ )
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -511,13 +511,14 @@
"""
def __init__(
- self, maker, monitor, dispenser, dispatcher, configPath,
+ self, maker, monitor, dispenser, dispatcher, stats, configPath,
inheritFDs=None, inheritSSLFDs=None
):
self.maker = maker
self.monitor = monitor
self.dispenser = dispenser
self.dispatcher = dispatcher
+ self.stats = stats
self.configPath = configPath
self.inheritFDs = inheritFDs
self.inheritSSLFDs = inheritSSLFDs
@@ -562,6 +563,9 @@
)
self.monitor.addProcessObject(process, PARENT_ENVIRONMENT)
+ # Hook up the stats service directory to the DPS server after a short delay
+ if self.stats is not None:
+ self.monitor._reactor.callLater(5, self.stats.makeDirectoryProxyClient)
@@ -1795,7 +1799,8 @@
)
statsService.setName("unix-stats")
statsService.setServiceParent(s)
- if config.Stats.EnableTCPStatsSocket:
+
+ elif config.Stats.EnableTCPStatsSocket:
stats = DashboardServer(logger.observer, cl if config.UseMetaFD else None)
statsService = TCPServer(
config.Stats.TCPStatsPort, stats, interface=""
@@ -1867,7 +1872,7 @@
multi = MultiService()
pool.setServiceParent(multi)
spawner = SlaveSpawnerService(
- self, monitor, dispenser, dispatcher, options["config"],
+ self, monitor, dispenser, dispatcher, stats, options["config"],
inheritFDs=inheritFDs, inheritSSLFDs=inheritSSLFDs
)
spawner.setServiceParent(multi)
Modified: CalendarServer/trunk/calendarserver/tools/dashboard.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/dashboard.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/calendarserver/tools/dashboard.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -946,7 +946,7 @@
nlines = len(defaultIfNone(self.readItem("directory"), {}))
self.rowCount = nlines
self._createWindow(
- "Directory Service", self.rowCount + 6, ncols=self.FORMAT_WIDTH,
+ "Directory Service", self.rowCount + 8, ncols=self.FORMAT_WIDTH,
begin_y=top, begin_x=left
)
return self
@@ -982,10 +982,16 @@
y += 2
overallCount = 0
+ overallCountCached = 0
+ overallCountUncached = 0
overallTimeSpent = 0.0
for methodName, (count, timeSpent) in sorted(records.items(), key=lambda x: x[0]):
overallCount += count
+ if methodName.endswith("-hit"):
+ overallCountCached += count
+ if "-" not in methodName:
+ overallCountUncached += count
overallTimeSpent += timeSpent
s = " {:<40}{:>15d}{:>15.1f}{:>15.3f} ".format(
@@ -1003,19 +1009,35 @@
pass
y += 1
- s = " {:<40}{:>15d}{:>15.1f}{:>15.5f} ".format(
+ s = " {:<40}{:>15d}{:>15.1f}{:>15.3f} ".format(
"Total:",
overallCount,
overallTimeSpent,
safeDivision(overallTimeSpent, overallCount, 1000.0)
)
+ s_cached = " {:<40}{:>15d}{:>14.1f}%{:>15s} ".format(
+ "Total Cached:",
+ overallCountCached,
+ safeDivision(overallCountCached, overallCount, 100.0),
+ "",
+ )
+ s_uncached = " {:<40}{:>15d}{:>14.1f}%{:>15s} ".format(
+ "Total Uncached:",
+ overallCountUncached,
+ safeDivision(overallCountUncached, overallCount, 100.0),
+ "",
+ )
if self.usesCurses:
self.window.hline(y, x, "-", self.FORMAT_WIDTH - 2)
y += 1
self.window.addstr(y, x, s)
+ self.window.addstr(y + 1, x, s_cached)
+ self.window.addstr(y + 2, x, s_uncached)
else:
print(s)
- y += 1
+ print(s1)
+ print(s2)
+ y += 3
if self.usesCurses:
self.window.refresh()
Modified: CalendarServer/trunk/txdav/dps/client.py
===================================================================
--- CalendarServer/trunk/txdav/dps/client.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/txdav/dps/client.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -27,6 +27,7 @@
from twext.who.util import ConstantsContainer
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twisted.internet.error import ConnectError
from twisted.internet.protocol import ClientCreator
from twisted.protocols import amp
from twisted.python.constants import Names, NamedConstant
@@ -40,7 +41,7 @@
RecordsMatchingTokensCommand, RecordsMatchingFieldsCommand,
MembersCommand, GroupsCommand, SetMembersCommand,
VerifyPlaintextPasswordCommand, VerifyHTTPDigestCommand,
- WikiAccessForUIDCommand, ContinuationCommand
+ WikiAccessForUIDCommand, ContinuationCommand, StatsCommand
)
from txdav.who.delegates import RecordType as DelegatesRecordType
from txdav.who.directory import (
@@ -345,7 +346,16 @@
)
+ @inlineCallbacks
+ def stats(self):
+ try:
+ result = yield self._sendCommand(StatsCommand)
+ returnValue(pickle.loads(result['stats']))
+ except ConnectError:
+ returnValue({})
+
+
@implementer(ICalendarStoreDirectoryRecord)
class DirectoryRecord(BaseDirectoryRecord, CalendarDirectoryRecordMixin):
Modified: CalendarServer/trunk/txdav/dps/commands.py
===================================================================
--- CalendarServer/trunk/txdav/dps/commands.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/txdav/dps/commands.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -199,3 +199,11 @@
response = [
('access', amp.String()),
]
+
+
+
+class StatsCommand(amp.Command):
+ arguments = []
+ response = [
+ ('stats', amp.String()),
+ ]
Modified: CalendarServer/trunk/txdav/dps/server.py
===================================================================
--- CalendarServer/trunk/txdav/dps/server.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/txdav/dps/server.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -37,7 +37,8 @@
RecordsMatchingTokensCommand, RecordsMatchingFieldsCommand,
MembersCommand, GroupsCommand, SetMembersCommand,
VerifyPlaintextPasswordCommand, VerifyHTTPDigestCommand,
- WikiAccessForUIDCommand, ContinuationCommand
+ WikiAccessForUIDCommand, ContinuationCommand,
+ StatsCommand,
# UpdateRecordsCommand, RemoveRecordsCommand
)
from txdav.who.cache import CachingDirectoryService
@@ -457,7 +458,17 @@
returnValue(response)
+ @StatsCommand.responder
+ @inlineCallbacks
+ def stats(self):
+ stats = yield self._directory.stats()
+ response = {
+ "stats": pickle.dumps(stats),
+ }
+ returnValue(response)
+
+
class DirectoryProxyAMPFactory(Factory):
"""
"""
@@ -583,7 +594,7 @@
setproctitle("CalendarServer Directory Proxy Service")
try:
- pool, txnFactory = getDBPool(config)
+ _ignore_pool, txnFactory = getDBPool(config)
store = storeFromConfig(config, txnFactory, None)
directory = directoryFromConfig(config)
if config.DirectoryProxy.InSidecarCachingSeconds:
Modified: CalendarServer/trunk/txdav/who/augment.py
===================================================================
--- CalendarServer/trunk/txdav/who/augment.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/txdav/who/augment.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -27,7 +27,7 @@
from zope.interface import implementer
-from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
from twistedcaldav.directory.augment import AugmentRecord
from twext.python.log import Logger
from twext.who.directory import DirectoryRecord
@@ -68,14 +68,7 @@
@param startTime: the start time of the call in seconds
@type startTime: C{float}
"""
- timings = AugmentedDirectoryService._timings
- if key not in timings:
- timings[key] = (0, 0.0)
- count, timeSpent = timings[key]
- count += 1
- duration = (time.time() - startTime)
- timeSpent += duration
- timings[key] = (count, timeSpent)
+ AugmentedDirectoryService._addTiming(key, time.time() - startTime)
return result
@@ -118,8 +111,18 @@
self._augmentDB = augmentDB
+ @classmethod
+ def _addTiming(cls, key, duration):
+ if key not in cls._timings:
+ cls._timings[key] = (0, 0.0)
+ count, timeSpent = cls._timings[key]
+ count += 1
+ timeSpent += duration
+ cls._timings[key] = (count, timeSpent)
+
+
def stats(self):
- return self._timings
+ return succeed(self._timings)
@property
Modified: CalendarServer/trunk/txdav/who/cache.py
===================================================================
--- CalendarServer/trunk/txdav/who/cache.py 2014-08-07 21:07:22 UTC (rev 13854)
+++ CalendarServer/trunk/txdav/who/cache.py 2014-08-07 21:30:05 UTC (rev 13855)
@@ -61,6 +61,7 @@
emailAddress = ValueConstant("emailAddress")
+
@implementer(IDirectoryService, IStoreDirectoryService)
class CachingDirectoryService(
BaseDirectoryService, CalendarDirectoryServiceMixin
@@ -81,10 +82,16 @@
def __init__(self, directory, expireSeconds=30):
BaseDirectoryService.__init__(self, directory.realmName)
self._directory = directory
+ self._directoryTiming = hasattr(self._directory, "_addTiming")
self._expireSeconds = expireSeconds
self.resetCache()
+ def _addTiming(self, key, duration):
+ if self._directoryTiming:
+ self._directory._addTiming(key, duration)
+
+
def resetCache(self):
"""
Clear the cache
@@ -156,12 +163,12 @@
now = time.time()
for indexType in self._cache:
- for key, (cachedTime, record) in self._cache[indexType].items():
+ for key, (cachedTime, _ignore_record) in self._cache[indexType].items():
if now - self._expireSeconds > cachedTime:
del self._cache[indexType][key]
- def lookupRecord(self, indexType, key):
+ def lookupRecord(self, indexType, key, name):
"""
Looks for a record in the specified index, under the specified key.
After every SCAN_AFTER_LOOKUP_COUNT lookups are done,
@@ -200,6 +207,7 @@
)
# This record has expired
del self._cache[indexType][key]
+ self._addTiming("{}-expired".format(name), 0)
return None
log.debug(
@@ -208,6 +216,7 @@
key=key
)
self._hitCount += 1
+ self._addTiming("{}-hit".format(name), 0)
return record
else:
log.debug(
@@ -215,6 +224,8 @@
index=indexType.value,
key=key
)
+
+ self._addTiming("{}-miss".format(name), 0)
return None
@@ -224,7 +235,7 @@
def recordWithUID(self, uid):
# First check our cache
- record = self.lookupRecord(IndexType.uid, uid)
+ record = self.lookupRecord(IndexType.uid, uid, "recordWithUID")
if record is None:
record = yield self._directory.recordWithUID(uid)
if record is not None:
@@ -241,7 +252,7 @@
def recordWithGUID(self, guid):
# First check our cache
- record = self.lookupRecord(IndexType.guid, guid)
+ record = self.lookupRecord(IndexType.guid, guid, "recordWithGUID")
if record is None:
record = yield self._directory.recordWithGUID(guid)
if record is not None:
@@ -260,7 +271,8 @@
# First check our cache
record = self.lookupRecord(
IndexType.shortName,
- (recordType.name, shortName)
+ (recordType.name, shortName),
+ "recordWithShortName"
)
if record is None:
record = yield self._directory.recordWithShortName(
@@ -280,7 +292,11 @@
def recordsWithEmailAddress(self, emailAddress):
# First check our cache
- record = self.lookupRecord(IndexType.emailAddress, emailAddress)
+ record = self.lookupRecord(
+ IndexType.emailAddress,
+ emailAddress,
+ "recordsWithEmailAddress"
+ )
if record is None:
records = yield self._directory.recordsWithEmailAddress(emailAddress)
if len(records) == 1:
@@ -367,3 +383,7 @@
return CalendarDirectoryServiceMixin.recordWithCalendarUserAddress(
self, cua
)
+
+
+ def stats(self):
+ return self._directory.stats()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140807/1e0b3b7d/attachment-0001.html>
More information about the calendarserver-changes
mailing list