[CalendarServer-changes] [4056] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Apr 21 20:40:46 PDT 2009
Revision: 4056
http://trac.macosforge.org/projects/calendarserver/changeset/4056
Author: cdaboo at apple.com
Date: 2009-04-21 20:40:46 -0700 (Tue, 21 Apr 2009)
Log Message:
-----------
Add a socket based protocol for getting stats from the server.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/conf/caldavd-apple.plist
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/conf/caldavd.plist
CalendarServer/trunk/setup.py
CalendarServer/trunk/support/Makefile.Apple
CalendarServer/trunk/twistedcaldav/accesslog.py
CalendarServer/trunk/twistedcaldav/admin/options.py
CalendarServer/trunk/twistedcaldav/admin/script.py
CalendarServer/trunk/twistedcaldav/admin/stats.py
CalendarServer/trunk/twistedcaldav/admin/util.py
CalendarServer/trunk/twistedcaldav/config.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -39,6 +39,7 @@
from twisted.plugin import IPlugin
from twisted.internet.reactor import callLater
from twisted.internet.process import ProcessExitedAlready
+from twisted.internet.protocol import Protocol, Factory
from twisted.internet.address import IPv4Address
from twisted.application.internet import TCPServer, SSLServer, UNIXServer
from twisted.application.service import Service, MultiService, IServiceMaker
@@ -93,6 +94,20 @@
log = Logger()
+class CalDAVStatisticsProtocol (Protocol):
+
+ def connectionMade(self):
+ stats = self.factory.logger.observer.getGlobalHits()
+ self.transport.write("%s\r\n" % (stats,))
+ self.transport.loseConnection()
+
+class CalDAVStatisticsServer (Factory):
+
+ protocol = CalDAVStatisticsProtocol
+
+ def __init__(self, logObserver):
+ self.logger = logObserver
+
class CalDAVService (MultiService):
def __init__(self, logObserver):
self.logObserver = logObserver
@@ -874,7 +889,7 @@
if sslPort[0] == 0:
sslPort = None
- # If the load balancer isn"t enabled, or if we only have one process
+ # If the load balancer isn't enabled, or if we only have one process
# We listen directly on the interfaces.
if (
@@ -1046,6 +1061,9 @@
]
monitor.addProcess("mailgateway", mailGatewayArgv, env=parentEnv)
+ stats = CalDAVStatisticsServer(logger)
+ statsService = UNIXServer(config.GlobalStatsSocket, stats)
+ statsService.setServiceParent(s)
return s
Modified: CalendarServer/trunk/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-apple.plist 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/conf/caldavd-apple.plist 2009-04-22 03:40:46 UTC (rev 4056)
@@ -104,7 +104,12 @@
<key>MaximumAttachmentSize</key>
<integer>1048576</integer><!-- 1Mb -->
+ <!-- Maximum number of unique attendees per entire event -->
+ <!-- 0 for no limit -->
+ <key>MaxAttendeesPerInstance</key>
+ <integer>100</integer>
+
<!--
Directory service
@@ -260,6 +265,27 @@
<key>DefaultLogLevel</key>
<string>warn</string> <!-- debug, info, warn, error -->
+ <!-- Global server stats -->
+ <key>GlobalStatsSocket</key>
+ <string>/var/run/caldavd-stats.sock</string>
+
+ <!--
+ To log every five minutes and keep stats for the last hour:
+
+ GlobalStatsLoggingPeriod : 60
+ GlobalStatsLoggingFrequency : 12
+
+ Set GlobalStatsLoggingFrequency to 0 to disable the stats
+ -->
+
+ <!-- Global server stats logging period -->
+ <key>GlobalStatsLoggingPeriod</key>
+ <integer>60</integer>
+
+ <!-- Global server stats logging frequency -->
+ <key>GlobalStatsLoggingFrequency</key>
+ <integer>12</integer>
+
<!-- Server statistics file -->
<key>ServerStatsFile</key>
<string>/var/run/caldavd/stats.plist</string>
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2009-04-22 03:40:46 UTC (rev 4056)
@@ -336,6 +336,27 @@
-->
</dict>
+ <!-- Global server stats -->
+ <key>GlobalStatsSocket</key>
+ <string>logs/caldavd-stats.sock</string>
+
+ <!--
+ To log every five minutes and keep stats for the last hour:
+
+ GlobalStatsLoggingPeriod : 60
+ GlobalStatsLoggingFrequency : 12
+
+ Set GlobalStatsLoggingFrequency to 0 to disable the stats
+ -->
+
+ <!-- Global server stats logging period -->
+ <key>GlobalStatsLoggingPeriod</key>
+ <integer>1</integer>
+
+ <!-- Global server stats logging frequency -->
+ <key>GlobalStatsLoggingFrequency</key>
+ <integer>1</integer>
+
<!-- Server statistics file -->
<key>ServerStatsFile</key>
<string>logs/stats.plist</string>
Modified: CalendarServer/trunk/conf/caldavd.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd.plist 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/conf/caldavd.plist 2009-04-22 03:40:46 UTC (rev 4056)
@@ -257,6 +257,27 @@
<key>DefaultLogLevel</key>
<string>warn</string> <!-- debug, info, warn, error -->
+ <!-- Global server stats -->
+ <key>GlobalStatsSocket</key>
+ <string>/var/run/caldavd-stats.sock</string>
+
+ <!--
+ To log every five minutes and keep stats for the last hour:
+
+ GlobalStatsLoggingPeriod : 60
+ GlobalStatsLoggingFrequency : 12
+
+ Set GlobalStatsLoggingFrequency to 0 to disable the stats
+ -->
+
+ <!-- Global server stats logging period -->
+ <key>GlobalStatsLoggingPeriod</key>
+ <integer>60</integer>
+
+ <!-- Global server stats logging frequency -->
+ <key>GlobalStatsLoggingFrequency</key>
+ <integer>12</integer>
+
<!-- Server statistics file -->
<key>ServerStatsFile</key>
<string>/var/run/caldavd/stats.plist</string>
Modified: CalendarServer/trunk/setup.py
===================================================================
--- CalendarServer/trunk/setup.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/setup.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,7 +1,7 @@
#!/usr/bin/env python
##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -105,7 +105,7 @@
"images/*/*.jpg",
],
},
- scripts = [ "bin/caldavd", "bin/caldav_export" ],
+ scripts = [ "bin/caldavd", "bin/caladmin", "bin/caldav_export" ],
data_files = [ ("caldavd", ["conf/caldavd.plist"]) ],
ext_modules = extensions,
py_modules = ["kqreactor", "memcacheclient"],
Modified: CalendarServer/trunk/support/Makefile.Apple
===================================================================
--- CalendarServer/trunk/support/Makefile.Apple 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/support/Makefile.Apple 2009-04-22 03:40:46 UTC (rev 4056)
@@ -4,7 +4,7 @@
#
# This is only useful internally at Apple, probably.
##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -103,6 +103,7 @@
$(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)$(MANDIR)/man8"
$(_v) $(INSTALL_FILE) "$(Sources)/doc/caldavd.8" "$(DSTROOT)$(MANDIR)/man8"
$(_v) $(INSTALL_FILE) "$(Sources)/doc/caldav_export.8" "$(DSTROOT)$(MANDIR)/man8"
+ $(_v) $(INSTALL_FILE) "$(Sources)/doc/caladmin.8" "$(DSTROOT)$(MANDIR)/man8"
$(_v) gzip -9 -f "$(DSTROOT)$(MANDIR)/man8/"*.[0-9]
$(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)$(NSLOCALDIR)/$(NSLIBRARYSUBDIR)/$(Project)"
$(_v) $(INSTALL_DIRECTORY) -m 0755 "$(DSTROOT)$(VARDIR)/log/caldavd"
Modified: CalendarServer/trunk/twistedcaldav/accesslog.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/accesslog.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/accesslog.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -26,10 +26,12 @@
]
import datetime
+import logging
import os
import time
from twisted.internet import protocol
+from twisted.protocols import amp
from twisted.web2 import iweb
from twisted.web2.dav import davxml
@@ -61,6 +63,9 @@
if eventDict.get('interface') is iweb.IRequest:
+ if config.GlobalStatsLoggingFrequency is not 0:
+ self.logGlobalHit()
+
request = eventDict['request']
response = eventDict['response']
loginfo = eventDict['loginfo']
@@ -143,6 +148,10 @@
def __init__(self, logpath):
self.logpath = logpath
+ self.globalHitCount = 0
+ self.globalHitHistory = []
+ for i in range(0, config.GlobalStatsLoggingFrequency + 1):
+ self.globalHitHistory.append({'time':int(time.time()), 'hits':0})
def logMessage(self, message, allowrotate=True):
"""
@@ -158,6 +167,21 @@
self.rotate()
self.f.write(message + '\n')
+ def rotateGlobalHitHistoryStats(self):
+ """
+ Roll the global hit history array: push the current stats as
+ the last element; pop the first (oldest) element and reschedule the task.
+ """
+
+ self.globalHitHistory.append({'time':int(time.time()), 'hits':self.globalHitCount})
+ del self.globalHitHistory[0]
+ log.msg("rotateGlobalHitHistoryStats: %s" % (self.globalHitHistory,), logLevel=logging.DEBUG)
+ if config.GlobalStatsLoggingFrequency is not 0:
+ self.reactor.callLater(
+ config.GlobalStatsLoggingPeriod * 60 / config.GlobalStatsLoggingFrequency,
+ self.rotateGlobalHitHistoryStats
+ )
+
def start(self):
"""
Start logging. Open the log file and log an 'open' message.
@@ -166,6 +190,11 @@
super(RotatingFileAccessLoggingObserver, self).start()
self._open()
self.logMessage("Log opened - server start: [%s]." % (datetime.datetime.now().ctime(),))
+
+ # Need a reactor for the callLater() support for rotateGlobalHitHistoryStats()
+ from twisted.internet import reactor
+ self.reactor = reactor
+ self.rotateGlobalHitHistoryStats()
def stop(self):
"""
@@ -250,13 +279,39 @@
self._open()
self.logMessage("Log opened - rotated: [%s]." % (datetime.datetime.now().ctime(),), False)
+ def logGlobalHit(self):
+ """
+ Increment the service-global hit counter
+ """
-from twisted.protocols import amp
+ self.globalHitCount += 1
+ def getGlobalHits(self):
+ """
+ Return the global hit stats
+ """
+ stats = '<?xml version="1.0" encoding="UTF-8"?><plist version="1.0">'
+ stats += "<dict><key>totalHits</key><integer>%d</integer>"
+ stats += "<key>recentHits</key><dict>"
+ stats += "<key>count</key><integer>%d</integer>"
+ stats += "<key>since</key><integer>%d</integer>"
+ stats += "<key>period</key><integer>%d</integer>"
+ stats += "<key>frequency</key><integer>%d</integer>"
+ stats += "</dict></dict></plist>"
+ return stats % (
+ self.globalHitCount,
+ self.globalHitCount - self.globalHitHistory[0]['hits'],
+ self.globalHitHistory[0]['time'],
+ config.GlobalStatsLoggingPeriod,
+ config.GlobalStatsLoggingFrequency
+ )
+
class LogMessage(amp.Command):
arguments = [('message', amp.String())]
+class LogGlobalHit(amp.Command):
+ arguments = []
class AMPCommonAccessLoggingObserver(CommonAccessLoggingObserverExtensions):
def __init__(self, mode, id):
@@ -302,7 +357,17 @@
else:
self._buffer.append(message)
+ def logGlobalHit(self):
+ """
+ Log a server hit via the remote AMP Protocol
+ """
+ if self.protocol is not None:
+ d = self.protocol.callRemote(LogGlobalHit)
+ d.addErrback(log.err)
+ else:
+ log.msg("logGlobalHit() only works with an AMP Protocol")
+
class AMPLoggingProtocol(amp.AMP):
"""
A server side protocol for logging to the given observer.
@@ -319,7 +384,12 @@
LogMessage.responder(logMessage)
+ def logGlobalHit(self):
+ self.observer.logGlobalHit()
+ return {}
+ LogGlobalHit.responder(logGlobalHit)
+
class AMPLoggingFactory(protocol.ServerFactory):
def __init__(self, observer):
self.observer = observer
Modified: CalendarServer/trunk/twistedcaldav/admin/options.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/options.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/options.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -87,7 +87,17 @@
registerCommand(StatsOptions)
+class StatsWatchOptions(SubCommand):
+ name = 'statswatch'
+ help = ('Watch the server hit stats.')
+ action = 'twistedcaldav.admin.stats.StatsWatchAction'
+ optParameters = [
+ ['refresh', 'r', None, 'Refresh stats every n seconds.'],
+ ]
+
+registerCommand(StatsWatchOptions)
+
from twisted.python import filepath
from twistedcaldav.config import config
Modified: CalendarServer/trunk/twistedcaldav/admin/script.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/script.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/script.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -83,6 +83,7 @@
if os.path.exists(self['config']):
config.loadConfig(self['config'])
+ self.masterConfig = config
self.root = filepath.FilePath(config.DocumentRoot)
self.calendarCollection = self.root.child('calendars')
self.principalCollection = self.root.child('principals')
Modified: CalendarServer/trunk/twistedcaldav/admin/stats.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/stats.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/stats.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -25,6 +25,9 @@
"""
import os
+import socket
+import plistlib
+import time
from twistedcaldav.admin import util
@@ -91,3 +94,68 @@
report['data'][stat] = value
return report
+
+class StatsWatchAction(object):
+ """
+ Pulls the current server stats from the main server process via a socket
+ For example:
+
+ bin/caladmin --config conf/caldavd-dev.plist statswatch
+ bin/caladmin --config conf/caldavd-dev.plist statswatch --refresh
+ """
+
+ def __init__(self, config):
+ self.config = config
+ self.refresh = None
+
+ def getHitStats(self):
+ response = ""
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ sock.connect(self.config.parent.masterConfig.GlobalStatsSocket)
+ while 1:
+ data = sock.recv(8192)
+ if not data: break
+ response += data
+ sock.close()
+ return response
+
+ def run(self):
+ if self.config['refresh'] is not None:
+ self.refresh = int(self.config['refresh'])
+
+ response = self.getHitStats()
+ plist = plistlib.readPlistFromString(response)
+ total = plist['totalHits']
+ since = time.time() - plist['recentHits']['since']
+ if self.refresh is None:
+ print "Total hits:\t%8d" % (total,)
+ if plist['recentHits']['frequency'] is not 0:
+ print "Last %dm%ds:\t%8d" % (since / 60, since % 60, plist['recentHits']['count'])
+ print "Rate:\t\t%8.2f" % (plist['recentHits']['count'] * 1.0 / (since if since != 0 else 1))
+ units = "second"
+ interval = plist['recentHits']['period'] * 60 / plist['recentHits']['frequency']
+ if interval % 60 is 0:
+ interval = interval / 60
+ units = "minute"
+ print "Update interval: %d %s%s" % (interval, units, ("", "s")[interval > 1])
+ else:
+ print "Recent stats are not updated."
+ else:
+ # attempt to gauge the recent hits for the first line of output
+ total_prev = (total - (plist['recentHits']['count'] / (since if since != 0 else 1) * self.refresh))
+ while 1:
+ print "Total hits: %10d\tLast %7s: %8d\tRate: %8.2f" % (
+ total,
+ "%dm%02ds" % (since / 60, since % 60),
+ plist['recentHits']['count'],
+ (total - total_prev) * 1.0 / (self.refresh if self.refresh != 0 else 1)
+ )
+ total_prev = total
+ time.sleep(self.refresh)
+ response = self.getHitStats()
+ plist = plistlib.readPlistFromString(response)
+ total = plist['totalHits']
+ since = time.time() - plist['recentHits']['since']
+
+ return None
+
Modified: CalendarServer/trunk/twistedcaldav/admin/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/util.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/util.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@
from twisted.web import microdom
-from twistedcaldav.directory.principal import RecordTypeProperty
from twistedcaldav.sql import db_prefix, AbstractSQLDatabase
from twistedcaldav.index import schema_version, collection_types
Modified: CalendarServer/trunk/twistedcaldav/config.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/config.py 2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/config.py 2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -186,12 +186,17 @@
"MoreAccessLogData" : True,
"DefaultLogLevel" : "",
"LogLevels" : {},
+
"AccountingCategories": {
"iTIP": False,
},
"AccountingPrincipals": [],
- "AccountingLogRoot": "/var/log/caldavd/accounting",
+ "AccountingLogRoot" : "/var/log/caldavd/accounting",
+ "GlobalStatsSocket" : "/var/run/caldavd-stats.sock",
+ "GlobalStatsLoggingPeriod" : 60,
+ "GlobalStatsLoggingFrequency" : 12,
+
#
# SSL/TLS
#
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090421/21bb43ad/attachment-0001.html>
More information about the calendarserver-changes
mailing list