[CalendarServer-changes] [4056] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Tue Apr 21 20:40:46 PDT 2009


Revision: 4056
          http://trac.macosforge.org/projects/calendarserver/changeset/4056
Author:   cdaboo at apple.com
Date:     2009-04-21 20:40:46 -0700 (Tue, 21 Apr 2009)
Log Message:
-----------
Add a socket based protocol for getting stats from the server.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/caldav.py
    CalendarServer/trunk/conf/caldavd-apple.plist
    CalendarServer/trunk/conf/caldavd-test.plist
    CalendarServer/trunk/conf/caldavd.plist
    CalendarServer/trunk/setup.py
    CalendarServer/trunk/support/Makefile.Apple
    CalendarServer/trunk/twistedcaldav/accesslog.py
    CalendarServer/trunk/twistedcaldav/admin/options.py
    CalendarServer/trunk/twistedcaldav/admin/script.py
    CalendarServer/trunk/twistedcaldav/admin/stats.py
    CalendarServer/trunk/twistedcaldav/admin/util.py
    CalendarServer/trunk/twistedcaldav/config.py

Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -39,6 +39,7 @@
 from twisted.plugin import IPlugin
 from twisted.internet.reactor import callLater
 from twisted.internet.process import ProcessExitedAlready
+from twisted.internet.protocol import Protocol, Factory
 from twisted.internet.address import IPv4Address
 from twisted.application.internet import TCPServer, SSLServer, UNIXServer
 from twisted.application.service import Service, MultiService, IServiceMaker
@@ -93,6 +94,20 @@
 log = Logger()
 
 
+class CalDAVStatisticsProtocol (Protocol): 
+
+    def connectionMade(self): 
+        stats = self.factory.logger.observer.getGlobalHits() 
+        self.transport.write("%s\r\n" % (stats,)) 
+        self.transport.loseConnection() 
+
+class CalDAVStatisticsServer (Factory): 
+
+    protocol = CalDAVStatisticsProtocol 
+
+    def __init__(self, logObserver): 
+        self.logger = logObserver 
+
 class CalDAVService (MultiService):
     def __init__(self, logObserver):
         self.logObserver = logObserver
@@ -874,7 +889,7 @@
         if sslPort[0] == 0:
             sslPort = None
 
-        # If the load balancer isn"t enabled, or if we only have one process
+        # If the load balancer isn't enabled, or if we only have one process
         # We listen directly on the interfaces.
 
         if (
@@ -1046,6 +1061,9 @@
             ]
             monitor.addProcess("mailgateway", mailGatewayArgv, env=parentEnv)
 
+        stats = CalDAVStatisticsServer(logger) 
+        statsService = UNIXServer(config.GlobalStatsSocket, stats) 
+        statsService.setServiceParent(s)
 
         return s
 

Modified: CalendarServer/trunk/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-apple.plist	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/conf/caldavd-apple.plist	2009-04-22 03:40:46 UTC (rev 4056)
@@ -104,7 +104,12 @@
     <key>MaximumAttachmentSize</key>
     <integer>1048576</integer><!-- 1Mb -->
 
+    <!-- Maximum number of unique attendees per entire event -->
+    <!-- 0 for no limit -->
+    <key>MaxAttendeesPerInstance</key>
+    <integer>100</integer>
 
+
     <!--
         Directory service
 
@@ -260,6 +265,27 @@
     <key>DefaultLogLevel</key>
     <string>warn</string> <!-- debug, info, warn, error -->
 
+	<!-- Global server stats --> 
+	<key>GlobalStatsSocket</key> 
+	<string>/var/run/caldavd-stats.sock</string> 
+
+	<!-- 
+		To log every five minutes and keep stats for the last hour: 
+
+			GlobalStatsLoggingPeriod : 60 
+			GlobalStatsLoggingFrequency : 12 
+
+		Set GlobalStatsLoggingFrequency to 0 to disable the stats 
+	--> 
+
+	<!-- Global server stats logging period --> 
+	<key>GlobalStatsLoggingPeriod</key> 
+	<integer>60</integer> 
+
+	<!-- Global server stats logging frequency --> 
+ 	<key>GlobalStatsLoggingFrequency</key> 
+	<integer>12</integer> 
+
     <!-- Server statistics file -->
     <key>ServerStatsFile</key>
     <string>/var/run/caldavd/stats.plist</string>

Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/conf/caldavd-test.plist	2009-04-22 03:40:46 UTC (rev 4056)
@@ -336,6 +336,27 @@
       -->
     </dict>
 
+	<!-- Global server stats --> 
+	<key>GlobalStatsSocket</key> 
+	<string>logs/caldavd-stats.sock</string> 
+
+	<!-- 
+		To log every five minutes and keep stats for the last hour: 
+
+			GlobalStatsLoggingPeriod : 60 
+			GlobalStatsLoggingFrequency : 12 
+
+		Set GlobalStatsLoggingFrequency to 0 to disable the stats 
+	--> 
+
+	<!-- Global server stats logging period --> 
+	<key>GlobalStatsLoggingPeriod</key> 
+	<integer>1</integer> 
+
+	<!-- Global server stats logging frequency --> 
+ 	<key>GlobalStatsLoggingFrequency</key> 
+	<integer>1</integer> 
+
     <!-- Server statistics file -->
     <key>ServerStatsFile</key>
     <string>logs/stats.plist</string>

Modified: CalendarServer/trunk/conf/caldavd.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd.plist	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/conf/caldavd.plist	2009-04-22 03:40:46 UTC (rev 4056)
@@ -257,6 +257,27 @@
     <key>DefaultLogLevel</key>
     <string>warn</string> <!-- debug, info, warn, error -->
 
+	<!-- Global server stats --> 
+	<key>GlobalStatsSocket</key> 
+	<string>/var/run/caldavd-stats.sock</string> 
+
+	<!-- 
+		To log every five minutes and keep stats for the last hour: 
+
+			GlobalStatsLoggingPeriod : 60 
+			GlobalStatsLoggingFrequency : 12 
+
+		Set GlobalStatsLoggingFrequency to 0 to disable the stats 
+	--> 
+
+	<!-- Global server stats logging period --> 
+	<key>GlobalStatsLoggingPeriod</key> 
+	<integer>60</integer> 
+
+	<!-- Global server stats logging frequency --> 
+ 	<key>GlobalStatsLoggingFrequency</key> 
+	<integer>12</integer> 
+
     <!-- Server statistics file -->
     <key>ServerStatsFile</key>
     <string>/var/run/caldavd/stats.plist</string>

Modified: CalendarServer/trunk/setup.py
===================================================================
--- CalendarServer/trunk/setup.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/setup.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -105,7 +105,7 @@
                            "images/*/*.jpg",
                          ],
                        },
-    scripts          = [ "bin/caldavd", "bin/caldav_export" ],
+    scripts          = [ "bin/caldavd", "bin/caladmin", "bin/caldav_export" ],
     data_files       = [ ("caldavd", ["conf/caldavd.plist"]) ],
     ext_modules      = extensions,
     py_modules       = ["kqreactor", "memcacheclient"],

Modified: CalendarServer/trunk/support/Makefile.Apple
===================================================================
--- CalendarServer/trunk/support/Makefile.Apple	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/support/Makefile.Apple	2009-04-22 03:40:46 UTC (rev 4056)
@@ -4,7 +4,7 @@
 #
 # This is only useful internally at Apple, probably.
 ##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -103,6 +103,7 @@
 	$(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)$(MANDIR)/man8"
 	$(_v) $(INSTALL_FILE) "$(Sources)/doc/caldavd.8" "$(DSTROOT)$(MANDIR)/man8"
 	$(_v) $(INSTALL_FILE) "$(Sources)/doc/caldav_export.8" "$(DSTROOT)$(MANDIR)/man8"
+	$(_v) $(INSTALL_FILE) "$(Sources)/doc/caladmin.8" "$(DSTROOT)$(MANDIR)/man8"
 	$(_v) gzip -9 -f "$(DSTROOT)$(MANDIR)/man8/"*.[0-9]
 	$(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)$(NSLOCALDIR)/$(NSLIBRARYSUBDIR)/$(Project)"
 	$(_v) $(INSTALL_DIRECTORY) -m 0755 "$(DSTROOT)$(VARDIR)/log/caldavd"

Modified: CalendarServer/trunk/twistedcaldav/accesslog.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/accesslog.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/accesslog.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -26,10 +26,12 @@
 ]
 
 import datetime
+import logging
 import os
 import time
 
 from twisted.internet import protocol
+from twisted.protocols import amp
 
 from twisted.web2 import iweb
 from twisted.web2.dav import davxml
@@ -61,6 +63,9 @@
 
         if eventDict.get('interface') is iweb.IRequest:
             
+            if config.GlobalStatsLoggingFrequency is not 0: 
+                self.logGlobalHit()
+
             request = eventDict['request']
             response = eventDict['response']
             loginfo = eventDict['loginfo']
@@ -143,6 +148,10 @@
 
     def __init__(self, logpath):
         self.logpath = logpath
+        self.globalHitCount = 0 
+        self.globalHitHistory = [] 
+        for i in range(0, config.GlobalStatsLoggingFrequency + 1): 
+            self.globalHitHistory.append({'time':int(time.time()), 'hits':0})
 
     def logMessage(self, message, allowrotate=True):
         """
@@ -158,6 +167,21 @@
             self.rotate()
         self.f.write(message + '\n')
 
+    def rotateGlobalHitHistoryStats(self): 
+        """ 
+        Roll the global hit history array: push the current stats as 
+        the last element; pop the first (oldest) element and reschedule the task. 
+        """ 
+
+        self.globalHitHistory.append({'time':int(time.time()), 'hits':self.globalHitCount}) 
+        del self.globalHitHistory[0] 
+        log.msg("rotateGlobalHitHistoryStats: %s" % (self.globalHitHistory,), logLevel=logging.DEBUG) 
+        if config.GlobalStatsLoggingFrequency is not 0: 
+            self.reactor.callLater(
+                config.GlobalStatsLoggingPeriod * 60 / config.GlobalStatsLoggingFrequency, 
+                self.rotateGlobalHitHistoryStats
+            ) 
+
     def start(self):
         """
         Start logging. Open the log file and log an 'open' message.
@@ -166,6 +190,11 @@
         super(RotatingFileAccessLoggingObserver, self).start()
         self._open()
         self.logMessage("Log opened - server start: [%s]." % (datetime.datetime.now().ctime(),))
+ 
+        # Need a reactor for the callLater() support for rotateGlobalHitHistoryStats() 
+        from twisted.internet import reactor 
+        self.reactor = reactor 
+        self.rotateGlobalHitHistoryStats() 
 
     def stop(self):
         """
@@ -250,13 +279,39 @@
         self._open()
         self.logMessage("Log opened - rotated: [%s]." % (datetime.datetime.now().ctime(),), False)
 
+    def logGlobalHit(self): 
+        """ 
+        Increment the service-global hit counter 
+        """ 
 
-from twisted.protocols import amp
+        self.globalHitCount += 1 
 
+    def getGlobalHits(self): 
+        """ 
+        Return the global hit stats 
+        """ 
 
+        stats = '<?xml version="1.0" encoding="UTF-8"?><plist version="1.0">' 
+        stats += "<dict><key>totalHits</key><integer>%d</integer>" 
+        stats += "<key>recentHits</key><dict>" 
+        stats += "<key>count</key><integer>%d</integer>" 
+        stats += "<key>since</key><integer>%d</integer>" 
+        stats += "<key>period</key><integer>%d</integer>" 
+        stats += "<key>frequency</key><integer>%d</integer>" 
+        stats += "</dict></dict></plist>" 
+        return stats % (
+            self.globalHitCount,
+            self.globalHitCount - self.globalHitHistory[0]['hits'], 
+            self.globalHitHistory[0]['time'],
+            config.GlobalStatsLoggingPeriod,
+            config.GlobalStatsLoggingFrequency
+        ) 
+
 class LogMessage(amp.Command):
     arguments = [('message', amp.String())]
 
+class LogGlobalHit(amp.Command): 
+    arguments = [] 
 
 class AMPCommonAccessLoggingObserver(CommonAccessLoggingObserverExtensions):
     def __init__(self, mode, id):
@@ -302,7 +357,17 @@
         else:
             self._buffer.append(message)
 
+    def logGlobalHit(self): 
+        """ 
+        Log a server hit via the remote AMP Protocol 
+        """ 
 
+        if self.protocol is not None: 
+            d = self.protocol.callRemote(LogGlobalHit) 
+            d.addErrback(log.err) 
+        else: 
+            log.msg("logGlobalHit() only works with an AMP Protocol") 
+
 class AMPLoggingProtocol(amp.AMP):
     """
     A server side protocol for logging to the given observer.
@@ -319,7 +384,12 @@
 
     LogMessage.responder(logMessage)
 
+    def logGlobalHit(self): 
+        self.observer.logGlobalHit() 
+        return {} 
 
+    LogGlobalHit.responder(logGlobalHit)
+
 class AMPLoggingFactory(protocol.ServerFactory):
     def __init__(self, observer):
         self.observer = observer

Modified: CalendarServer/trunk/twistedcaldav/admin/options.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/options.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/options.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -87,7 +87,17 @@
 
 registerCommand(StatsOptions)
 
+class StatsWatchOptions(SubCommand): 
+    name = 'statswatch' 
+    help = ('Watch the server hit stats.') 
+    action = 'twistedcaldav.admin.stats.StatsWatchAction' 
 
+    optParameters = [ 
+        ['refresh', 'r', None, 'Refresh stats every n seconds.'], 
+    ] 
+
+registerCommand(StatsWatchOptions)
+
 from twisted.python import filepath
 from twistedcaldav.config import config
 

Modified: CalendarServer/trunk/twistedcaldav/admin/script.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/script.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/script.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -83,6 +83,7 @@
             if os.path.exists(self['config']):
                 config.loadConfig(self['config'])
 
+        self.masterConfig = config
         self.root = filepath.FilePath(config.DocumentRoot)
         self.calendarCollection = self.root.child('calendars')
         self.principalCollection = self.root.child('principals')

Modified: CalendarServer/trunk/twistedcaldav/admin/stats.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/stats.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/stats.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,6 +25,9 @@
 """
 
 import os
+import socket 
+import plistlib 
+import time 
 
 from twistedcaldav.admin import util        
 
@@ -91,3 +94,68 @@
             report['data'][stat] = value
 
         return report
+
+class StatsWatchAction(object):
+    """
+    Pulls the current server stats from the main server process via a socket
+    For example:
+
+    bin/caladmin --config conf/caldavd-dev.plist statswatch
+    bin/caladmin --config conf/caldavd-dev.plist statswatch --refresh
+    """
+ 
+    def __init__(self, config):
+        self.config = config
+        self.refresh = None
+ 
+    def getHitStats(self):
+        response = ""
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.connect(self.config.parent.masterConfig.GlobalStatsSocket)
+        while 1:
+            data = sock.recv(8192)
+            if not data: break
+            response += data
+            sock.close()
+            return response
+ 
+    def run(self):
+        if self.config['refresh'] is not None:
+            self.refresh = int(self.config['refresh'])
+ 
+        response = self.getHitStats()
+        plist = plistlib.readPlistFromString(response)
+        total = plist['totalHits']
+        since = time.time() - plist['recentHits']['since']
+        if self.refresh is None:
+            print "Total hits:\t%8d" % (total,)
+            if plist['recentHits']['frequency'] is not 0:
+                print "Last %dm%ds:\t%8d" % (since / 60, since % 60, plist['recentHits']['count'])
+                print "Rate:\t\t%8.2f" % (plist['recentHits']['count'] * 1.0 / (since if since != 0 else 1))
+                units = "second"
+                interval = plist['recentHits']['period'] * 60 / plist['recentHits']['frequency']
+                if interval % 60 is 0:
+                    interval = interval / 60
+                    units = "minute"
+                    print "Update interval: %d %s%s" % (interval, units, ("", "s")[interval > 1])
+                else:
+                    print "Recent stats are not updated."
+        else:
+            # attempt to gauge the recent hits for the first line of output
+            total_prev = (total - (plist['recentHits']['count'] / (since if since != 0 else 1) * self.refresh))
+            while 1:
+                print "Total hits: %10d\tLast %7s: %8d\tRate: %8.2f" % (
+                    total,
+                    "%dm%02ds" % (since / 60, since % 60),
+                    plist['recentHits']['count'],
+                    (total - total_prev) * 1.0 / (self.refresh if self.refresh != 0 else 1)
+                )
+                total_prev = total
+                time.sleep(self.refresh)
+                response = self.getHitStats()
+                plist = plistlib.readPlistFromString(response)
+                total = plist['totalHits']
+                since = time.time() - plist['recentHits']['since']
+ 
+        return None
+

Modified: CalendarServer/trunk/twistedcaldav/admin/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/admin/util.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/admin/util.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@
 
 from twisted.web import microdom
 
-from twistedcaldav.directory.principal import RecordTypeProperty
 from twistedcaldav.sql import db_prefix, AbstractSQLDatabase
 from twistedcaldav.index import schema_version, collection_types
 

Modified: CalendarServer/trunk/twistedcaldav/config.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/config.py	2009-04-22 00:20:05 UTC (rev 4055)
+++ CalendarServer/trunk/twistedcaldav/config.py	2009-04-22 03:40:46 UTC (rev 4056)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -186,12 +186,17 @@
     "MoreAccessLogData" : True,
     "DefaultLogLevel"   : "",
     "LogLevels"         : {},
+
     "AccountingCategories": {
         "iTIP": False,
     },
     "AccountingPrincipals": [],
-    "AccountingLogRoot": "/var/log/caldavd/accounting",
+    "AccountingLogRoot"   : "/var/log/caldavd/accounting",
 
+    "GlobalStatsSocket"           : "/var/run/caldavd-stats.sock", 
+    "GlobalStatsLoggingPeriod"    : 60, 
+    "GlobalStatsLoggingFrequency" : 12, 
+
     #
     # SSL/TLS
     #
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090421/21bb43ad/attachment-0001.html>


More information about the calendarserver-changes mailing list