[CalendarServer-changes] [4925] CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch

[source_changes at macosforge.org]

Revision: 4925
          http://trac.macosforge.org/projects/calendarserver/changeset/4925
Author:   sagen at apple.com
Date:     2010-01-14 22:01:13 -0800 (Thu, 14 Jan 2010)
Log Message:
-----------
A utility to monitor the inspection ports on a running calendar server, watch for stalled processes, and automatically run dtruss and sample on them

Added Paths:
-----------
    CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch

Added: CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch	                        (rev 0)
+++ CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch	2010-01-15 06:01:13 UTC (rev 4925)
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import sys
+
+from twisted.internet import reactor
+from twisted.protocols.basic import LineReceiver
+from twisted.internet.protocol import ClientFactory
+from subprocess import Popen, PIPE, STDOUT
+import time
+
+INACTIVITY_SECONDS = 60
+
+class WatchProtocol(LineReceiver):
+    def lineReceived(self, line):
+        self.history.append(line)
+        if len(self.history) > 40:
+            del self.history[0]
+
+        if self.later.active():
+            self.later.reset(INACTIVITY_SECONDS)
+
+    def inactivity(self):
+        print "Server process %d inactive for over %d seconds" % (self.id, INACTIVITY_SECONDS)
+        print "Inspection history:"
+        for line in self.history:
+            print line
+        pid = pidForID(self.id)
+        if pid is None:
+            print "Error determining PID from ID"
+        else:
+            print "Server process for ID %d is %d" % (self.id, pid)
+            print "Running dtruss..."
+            dtruss(pid)
+            print "Running sample..."
+            sample(pid)
+        self.later = reactor.callLater(INACTIVITY_SECONDS, self.inactivity)
+        print "--------------------------------------------------------------"
+
+class WatchFactory(ClientFactory):
+    protocol = WatchProtocol
+
+    def __init__(self, id):
+        self.id = id
+
+    def buildProtocol(self, addr):
+        p = ClientFactory.buildProtocol(self, addr)
+        p.id = self.id
+        p.history = []
+        p.later = reactor.callLater(INACTIVITY_SECONDS, p.inactivity)
+        return p
+
+
+def pidForID(id):
+    child = Popen(args=['/bin/ps', 'axo', 'pid,command'],
+        stdout=PIPE, stderr=STDOUT)
+    try:
+        output, error = child.communicate()
+    except IOError, e:
+        print "Couldn't determine PID of server process", e
+        return None
+    for line in output.split("\n"):
+        if line:
+            pid, command = line.split(" ", 1)
+            if "caldavd" in command and " LogID=%d " % (id,) in command:
+                return int(pid)
+    return None
+
+def sample(pid):
+    child = Popen(args=['/usr/bin/sample', str(pid)],
+        stdout=PIPE, stderr=STDOUT)
+    output, error = child.communicate()
+    print output
+
+def dtruss(pid):
+    filename = "/tmp/auto_dtrace_%d_%f.out" % (pid, time.time())
+    output = open(filename, "w")
+    try:
+        child = Popen(args=['/usr/bin/dtruss', '-p', str(pid)],
+            stdout=output, stderr=STDOUT)
+        time.sleep(5)
+        child.terminate()
+        print "Dtrace of %d captured to %s" % (pid, filename)
+    except OSError, e:
+        print "Couldn't run dtruss:", e
+    finally:
+        output.close()
+
+def main():
+    for id in range(0, 75):
+        port = id + 10000
+        factory = WatchFactory(id)
+        reactor.connectTCP("localhost", port, factory)
+    print "Monitoring..."
+    reactor.run()
+
+main()


Property changes on: CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch
___________________________________________________________________
Added: svn:executable
   + *
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100114/e3d61343/attachment.html>