[CalendarServer-changes] [5906] CalendarServer/trunk/contrib/tools
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jul 15 12:47:18 PDT 2010
Revision: 5906
http://trac.macosforge.org/projects/calendarserver/changeset/5906
Author: cdaboo at apple.com
Date: 2010-07-15 12:47:18 -0700 (Thu, 15 Jul 2010)
Log Message:
-----------
dtrace helper stuff.
Added Paths:
-----------
CalendarServer/trunk/contrib/tools/dtraceanalyze.py
CalendarServer/trunk/contrib/tools/trace.d
Added: CalendarServer/trunk/contrib/tools/dtraceanalyze.py
===================================================================
--- CalendarServer/trunk/contrib/tools/dtraceanalyze.py (rev 0)
+++ CalendarServer/trunk/contrib/tools/dtraceanalyze.py 2010-07-15 19:47:18 UTC (rev 5906)
@@ -0,0 +1,394 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from __future__ import with_statement
+import collections
+import getopt
+import os
+import re
+import sys
+import tables
+
+class Dtrace(object):
+
+ class DtraceLine(object):
+
+ prefix_maps = {
+ "/usr/share/caldavd/lib/python/": "{caldavd}/",
+ "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6": "{Python}",
+ "/System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5": "{Python}",
+ "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python": "{Extras}",
+ "/System/Library/Frameworks/Python.framework/Versions/2.5/Extras/lib/python": "{Extras}",
+ }
+ contains_maps = {
+ "/CalendarServer": "{caldavd}",
+ "/Twisted": "{Twisted}",
+ "/vobject": "{vobject}",
+ }
+
+ def __init__(self, line, lineno):
+
+ self.entering = True
+ self.function_name = ""
+ self.file_location = ""
+ self.parent = None
+ self.children = []
+
+ re_matched = re.match("(..) ([^ ]+) \(([^\)]+)\)", line)
+ if re_matched is None:
+ print line
+ results = re_matched.groups()
+ if results[0] == "<-":
+ self.entering = False
+ elif results[0] == "->":
+ self.entering = True
+ else:
+ raise ValueError("Invalid start of line at %d" % (lineno,))
+
+ self.function_name = results[1]
+ self.file_location = results[2]
+ for key, value in Dtrace.DtraceLine.prefix_maps.iteritems():
+ if self.file_location.startswith(key):
+ self.file_location = value + self.file_location[len(key):]
+ break
+ else:
+ for key, value in Dtrace.DtraceLine.contains_maps.iteritems():
+ found1 = self.file_location.find(key)
+ if found1 != -1:
+ found2 = self.file_location[found1+1:].find('/')
+ if found2 != -1:
+ self.file_location = value + self.file_location[found1+found2+1:]
+ else:
+ self.file_location = value
+ break
+
+
+ def getKey(self):
+ return (self.file_location, self.function_name,)
+
+ def getPartialKey(self):
+ return (self.filePath(), self.function_name,)
+
+ def addChild(self, child):
+ child.parent = self
+ self.children.append(child)
+
+ def checkForCollapse(self, other):
+ if self.entering and not other.entering:
+ if self.function_name == other.function_name:
+ if self.filePath() == other.filePath():
+ return True
+ return False
+
+ def filePath(self):
+ return self.file_location[0][:self.file_location[0].rfind(":")]
+
+ def prettyPrint(self, indent, sout):
+ sout.write("%s%s (%s)\n" % (" " * indent, self.function_name, self.file_location,))
+
+ class DtraceStack(object):
+
+ def __init__(self, lines, no_collapse):
+ self.start_indent = 0
+ self.stack = []
+ self.called_by = {}
+ self.call_into = {}
+
+ self.processLines(lines, no_collapse)
+
+ def processLines(self, lines, no_collapse):
+
+ new_lines = []
+ last_line = None
+ for line in lines:
+ if last_line:
+ if not no_collapse and line.checkForCollapse(last_line):
+ new_lines.pop()
+ last_line = None
+ continue
+ new_lines.append(line)
+ last_line = line
+
+ indent = 0
+ min_indent = 0
+ current_line = None
+ for line in new_lines:
+ if line.entering:
+ indent += 1
+ self.stack.append((indent, line,))
+ if current_line:
+ current_line.addChild(line)
+ current_line = line
+ else:
+ indent -= 1
+ current_line = current_line.parent if current_line else None
+ min_indent = min(min_indent, indent)
+
+ if min_indent < 0:
+ self.start_indent = -min_indent
+ else:
+ self.start_indent = 0
+
+ self.generateCallInfo()
+
+ def generateCallInfo(self):
+
+ for _ignore, line in self.stack:
+ key = line.getKey()
+
+ if line.parent:
+ parent_key = line.parent.getKey()
+ parent_calls = self.called_by.setdefault(key, {}).get(parent_key, 0)
+ self.called_by[key][parent_key] = parent_calls + 1
+
+ for child in line.children:
+ child_key = child.getKey()
+ child_calls = self.call_into.setdefault(key, {}).get(child_key, 0)
+ self.call_into[key][child_key] = child_calls + 1
+
+ def prettyPrint(self, sout):
+ for indent, line in self.stack:
+ line.prettyPrint(self.start_indent + indent, sout)
+
+ def __init__(self, filepath):
+
+ self.filepath = filepath
+ self.calltimes = collections.defaultdict(lambda: [0, 0, 0])
+ self.exclusiveTotal = 0
+
+ def analyze(self, do_stack, no_collapse):
+
+ print "Parsing dtrace output."
+
+ # Parse the trace lines first and look for the start of the call times
+ lines = []
+ traces = True
+ index = -1
+ with file(filepath) as f:
+ for lineno, line in enumerate(f):
+ if traces:
+ if line.strip() and line[0:3] in ("-> ", "<- "):
+ lines.append(Dtrace.DtraceLine(line, lineno + 1))
+ elif line.startswith("Count,"):
+ traces = False
+ else:
+ if line[0] != ' ':
+ continue
+ line = line.strip()
+ if line.startswith("FILE"):
+ index += 1
+ if index >= 0:
+ self.parseCallTimeLine(line, index)
+
+ self.printTraceDetails(lines, do_stack, no_collapse)
+
+ for ctr, title in enumerate(("Sorted by Count", "Sorted by Exclusive", "Sorted by Inclusive",)):
+ print title
+ self.printCallTimeTotals(ctr)
+
+ def printTraceDetails(self, lines, do_stack, no_collapse):
+
+ print "Found %d lines" % (len(lines),)
+ print "============================"
+ print ""
+
+ self.stack = Dtrace.DtraceStack(lines, no_collapse)
+ if do_stack:
+ with file("stacked.txt", "w") as f:
+ self.stack.prettyPrint(f)
+ print "Wrote stack calls to 'stacked.txt'"
+ print "============================"
+ print ""
+
+ # Get stats for each call
+ stats = {}
+ last_exit = None
+ for line in lines:
+ key = line.getKey()
+ if line.entering:
+ counts = stats.get(key, (0, 0))
+ counts = (counts[0] + (1 if no_collapse else 0), counts[1] + (0 if no_collapse else 1))
+ if line.getPartialKey() != last_exit:
+ counts = (counts[0] + (0 if no_collapse else 1), counts[1] + (1 if no_collapse else 0))
+ stats[key] = counts
+ else:
+ last_exit = line.getPartialKey()
+
+ print "Function Call Counts"
+ print ""
+ table = tables.Table()
+ table.addHeader(("Count", "Function", "File",))
+ for key, value in sorted(stats.iteritems(), key=lambda x: x[1][0], reverse=True):
+ table.addRow(("%d (%d)" % value, key[1], key[0],))
+ table.printTable()
+
+ print ""
+ print "Called By Counts"
+ print ""
+ table = tables.Table()
+ table.addHeader(("Function", "Caller", "Count",))
+ for main_key in sorted(self.stack.called_by.keys(), key=lambda x: x[1] + x[0]):
+ first = True
+ for key, value in sorted(self.stack.called_by[main_key].iteritems(), key=lambda x: x[1], reverse=True):
+ table.addRow((
+ ("%s (%s)" % (main_key[1], main_key[0],)) if first else "",
+ "%s (%s)" % (key[1], key[0],),
+ str(value),
+ ))
+ first = False
+ table.printTable()
+
+ print ""
+ print "Call Into Counts"
+ print ""
+ table = tables.Table()
+ table.addHeader(("Function", "Calls", "Count",))
+ for main_key in sorted(self.stack.call_into.keys(), key=lambda x: x[1] + x[0]):
+ first = True
+ for key, value in sorted(self.stack.call_into[main_key].iteritems(), key=lambda x: x[1], reverse=True):
+ table.addRow((
+ ("%s (%s)" % (main_key[1], main_key[0],)) if first else "",
+ "%s (%s)" % (key[1], key[0],),
+ str(value),
+ ))
+ first = False
+ table.printTable()
+ print ""
+
+ def parseCallTimeLine(self, line, index):
+
+ file, type, name, value = line.split()
+ if file in ("-", "FILE"):
+ return
+ else:
+ self.calltimes[(file, name)][index] = int(value)
+ if index == 1:
+ self.exclusiveTotal += int(value)
+
+ def printCallTimeTotals(self, sortIndex):
+
+ table = tables.Table()
+
+ table.setDefaultColumnFormats((
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ ))
+
+ table.addHeader(("File", "Name", "Count", "Inclusive", "Exclusive", "Children",))
+ for key, value in sorted(self.calltimes.items(), key=lambda x:x[1][sortIndex], reverse=True):
+ table.addRow((
+ key[0],
+ key[1],
+ value[0],
+ value[2],
+ "%s (%6.3f%%)" % (value[1], (100.0 * value[1]) / self.exclusiveTotal),
+ value[2] - value[1],
+ ))
+ table.addRow()
+ table.addRow((
+ "Total:",
+ "",
+ "",
+ "",
+ self.exclusiveTotal,
+ "",
+ ))
+
+ table.printTable()
+ print ""
+
+def usage(error_msg=None):
+ if error_msg:
+ print error_msg
+
+ print """Usage: dtraceanalyze [options] FILE
+Options:
+ -h Print this help and exit
+ --stack Save indented stack to file
+ --raw-count Display call counts based on full trace,
+ else display counts on collapsed values.
+
+Arguments:
+ FILE File name containing dtrace output to analyze
+
+Description:
+ This utility will analyze the output of the trace.d dtrace script to produce
+ useful statistics, and other performance related data.
+
+ To use this do the following:
+
+ > sudo ./trace.d > results.txt
+ ...
+ > ./dtraceanalyze.py results.txt
+"""
+
+ if error_msg:
+ raise ValueError(error_msg)
+ else:
+ sys.exit(0)
+
+if __name__ == "__main__":
+
+ sys.setrecursionlimit(10000)
+ do_stack = False
+ no_collapse = False
+ try:
+ options, args = getopt.getopt(sys.argv[1:], "h", ["stack", "no-collapse"])
+
+ for option, value in options:
+ if option == "-h":
+ usage()
+ elif option == "--stack":
+ do_stack = True
+ elif option == "--no-collapse":
+ no_collapse = True
+ else:
+ usage("Unrecognized option: %s" % (option,))
+
+ if len(args) == 0:
+ fname = "results.txt"
+ elif len(args) != 1:
+ usage("Must have one argument")
+ else:
+ fname = args[0]
+
+ filepath = os.path.expanduser(fname)
+ if not os.path.exists(filepath):
+ usage("File '%s' does not exist" % (filepath,))
+
+ print "CalendarServer dtrace analysis tool tool"
+ print "====================================="
+ print ""
+ if do_stack:
+ print "Generating nested stack call file."
+ if no_collapse:
+ print "Consecutive function calls will not be removed."
+ else:
+ print "Consecutive function calls will be removed."
+ print "============================"
+ print ""
+
+ Dtrace(filepath).analyze(do_stack, no_collapse)
+
+ except Exception, e:
+ raise
+ sys.exit(str(e))
Property changes on: CalendarServer/trunk/contrib/tools/dtraceanalyze.py
___________________________________________________________________
Added: svn:executable
+ *
Added: CalendarServer/trunk/contrib/tools/trace.d
===================================================================
--- CalendarServer/trunk/contrib/tools/trace.d (rev 0)
+++ CalendarServer/trunk/contrib/tools/trace.d 2010-07-15 19:47:18 UTC (rev 5906)
@@ -0,0 +1,96 @@
+#!/usr/sbin/dtrace -Zs
+/*
+ * trace.d - measure Python on-CPU times and flow for functions.
+ *
+ * This traces Python activity from a specified process.
+ *
+ * USAGE: trace.d <PID>
+ *
+ * FIELDS:
+ * FILE Filename of the Python program
+ * TYPE Type of call (func/total)
+ * NAME Name of call (function name)
+ * TOTAL Total on-CPU time for calls (us)
+ *
+ * Filename and function names are printed if available.
+ *
+ * COPYRIGHT: Copyright (c) 2007 Brendan Gregg.
+ *
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at Docs/cddl1.txt
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * CDDL HEADER END
+ *
+ * 09-Sep-2007 Brendan Gregg Created original py_ scripts.
+ *
+ * This is a combination of py_flow.d and py_cputime.d that takes a pid
+ * as an argument and runs for 10 seconds.
+ */
+
+#pragma D option quiet
+
+dtrace:::BEGIN
+{
+ printf("Tracing... Hit Ctrl-C to end.\n");
+}
+
+python*:::function-entry
+/pid == $1/
+{
+ self->depth++;
+ self->exclude[self->depth] = 0;
+ self->function[self->depth] = vtimestamp;
+
+ printf("%s %s (%s:%d)\n", "->",
+ copyinstr(arg1), copyinstr(arg0), arg2);
+}
+
+python*:::function-return
+/pid == $1 && self->function[self->depth]/
+{
+ this->oncpu_incl = vtimestamp - self->function[self->depth];
+ this->oncpu_excl = this->oncpu_incl - self->exclude[self->depth];
+ self->function[self->depth] = 0;
+ self->exclude[self->depth] = 0;
+ this->file = basename(copyinstr(arg0));
+ this->name = copyinstr(arg1);
+
+ @num[this->file, "func", this->name] = count();
+ @num["-", "total", "-"] = count();
+ @types_incl[this->file, "func", this->name] = sum(this->oncpu_incl);
+ @types_excl[this->file, "func", this->name] = sum(this->oncpu_excl);
+ @types_excl["-", "total", "-"] = sum(this->oncpu_excl);
+
+ self->depth--;
+ self->exclude[self->depth] += this->oncpu_incl;
+
+ printf("%s %s (%s:%d)\n", "<-",
+ copyinstr(arg1), copyinstr(arg0), arg2);
+}
+
+profile:::tick-10s
+{
+ printf("\nCount,\n");
+ printf(" %-20s %-10s %-32s %8s\n", "FILE", "TYPE", "NAME", "COUNT");
+ printa(" %-20s %-10s %-32s %@8d\n", @num);
+
+ normalize(@types_excl, 1000);
+ printf("\nExclusive function on-CPU times (us),\n");
+ printf(" %-20s %-10s %-32s %8s\n", "FILE", "TYPE", "NAME", "TOTAL");
+ printa(" %-20s %-10s %-32s %@8d\n", @types_excl);
+
+ normalize(@types_incl, 1000);
+ printf("\nInclusive function on-CPU times (us),\n");
+ printf(" %-20s %-10s %-32s %8s\n", "FILE", "TYPE", "NAME", "TOTAL");
+ printa(" %-20s %-10s %-32s %@8d\n", @types_incl);
+ exit(0);
+}
Property changes on: CalendarServer/trunk/contrib/tools/trace.d
___________________________________________________________________
Added: svn:executable
+ *
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100715/01135e36/attachment-0001.html>
More information about the calendarserver-changes
mailing list