[CalendarServer-changes] [10119] CalendarServer/trunk/contrib/tools/readStats.py
source_changes at macosforge.org
source_changes at macosforge.org
Mon Dec 3 08:17:17 PST 2012
Revision: 10119
http://trac.calendarserver.org//changeset/10119
Author: cdaboo at apple.com
Date: 2012-12-03 08:17:17 -0800 (Mon, 03 Dec 2012)
Log Message:
-----------
More improvements to stats logging.
Modified Paths:
--------------
CalendarServer/trunk/contrib/tools/readStats.py
Modified: CalendarServer/trunk/contrib/tools/readStats.py
===================================================================
--- CalendarServer/trunk/contrib/tools/readStats.py 2012-12-03 00:08:43 UTC (rev 10118)
+++ CalendarServer/trunk/contrib/tools/readStats.py 2012-12-03 16:17:17 UTC (rev 10119)
@@ -16,13 +16,14 @@
##
from StringIO import StringIO
+import collections
import datetime
+import getopt
import json
import socket
+import sys
import tables
import time
-import sys
-import getopt
"""
This tool reads data from the server's statistics socket and prints a summary.
@@ -53,23 +54,23 @@
-def printStats(stats, multimode):
+def printStats(stats, multimode, showMethods, topUsers):
if len(stats) == 1:
if "Failed" in stats[0]:
printFailedStats(stats[0]["Failed"])
else:
try:
- printStat(stats[0])
+ printStat(stats[0], multimode[0], showMethods, topUsers)
except KeyError, e:
printFailedStats("Unable to find key '%s' in statistics from server socket" % (e,))
sys.exit(1)
else:
- printMultipleStats(stats, multimode)
+ printMultipleStats(stats, multimode, showMethods, topUsers)
-def printStat(stats):
+def printStat(stats, index, showMethods, topUsers):
print "- " * 40
print "Server: %s" % (stats["Server"],)
@@ -90,17 +91,19 @@
print "Current Memory Used: Unavailable"
print
printRequestSummary(stats)
- printHistogramSummary(stats["5 Minutes"])
+ printHistogramSummary(stats[index])
+ if showMethods:
+ printMethodCounts(stats[index])
+ if topUsers:
+ printUserCounts(stats[index], topUsers)
-def printMultipleStats(stats, multimode):
+def printMultipleStats(stats, multimode, showMethods, topUsers):
labels = serverLabels(stats)
print "- " * 40
- print "Servers: %s" % (", ".join(labels),)
-
print datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
times = []
@@ -110,30 +113,50 @@
except KeyError:
t = "-"
times.append(t)
- print "Service Uptime: %s" % (", ".join(times),)
cpus = []
memories = []
for stat in stats:
if stat["System"]["cpu count"] > 0:
- cpus.append("%.1f%%" % (stat["System"]["cpu use"],))
- memories.append("%.1f%%" % (stat["System"]["memory percent"],))
+ cpus.append(stat["System"]["cpu use"])
+ memories.append(stat["System"]["memory percent"])
else:
- cpus.append("-")
- memories.append("-")
- print "Current CPU: %s" % (", ".join(cpus),)
- print "Current Memory Used: %s" % (", ".join(memories),)
- print
- printMultiRequestSummary(stats, labels, multimode)
+ cpus.append(-1)
+ memories.append(-1)
+
+ printMultiRequestSummary(stats, cpus, memories, times, labels, multimode)
printMultiHistogramSummary(stats, multimode[0])
+ if showMethods:
+ printMultiMethodCounts(stats, multimode[0])
+ if topUsers:
+ printMultiUserCounts(stats, multimode[0], topUsers)
def serverLabels(stats):
- return [str(stat["Server"]) for stat in stats]
+ servers = [stat["Server"] for stat in stats]
+ if isinstance(servers[0], tuple):
+ hosts = set([item[0] for item in servers])
+ ports = set([item[1] for item in servers])
+ if len(ports) == 1:
+ servers = [item[0] for item in servers]
+ elif len(hosts) == 1:
+ servers = [":%d" % item[1] for item in servers]
+ elif len(hosts) == len(servers):
+ servers = [item[0] for item in servers]
+ else:
+ servers = ["%s:%s" % item for item in servers]
+ servers = [item.split(".") for item in servers]
+ while True:
+ if all([item[-1] == servers[0][-1] for item in servers]):
+ servers = [item[:-1] for item in servers]
+ else:
+ break
+ return [".".join(item) for item in servers]
+
def printFailedStats(message):
print "- " * 40
@@ -186,17 +209,18 @@
-def printMultiRequestSummary(stats, labels, index):
+def printMultiRequestSummary(stats, cpus, memories, times, labels, index):
key, seconds = index
table = tables.Table()
table.addHeader(
- ("Server", "Requests", "Av. Requests", "Av. Response", "Av. Response", "Max. Response", "Slot", "CPU", "500's"),
+ ("Server", "Requests", "Av. Requests", "Av. Response", "Av. Response", "Max. Response", "Slot", "CPU", "CPU", "Memory", "500's", "Uptime",),
)
table.addHeader(
- (key, "", "per second", "(ms)", "no write(ms)", "(ms)", "Average", "Average", ""),
+ (key, "", "per second", "(ms)", "no write(ms)", "(ms)", "Average", "Average", "Current", "Current", "", "",),
)
+ max_column = 5
table.setDefaultColumnFormats(
(
tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
@@ -207,11 +231,14 @@
tables.Table.ColumnFormat("%.1f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.2f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
)
)
- totals = ["Overall:", 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0]
+ totals = ["Overall:", 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, "", ]
for ctr, stat in enumerate(stats):
stat = stat[key]
@@ -225,12 +252,18 @@
col.append(stat["T-MAX"])
col.append(safeDivision(float(stat["slots"]), stat["requests"]))
col.append(safeDivision(stat["cpu"], stat["requests"]))
+ col.append(cpus[ctr])
+ col.append(memories[ctr])
col.append(stat["500"])
+ col.append(times[ctr])
table.addRow(col)
- for item in xrange(1, len(col)):
- totals[item] += col[item]
+ for item in xrange(1, len(col) - 1):
+ if item == max_column:
+ totals[item] = max(totals[item], col[item])
+ else:
+ totals[item] += col[item]
- for item in (2, 3, 4, 6, 7):
+ for item in (3, 4, 6, 7, 8, 9):
totals[item] /= len(stats)
table.addFooter(totals)
@@ -250,7 +283,7 @@
)
table.setDefaultColumnFormats(
(
- tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.CENTER_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
@@ -264,7 +297,7 @@
)
for i in ("T", "T-RESP-WR",):
table.addRow((
- "Overall Response" if i == "T" else "Response without Write",
+ "Overall Response" if i == "T" else "Response Write",
(stat[i]["<10ms"], safeDivision(stat[i]["<10ms"], stat["requests"], 100.0)),
(stat[i]["10ms<->100ms"], safeDivision(stat[i]["10ms<->100ms"], stat["requests"], 100.0)),
(stat[i]["100ms<->1s"], safeDivision(stat[i]["100ms<->1s"], stat["requests"], 100.0)),
@@ -303,7 +336,7 @@
)
table.setDefaultColumnFormats(
(
- tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.CENTER_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
@@ -317,7 +350,7 @@
)
for i in ("T", "T-RESP-WR",):
table.addRow((
- "Overall Response" if i == "T" else "Response without Write",
+ "Overall Response" if i == "T" else "Response Write",
(totals[i]["<10ms"], safeDivision(totals[i]["<10ms"], totals[i]["requests"], 100.0)),
(totals[i]["10ms<->100ms"], safeDivision(totals[i]["10ms<->100ms"], totals[i]["requests"], 100.0)),
(totals[i]["100ms<->1s"], safeDivision(totals[i]["100ms<->1s"], totals[i]["requests"], 100.0)),
@@ -334,6 +367,128 @@
+def printMethodCounts(stat):
+
+ print "Method Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("Method", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(stat["method"].values())
+ for method in sorted(stat["method"].keys()):
+ table.addRow((
+ method,
+ stat["method"][method],
+ safeDivision(stat["method"][method], total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
+def printMultiMethodCounts(stats, index):
+
+ methods = collections.defaultdict(int)
+ for stat in stats:
+ for method in stat[index]["method"]:
+ methods[method] += stat[index]["method"][method]
+
+ print "Method Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("Method", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(methods.values())
+ for method in sorted(methods.keys()):
+ table.addRow((
+ method,
+ methods[method],
+ safeDivision(methods[method], total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
+def printUserCounts(stat, topUsers):
+
+ print "User Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("User", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(stat["uid"].values())
+ for uid in sorted(stat["uid"].items(), key=lambda x: x[1], reverse=True)[:topUsers]:
+ table.addRow((
+ uid,
+ stat["uid"][uid],
+ safeDivision(stat["uid"][uid], total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
+def printMultiUserCounts(stats, index, topUsers):
+
+ uids = collections.defaultdict(int)
+ for stat in stats:
+ for uid in stat[index]["uid"]:
+ uids[uid] += stat[index]["uid"][uid]
+
+ print "User Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("User", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(uids.values())
+ for uid, count in sorted(uids.items(), key=lambda x: x[1], reverse=True)[:topUsers]:
+ table.addRow((
+ uid,
+ count,
+ safeDivision(count, total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
def usage(error_msg=None):
if error_msg:
print error_msg
@@ -348,6 +503,8 @@
--1 Display multiserver 1 minute average
--5 Display multiserver 5 minute average (the default)
--60 Display multiserver 1 hour average
+ --methods Include details about HTTP method usage
+ --users N Include details about top N users
Description:
This utility will print a summary of statistics read from a
@@ -366,11 +523,13 @@
delay = 10
servers = ("data/Logs/state/caldavd-stats.sock",)
useTCP = False
+ showMethods = False
+ topUsers = 0
multimodes = (("Current", 60,), ("1 Minute", 60,), ("5 Minutes", 5 * 60,), ("1 Hour", 60 * 60,),)
multimode = multimodes[2]
- options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", "0", "1", "5", "60"])
+ options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", "0", "1", "5", "60", "methods", "users="])
for option, value in options:
if option == "-h":
@@ -390,7 +549,11 @@
multimode = multimodes[2]
elif option == "--60":
multimode = multimodes[3]
+ elif option == "--methods":
+ showMethods = True
+ elif option == "--users":
+ topUsers = int(value)
while True:
- printStats([readSock(server, useTCP) for server in servers], multimode)
+ printStats([readSock(server, useTCP) for server in servers], multimode, showMethods, topUsers)
time.sleep(delay)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121203/d3334ddd/attachment-0001.html>
More information about the calendarserver-changes
mailing list