[CalendarServer-changes] [13779] CalendarServer/trunk/contrib/tools
source_changes at macosforge.org
source_changes at macosforge.org
Mon Jul 21 12:28:13 PDT 2014
Revision: 13779
http://trac.calendarserver.org//changeset/13779
Author: cdaboo at apple.com
Date: 2014-07-21 12:28:13 -0700 (Mon, 21 Jul 2014)
Log Message:
-----------
Add option to write stats JSON data to a file. Enable stats analysis plotting tool to read the JSON data.
Modified Paths:
--------------
CalendarServer/trunk/contrib/tools/readStats.py
CalendarServer/trunk/contrib/tools/statsanalysis.py
Modified: CalendarServer/trunk/contrib/tools/readStats.py
===================================================================
--- CalendarServer/trunk/contrib/tools/readStats.py 2014-07-21 19:26:20 UTC (rev 13778)
+++ CalendarServer/trunk/contrib/tools/readStats.py 2014-07-21 19:28:13 UTC (rev 13779)
@@ -63,7 +63,7 @@
-def printStats(stats, multimode, showMethods, topUsers, showAgents):
+def printStats(stats, multimode, showMethods, topUsers, showAgents, dumpFile):
if len(stats) == 1:
if "Failed" in stats[0]:
printFailedStats(stats[0]["Failed"])
@@ -501,6 +501,7 @@
--methods Include details about HTTP method usage
--users N Include details about top N users
--agents Include details about HTTP User-Agent usage
+ --dump FILE File to dump JSON data to
Description:
This utility will print a summary of statistics read from a
@@ -522,11 +523,12 @@
showMethods = False
topUsers = 0
showAgents = False
+ dumpFile = None
multimodes = (("current", 60,), ("1m", 60,), ("5m", 5 * 60,), ("1h", 60 * 60,),)
multimode = multimodes[2]
- options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", "0", "1", "5", "60", "methods", "users=", "agents"])
+ options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", "0", "1", "5", "60", "methods", "users=", "agents", "dump="])
for option, value in options:
if option == "-h":
@@ -552,7 +554,16 @@
topUsers = int(value)
elif option == "--agents":
showAgents = True
+ elif option == "--dump":
+ dumpFile = value
while True:
- printStats([readSock(server, useTCP) for server in servers], multimode, showMethods, topUsers, showAgents)
+ stats = [readSock(server, useTCP) for server in servers]
+ printStats(stats, multimode, showMethods, topUsers, showAgents, dumpFile)
+ if dumpFile is not None:
+ stats = dict([("{}:{}".format(*servers[ctr]), stat) for ctr, stat in enumerate(stats)])
+ stats["timestamp"] = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+ with open(dumpFile, "a") as f:
+ f.write(json.dumps(stats))
+ f.write("\n")
time.sleep(delay)
Modified: CalendarServer/trunk/contrib/tools/statsanalysis.py
===================================================================
--- CalendarServer/trunk/contrib/tools/statsanalysis.py 2014-07-21 19:26:20 UTC (rev 13778)
+++ CalendarServer/trunk/contrib/tools/statsanalysis.py 2014-07-21 19:28:13 UTC (rev 13779)
@@ -16,6 +16,9 @@
##
from __future__ import print_function
+from json import dumps, loads
+from matplotlib.ticker import AutoMinorLocator
+import collections
import getopt
import matplotlib.pyplot as plt
import os
@@ -23,6 +26,9 @@
dataset = {}
+def safeDivision(value, total, factor=1):
+ return value * factor / total if total else 0
+
def analyze(fpath, title):
"""
Analyze a readStats data file.
@@ -34,17 +40,46 @@
"""
print("Analyzing data for %s" % (title,))
- dataset[title] = {}
- with open(fpath) as f:
- for line in f:
+ fpath_cached = fpath + ".cached"
+ if os.path.exists(fpath_cached):
+ with open(fpath_cached) as f:
+ d = loads(f.read())
+ dataset[title] = dict([(int(k), v) for k, v in d.items()])
+ else:
+ dataset[title] = {}
+ json = False
+ with open(fpath) as f:
+ line = f.next()
if line.startswith("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -"):
- analyzeRecord(f, title)
+ analyzeTableFormat(f, title)
+ elif line.startswith("{"):
+ analyzeJSONFormat(f, line, title)
+ json = True
+ if not json:
+ with open(fpath_cached, "w") as f:
+ f.write(dumps(dataset[title]))
+
print("Read %d data points\n" % (len(dataset[title]),))
-def analyzeRecord(liter, title):
+def analyzeTableFormat(f, title):
"""
+ Analyze a "table" format output file. First line has already been tested.
+
+ @param f: file object to read
+ @type f: L{File}
+ @param title: title to use for data set
+ @type title: L{str}
+ """
+ analyzeTableRecord(f, title)
+ for line in f:
+ if line.startswith("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -"):
+ analyzeTableRecord(f, title)
+
+
+def analyzeTableRecord(liter, title):
+ """
Analyze one entry from the readStats data file.
@param liter: iterator over lines in the file
@@ -125,6 +160,114 @@
return methods
+def analyzeJSONFormat(f, first, title):
+ """
+ Analyze a JSON format output file. First line has already been tested.
+
+ @param f: file object to read
+ @type f: L{File}
+ @param title: title to use for data set
+ @type title: L{str}
+ """
+ analyzeJSONRecord(first, title)
+ for line in f:
+ if line:
+ analyzeJSONRecord(line, title)
+
+
+def analyzeJSONRecord(line, title):
+ """
+ Analyze a JSON record.
+
+ @param line: line of JSON data to parse
+ @type line: L{str}
+ @param title: title to use for data set
+ @type title: L{str}
+ """
+
+ j = loads(line)
+
+ time = j["timestamp"][11:]
+ seconds = 60 * (60 * int(time[0:2]) + int(time[3:5])) + int(time[6:8])
+ dataset[title][seconds] = {}
+
+ allstats = [stat for server, stat in j.items() if server != "timestamp"]
+
+ analyzeJSONStatsSummary(allstats, title, seconds)
+ analyzeJSONStatsMethods(allstats, title, seconds)
+
+
+def analyzeJSONStatsSummary(allstats, title, seconds):
+ """
+ Analyze all server JSON summary stats.
+
+ @param allstats: set of JSON stats to analyze
+ @type allstats: L{dict}
+ @param title: title to use for data set
+ @type title: L{str}
+ @param seconds: timestamp for stats record
+ @type seconds: L{int}
+ """
+
+ results = collections.defaultdict(list)
+ for stats in allstats:
+ stat = stats["1m"]
+
+ results["Requests"].append(stat["requests"])
+ results["Av. Requests per second"].append(safeDivision(float(stat["requests"]), 60.0))
+ results["Av. Response"].append(safeDivision(stat["t"], stat["requests"]))
+ results["Av. Response no write"].append(safeDivision(stat["t"] - stat["t-resp-wr"], stat["requests"]))
+ results["Max. Response"].append(stat["T-MAX"])
+ results["Slot Average"].append(safeDivision(float(stat["slots"]), stat["requests"]))
+ results["CPU Average"].append(safeDivision(stat["cpu"], stat["requests"]))
+ results["CPU Current"].append(stats["system"]["cpu use"])
+ results["Memory Current"].append(stats["system"]["memory percent"])
+ results["500's"].append(stat["500"])
+
+ for key in ("Requests", "Av. Requests per second", "500's",):
+ dataset[title][seconds]["Overall:{}".format(key)] = sum(results[key])
+
+ for key in ("Av. Response", "Av. Response no write", "Slot Average", "CPU Average", "CPU Current", "Memory Current",):
+ dataset[title][seconds]["Overall:{}".format(key)] = sum(results[key]) / len(allstats)
+
+ dataset[title][seconds]["Overall:Max. Response"] = max(results["Max. Response"])
+
+
+def analyzeJSONStatsMethods(allstats, title, seconds):
+ """
+ Analyze all server JSON method stats.
+
+ @param allstats: set of JSON stats to analyze
+ @type allstats: L{dict}
+ @param title: title to use for data set
+ @type title: L{str}
+ @param seconds: timestamp for stats record
+ @type seconds: L{int}
+ """
+
+ methods = collections.defaultdict(int)
+ method_times = collections.defaultdict(float)
+ response_average = {}
+ for stat in allstats:
+ for method in stat["1m"]["method"]:
+ methods[method] += stat["1m"]["method"][method]
+ if "method-t" in stat["1m"]:
+ for method_time in stat["1m"]["method-t"]:
+ method_times[method_time] += stat["1m"]["method-t"][method_time]
+ response_average[method_time] = method_times[method_time] / methods[method_time]
+
+ total_count = sum(methods.values())
+ total_avresponse = sum(response_average.values())
+ total_response = sum(method_times.values())
+
+ for method, count in methods.items():
+ dataset[title][seconds]["Method:{}:Count".format(method)] = count
+ dataset[title][seconds]["Method:{}:Count %".format(method)] = safeDivision(methods[method], total_count, 100.0)
+ dataset[title][seconds]["Method:{}:Av. Response".format(method)] = response_average[method]
+ dataset[title][seconds]["Method:{}:Av. Response %".format(method)] = safeDivision(response_average[method], total_avresponse, 100.0)
+ dataset[title][seconds]["Method:{}:Total Resp. %".format(method)] = safeDivision(method_times[method], total_response, 100.0)
+
+
def plotSeries(key, ymin=None, ymax=None):
"""
Plot the chosen dataset key for each scanned data file.
@@ -155,6 +298,10 @@
(1, 4, 7, 10, 13, 16, 19, 22,),
(18, 21, 0, 3, 6, 9, 12, 15,),
)
+ plt.minorticks_on()
+ plt.gca().xaxis.set_minor_locator(AutoMinorLocator(n=3))
+ plt.grid(True, "major", "x", alpha=0.5, linewidth=0.5)
+ plt.grid(True, "minor", "x", alpha=0.5, linewidth=0.5)
plt.legend(titles, 'upper left', shadow=True, fancybox=True)
plt.show()
@@ -202,7 +349,7 @@
fnames = os.listdir(scanDir)
count = 1
for name in fnames:
- if name.startswith("stats_all.log"):
+ if name.startswith("stats_all.log") and not name.endswith(".cached"):
print("Found file: %s" % (os.path.join(scanDir, name),))
trailer = name[len("stats_all.log"):]
if trailer.startswith("-"):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140721/92f8f0d3/attachment-0001.html>
More information about the calendarserver-changes
mailing list