<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[15658] CalendarServer/trunk/calendarserver/tools/dashview.py</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/15658">15658</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2016-06-06 11:21:21 -0700 (Mon, 06 Jun 2016)</dd>
</dl>
<h3>Log Message</h3>
<pre>Add option to show aggregated data for all hosts in a pod. Turn off logging.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkcalendarservertoolsdashviewpy">CalendarServer/trunk/calendarserver/tools/dashview.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkcalendarservertoolsdashviewpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tools/dashview.py (15657 => 15658)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tools/dashview.py 2016-06-04 15:58:14 UTC (rev 15657)
+++ CalendarServer/trunk/calendarserver/tools/dashview.py 2016-06-06 18:21:21 UTC (rev 15658)
</span><span class="lines">@@ -20,6 +20,8 @@
</span><span class="cx"> server as exposed by the L{DashboardProtocol} stats socket.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+from collections import OrderedDict
+from operator import itemgetter
</ins><span class="cx"> import argparse
</span><span class="cx"> import collections
</span><span class="cx"> import curses.panel
</span><span class="lines">@@ -35,7 +37,7 @@
</span><span class="cx"> import time
</span><span class="cx">
</span><span class="cx"> LOG_FILENAME = "db.log"
</span><del>-logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)
</del><ins>+#logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -65,8 +67,12 @@
</span><span class="cx"> d.run()
</span><span class="cx"> curses.wrapper(_wrapped)
</span><span class="cx">
</span><ins>+# client = DashboardClient(self, server)
+# client.getOneItem("podA", "localhost:8100", "jobcount")
+# print(json.dumps(client.currentData["pods"]["podA"]["aggregate"], indent=1))
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def safeDivision(value, total, factor=1):
</span><span class="cx"> return value * factor / total if total else 0
</span><span class="cx">
</span><span class="lines">@@ -109,10 +115,11 @@
</span><span class="cx"> self.seconds = 1.0
</span><span class="cx"> self.sched = sched.scheduler(time.time, time.sleep)
</span><span class="cx">
</span><ins>+ self.aggregate = False
</ins><span class="cx"> self.selected_server = Point()
</span><span class="cx"> self.server_window = None
</span><span class="cx">
</span><del>- self.client = DashboardClient(server)
</del><ins>+ self.client = DashboardClient(self, server)
</ins><span class="cx"> self.client_error = False
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -281,6 +288,11 @@
</span><span class="cx"> self.windows = []
</span><span class="cx"> self.displayWindow(self.registered_windows["h"])
</span><span class="cx">
</span><ins>+ elif c == "x":
+ self.aggregate = not self.aggregate
+ self.client.update()
+ self.displayWindow(None)
+
</ins><span class="cx"> elif c in self.registered_windows:
</span><span class="cx"> self.displayWindow(self.registered_windows[c])
</span><span class="cx">
</span><span class="lines">@@ -336,7 +348,8 @@
</span><span class="cx"> Client that connects to a server and fetches information.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def __init__(self, sockname):
</del><ins>+ def __init__(self, dashboard, sockname):
+ self.dashboard = dashboard
</ins><span class="cx"> self.socket = None
</span><span class="cx"> if isinstance(sockname, str):
</span><span class="cx"> self.sockname = sockname[5:]
</span><span class="lines">@@ -391,6 +404,8 @@
</span><span class="cx">
</span><span class="cx"> # Only read each item once
</span><span class="cx"> self.currentData = self.readSock()
</span><ins>+ if self.dashboard.aggregate:
+ self.aggregateData()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def getOneItem(self, pod, server, item):
</span><span class="lines">@@ -409,7 +424,195 @@
</span><span class="cx"> return self.currentData["pods"][pod][server].get(item)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def aggregateData(self):
+ """
+ Aggregate the data from all hosts into one for each pod.
+ """
+ results = OrderedDict()
+ results["timestamp"] = self.currentData["timestamp"]
+ results["pods"] = OrderedDict()
+ for pod in self.currentData["pods"].keys():
+ results["pods"][pod] = OrderedDict()
+ results["pods"][pod]["aggregate"] = self.aggregatePodData(self.currentData["pods"][pod])
+ self.currentData = results
</ins><span class="cx">
</span><ins>+
+ def aggregatePodData(self, data):
+ """
+ Aggregate the data from a pod from all hosts into one.
+
+ @param data: host data to aggregate
+ @type data: L{dict}
+ """
+
+ results = OrderedDict()
+
+ # Get all items available in all servers first
+ items = collections.defaultdict(list)
+ for server in data.keys():
+ for item in data[server].keys():
+ items[item].append(data[server][item])
+
+ # Iterate each item to get the data sets and aggregate
+ for item, allHostData in items.items():
+ method_name = "aggregator_{}".format(item)
+ if not hasattr(Aggregator, method_name):
+ print("Missing aggregator for {}".format(item))
+ logging.error("Missing aggregator for {}".format(item))
+ results[item] = allHostData[0]
+ else:
+ results[item] = getattr(Aggregator, method_name)(allHostData)
+
+ return results
+
+
+
+class Aggregator(object):
+
+ @staticmethod
+ def aggregator_directory(serversdata):
+ results = OrderedDict()
+ for server_data in serversdata:
+ for operation_name, operation_details in server_data.items():
+ if operation_name not in results:
+ results[operation_name] = operation_details
+ elif isinstance(results[operation_name], list):
+ results[operation_name][0] += operation_details[0]
+ results[operation_name][1] += operation_details[1]
+ else:
+ results[operation_name] += operation_details
+
+ return results
+
+
+ @staticmethod
+ def aggregator_job_assignments(serversdata):
+
+ results = OrderedDict()
+ results["workers"] = [[0, 0, 0]] * len(serversdata[0]["workers"])
+ results["level"] = sum(map(itemgetter("level"), serversdata))
+
+ for server_data in serversdata:
+ for ctr, item in enumerate(server_data["workers"]):
+ for i in range(3):
+ results["workers"][ctr][i] += item[i]
+ return results
+
+
+ @staticmethod
+ def aggregator_jobcount(serversdata):
+ return serversdata[0]
+
+
+ @staticmethod
+ def aggregator_jobs(serversdata):
+# results = OrderedDict()
+# for server_data in serversdata:
+# for job_name, job_details in server_data.items():
+# if job_name not in results:
+# results[job_name] = OrderedDict()
+# for detail_name, detail_value in job_details.items():
+# if detail_name in results[job_name]:
+# results[job_name][detail_name] += detail_value
+# else:
+# results[job_name][detail_name] = detail_value
+# return results
+ return serversdata[0]
+
+
+ @staticmethod
+ def aggregator_stats(serversdata):
+ results = OrderedDict()
+ for stat in ("current", "1m", "5m", "1h"):
+ results[stat] = Aggregator.serverStat(map(itemgetter(stat), serversdata))
+
+ # NB ignore the "system" key as it is not used
+
+ return results
+
+
+ @staticmethod
+ def serverStat(serversdata):
+ results = OrderedDict()
+
+ # Values that are summed
+ for key in ("requests", "t", "t-resp-wr", "401", "500", "cpu", "slots", "max-slots",):
+ results[key] = sum(map(itemgetter(key), serversdata))
+
+ # Averaged
+ for key in ("cpu", "max-slots",):
+ results[key] = safeDivision(results[key], len(serversdata))
+
+ # Values that are maxed
+ for key in ("T-MAX",):
+ results[key] = max(map(itemgetter(key), serversdata))
+
+ # Values that are summed dict values
+ for key in ("method", "method-t", "uid", "user-agent", "T", "T-RESP-WR",):
+ results[key] = Aggregator.dictValueSums(map(itemgetter(key), serversdata))
+
+ return results
+
+
+ @staticmethod
+ def aggregator_slots(serversdata):
+ results = OrderedDict()
+
+ # Sum all items in each dict
+ slot_count = len(serversdata[0]["slots"])
+ results["slots"] = []
+ for i in range(slot_count):
+ results["slots"].append(Aggregator.dictValueSums(map(
+ itemgetter(i),
+ map(itemgetter("slots"), serversdata)
+ )))
+ results["slots"][i]["slot"] = i
+
+ # Check for any one being overloaded
+ results["overloaded"] = any(map(itemgetter("overloaded"), serversdata))
+
+ return results
+
+
+ @staticmethod
+ def aggregator_stats_system(serversdata):
+ results = OrderedDict()
+ for server_data in serversdata:
+ for stat_name, stat_value in server_data.items():
+ if stat_name not in results:
+ results[stat_name] = stat_value
+ else:
+ results[stat_name] += stat_value
+
+ # Some values should be averaged
+ for stat_name in ("memory used", "cpu use", "memory percent"):
+ results[stat_name] = safeDivision(results[stat_name], len(serversdata))
+
+ # Want the earliest time
+ results["start time"] = min(map(itemgetter("start time"), serversdata))
+
+ return results
+
+
+ @staticmethod
+ def dictValueSums(listOfDicts):
+ """
+ Sum the values of a list of dicts.
+
+ @param listOfDicts: list of dicts to sum
+ @type listOfDicts: L{list} of L{dict}
+ """
+ results = OrderedDict()
+ for result in listOfDicts:
+ for key, value in result.items():
+ if key not in results:
+ results[key] = value
+ else:
+ results[key] += value
+ return results
+
+
+
</ins><span class="cx"> class Point(object):
</span><span class="cx">
</span><span class="cx"> def __init__(self, x=0, y=0):
</span><span class="lines">@@ -632,7 +835,7 @@
</span><span class="cx"> if selected:
</span><span class="cx"> selected_server = server
</span><span class="cx">
</span><del>- self.window.addstr(pt.y, pt.x, " {}".format(selected_server))
</del><ins>+ self.window.addstr(pt.y, pt.x, " {}".format(selected_server if selected_server else ""))
</ins><span class="cx"> pt.yplus()
</span><span class="cx">
</span><span class="cx"> self.window.refresh()
</span><span class="lines">@@ -652,6 +855,7 @@
</span><span class="cx"> "",
</span><span class="cx"> " - (space) Pause",
</span><span class="cx"> " t - Toggle Update Speed",
</span><ins>+ " x - Toggle Aggregate Mode",
</ins><span class="cx"> "",
</span><span class="cx"> " q - Quit",
</span><span class="cx"> )
</span></span></pre>
</div>
</div>
</body>
</html>