[CalendarServer-changes] [15680] CalendarServer/trunk/calendarserver/tools/dashtime.py
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jun 17 06:51:36 PDT 2016
Revision: 15680
http://trac.calendarserver.org//changeset/15680
Author: cdaboo at apple.com
Date: 2016-06-17 06:51:36 -0700 (Fri, 17 Jun 2016)
Log Message:
-----------
Add per-line zlib decompress. Per-host plots are now scaled so that each host plot has the same y-axis range for easier comparison.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/dashtime.py
Modified: CalendarServer/trunk/calendarserver/tools/dashtime.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/dashtime.py 2016-06-17 00:26:21 UTC (rev 15679)
+++ CalendarServer/trunk/calendarserver/tools/dashtime.py 2016-06-17 13:51:36 UTC (rev 15680)
@@ -20,6 +20,7 @@
from bz2 import BZ2File
from collections import OrderedDict, defaultdict
+from zlib import decompress
import argparse
import json
import matplotlib.pyplot as plt
@@ -65,6 +66,13 @@
@staticmethod
+ def getMaxY(measurement, numHosts):
+ if "-" in measurement:
+ measurement = measurement.split("-", 1)[0]
+ return DataType.allTypes[measurement].maxY(numHosts)
+
+
+ @staticmethod
def skip(measurement):
if "-" in measurement:
measurement = measurement.split("-", 1)[0]
@@ -86,7 +94,7 @@
@staticmethod
- def maxY(stats, item):
+ def maxY(numHosts):
raise NotImplementedError
@@ -119,14 +127,12 @@
@staticmethod
- def maxY(stats, item, host):
- return 100 * len(stats) if host is None else 100
+ def maxY(numHosts):
+ return 100 * numHosts
@staticmethod
def calculate(stats, item, hosts):
- if hosts is None:
- hosts = stats.keys()
return sum([stats[onehost]["stats_system"]["cpu use"] for onehost in hosts])
@@ -145,14 +151,12 @@
@staticmethod
- def maxY(stats, item, host):
+ def maxY(numHosts):
return None
@staticmethod
def calculate(stats, item, hosts):
- if hosts is None:
- hosts = stats.keys()
return sum([stats[onehost]["stats"]["1m"]["requests"] for onehost in hosts]) / 60.0
@@ -171,14 +175,12 @@
@staticmethod
- def maxY(stats, item, host):
+ def maxY(numHosts):
return None
@staticmethod
def calculate(stats, item, hosts):
- if hosts is None:
- hosts = stats.keys()
tsum = sum([stats[onehost]["stats"]["1m"]["t"] for onehost in hosts])
rsum = sum([stats[onehost]["stats"]["1m"]["requests"] for onehost in hosts])
return safeDivision(tsum, rsum)
@@ -200,14 +202,12 @@
@staticmethod
- def maxY(stats, item, host):
+ def maxY(numHosts):
return None
@staticmethod
def calculate(stats, item, hosts):
- if hosts is None:
- hosts = stats.keys()
result = 0
for onehost in hosts:
completed = sum(map(operator.itemgetter(2), stats[onehost]["job_assignments"]["workers"]))
@@ -232,14 +232,12 @@
@staticmethod
- def maxY(stats, item, host):
+ def maxY(numHosts):
return None
@staticmethod
def calculate(stats, item, hosts):
- if hosts is None:
- hosts = stats.keys()
return sum([stats[onehost]["stats"]["1m"]["method"].get(item, 0) for onehost in hosts])
@@ -259,14 +257,12 @@
@staticmethod
- def maxY(stats, item, host):
+ def maxY(numHosts):
return None
@staticmethod
def calculate(stats, item, hosts):
- if hosts is None:
- hosts = stats.keys()
tsum = sum([stats[onehost]["stats"]["1m"]["method-t"].get(item, 0) for onehost in hosts])
rsum = sum([stats[onehost]["stats"]["1m"]["method"].get(item, 0) for onehost in hosts])
return safeDivision(tsum, rsum)
@@ -287,12 +283,13 @@
@staticmethod
- def maxY(stats, item, host):
+ def maxY(numHosts):
return None
@staticmethod
def calculate(stats, item, hosts):
+ # Job queue stat only read for first host
onehost = sorted(stats.keys())[0]
if item:
@@ -370,12 +367,6 @@
@type hosts: L{list} or L{str}
"""
- # Initialize the plot arrays
- for measurement in valuekeys:
- y[measurement] = []
- titles[measurement] = DataType.getTitle(measurement)
- ymaxes[measurement] = None
-
# For each log file line, process the data for each required measurement
with logfile:
line = logfile.readline()
@@ -388,11 +379,23 @@
if line[0] == "\x1e":
line = line[1:]
+ if line[0] != "{":
+ line = decompress(line.decode("base64"))
jline = json.loads(line)
x.append(ctr)
ctr += 1
+ # Initialize the plot arrays when we know how many hosts there are
+ if len(y) == 0:
+ if hosts is None:
+ hosts = sorted(jline["pods"][args.p].keys())
+ for measurement in valuekeys:
+ y[measurement] = []
+ titles[measurement] = DataType.getTitle(measurement)
+ ymaxes[measurement] = DataType.getMaxY(measurement, len(hosts))
+
+
for measurement in valuekeys:
stats = jline["pods"][args.p]
y[measurement].append(DataType.process(measurement, stats, hosts))
@@ -431,6 +434,8 @@
if line[0] == "\x1e":
line = line[1:]
+ if line[0] != "{":
+ line = decompress(line.decode("base64"))
jline = json.loads(line)
x.append(ctr)
@@ -445,12 +450,12 @@
ykey = "{}={}".format(measurement, host)
y[ykey] = []
titles[ykey] = DataType.getTitle(measurement)
- ymaxes[ykey] = None
+ ymaxes[ykey] = DataType.getMaxY(measurement, 1)
for measurement in combinedkeys:
y[measurement] = []
titles[measurement] = DataType.getTitle(measurement)
- ymaxes[measurement] = None
+ ymaxes[measurement] = DataType.getMaxY(measurement, len(hosts))
# Get actual measurement data
for host in hosts:
@@ -467,19 +472,29 @@
if ctr > line_start + line_count:
break
- # Offset data that is averaged over the previous minute
+ # Offset data that is averaged over the previous minute. Also determine
+ # the highest max value of all the per-host measurements and scale each
+ # per-host plot to the same range.
+ overall_ymax = 0
for host in hosts:
for measurement in perhostkeys:
+ ykey = "{}={}".format(measurement, host)
+ overall_ymax = max(overall_ymax, max(y[ykey]))
if DataType.skip(measurement):
- ykey = "{}={}".format(measurement, host)
y[ykey] = y[ykey][60:]
y[ykey].extend([None] * 60)
+ for host in hosts:
+ for measurement in perhostkeys:
+ ykey = "{}={}".format(measurement, host)
+ ymaxes[ykey] = overall_ymax
+
for measurement in combinedkeys:
if DataType.skip(measurement):
y[measurement] = y[measurement][60:]
y[measurement].extend([None] * 60)
+ # Data for a single host, with jobs queued detail for all hosts
# singleHost((
# CPUDataType.key,
# RequestsDataType.key,
@@ -489,6 +504,8 @@
# JobQueueDataType.key + "-PUSH",
# JobQueueDataType.key,
# ))
+
+ # Data aggregated for all hosts - job detail
# combinedHosts((
# CPUDataType.key,
# RequestsDataType.key,
@@ -498,22 +515,39 @@
# JobQueueDataType.key + "-PUSH",
# JobQueueDataType.key,
# ))
- combinedHosts((
- CPUDataType.key,
- RequestsDataType.key,
- ResponseDataType.key,
- MethodCountDataType.key + "-PUT ics",
- MethodCountDataType.key + "-REPORT cal-home-sync",
- MethodCountDataType.key + "-PROPFIND Calendar Home",
- MethodCountDataType.key + "-REPORT cal-sync",
- MethodCountDataType.key + "-PROPFIND Calendar",
- ))
+
+
+ # Data aggregated for all hosts - method detail
+# combinedHosts((
+# CPUDataType.key,
+# RequestsDataType.key,
+# ResponseDataType.key,
+# MethodCountDataType.key + "-PUT ics",
+# MethodCountDataType.key + "-REPORT cal-home-sync",
+# MethodCountDataType.key + "-PROPFIND Calendar Home",
+# MethodCountDataType.key + "-REPORT cal-sync",
+# MethodCountDataType.key + "-PROPFIND Calendar",
+# ))
+
+ # Per-host CPU, and total CPU
# perHost((
# RequestsDataType.key,
# ), (
# CPUDataType.key,
+# ))
+
+ # Per-host job completion, and total CPU, total jobs queued
+# singleHost((
+# CPUDataType.key,
+# JobsCompletedDataType.key,
# JobQueueDataType.key,
# ))
+ perHost((
+ JobsCompletedDataType.key,
+ ), (
+ CPUDataType.key,
+ JobQueueDataType.key,
+ ))
# Generate a single stacked plot of the data
for plotnum, measurement in enumerate(y.keys()):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160617/b5250f18/attachment-0001.html>
More information about the calendarserver-changes
mailing list