[CalendarServer-changes] [15680] CalendarServer/trunk/calendarserver/tools/dashtime.py

source_changes at macosforge.org source_changes at macosforge.org
Fri Jun 17 06:51:36 PDT 2016


Revision: 15680
          http://trac.calendarserver.org//changeset/15680
Author:   cdaboo at apple.com
Date:     2016-06-17 06:51:36 -0700 (Fri, 17 Jun 2016)
Log Message:
-----------
Add per-line zlib decompress. Per-host plots are now scaled so that each host plot has the same y-axis range for easier comparison.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tools/dashtime.py

Modified: CalendarServer/trunk/calendarserver/tools/dashtime.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/dashtime.py	2016-06-17 00:26:21 UTC (rev 15679)
+++ CalendarServer/trunk/calendarserver/tools/dashtime.py	2016-06-17 13:51:36 UTC (rev 15680)
@@ -20,6 +20,7 @@
 
 from bz2 import BZ2File
 from collections import OrderedDict, defaultdict
+from zlib import decompress
 import argparse
 import json
 import matplotlib.pyplot as plt
@@ -65,6 +66,13 @@
 
 
     @staticmethod
+    def getMaxY(measurement, numHosts):
+        if "-" in measurement:
+            measurement = measurement.split("-", 1)[0]
+        return DataType.allTypes[measurement].maxY(numHosts)
+
+
+    @staticmethod
     def skip(measurement):
         if "-" in measurement:
             measurement = measurement.split("-", 1)[0]
@@ -86,7 +94,7 @@
 
 
     @staticmethod
-    def maxY(stats, item):
+    def maxY(numHosts):
         raise NotImplementedError
 
 
@@ -119,14 +127,12 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
-        return 100 * len(stats) if host is None else 100
+    def maxY(numHosts):
+        return 100 * numHosts
 
 
     @staticmethod
     def calculate(stats, item, hosts):
-        if hosts is None:
-            hosts = stats.keys()
         return sum([stats[onehost]["stats_system"]["cpu use"] for onehost in hosts])
 
 
@@ -145,14 +151,12 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
+    def maxY(numHosts):
         return None
 
 
     @staticmethod
     def calculate(stats, item, hosts):
-        if hosts is None:
-            hosts = stats.keys()
         return sum([stats[onehost]["stats"]["1m"]["requests"] for onehost in hosts]) / 60.0
 
 
@@ -171,14 +175,12 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
+    def maxY(numHosts):
         return None
 
 
     @staticmethod
     def calculate(stats, item, hosts):
-        if hosts is None:
-            hosts = stats.keys()
         tsum = sum([stats[onehost]["stats"]["1m"]["t"] for onehost in hosts])
         rsum = sum([stats[onehost]["stats"]["1m"]["requests"] for onehost in hosts])
         return safeDivision(tsum, rsum)
@@ -200,14 +202,12 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
+    def maxY(numHosts):
         return None
 
 
     @staticmethod
     def calculate(stats, item, hosts):
-        if hosts is None:
-            hosts = stats.keys()
         result = 0
         for onehost in hosts:
             completed = sum(map(operator.itemgetter(2), stats[onehost]["job_assignments"]["workers"]))
@@ -232,14 +232,12 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
+    def maxY(numHosts):
         return None
 
 
     @staticmethod
     def calculate(stats, item, hosts):
-        if hosts is None:
-            hosts = stats.keys()
         return sum([stats[onehost]["stats"]["1m"]["method"].get(item, 0) for onehost in hosts])
 
 
@@ -259,14 +257,12 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
+    def maxY(numHosts):
         return None
 
 
     @staticmethod
     def calculate(stats, item, hosts):
-        if hosts is None:
-            hosts = stats.keys()
         tsum = sum([stats[onehost]["stats"]["1m"]["method-t"].get(item, 0) for onehost in hosts])
         rsum = sum([stats[onehost]["stats"]["1m"]["method"].get(item, 0) for onehost in hosts])
         return safeDivision(tsum, rsum)
@@ -287,12 +283,13 @@
 
 
     @staticmethod
-    def maxY(stats, item, host):
+    def maxY(numHosts):
         return None
 
 
     @staticmethod
     def calculate(stats, item, hosts):
+        # Job queue stat only read for first host
         onehost = sorted(stats.keys())[0]
 
         if item:
@@ -370,12 +367,6 @@
         @type hosts: L{list} or L{str}
         """
 
-        # Initialize the plot arrays
-        for measurement in valuekeys:
-            y[measurement] = []
-            titles[measurement] = DataType.getTitle(measurement)
-            ymaxes[measurement] = None
-
         # For each log file line, process the data for each required measurement
         with logfile:
             line = logfile.readline()
@@ -388,11 +379,23 @@
 
                 if line[0] == "\x1e":
                     line = line[1:]
+                if line[0] != "{":
+                    line = decompress(line.decode("base64"))
                 jline = json.loads(line)
 
                 x.append(ctr)
                 ctr += 1
 
+                # Initialize the plot arrays when we know how many hosts there are
+                if len(y) == 0:
+                    if hosts is None:
+                        hosts = sorted(jline["pods"][args.p].keys())
+                    for measurement in valuekeys:
+                        y[measurement] = []
+                        titles[measurement] = DataType.getTitle(measurement)
+                        ymaxes[measurement] = DataType.getMaxY(measurement, len(hosts))
+
+
                 for measurement in valuekeys:
                     stats = jline["pods"][args.p]
                     y[measurement].append(DataType.process(measurement, stats, hosts))
@@ -431,6 +434,8 @@
 
                 if line[0] == "\x1e":
                     line = line[1:]
+                if line[0] != "{":
+                    line = decompress(line.decode("base64"))
                 jline = json.loads(line)
 
                 x.append(ctr)
@@ -445,12 +450,12 @@
                             ykey = "{}={}".format(measurement, host)
                             y[ykey] = []
                             titles[ykey] = DataType.getTitle(measurement)
-                            ymaxes[ykey] = None
+                            ymaxes[ykey] = DataType.getMaxY(measurement, 1)
 
                     for measurement in combinedkeys:
                         y[measurement] = []
                         titles[measurement] = DataType.getTitle(measurement)
-                        ymaxes[measurement] = None
+                        ymaxes[measurement] = DataType.getMaxY(measurement, len(hosts))
 
                 # Get actual measurement data
                 for host in hosts:
@@ -467,19 +472,29 @@
                 if ctr > line_start + line_count:
                     break
 
-        # Offset data that is averaged over the previous minute
+        # Offset data that is averaged over the previous minute. Also determine
+        # the highest max value of all the per-host measurements and scale each
+        # per-host plot to the same range.
+        overall_ymax = 0
         for host in hosts:
             for measurement in perhostkeys:
+                ykey = "{}={}".format(measurement, host)
+                overall_ymax = max(overall_ymax, max(y[ykey]))
                 if DataType.skip(measurement):
-                    ykey = "{}={}".format(measurement, host)
                     y[ykey] = y[ykey][60:]
                     y[ykey].extend([None] * 60)
+        for host in hosts:
+            for measurement in perhostkeys:
+                ykey = "{}={}".format(measurement, host)
+                ymaxes[ykey] = overall_ymax
+
         for measurement in combinedkeys:
             if DataType.skip(measurement):
                 y[measurement] = y[measurement][60:]
                 y[measurement].extend([None] * 60)
 
 
+    # Data for a single host, with jobs queued detail for all hosts
 #    singleHost((
 #        CPUDataType.key,
 #        RequestsDataType.key,
@@ -489,6 +504,8 @@
 #        JobQueueDataType.key + "-PUSH",
 #        JobQueueDataType.key,
 #    ))
+
+    # Data aggregated for all hosts - job detail
 #    combinedHosts((
 #        CPUDataType.key,
 #        RequestsDataType.key,
@@ -498,22 +515,39 @@
 #        JobQueueDataType.key + "-PUSH",
 #        JobQueueDataType.key,
 #    ))
-    combinedHosts((
-        CPUDataType.key,
-        RequestsDataType.key,
-        ResponseDataType.key,
-        MethodCountDataType.key + "-PUT ics",
-        MethodCountDataType.key + "-REPORT cal-home-sync",
-        MethodCountDataType.key + "-PROPFIND Calendar Home",
-        MethodCountDataType.key + "-REPORT cal-sync",
-        MethodCountDataType.key + "-PROPFIND Calendar",
-    ))
+
+
+    # Data aggregated for all hosts - method detail
+#    combinedHosts((
+#        CPUDataType.key,
+#        RequestsDataType.key,
+#        ResponseDataType.key,
+#        MethodCountDataType.key + "-PUT ics",
+#        MethodCountDataType.key + "-REPORT cal-home-sync",
+#        MethodCountDataType.key + "-PROPFIND Calendar Home",
+#        MethodCountDataType.key + "-REPORT cal-sync",
+#        MethodCountDataType.key + "-PROPFIND Calendar",
+#    ))
+
+    # Per-host CPU, and total CPU
 #    perHost((
 #        RequestsDataType.key,
 #    ), (
 #        CPUDataType.key,
+#    ))
+
+    # Per-host job completion, and total CPU, total jobs queued
+#    singleHost((
+#        CPUDataType.key,
+#        JobsCompletedDataType.key,
 #        JobQueueDataType.key,
 #    ))
+    perHost((
+        JobsCompletedDataType.key,
+    ), (
+        CPUDataType.key,
+        JobQueueDataType.key,
+    ))
 
     # Generate a single stacked plot of the data
     for plotnum, measurement in enumerate(y.keys()):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160617/b5250f18/attachment-0001.html>


More information about the calendarserver-changes mailing list