[CalendarServer-changes] [11402] CalendarServer/trunk/contrib/performance/loadtest
source_changes at macosforge.org
source_changes at macosforge.org
Tue Jun 25 11:48:16 PDT 2013
Revision: 11402
http://trac.calendarserver.org//changeset/11402
Author: cdaboo at apple.com
Date: 2013-06-25 11:48:16 -0700 (Tue, 25 Jun 2013)
Log Message:
-----------
Add a quality of service score to the overall results. The benchmarks.json file needs to be adjusted for the test system to
indicate the expected responsiveness.
Modified Paths:
--------------
CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist
CalendarServer/trunk/contrib/performance/loadtest/config.plist
CalendarServer/trunk/contrib/performance/loadtest/population.py
Added Paths:
-----------
CalendarServer/trunk/contrib/performance/loadtest/benchmarks.json
Added: CalendarServer/trunk/contrib/performance/loadtest/benchmarks.json
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/benchmarks.json (rev 0)
+++ CalendarServer/trunk/contrib/performance/loadtest/benchmarks.json 2013-06-25 18:48:16 UTC (rev 11402)
@@ -0,0 +1,42 @@
+{
+ "GET{event}" : {"mean": 0.100, "weight": 1.000},
+
+ "PUT{event}" : {"mean": 0.250, "weight": 1.000},
+ "PUT{attendee-small}" : {"mean": 0.500, "weight": 1.000},
+ "PUT{attendee-medium}" : {"mean": 1.000, "weight": 0.900},
+ "PUT{attendee-large}" : {"mean": 1.500, "weight": 0.750},
+ "PUT{attendee-huge}" : {"mean": 2.000, "weight": 0.500},
+ "PUT{organizer-small}" : {"mean": 0.250, "weight": 1.000},
+ "PUT{organizer-medium}" : {"mean": 0.500, "weight": 0.900},
+ "PUT{organizer-large}" : {"mean": 1.000, "weight": 0.750},
+ "PUT{organizer-huge}" : {"mean": 3.000, "weight": 0.500},
+
+ "DELETE{event}" : {"mean": 0.500, "weight": 0.750},
+
+ "POST{fb-small}" : {"mean": 0.500, "weight": 1.000},
+ "POST{fb-medium}" : {"mean": 0.750, "weight": 1.000},
+ "POST{fb-large}" : {"mean": 1.000, "weight": 0.800},
+ "POST{fb-huge}" : {"mean": 2.000, "weight": 0.700},
+
+ "PROPFIND{well-known}" : {"mean": 0.100, "weight": 0.100},
+ "PROPFIND{find-principal}" : {"mean": 0.250, "weight": 0.100},
+ "PROPFIND{principal}" : {"mean": 0.250, "weight": 0.100},
+ "PROPFIND{home}" : {"mean": 0.250, "weight": 1.000},
+ "PROPFIND{calendar}" : {"mean": 0.250, "weight": 1.000},
+ "PROPFIND{notification}" : {"mean": 0.100, "weight": 0.500},
+ "PROPFIND{notification-items}" : {"mean": 0.100, "weight": 0.500},
+
+ "PROPPATCH{calendar}" : {"mean": 0.250, "weight": 0.500},
+
+ "REPORT{pset}" : {"mean": 0.250, "weight": 0.100},
+ "REPORT{expand}" : {"mean": 0.250, "weight": 1.000},
+ "REPORT{psearch}" : {"mean": 0.250, "weight": 0.100},
+ "REPORT{sync-init}" : {"mean": 0.250, "weight": 0.500},
+ "REPORT{sync}" : {"mean": 0.250, "weight": 1.000},
+ "REPORT{vevent}" : {"mean": 0.250, "weight": 1.000},
+ "REPORT{vtodo}" : {"mean": 0.250, "weight": 1.000},
+ "REPORT{multiget-small}" : {"mean": 0.250, "weight": 1.000},
+ "REPORT{multiget-medium}" : {"mean": 0.500, "weight": 0.900},
+ "REPORT{multiget-large}" : {"mean": 1.000, "weight": 0.750},
+ "REPORT{multiget-huge}" : {"mean": 2.000, "weight": 0.500}
+}
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2013-06-25 16:09:43 UTC (rev 11401)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.dist.plist 2013-06-25 18:48:16 UTC (rev 11402)
@@ -598,6 +598,10 @@
<key>thresholdsPath</key>
<string>contrib/performance/loadtest/thresholds.json</string>
+ <!-- The benchmarks for overall QoS -->
+ <key>benchmarksPath</key>
+ <string>contrib/performance/loadtest/benchmarks.json</string>
+
<!-- The % of failures that constitute a failed test -->
<key>failCutoff</key>
<real>1.0</real>
Modified: CalendarServer/trunk/contrib/performance/loadtest/config.plist
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/config.plist 2013-06-25 16:09:43 UTC (rev 11401)
+++ CalendarServer/trunk/contrib/performance/loadtest/config.plist 2013-06-25 18:48:16 UTC (rev 11402)
@@ -543,6 +543,10 @@
<key>thresholdsPath</key>
<string>contrib/performance/loadtest/thresholds.json</string>
+ <!-- The benchmarks for overall QoS -->
+ <key>benchmarksPath</key>
+ <string>contrib/performance/loadtest/benchmarks.json</string>
+
<!-- The % of failures that constitute a failed test -->
<key>failCutoff</key>
<real>1.0</real>
Modified: CalendarServer/trunk/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2013-06-25 16:09:43 UTC (rev 11401)
+++ CalendarServer/trunk/contrib/performance/loadtest/population.py 2013-06-25 18:48:16 UTC (rev 11402)
@@ -385,6 +385,7 @@
('mean', 8, '%8.4f'),
('median', 8, '%8.4f'),
('stddev', 8, '%8.4f'),
+ ('QoS', 8, '%8.4f'),
('STATUS', 8, '%8s'),
]
@@ -413,6 +414,11 @@
self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
self._fields.extend(self._fields_extend)
+ if "benchmarksPath" in params:
+ self.benchmarks = json.load(open(params["benchmarksPath"]))
+ else:
+ self.benchmarks = {}
+
if "failCutoff" in params:
self._fail_cut_off = params["failCutoff"]
@@ -455,6 +461,44 @@
output.write(fmt % (k.title(), items[k],))
+ def qos(self):
+ """
+ Determine a "quality of service" value that can be used for comparisons between runs. This value
+ is based on the percentage deviation of means of each request from a set of "benchmarks" for each
+ type of request.
+ """
+
+ # Get means for each type of method
+ means = {}
+ for method, results in self._perMethodTimes.items():
+ means[method] = mean([duration for success, duration in results if success])
+
+ # Determine percentage differences with weighting
+ differences = []
+ for method, value in means.items():
+ result = self.qos_value(method, value)
+ if result is not None:
+ differences.append(result)
+
+ return mean(differences) if differences else "None"
+
+
+ def qos_value(self, method, value):
+ benchmark = self.benchmarks.get(method)
+ if benchmark is None:
+ return None
+ test_mean, weight = (benchmark["mean"], benchmark["weight"],)
+ return ((value / test_mean) - 1.0) * weight + 1.0
+
+
+ def _summarizeData(self, operation, data):
+ data = SummarizingMixin._summarizeData(self, operation, data)
+ value = self.qos_value(operation, data[-4])
+ if value is None:
+ value = 0.0
+ return data[:-1] + (value,) + data[-1:]
+
+
def report(self, output):
output.write("\n")
output.write("** REPORT **\n")
@@ -473,7 +517,8 @@
'Clients': self.countClients(),
'Start time': self._startTime.strftime('%m/%d %H:%M:%S'),
'Run time': "%02d:%02d:%02d" % (runHours, runMinutes, runSeconds),
- 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,)
+ 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,),
+ 'QoS': "%-8.4f" % (self.qos(),),
}
if self.countClientFailures() > 0:
items['Failed clients'] = self.countClientFailures()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130625/f53f8bcf/attachment-0001.html>
More information about the calendarserver-changes
mailing list