[CalendarServer-changes] [5838] CalendarServer/trunk/contrib/tools
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jul 2 09:35:57 PDT 2010
Revision: 5838
http://trac.macosforge.org/projects/calendarserver/changeset/5838
Author: cdaboo at apple.com
Date: 2010-07-02 09:35:56 -0700 (Fri, 02 Jul 2010)
Log Message:
-----------
More improvements to the monitor script analysis tool. Also added a tool to split monitor
output into weekly intervals.
Modified Paths:
--------------
CalendarServer/trunk/contrib/tools/monitoranalysis.py
Added Paths:
-----------
CalendarServer/trunk/contrib/tools/monitorsplit.py
Modified: CalendarServer/trunk/contrib/tools/monitoranalysis.py
===================================================================
--- CalendarServer/trunk/contrib/tools/monitoranalysis.py 2010-07-02 05:15:53 UTC (rev 5837)
+++ CalendarServer/trunk/contrib/tools/monitoranalysis.py 2010-07-02 16:35:56 UTC (rev 5838)
@@ -19,13 +19,17 @@
import getopt
import sys
import os
+import datetime
dataset = []
+initialDate = None
-def analyze(fpath, startDate, endDate, title=None):
+def analyze(fpath, noweekends, startDate=None, endDate=None, title=None):
print "Analyzing data for %s" % (fpath,)
data = []
+ firstDate = None
+ global initialDate
with open(fpath) as f:
for line in f:
try:
@@ -34,13 +38,24 @@
date = line[:10]
if startDate and date < startDate or endDate and date > endDate:
continue
+
+ if noweekends:
+ dt = datetime.date(int(date[0:4]), int(date[5:7]), int(date[8:10]))
+ if dt.weekday() > 4:
+ continue
digits = line[11:13]
if digits in ("05", "06"):
for _ignore in range(3):
f.next()
continue
- datetime = line[:19]
+ dtstamp = line[:19]
+
+ if firstDate is None:
+ firstDate = date.replace("/", "")
+ if initialDate is None:
+ initialDate = firstDate
+
if "Listenq" in line:
lqnon = line[len("2010/05/12 22:27:24 Listenq (ssl+non): "):].split("+", 1)[1]
else:
@@ -59,8 +74,8 @@
resp = int(float(resp)/10.0) * 10
if reqs <= 80:
- data.append((datetime, reqs, resp, lqnon, cpu))
- #print "%s %d %d %d %d" % (datetime, reqs, resp, lqnon, cpu)
+ data.append((dtstamp, reqs, resp, lqnon, cpu))
+ #print "%s %d %d %d %d" % (dtstamp, reqs, resp, lqnon, cpu)
except StopIteration:
break
@@ -72,13 +87,13 @@
elif endDate:
title = "Up to %s" % (endDate,)
else:
- title = None
+ title = "Start at %s" % (firstDate,)
dataset.append((title, data,))
print "Stored %d data points" % (len(data),)
-def plotListenQBands(data, first, last):
+def plotListenQBands(data, first, last, xlim, ylim):
x1 = []
y1 = []
@@ -105,10 +120,10 @@
if last:
plt.xlabel("Requests/second")
plt.ylabel("Av. Response Time (ms)")
- plt.xlim(0, 80)
- plt.ylim(0, 4000)
+ plt.xlim(0, xlim)
+ plt.ylim(0, ylim)
-def plotCPUBands(data, first, last):
+def plotCPUBands(data, first, last, xlim, ylim):
x = [[], [], [], []]
y = [[], [], [], []]
@@ -139,14 +154,14 @@
if last:
plt.xlabel("Requests/second")
plt.ylabel("Av. Response Time (ms)")
- plt.xlim(0, 80)
- plt.ylim(0, 4000)
+ plt.xlim(0, xlim)
+ plt.ylim(0, ylim)
-def plot():
+def plot(figure, noshow, nosave, pngDir, xlim, ylim):
print "Plotting data"
- plt.figure(1)
+ plt.figure(figure, figsize=(16, 5 * len(dataset)))
nplots = len(dataset)
subplot = nplots*100 + 20
@@ -158,14 +173,12 @@
title = "#%d" % (ctr+1,)
plt.subplot(subplot + 2*ctr + 1)
- plotListenQBands(data, first=(ctr == 0), last=(ctr+1 == len(dataset)))
+ plotListenQBands(data, first=(ctr == 0), last=(ctr+1 == len(dataset)), xlim=xlim, ylim=ylim)
plt.title("ListenQ %s" % (title,))
plt.subplot(subplot + 2*ctr + 2)
- plotCPUBands(data, first=(ctr == 0), last=(ctr+1 == len(dataset)))
+ plotCPUBands(data, first=(ctr == 0), last=(ctr+1 == len(dataset)), xlim=xlim, ylim=ylim)
plt.title("CPU %s" % (title,))
-
- plt.show()
def argPath(path):
fpath = os.path.expanduser(path)
@@ -180,9 +193,16 @@
if error_msg:
print error_msg
- print """Usage: monitoranalysis [options] FILE+
+ print """Usage: monitoranalysis [options] [FILE+]
Options:
- -h Print this help and exit
+ -h Print this help and exit
+ -d Directory to save PNGs to
+ -s Directory to scan for data instead of FILEs
+ --no-weekends Ignore data for Saturday and Sunday
+ --no-show Do not show plots on screen
+ --no-save Do not save plots to file
+ --xlim x-axis limit [80]
+ --ylim y-axim limit [4000]
Arguments:
FILE File names for the requests.log to analyze. A date
@@ -203,42 +223,95 @@
if __name__ == "__main__":
- options, args = getopt.getopt(sys.argv[1:], "h", [])
+ pngDir = None
+ scanDir = None
+ noweekends = False
+ noshow = False
+ nosave = False
+ xlim = 80
+ ylim = 4000
+ options, args = getopt.getopt(sys.argv[1:], "hd:s:", ["no-weekends", "no-show", "no-save", "xlim=", "ylim="])
for option, value in options:
if option == "-h":
usage()
+ elif option == "-d":
+ pngDir = os.path.expanduser(value)
+ elif option == "-s":
+ scanDir = os.path.expanduser(value)
+ elif option == "--no-show":
+ noshow = True
+ elif option == "--no-save":
+ nosave = True
+ elif option == "--no-weekends":
+ noweekends = True
+ elif option == "--xlim":
+ xlim = int(value)
+ elif option == "--ylim":
+ ylim = int(value)
else:
usage("Unrecognized option: %s" % (option,))
+ if pngDir is None and scanDir:
+ pngDir = scanDir
+
+ if not nosave and not os.path.isdir(pngDir):
+ usage("Must have a valid -d path for saving images")
+
# Process arguments
- if len(args) == 0:
+ if len(args) == 0 and scanDir is None:
usage("Must have arguments")
+ elif scanDir and len(args) != 0:
+ usage("No arguments allowed when scanning a directory")
pwd = os.getcwd()
- for arg in args:
- if "," in arg:
- items = arg.split(",")
- arg = items[0]
- start = []
- end = []
- for daterange in items[1:]:
- splits = daterange.split("-")
- if len(splits) == 1:
- start.append(expandDate(splits[0]))
- end.append(None)
- elif len(splits) == 2:
- start.append(expandDate(splits[0]))
- end.append(expandDate(splits[1]))
- else:
- start.append(None)
- end.append(None)
- else:
- start = (None,)
- end = (None,)
-
- for i in range(len(start)):
- analyze(argPath(arg), start[i], end[i])
-
- plot()
+ if scanDir:
+ fnames = os.listdir(scanDir)
+ count = 1
+ for name in fnames:
+ if name.startswith("request.log"):
+ print "Found file: %s" % (os.path.join(scanDir, name),)
+ trailer = name[len("request.log"):]
+ if trailer.startswith("."):
+ trailer = trailer[1:]
+ initialDate = None
+ dataset = []
+ analyze(os.path.join(scanDir, name), noweekends)
+ plot(count, noshow, nosave, pngDir, xlim, ylim)
+ if not nosave:
+ plt.savefig(os.path.expanduser(os.path.join(pngDir, "Monitor-%s" % (trailer,))))
+ count += 1
+
+ if not noshow:
+ plt.show()
+ else:
+ for arg in args:
+ if "," in arg:
+ items = arg.split(",")
+ arg = items[0]
+ start = []
+ end = []
+ for daterange in items[1:]:
+ splits = daterange.split("-")
+ if len(splits) == 1:
+ start.append(expandDate(splits[0]))
+ end.append(None)
+ elif len(splits) == 2:
+ start.append(expandDate(splits[0]))
+ end.append(expandDate(splits[1]))
+ else:
+ start.append(None)
+ end.append(None)
+ else:
+ start = (None,)
+ end = (None,)
+
+ for i in range(len(start)):
+ analyze(argPath(arg), noweekends, start[i], end[i])
+
+ plot(1, noshow, nosave, pngDir, xlim, ylim)
+ if not nosave:
+ plt.savefig(os.path.expanduser(os.path.join(pngDir, "Monitor-%s" % (initialDate,))))
+ if not noshow:
+ plt.show()
Added: CalendarServer/trunk/contrib/tools/monitorsplit.py
===================================================================
--- CalendarServer/trunk/contrib/tools/monitorsplit.py (rev 0)
+++ CalendarServer/trunk/contrib/tools/monitorsplit.py 2010-07-02 16:35:56 UTC (rev 5838)
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import getopt
+import sys
+import os
+from gzip import GzipFile
+import datetime
+
+outputFile = None
+fileCount = 0
+lastWeek = None
+
+def split(fpath, outputDir):
+
+ global outputFile, fileCount, lastWeek
+
+ print "Splitting data for %s" % (fpath,)
+ f = GzipFile(fpath) if fpath.endswith(".gz") else open(fpath)
+ for line in f:
+ if line.startswith("2010/0"):
+ date = line[:10]
+ date = date.replace("/", "")
+ hours = line[11:13]
+
+ dt = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8]))
+
+ currentWeek = dt.isocalendar()[1]
+ if dt.weekday() == 0 and hours <= "06":
+ currentWeek -= 1
+ if lastWeek != currentWeek:
+ if outputFile:
+ outputFile.close()
+ outputFile = open(os.path.join(outputDir, "request.log.%s" % (date,)), "w")
+ fileCount += 1
+ lastWeek = currentWeek
+ print "Changed to week of %s" % (date,)
+
+ output = ["-----\n"]
+ output.append(line)
+ try:
+ output.append(f.next())
+ output.append(f.next())
+ output.append(f.next())
+ except StopIteration:
+ break
+ outputFile.write("".join(output))
+ f.close()
+
+def argPath(path):
+ fpath = os.path.expanduser(path)
+ if not fpath.startswith("/"):
+ fpath = os.path.join(pwd, fpath)
+ return fpath
+
+def expandDate(date):
+ return "%s/%s/%s" % (date[0:4], date[4:6], date[6:8],)
+
+def usage(error_msg=None):
+ if error_msg:
+ print error_msg
+
+ print """Usage: monitoranalysis [options] FILE+
+Options:
+ -h Print this help and exit
+ -d Directory to store split files in
+
+Arguments:
+ FILE File names for the requests.log to analyze. A date
+ range can be specified by append a comma, then a
+ dash seperated pair of YYYYMMDD dates, e.g.:
+ ~/request.log,20100614-20100619. Multiple
+ ranges can be specified for multiple plots.
+
+Description:
+This utility will analyze the output of the request monitor tool and
+generate some pretty plots of data.
+"""
+
+ if error_msg:
+ raise ValueError(error_msg)
+ else:
+ sys.exit(0)
+
+if __name__ == "__main__":
+
+ outputDir = None
+
+ options, args = getopt.getopt(sys.argv[1:], "hd:", [])
+
+ for option, value in options:
+ if option == "-h":
+ usage()
+ elif option == "-d":
+ outputDir = argPath(value)
+ else:
+ usage("Unrecognized option: %s" % (option,))
+
+ if not outputDir or not os.path.isdir(outputDir):
+ usage("Must specify a valid output directory.")
+
+ # Process arguments
+ if len(args) == 0:
+ usage("Must have arguments")
+
+ pwd = os.getcwd()
+
+ for arg in args:
+ split(argPath(arg), outputDir)
+
+ print "Created %d files" % (fileCount,)
Property changes on: CalendarServer/trunk/contrib/tools/monitorsplit.py
___________________________________________________________________
Added: svn:executable
+ *
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100702/70bb7bf6/attachment-0001.html>
More information about the calendarserver-changes
mailing list