Revision: 1568 http://trac.macosforge.org/projects/calendarserver/changeset/1568 Author: cdaboo@apple.com Date: 2007-05-24 13:55:06 -0700 (Thu, 24 May 2007) Log Message: ----------- Add time-of-day analysis for requests. Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/admin/formatters.py CalendarServer/trunk/twistedcaldav/admin/logs.py Modified: CalendarServer/trunk/twistedcaldav/admin/formatters.py =================================================================== --- CalendarServer/trunk/twistedcaldav/admin/formatters.py 2007-05-24 16:55:04 UTC (rev 1567) +++ CalendarServer/trunk/twistedcaldav/admin/formatters.py 2007-05-24 20:55:06 UTC (rev 1568) @@ -119,6 +119,28 @@ values += tuple(value[f] for f in fields[1:]) self.writeLine(values) + def writeFrequencies(self, frequencies): + + width = len(frequencies) + plot = [[" "] * width for ignore in range(20)] + plot.append(["|---"] * 24) + plot.append(["%02d " % d for d in range(24)]) + + max_count = 0 + for freq in frequencies: + max_count = max(freq, max_count) + + for column, freq in enumerate(frequencies): + if freq == 0: + continue + scaled = (20 * freq) / max_count + for row in range(20): + if row <= scaled: + plot[19 - row][column] = "*" + + self.write("\n".join(["".join(p) for p in plot])) + self.write("\n") + def writeReport(self, report, name, fields, headings): if self.options.has_key('fields'): fields = self.options.get('fields', '').split(',') @@ -169,9 +191,11 @@ self.writeReport(report, 'Statistics', fields, headings) def report_logs(self, report): - self.write('Log Statistics:\n') + self.write('Log Statistics:\n\n') - self.write(' Bytes Out: %s\n' % (report['data']['bytesOut'],)) + self.write(' Start Date: %s\n End Date :%s\n\n' % report['data']['dateRange']) + + self.write(' Bytes Out: %s\n\n' % (report['data']['bytesOut'],)) self.write(' # Requests:\n') fields = ( @@ -203,7 +227,12 @@ 'maxtime': 'Max. time', } self.writeMap(report['data']['requestStats'], fields, headings) + self.write('\n') + self.write(' # Requests by time of day:\n') + self.writeFrequencies(report['data']['timeOfDayStats']) + self.write('\n') + self.write(' User Agents:\n') for ua, count in report['data']['userAgents'].iteritems(): Modified: CalendarServer/trunk/twistedcaldav/admin/logs.py =================================================================== --- CalendarServer/trunk/twistedcaldav/admin/logs.py 2007-05-24 16:55:04 UTC (rev 1567) +++ CalendarServer/trunk/twistedcaldav/admin/logs.py 2007-05-24 20:55:06 UTC (rev 1568) @@ -28,10 +28,16 @@ from twistedcaldav.admin import util +PLIST_VERSION = 1 + statsTemplate = plistlib.Dict( - bytesOut=0, + version=PLIST_VERSION, + bytesOut="0", + startDate="", + endDate="", requestStats=plistlib.Dict( - ), + ), + timeOfDayStats=[0] * 96, invitations=plistlib.Dict( day=0, week=0, @@ -43,18 +49,30 @@ class Stats(object): def __init__(self, fp): self.fp = fp + self._data = None if self.fp.exists(): self._data = plistlib.readPlist(self.fp.path) - else: + if self._data.version != PLIST_VERSION: + self._data = None + + if self._data is None: self._data = statsTemplate self.save() + def addDate(self, date): + if not self._data.startDate: + self._data.startDate = date + self._data.endDate = date + + def getDateRange(self): + return (self._data.startDate, self._data.endDate) + def getBytes(self): - return self._data.bytesOut + return long(self._data.bytesOut) def addBytes(self, bytes): - self._data.bytesOut += bytes + self._data.bytesOut = str(long(self._data.bytesOut) + bytes) def addRequestStats(self, request, status, bytes, time): if request in self._data.requestStats: @@ -103,6 +121,14 @@ def getRequestStats(self): return self._data.requestStats + def addTimeOfDayStats(self, request, time): + hour, minute = time.split(":") + bucket = int(hour) * 4 + divmod(int(minute), 15)[0] + self._data.timeOfDayStats[bucket] = self._data.timeOfDayStats[bucket] + 1 + + def getTimeOfDayStats(self): + return self._data.timeOfDayStats + def addUserAgent(self, useragent): if useragent in self._data.userAgents: self._data.userAgents[useragent] += 1 @@ -171,28 +197,45 @@ def run(self): if not self.readOnly: - for line in self.logfile.open(): + total_count = -1 + for total_count, line in enumerate(self.logfile.open()): + pass + total_count += 1 + print "Reading file: %s (%d lines)" % (self.logfile.basename(), total_count,) + print "|" + "--" * 48 + "|" + last_count = 0 + for line_count, line in enumerate(self.logfile.open()): if (line.startswith('Log opened') or line.startswith('Log closed')): continue else: pline = parseCLFLine(line) + self.stats.addDate(pline[3]) self.stats.addBytes(int(pline[6])) self.stats.addRequestStats(pline[4].split(' ')[0], int(pline[5]), int(pline[6]), float(pline[9][:-3])) + self.stats.addTimeOfDayStats(pline[4].split(' ')[0], pline[3][pline[3].find(":") + 1:][:5]) if len(pline) > 7: self.stats.addUserAgent(pline[8]) + + if (50 * line_count) / total_count > last_count: + print ".", + last_count = (50 * line_count) / total_count + print "\n\n" + self.stats.save() if not self.noOutput: report = { 'type': 'logs', 'data': { + 'dateRange': self.stats.getDateRange(), 'bytesOut': util.prepareByteValue(self.config, self.stats.getBytes()), 'requestStats': self.stats.getRequestStats(), + 'timeOfDayStats': self.stats.getTimeOfDayStats(), 'userAgents': self.stats.getUserAgents(), } }