[CalendarServer-changes] [12892] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Thu Mar 13 11:09:35 PDT 2014
Revision: 12892
http://trac.calendarserver.org//changeset/12892
Author: cdaboo at apple.com
Date: 2014-03-13 11:09:35 -0700 (Thu, 13 Mar 2014)
Log Message:
-----------
Rework stats socket into a bi-directional request/response protocol. Turn the job queue viewer into a more generic
"dashboard" tool that can make use of the new protocol. Add master/child slots as another type of data to view in
the dashboard.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/accesslog.py
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/contrib/tools/readStats.py
CalendarServer/trunk/txweb2/metafd.py
Added Paths:
-----------
CalendarServer/trunk/bin/calendarserver_dashboard
CalendarServer/trunk/calendarserver/dashboard_service.py
CalendarServer/trunk/calendarserver/tools/dashboard.py
Removed Paths:
-------------
CalendarServer/trunk/bin/calendarserver_monitor_jobs
CalendarServer/trunk/calendarserver/tools/jobitems.py
Added: CalendarServer/trunk/bin/calendarserver_dashboard
===================================================================
--- CalendarServer/trunk/bin/calendarserver_dashboard (rev 0)
+++ CalendarServer/trunk/bin/calendarserver_dashboard 2014-03-13 18:09:35 UTC (rev 12892)
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2006-2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+# This tool can run standalone, so does not need any special PYTHONPATH or preamble handling
+
+if __name__ == "__main__":
+ from calendarserver.tools.dashboard import main
+ main()
Property changes on: CalendarServer/trunk/bin/calendarserver_dashboard
___________________________________________________________________
Added: svn:executable
+ *
Deleted: CalendarServer/trunk/bin/calendarserver_monitor_jobs
===================================================================
--- CalendarServer/trunk/bin/calendarserver_monitor_jobs 2014-03-13 00:51:31 UTC (rev 12891)
+++ CalendarServer/trunk/bin/calendarserver_monitor_jobs 2014-03-13 18:09:35 UTC (rev 12892)
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-##
-# Copyright (c) 2006-2014 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-import os
-import sys
-
-# In OS X Server context, add to PATH to find Postgres utilities (initdb, pg_ctl)
-if "Server.app" in sys.argv[0]:
- os.environ["PATH"] += ":" + os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), "bin")
-
-#PYTHONPATH
-
-if __name__ == "__main__":
- if "PYTHONPATH" in globals():
- sys.path.insert(0, PYTHONPATH)
- else:
- try:
- import _calendarserver_preamble
- except ImportError:
- sys.exc_clear()
-
- from calendarserver.tools.jobitems import main
- main()
Modified: CalendarServer/trunk/calendarserver/accesslog.py
===================================================================
--- CalendarServer/trunk/calendarserver/accesslog.py 2014-03-13 00:51:31 UTC (rev 12891)
+++ CalendarServer/trunk/calendarserver/accesslog.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -428,13 +428,13 @@
self.mergeStats(oneHour, stat)
printStats = {
- "System": self.systemStats.items,
- "Current": currentStats,
- "1 Minute": previousMinute,
- "5 Minutes": fiveMinutes,
- "1 Hour": oneHour,
+ "system": self.systemStats.items,
+ "current": currentStats,
+ "1m": previousMinute,
+ "5m": fiveMinutes,
+ "1h": oneHour,
}
- return json.dumps(printStats)
+ return printStats
def ensureSequentialStats(self):
Added: CalendarServer/trunk/calendarserver/dashboard_service.py
===================================================================
--- CalendarServer/trunk/calendarserver/dashboard_service.py (rev 0)
+++ CalendarServer/trunk/calendarserver/dashboard_service.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -0,0 +1,144 @@
+##
+# Copyright (c) 2012-2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.jobqueue import JobItem
+
+from twisted.internet.defer import inlineCallbacks, succeed, returnValue
+from twisted.internet.protocol import Factory
+from twisted.protocols.basic import LineReceiver
+
+import json
+
+"""
+Protocol and other items that enable introspection of the server. This will include
+server protocol analysis statistics, job queue load, and other useful information that
+a server admin or developer would like to keep an eye on.
+"""
+
+class DashboardProtocol (LineReceiver):
+ """
+ A protocol that receives a line containing a JSON object representing a request,
+ and returns a line containing a JSON object as a response.
+ """
+
+ unknown_cmd = json.dumps({"result": "unknown command"})
+ bad_cmd = json.dumps({"result": "bad command"})
+
+ def lineReceived(self, line):
+ """
+ Process a request which is expected to be a JSON object.
+
+ @param line: The line which was received with the delimiter removed.
+ @type line: C{bytes}
+ """
+
+ # Look for exit
+ if line in ("exit", "quit"):
+ self.sendLine("Done")
+ self.transport.loseConnection()
+ return
+
+ def _write(result):
+ self.sendLine(result)
+
+ try:
+ j = json.loads(line)
+ if isinstance(j, list):
+ self.process_data(j)
+ except (ValueError, KeyError):
+ _write(self.bad_cmd)
+
+
+ @inlineCallbacks
+ def process_data(self, j):
+ results = {}
+ for data in j:
+ if hasattr(self, "data_{}".format(data)):
+ result = yield getattr(self, "data_{}".format(data))()
+ elif data.startswith("stats_"):
+ result = yield self.data_stats()
+ result = result.get(data[6:], "")
+ else:
+ result = ""
+ results[data] = result
+
+ self.sendLine(json.dumps(results))
+
+
+ def data_stats(self):
+ """
+ Return the logging protocol statistics.
+
+ @return: a string containing the JSON result.
+ @rtype: L{str}
+ """
+ return succeed(self.factory.logger.observer.getStats())
+
+
+ def data_slots(self):
+ """
+ Return the logging protocol statistics.
+
+ @return: a string containing the JSON result.
+ @rtype: L{str}
+ """
+ if self.factory.limiter is None:
+ raise ValueError()
+ states = tuple(self.factory.limiter.dispatcher.slavestates)
+ results = []
+ for num, status in states:
+ result = {"slot": num}
+ result.update(status.items())
+ results.append(result)
+ return succeed({"slots": results, "overloaded": self.factory.limiter.overloaded})
+
+
+ def data_jobcount(self):
+ """
+ Return a count of job types.
+
+ @return: the JSON result.
+ @rtype: L{int}
+ """
+
+ return succeed(JobItem.numberOfWorkTypes())
+
+
+ @inlineCallbacks
+ def data_jobs(self):
+ """
+ Return a summary of the job queue.
+
+ @return: a string containing the JSON result.
+ @rtype: L{str}
+ """
+
+ txn = self.factory.store.newTransaction()
+ records = (yield JobItem.histogram(txn))
+ yield txn.commit()
+
+ returnValue(records)
+
+
+
+class DashboardServer(Factory):
+
+ protocol = DashboardProtocol
+
+ def __init__(self, logObserver, limiter):
+ self.logger = logObserver
+ self.limiter = limiter
+ self.store = None
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2014-03-13 00:51:31 UTC (rev 12891)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -46,7 +46,7 @@
from twisted.internet.endpoints import UNIXClientEndpoint, TCP4ClientEndpoint
from twisted.internet.process import ProcessExitedAlready
from twisted.internet.protocol import ProcessProtocol
-from twisted.internet.protocol import Protocol, Factory
+from twisted.internet.protocol import Factory
from twisted.plugin import IPlugin
from twisted.protocols.amp import AMP
from twisted.python.log import FileLogObserver, ILogObserver
@@ -110,6 +110,7 @@
from calendarserver.accesslog import RotatingFileAccessLoggingObserver
from calendarserver.controlsocket import ControlSocket
from calendarserver.controlsocket import ControlSocketConnectingService
+from calendarserver.dashboard_service import DashboardServer
from calendarserver.push.amppush import AMPPushMaster, AMPPushForwarder
from calendarserver.push.applepush import ApplePushNotifierService
from calendarserver.push.notifier import PushDistributor
@@ -216,24 +217,6 @@
-class CalDAVStatisticsProtocol (Protocol):
-
- def connectionMade(self):
- stats = self.factory.logger.observer.getStats()
- self.transport.write("{}\r\n".format(stats))
- self.transport.loseConnection()
-
-
-
-class CalDAVStatisticsServer (Factory):
-
- protocol = CalDAVStatisticsProtocol
-
- def __init__(self, logObserver):
- self.logger = logObserver
-
-
-
class ErrorLoggingMultiService(MultiService, object):
""" Registers a rotating file logger for error logging, if
config.ErrorLogEnabled is True. """
@@ -1834,14 +1817,14 @@
# Start listening on the stats socket, for administrators to inspect
# the current stats on the server.
if config.Stats.EnableUnixStatsSocket:
- stats = CalDAVStatisticsServer(logger)
+ stats = DashboardServer(logger, cl if config.UseMetaFD else None)
statsService = GroupOwnedUNIXServer(
gid, config.Stats.UnixStatsSocket, stats, mode=0660
)
statsService.setName("unix-stats")
statsService.setServiceParent(s)
if config.Stats.EnableTCPStatsSocket:
- stats = CalDAVStatisticsServer(logger)
+ stats = DashboardServer(logger, cl if config.UseMetaFD else None)
statsService = TCPServer(
config.Stats.TCPStatsPort, stats, interface=""
)
@@ -1888,6 +1871,8 @@
if store is None:
raise StoreNotAvailable()
+ stats.store = store
+
from twisted.internet import reactor
pool = PeerConnectionPool(
reactor, store.newTransaction, config.WorkQueue.ampPort
Added: CalendarServer/trunk/calendarserver/tools/dashboard.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/dashboard.py (rev 0)
+++ CalendarServer/trunk/calendarserver/tools/dashboard.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -0,0 +1,538 @@
+##
+# Copyright (c) 2012-2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+A curses (or plain text) based dashboard for viewing various aspects of the server as exposed by
+the L{DashboardProtocol} stats socket.
+"""
+
+from getopt import getopt, GetoptError
+
+import curses
+import json
+import os
+import sched
+import sys
+import time
+import socket
+import errno
+
+def usage(e=None):
+
+ name = os.path.basename(sys.argv[0])
+ print("usage: %s [options]" % (name,))
+ print("")
+ print(" TODO: describe usage")
+ print("")
+ print("options:")
+ print(" -h --help: print this help and exit")
+ print(" -t: text output, not curses")
+ print("")
+
+ if e:
+ sys.exit(64)
+ else:
+ sys.exit(0)
+
+
+BOX_WIDTH = 52
+
+def main():
+
+ try:
+ (optargs, _ignore_args) = getopt(
+ sys.argv[1:], "ht", [
+ "help",
+ ],
+ )
+ except GetoptError, e:
+ usage(e)
+
+ #
+ # Get configuration
+ #
+ useCurses = True
+
+ for opt, _ignore_arg in optargs:
+ if opt in ("-h", "--help"):
+ usage()
+
+ elif opt in ("-t"):
+ useCurses = False
+
+ else:
+ raise NotImplementedError(opt)
+
+ if useCurses:
+ def _wrapped(stdscrn):
+ curses.curs_set(0)
+ curses.use_default_colors()
+ d = Dashboard(stdscrn, True)
+ d.run()
+ curses.wrapper(_wrapped)
+ else:
+ d = Dashboard(None, False)
+ d.run()
+
+
+
+class Dashboard(object):
+ """
+ Main dashboard controller. Use Python's L{sched} feature to schedule updates.
+ """
+
+ screen = None
+ registered_windows = {}
+
+ def __init__(self, screen, usesCurses):
+ self.screen = screen
+ self.usesCurses = usesCurses
+ self.paused = False
+ self.seconds = 0.1
+ self.sched = sched.scheduler(time.time, time.sleep)
+
+
+ @classmethod
+ def registerWindow(cls, wtype, keypress):
+ """
+ Register a window type along with a key press action. This allows the controller to
+ select the appropriate window when its key is pressed, and also provides help information
+ to the L{HelpWindow} for each available window type.
+ """
+ cls.registered_windows[keypress] = wtype
+
+
+ def run(self):
+ """
+ Create the initial window and run the L{scheduler}.
+ """
+ self.windows = []
+ self.displayWindow(None)
+ self.sched.enter(0, 0, self.updateDisplay, ())
+ self.sched.run()
+
+
+ def displayWindow(self, wtype):
+ """
+ Display a new window type, clearing out the old one first.
+ """
+ if self.windows:
+ for window in self.windows:
+ window.clear()
+ self.windows = []
+
+ if wtype is not None:
+ self.windows.append(wtype(self.usesCurses).makeWindow())
+ self.windows[-1].update()
+ else:
+ top = 0
+ for wtype in filter(lambda x: x.all, self.registered_windows.values()):
+ self.windows.append(wtype(self.usesCurses).makeWindow(top=top))
+ self.windows[-1].update()
+ top += self.windows[-1].nlines
+
+
+ def updateDisplay(self):
+ """
+ Periodic update of the current window and check for a key press.
+ """
+ try:
+ if not self.paused:
+ for window in filter(lambda x: x.requiresUpdate(), self.windows):
+ window.update()
+ except Exception as e:
+ print(str(e))
+ if not self.usesCurses:
+ print("-------------")
+
+ # Check keystrokes
+ if self.usesCurses:
+ try:
+ c = self.windows[-1].window.getkey()
+ except:
+ c = -1
+ if c == "q":
+ sys.exit(0)
+ elif c == " ":
+ self.paused = not self.paused
+ elif c == "t":
+ self.seconds = 1.0 if self.seconds == 0.1 else 0.1
+ elif c == "a":
+ self.displayWindow(None)
+ elif c in self.registered_windows:
+ self.displayWindow(self.registered_windows[c])
+
+ self.sched.enter(self.seconds, 0, self.updateDisplay, ())
+
+
+
+class BaseWindow(object):
+ """
+ Common behavior for window types.
+ """
+
+ help = "Not Implemented"
+ all = True
+
+ def __init__(self, usesCurses):
+ self.usesCurses = usesCurses
+
+
+ def makeWindow(self, top=0, left=0):
+ raise NotImplementedError
+
+
+ def _createWindow(self, title, nlines, ncols=BOX_WIDTH, begin_y=0, begin_x=0):
+ """
+ Initialize a curses window based on the sizes required.
+ """
+ if self.usesCurses:
+ self.window = curses.newwin(nlines, ncols, begin_y, begin_x)
+ self.window.nodelay(1)
+ else:
+ self.window = None
+ self.title = title
+ self.nlines = nlines
+ self.ncols = ncols
+ self.iter = 0
+ self.lastResult = {}
+
+
+ def requiresUpdate(self):
+ """
+ Indicates whether a window type has dynamic data that should be refreshed on each update,
+ or whether it is static data (e.g., L{HelpWindow}) that only needs to be drawn once.
+ """
+ return True
+
+
+ def clear(self):
+ """
+ Clear any drawing done by the current window type.
+ """
+ if self.usesCurses:
+ self.window.erase()
+ self.window.refresh()
+
+
+ def update(self):
+ """
+ Periodic window update - redraw the window.
+ """
+ raise NotImplementedError
+
+
+
+class BaseSocketWindow(BaseWindow):
+ """
+ Common behavior for a window that reads from the server's stats socket.
+ """
+
+ def __init__(self, usesCurses):
+ super(BaseSocketWindow, self).__init__(usesCurses)
+ self.socket = None
+ self.sockname = ("localhost", 8100)
+ self.useTCP = True
+
+
+ def readSock(self, item):
+ """
+ Open a socket, send the specified request, and retrieve the response. Keep the socket open.
+ """
+ try:
+ if self.socket is None:
+ self.socket = socket.socket(socket.AF_INET if self.useTCP else socket.AF_UNIX, socket.SOCK_STREAM)
+ self.socket.connect(self.sockname)
+ self.socket.setblocking(0)
+ self.socket.sendall(json.dumps([item]) + "\r\n")
+ data = ""
+ while not data.endswith("\n"):
+ try:
+ d = self.socket.recv(1024)
+ except socket.error as se:
+ if se.args[0] != errno.EWOULDBLOCK:
+ raise
+ continue
+ if d:
+ data += d
+ else:
+ break
+ data = json.loads(data)[item]
+ except socket.error as e:
+ data = {"Failed": "Unable to read statistics from server: %s %s" % (self.sockname, e)}
+ self.socket = None
+ return data
+
+
+
+class HelpWindow(BaseWindow):
+ """
+ Display help for the dashboard.
+ """
+
+ help = "display dashboard help"
+ all = False
+
+ def __init__(self, usesCurses):
+ super(HelpWindow, self).__init__(usesCurses)
+ self.help = (
+ "",
+ "a - all windows",
+ " - (space) pause dashboard polling",
+ "t - toggle update between 0.1 and 1.0 seconds",
+ "",
+ "q - exit the dashboard",
+ )
+
+
+ def makeWindow(self, top=0, left=0):
+ self._createWindow("Help", len(self.help) + len(Dashboard.registered_windows) + 2, begin_y=top, begin_x=left)
+ return self
+
+
+ def requiresUpdate(self):
+ return False
+
+
+ def update(self):
+
+ if self.usesCurses:
+ self.window.erase()
+ self.window.border()
+ self.window.addstr(0, 2, "Help for Dashboard")
+
+ x = 1
+ y = 1
+
+ items = []
+ for keypress, wtype in sorted(Dashboard.registered_windows.items(), key=lambda x: x[0]):
+ items.append("{} - {}".format(keypress, wtype.help))
+ items.extend(self.help)
+ for item in items:
+ if self.usesCurses:
+ self.window.addstr(y, x, item)
+ else:
+ print(item)
+ y += 1
+
+ if self.usesCurses:
+ self.window.refresh()
+
+
+
+class WorkWindow(BaseSocketWindow):
+ """
+ Display the status of the server's job queue.
+ """
+
+ help = "display server jobs"
+
+ def makeWindow(self, top=0, left=0):
+ nlines = self.readSock("jobcount")
+ self._createWindow("Jobs", nlines + 5, begin_y=top, begin_x=left)
+ return self
+
+
+ def update(self):
+ records = self.readSock("jobs")
+ self.iter += 1
+
+ if self.usesCurses:
+ self.window.erase()
+ self.window.border()
+ self.window.addstr(0, 2, self.title + " {} ({})".format(len(records), self.iter,))
+
+ x = 1
+ y = 1
+ s = " {:<40}{:>8} ".format("Work Type", "Count")
+ if self.usesCurses:
+ self.window.addstr(y, x, s, curses.A_REVERSE)
+ else:
+ print(s)
+ y += 1
+ for work_type, count in sorted(records.items(), key=lambda x: x[0]):
+ changed = work_type in self.lastResult and self.lastResult[work_type] != count
+ s = "{}{:<40}{:>8} ".format(">" if count else " ", work_type, count)
+ try:
+ if self.usesCurses:
+ self.window.addstr(y, x, s, curses.A_REVERSE if changed else (curses.A_BOLD if count else curses.A_NORMAL))
+ else:
+ print(s)
+ except curses.error:
+ pass
+ y += 1
+
+ s = " {:<40}{:>8} ".format("Total:", sum(records.values()))
+ if self.usesCurses:
+ self.window.hline(y, x, "-", BOX_WIDTH - 2)
+ y += 1
+ self.window.addstr(y, x, s)
+ else:
+ print(s)
+ y += 1
+
+ if self.usesCurses:
+ self.window.refresh()
+
+ self.lastResult = records
+
+
+
+class SlotsWindow(BaseSocketWindow):
+ """
+ Displays the status of the server's master process worker slave slots.
+ """
+
+ help = "display server child slots"
+ FORMAT_WIDTH = 72
+
+ def makeWindow(self, top=0, left=0):
+ slots = self.readSock("slots")["slots"]
+ self._createWindow("Slots", len(slots) + 5, self.FORMAT_WIDTH, begin_y=top, begin_x=left)
+ return self
+
+
+ def update(self):
+ data = self.readSock("slots")
+ records = data["slots"]
+ self.iter += 1
+
+ if self.usesCurses:
+ self.window.erase()
+ self.window.border()
+ self.window.addstr(0, 2, self.title + " {} ({})".format(len(records), self.iter,))
+
+ x = 1
+ y = 1
+ s = " {:>4}{:>8}{:>8}{:>8}{:>8}{:>8}{:>8}{:>8}{:>8} ".format("Slot", "unack", "ack", "uncls", "total", "start", "strting", "stopped", "abd")
+ if self.usesCurses:
+ self.window.addstr(y, x, s, curses.A_REVERSE)
+ else:
+ print(s)
+ y += 1
+ for record in sorted(records, key=lambda x: x["slot"]):
+ changed = record["slot"] in self.lastResult and self.lastResult[record["slot"]] != record
+ s = " {:>4}{:>8}{:>8}{:>8}{:>8}{:>8}{:>8}{:>8}{:>8} ".format(
+ record["slot"],
+ record["unacknowledged"],
+ record["acknowledged"],
+ record["unclosed"],
+ record["total"],
+ record["started"],
+ record["starting"],
+ record["stopped"],
+ record["abandoned"],
+ )
+ try:
+ count = record["unacknowledged"] + record["acknowledged"]
+ if self.usesCurses:
+ self.window.addstr(y, x, s, curses.A_REVERSE if changed else (curses.A_BOLD if count else curses.A_NORMAL))
+ else:
+ print(s)
+ except curses.error:
+ pass
+ y += 1
+
+ s = " {:<12}{:>8}{:>16}".format(
+ "Total:",
+ sum([record["unacknowledged"] + record["acknowledged"] for record in records]),
+ sum([record["total"] for record in records]),
+ )
+ if self.usesCurses:
+ self.window.hline(y, x, "-", self.FORMAT_WIDTH - 2)
+ y += 1
+ self.window.addstr(y, x, s)
+ x += len(s) + 4
+ s = "{:>10}".format("OVERLOADED" if data["overloaded"] else "")
+ self.window.addstr(y, x, s, curses.A_REVERSE if data["overloaded"] else curses.A_NORMAL)
+ else:
+ if data["overloaded"]:
+ s += " OVERLOADED"
+ print(s)
+ y += 1
+
+ if self.usesCurses:
+ self.window.refresh()
+
+ self.lastResult = records
+
+
+
+class SystemWindow(BaseSocketWindow):
+ """
+ Displays the system information provided by the server.
+ """
+
+ help = "display system details"
+
+ def makeWindow(self, top=0, left=0):
+ slots = self.readSock("stats")["system"]
+ self._createWindow("System", len(slots) + 3, begin_y=top, begin_x=left)
+ return self
+
+
+ def update(self):
+ records = self.readSock("stats")["system"]
+ self.iter += 1
+
+ if self.usesCurses:
+ self.window.erase()
+ self.window.border()
+ self.window.addstr(0, 2, self.title + " {} ({})".format(len(records), self.iter,))
+
+ x = 1
+ y = 1
+ s = " {:<30}{:>18} ".format("Item", "Value")
+ if self.usesCurses:
+ self.window.addstr(y, x, s, curses.A_REVERSE)
+ else:
+ print(s)
+ y += 1
+
+ records["cpu use"] = "{:.2f}".format(records["cpu use"])
+ records["memory percent"] = "{:.1f}".format(records["memory percent"])
+ records["memory used"] = "{:.2f} GB".format(records["memory used"] / (1000.0 * 1000.0 * 1000.0))
+ records["uptime"] = int(time.time() - records["start time"])
+ hours, mins = divmod(records["uptime"] / 60, 60)
+ records["uptime"] = "{}:{:02d} hours".format(hours, mins)
+ del records["start time"]
+
+ for item, value in sorted(records.items(), key=lambda x: x[0]):
+ changed = item in self.lastResult and self.lastResult[item] != value
+ s = " {:<30}{:>18} ".format(item, value)
+ try:
+ if self.usesCurses:
+ self.window.addstr(y, x, s, curses.A_REVERSE if changed else curses.A_NORMAL)
+ else:
+ print(s)
+ except curses.error:
+ pass
+ y += 1
+
+ if self.usesCurses:
+ self.window.refresh()
+
+ self.lastResult = records
+
+
+Dashboard.registerWindow(HelpWindow, "h")
+Dashboard.registerWindow(WorkWindow, "j")
+Dashboard.registerWindow(SlotsWindow, "c")
+Dashboard.registerWindow(SystemWindow, "s")
+
+
+if __name__ == "__main__":
+ main()
Deleted: CalendarServer/trunk/calendarserver/tools/jobitems.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/jobitems.py 2014-03-13 00:51:31 UTC (rev 12891)
+++ CalendarServer/trunk/calendarserver/tools/jobitems.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -1,293 +0,0 @@
-#!/usr/bin/env python
-
-##
-# Copyright (c) 2006-2014 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-from __future__ import print_function
-
-from calendarserver.tools.cmdline import utilityMain, WorkerService
-
-from getopt import getopt, GetoptError
-
-from twext.enterprise.jobqueue import JobItem
-
-from twisted.internet.defer import inlineCallbacks, succeed
-
-import curses
-import os
-import sys
-
-useCurses = True
-
-def usage(e=None):
-
- name = os.path.basename(sys.argv[0])
- print("usage: %s [options]" % (name,))
- print("")
- print(" TODO: describe usage")
- print("")
- print("options:")
- print(" -h --help: print this help and exit")
- print(" -e --error: send stderr to stdout")
- print(" -f --config <path>: specify caldavd.plist configuration path")
- print(" -t: text output, not curses")
- print("")
-
- if e:
- sys.exit(64)
- else:
- sys.exit(0)
-
-
-BOX_WIDTH = 52
-
-def main():
-
- try:
- (optargs, _ignore_args) = getopt(
- sys.argv[1:], "hef:t", [
- "help",
- "error",
- "config=",
- ],
- )
- except GetoptError, e:
- usage(e)
-
- #
- # Get configuration
- #
- global useCurses
- configFileName = None
- debug = False
-
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage()
-
- if opt in ("-e", "--error"):
- debug = True
-
- elif opt in ("-f", "--config"):
- configFileName = arg
-
- elif opt in ("-t"):
- useCurses = False
-
- else:
- raise NotImplementedError(opt)
-
- if useCurses:
- def _wrapped(stdscrn):
- JobItemMonitorService.screen = stdscrn
- curses.curs_set(0)
- curses.use_default_colors()
- utilityMain(configFileName, JobItemMonitorService, verbose=debug)
- curses.wrapper(_wrapped)
- else:
- utilityMain(configFileName, JobItemMonitorService, verbose=debug)
-
-
-
-class JobItemMonitorService(WorkerService, object):
-
- screen = None
-
- def __init__(self, store):
- super(JobItemMonitorService, self).__init__(store)
- from twisted.internet import reactor
- self.reactor = reactor
- self.paused = False
- self.seconds = 0.1
-
-
- @inlineCallbacks
- def doWork(self):
- self.window = None
- yield self.displayJobs()
- self.reactor.callLater(0, self.updateDisplay)
-
-
- def postStartService(self):
- """
- Don't quit right away
- """
- pass
-
-
- @inlineCallbacks
- def displayHelp(self):
- if self.window is not None:
- self.window.clear()
- self.window = HelpWindow(10, BOX_WIDTH, 0, 0, self.store, "Help")
- yield self.window.update()
-
-
- @inlineCallbacks
- def displayJobs(self):
- if self.window is not None:
- self.window.clear()
- self.window = WorkWindow(JobItem.numberOfWorkTypes() + 5, BOX_WIDTH, 0, 0, self.store, "Jobs")
- yield self.window.update()
-
-
- @inlineCallbacks
- def updateDisplay(self):
- try:
- if not self.paused and self.window.requiresUpdate():
- yield self.window.update()
- except Exception as e:
- print(str(e))
- if not useCurses:
- print("-------------")
-
- # Check keystrokes
- if useCurses:
- try:
- c = self.window.window.getkey()
- except:
- c = -1
- if c == "q":
- self.reactor.stop()
- elif c == " ":
- self.paused = not self.paused
- elif c == "t":
- self.seconds = 1.0 if self.seconds == 0.1 else 1.0
- elif c == "h":
- yield self.displayHelp()
- elif c == "j":
- yield self.displayJobs()
-
- self.reactor.callLater(self.seconds, self.updateDisplay)
-
-
-
-class BaseWindow(object):
- def __init__(self, nlines, ncols, begin_y, begin_x, store, title):
- self.window = curses.newwin(nlines, ncols, begin_y, begin_x) if useCurses else None
- if useCurses:
- self.window.nodelay(1)
- self.ncols = ncols
- self.store = store
- self.title = title
- self.iter = 0
- self.lastResult = {}
-
-
- def requiresUpdate(self):
- return True
-
-
- def clear(self):
-
- if useCurses:
- self.window.erase()
- self.window.refresh()
-
-
- def update(self):
- return succeed(True)
-
-
-
-class HelpWindow(BaseWindow):
-
- def requiresUpdate(self):
- return False
-
-
- def update(self):
-
- if useCurses:
- self.window.erase()
- self.window.border()
- self.window.addstr(0, 2, "Help for Dashboard")
-
- x = 1
- y = 1
-
- for title in (
- "j - display server jobs",
- "h - display dashboard help",
- "",
- " - (space) pause dashboard polling",
- "t - toggle update between 0.1 and 1.0 seconds",
- "",
- "q - exit the dashboard",
- ):
- if useCurses:
- self.window.addstr(y, x, title)
- else:
- print(title)
- y += 1
-
- if useCurses:
- self.window.refresh()
-
- return succeed(True)
-
-
-
-class WorkWindow(BaseWindow):
-
- @inlineCallbacks
- def update(self):
- txn = self.store.newTransaction()
- records = (yield JobItem.histogram(txn))
- self.iter += 1
-
- if useCurses:
- self.window.erase()
- self.window.border()
- self.window.addstr(0, 2, self.title + " {} ({})".format(len(records), self.iter,))
-
- x = 1
- y = 1
- s = " {:<40}{:>8} ".format("Work Type", "Count")
- if useCurses:
- self.window.addstr(y, x, s, curses.A_REVERSE)
- else:
- print(s)
- y += 1
- for work_type, count in sorted(records.items(), key=lambda x: x[0]):
- changed = work_type in self.lastResult and self.lastResult[work_type] != count
- s = "{}{:<40}{:>8} ".format(">" if count else " ", work_type, count)
- try:
- if useCurses:
- self.window.addstr(y, x, s, curses.A_REVERSE if changed else (curses.A_BOLD if count else curses.A_NORMAL))
- else:
- print(s)
- except curses.error:
- pass
- y += 1
-
- s = " {:<40}{:>8} ".format("Total:", sum(records.values()))
- if useCurses:
- self.window.hline(y, x, "-", BOX_WIDTH - 2)
- y += 1
- self.window.addstr(y, x, s)
- else:
- print(s)
- y += 1
-
- if useCurses:
- self.window.refresh()
-
- self.lastResult = records
- yield txn.commit()
-
-
-if __name__ == "__main__":
- main()
Modified: CalendarServer/trunk/contrib/tools/readStats.py
===================================================================
--- CalendarServer/trunk/contrib/tools/readStats.py 2014-03-13 00:51:31 UTC (rev 12891)
+++ CalendarServer/trunk/contrib/tools/readStats.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -25,6 +25,7 @@
import sys
import tables
import time
+import errno
"""
This tool reads data from the server's statistics socket and prints a summary.
@@ -39,18 +40,25 @@
try:
s = socket.socket(socket.AF_INET if useTCP else socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(sockname)
+ s.sendall(json.dumps(["stats"]) + "\r\n")
+ s.setblocking(0)
data = ""
- while True:
- d = s.recv(1024)
+ while not data.endswith("\n"):
+ try:
+ d = s.recv(1024)
+ except socket.error as se:
+ if se.args[0] != errno.EWOULDBLOCK:
+ raise
+ continue
if d:
data += d
else:
break
s.close()
- data = json.loads(data)
- except socket.error:
- data = {"Failed": "Unable to read statistics from server: %s" % (sockname,)}
- data["Server"] = sockname
+ data = json.loads(data)["stats"]
+ except socket.error as e:
+ data = {"Failed": "Unable to read statistics from server: %s %s" % (sockname, e)}
+ data["server"] = sockname
return data
@@ -74,18 +82,18 @@
def printStat(stats, index, showMethods, topUsers, showAgents):
print("- " * 40)
- print("Server: %s" % (stats["Server"],))
+ print("Server: %s" % (stats["server"],))
print(datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
- print("Service Uptime: %s" % (datetime.timedelta(seconds=(int(time.time() - stats["System"]["start time"]))),))
- if stats["System"]["cpu count"] > 0:
+ print("Service Uptime: %s" % (datetime.timedelta(seconds=(int(time.time() - stats["system"]["start time"]))),))
+ if stats["system"]["cpu count"] > 0:
print("Current CPU: %.1f%% (%d CPUs)" % (
- stats["System"]["cpu use"],
- stats["System"]["cpu count"],
+ stats["system"]["cpu use"],
+ stats["system"]["cpu count"],
))
print("Current Memory Used: %d bytes (%.1f GB) (%.1f%% of total)" % (
- stats["System"]["memory used"],
- stats["System"]["memory used"] / (1024.0 * 1024 * 1024),
- stats["System"]["memory percent"],
+ stats["system"]["memory used"],
+ stats["system"]["memory used"] / (1024.0 * 1024 * 1024),
+ stats["system"]["memory percent"],
))
else:
print("Current CPU: Unavailable")
@@ -112,7 +120,7 @@
times = []
for stat in stats:
try:
- t = str(datetime.timedelta(seconds=int(time.time() - stat["System"]["start time"])))
+ t = str(datetime.timedelta(seconds=int(time.time() - stat["system"]["start time"])))
except KeyError:
t = "-"
times.append(t)
@@ -120,9 +128,9 @@
cpus = []
memories = []
for stat in stats:
- if stat["System"]["cpu count"] > 0:
- cpus.append(stat["System"]["cpu use"])
- memories.append(stat["System"]["memory percent"])
+ if stat["system"]["cpu count"] > 0:
+ cpus.append(stat["system"]["cpu use"])
+ memories.append(stat["system"]["memory percent"])
else:
cpus.append(-1)
memories.append(-1)
@@ -139,7 +147,7 @@
def serverLabels(stats):
- servers = [stat["Server"] for stat in stats]
+ servers = [stat["server"] for stat in stats]
if isinstance(servers[0], tuple):
hosts = set([item[0] for item in servers])
ports = set([item[1] for item in servers])
@@ -193,7 +201,7 @@
)
)
- for key, seconds in (("Current", 60,), ("1 Minute", 60,), ("5 Minutes", 5 * 60,), ("1 Hour", 60 * 60,),):
+ for key, seconds in (("current", 60,), ("1m", 60,), ("5m", 5 * 60,), ("1h", 60 * 60,),):
stat = stats[key]
table.addRow((
@@ -515,7 +523,7 @@
topUsers = 0
showAgents = False
- multimodes = (("Current", 60,), ("1 Minute", 60,), ("5 Minutes", 5 * 60,), ("1 Hour", 60 * 60,),)
+ multimodes = (("current", 60,), ("1m", 60,), ("5m", 5 * 60,), ("1h", 60 * 60,),)
multimode = multimodes[2]
options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", "0", "1", "5", "60", "methods", "users=", "agents"])
Modified: CalendarServer/trunk/txweb2/metafd.py
===================================================================
--- CalendarServer/trunk/txweb2/metafd.py 2014-03-13 00:51:31 UTC (rev 12891)
+++ CalendarServer/trunk/txweb2/metafd.py 2014-03-13 18:09:35 UTC (rev 12892)
@@ -223,6 +223,10 @@
self.stopped = stopped
+ def items(self):
+ return dict([(attr, getattr(self, attr)) for attr in self.showAttributes])
+
+
def effective(self):
"""
The current effective load.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140313/12674fa7/attachment-0001.html>
More information about the calendarserver-changes
mailing list