[CalendarServer-changes] [11527] CalendarServer/trunk/contrib/performance/sqlusage

source_changes at macosforge.org source_changes at macosforge.org
Wed Jul 17 12:22:24 PDT 2013


Revision: 11527
          http://trac.calendarserver.org//changeset/11527
Author:   cdaboo at apple.com
Date:     2013-07-17 12:22:24 -0700 (Wed, 17 Jul 2013)
Log Message:
-----------
Add ability to check scaling vs number of sharees.

Modified Paths:
--------------
    CalendarServer/trunk/contrib/performance/sqlusage/sqlusage.py

Added Paths:
-----------
    CalendarServer/trunk/contrib/performance/sqlusage/requests/propfind_invite.py

Added: CalendarServer/trunk/contrib/performance/sqlusage/requests/propfind_invite.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/propfind_invite.py	                        (rev 0)
+++ CalendarServer/trunk/contrib/performance/sqlusage/requests/propfind_invite.py	2013-07-17 19:22:24 UTC (rev 11527)
@@ -0,0 +1,54 @@
+##
+# Copyright (c) 2012-2013 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from caldavclientlibrary.protocol.http.data.string import ResponseDataString
+from caldavclientlibrary.protocol.webdav.definitions import statuscodes, \
+    headers
+from caldavclientlibrary.protocol.webdav.propfind import PropFind
+from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
+from caldavclientlibrary.protocol.caldav.definitions import csxml
+
+class PropfindInviteTest(HTTPTestBase):
+    """
+    A propfind operation
+    """
+
+    def __init__(self, label, sessions, logFilePath, depth=1):
+        super(PropfindInviteTest, self).__init__(label, sessions, logFilePath)
+        self.depth = headers.Depth1 if depth == 1 else headers.Depth0
+
+
+    def doRequest(self):
+        """
+        Execute the actual HTTP request.
+        """
+        props = (
+            csxml.invite,
+        )
+
+        # Create WebDAV propfind
+        request = PropFind(self.sessions[0], self.sessions[0].calendarHref, self.depth, props)
+        result = ResponseDataString()
+        request.setOutput(result)
+
+        # Process it
+        self.sessions[0].runSession(request)
+
+        # If its a 207 we want to parse the XML
+        if request.getStatusCode() == statuscodes.MultiStatus:
+            pass
+        else:
+            raise RuntimeError("Propfind request failed: %s" % (request.getStatusCode(),))

Modified: CalendarServer/trunk/contrib/performance/sqlusage/sqlusage.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/sqlusage.py	2013-07-17 19:20:00 UTC (rev 11526)
+++ CalendarServer/trunk/contrib/performance/sqlusage/sqlusage.py	2013-07-17 19:22:24 UTC (rev 11527)
@@ -23,6 +23,7 @@
 from contrib.performance.sqlusage.requests.invite import InviteTest
 from contrib.performance.sqlusage.requests.multiget import MultigetTest
 from contrib.performance.sqlusage.requests.propfind import PropfindTest
+from contrib.performance.sqlusage.requests.propfind_invite import PropfindInviteTest
 from contrib.performance.sqlusage.requests.put import PutTest
 from contrib.performance.sqlusage.requests.query import QueryTest
 from contrib.performance.sqlusage.requests.sync import SyncTest
@@ -31,6 +32,7 @@
 import getopt
 import itertools
 import sys
+from caldavclientlibrary.client.principal import principalCache
 
 """
 This tool is designed to analyze how SQL is being used for various HTTP requests.
@@ -41,7 +43,8 @@
 with calendar size can be plotted.
 """
 
-EVENT_COUNTS = (0, 1, 5, 10, 50, 100, 500, 1000, 5000)
+EVENT_COUNTS = (0, 1, 5, 10, 50, 100, 500, 1000,)
+SHAREE_COUNTS = (0, 1, 5, 10, 50, 100,)
 
 ICAL = """BEGIN:VCALENDAR
 CALSCALE:GREGORIAN
@@ -78,16 +81,17 @@
 
 class SQLUsageSession(CalDAVSession):
 
-    def __init__(self, server, port=None, ssl=False, user="", pswd="", principal=None, root=None, logging=False):
+    def __init__(self, server, port=None, ssl=False, user="", pswd="", principal=None, root=None, calendar="calendar", logging=False):
 
         super(SQLUsageSession, self).__init__(server, port, ssl, user, pswd, principal, root, logging)
         self.homeHref = "/calendars/users/%s/" % (self.user,)
-        self.calendarHref = "/calendars/users/%s/calendar/" % (self.user,)
+        self.calendarHref = "/calendars/users/%s/%s/" % (self.user, calendar,)
         self.inboxHref = "/calendars/users/%s/inbox/" % (self.user,)
+        self.notificationHref = "/calendars/users/%s/notification/" % (self.user,)
 
 
 
-class SQLUsage(object):
+class EventSQLUsage(object):
 
     def __init__(self, server, port, users, pswds, logFilePath):
         self.server = server
@@ -100,7 +104,7 @@
         self.currentCount = 0
 
 
-    def runLoop(self, counts):
+    def runLoop(self, event_counts):
 
         # Make the sessions
         sessions = [
@@ -129,7 +133,7 @@
             session.getPropertiesOnHierarchy(URL(path=session.calendarHref), props)
 
         # Now loop over sets of events
-        for count in counts:
+        for count in event_counts:
             print("Testing count = %d" % (count,))
             self.ensureEvents(sessions[0], sessions[0].calendarHref, count)
             result = {}
@@ -181,6 +185,112 @@
 
 
 
+class SharerSQLUsage(object):
+
+    def __init__(self, server, port, users, pswds, logFilePath):
+        self.server = server
+        self.port = port
+        self.users = users
+        self.pswds = pswds
+        self.logFilePath = logFilePath
+        self.requestLabels = []
+        self.results = {}
+        self.currentCount = 0
+
+
+    def runLoop(self, sharee_counts):
+
+        # Make the sessions
+        sessions = [
+            SQLUsageSession(self.server, self.port, user=user, pswd=pswd, root="/", calendar="shared")
+            for user, pswd in itertools.izip(self.users, self.pswds)
+        ]
+        sessions = sessions[0:1]
+
+        # Create the calendar first
+        sessions[0].makeCalendar(URL(path=sessions[0].calendarHref))
+
+        # Set of requests to execute
+        requests = [
+            MultigetTest("multiget-1", sessions, self.logFilePath, 1),
+            MultigetTest("multiget-50", sessions, self.logFilePath, 50),
+            PropfindInviteTest("propfind", sessions, self.logFilePath, 1),
+            SyncTest("sync-full", sessions, self.logFilePath, True, 0),
+            SyncTest("sync-1", sessions, self.logFilePath, False, 1),
+            QueryTest("query-1", sessions, self.logFilePath, 1),
+            QueryTest("query-10", sessions, self.logFilePath, 10),
+            PutTest("put", sessions, self.logFilePath),
+        ]
+        self.requestLabels = [request.label for request in requests]
+
+        # Warm-up server by doing shared calendar propfinds
+        props = (davxml.resourcetype,)
+        for session in sessions:
+            session.getPropertiesOnHierarchy(URL(path=session.calendarHref), props)
+
+        # Now loop over sets of events
+        for count in sharee_counts:
+            print("Testing count = %d" % (count,))
+            self.ensureSharees(sessions[0], sessions[0].calendarHref, count)
+            result = {}
+            for request in requests:
+                print("  Test = %s" % (request.label,))
+                result[request.label] = request.execute(count)
+            self.results[count] = result
+
+
+    def report(self):
+
+        self._printReport("SQL Statement Count", "count", "%d")
+        self._printReport("SQL Rows Returned", "rows", "%d")
+        self._printReport("SQL Time", "timing", "%.1f")
+
+
+    def _printReport(self, title, attr, colFormat):
+        table = tables.Table()
+
+        print(title)
+        headers = ["Sharees"] + self.requestLabels
+        table.addHeader(headers)
+        formats = [tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY)] + \
+            [tables.Table.ColumnFormat(colFormat, tables.Table.ColumnFormat.RIGHT_JUSTIFY)] * len(self.requestLabels)
+        table.setDefaultColumnFormats(formats)
+        for k in sorted(self.results.keys()):
+            row = [k] + [getattr(self.results[k][item], attr) for item in self.requestLabels]
+            table.addRow(row)
+        os = StringIO()
+        table.printTable(os=os)
+        print(os.getvalue())
+        print("")
+
+
+    def ensureSharees(self, session, calendarhref, n):
+        """
+        Make sure the required number of sharees are present in the calendar.
+
+        @param n: number of sharees
+        @type n: C{int}
+        """
+
+        users = []
+        uids = []
+        for i in range(n - self.currentCount):
+            index = self.currentCount + i + 2
+            users.append("user%02d" % (index,))
+            uids.append("urn:uuid:user%02d" % (index,))
+        session.addInvitees(URL(path=calendarhref), uids, True)
+
+        # Now accept each one
+        for user in users:
+            acceptor = SQLUsageSession(self.server, self.port, user=user, pswd=user, root="/", calendar="shared")
+            notifications = acceptor.getNotifications(URL(path=acceptor.notificationHref))
+            principal = principalCache.getPrincipal(acceptor, acceptor.principalPath)
+            acceptor.processNotification(principal, notifications[0], True)
+
+        self.currentCount = n
+
+
+
 def usage(error_msg=None):
     if error_msg:
         print(error_msg)
@@ -192,7 +302,7 @@
     --port         Server port
     --user         User name
     --pswd         Password
-    --counts       Comma-separated list of event counts to test
+    --event_counts       Comma-separated list of event event_counts to test
 
 Arguments:
     FILE           File name for sqlstats.log to analyze.
@@ -213,9 +323,14 @@
     users = ("user01", "user02",)
     pswds = ("user01", "user02",)
     file = "sqlstats.logs"
-    counts = EVENT_COUNTS
+    event_counts = EVENT_COUNTS
+    sharee_counts = SHAREE_COUNTS
 
-    options, args = getopt.getopt(sys.argv[1:], "h", ["server=", "port=", "user=", "pswd=", "counts=", ])
+    options, args = getopt.getopt(
+        sys.argv[1:],
+        "h",
+        ["server=", "port=", "user=", "pswd=", "event-counts=", "sharee-counts=", ]
+    )
 
     for option, value in options:
         if option == "-h":
@@ -228,8 +343,10 @@
             users = value.split(",")
         elif option == "--pswd":
             pswds = value.split(",")
-        elif option == "--counts":
-            counts = [int(i) for i in value.split(",")]
+        elif option == "--event-counts":
+            event_counts = [int(i) for i in value.split(",")]
+        elif option == "--sharee-counts":
+            sharee_counts = [int(i) for i in value.split(",")]
         else:
             usage("Unrecognized option: %s" % (option,))
 
@@ -239,6 +356,10 @@
     elif len(args) != 0:
         usage("Must zero or one file arguments")
 
-    sql = SQLUsage(server, port, users, pswds, file)
-    sql.runLoop(counts)
+    sql = EventSQLUsage(server, port, users, pswds, file)
+    sql.runLoop(event_counts)
     sql.report()
+
+    sql = SharerSQLUsage(server, port, users, pswds, file)
+    sql.runLoop(sharee_counts)
+    sql.report()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130717/cc46a863/attachment-0001.html>


More information about the calendarserver-changes mailing list