<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[11826] CalendarServer/branches/users/cdaboo/performance-tweaks</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/11826">11826</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2013-10-18 12:38:34 -0700 (Fri, 18 Oct 2013)</dd>
</dl>
<h3>Log Message</h3>
<pre>Various performance tweaks and fixes to extended log items.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakscalendarserveraccesslogpy">CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavresourcepy">CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavscheduling_storecaldavresourcepy">CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavbasedatastoreutilpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastorefilepy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulepy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingimplicitpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingutilspy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdabooperformancetweakscalendarserveraccesslogpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -173,7 +173,7 @@
</span><span class="cx"> formatArgs["t"] = (nowtime - request.timeStamps[0][1]) * 1000
</span><span class="cx">
</span><span class="cx"> if hasattr(request, "extendedLogItems"):
</span><del>- for k, v in request.extendedLogItems.iteritems():
</del><ins>+ for k, v in sorted(request.extendedLogItems.iteritems(), key=lambda x: x[0]):
</ins><span class="cx"> k = str(k).replace('"', "%22")
</span><span class="cx"> v = str(v).replace('"', "%22")
</span><span class="cx"> if " " in v:
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -333,6 +333,12 @@
</span><span class="cx"> else:
</span><span class="cx"> yield transaction.commit()
</span><span class="cx">
</span><ins>+ # Log extended item
+ if transaction.logItems:
+ if not hasattr(request, "extendedLogItems"):
+ request.extendedLogItems = {}
+ request.extendedLogItems.update(transaction.logItems)
+
</ins><span class="cx"> # May need to reset the last-modified header in the response as txn.commit() can change it due to pre-commit hooks
</span><span class="cx"> if response.headers.hasHeader("last-modified"):
</span><span class="cx"> response.headers.setHeader("last-modified", self.lastModified())
</span><span class="lines">@@ -2551,15 +2557,6 @@
</span><span class="cx"> return self._newStoreHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object._newStoreObject, mode)
</span><span class="cx">
</span><span class="cx">
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
- """
- Return all child object resources with the specified UID.
-
- Pass through direct to store.
- """
- return self._newStoreHome.getCalendarResourcesForUID(uid, allow_shared)
-
-
</del><span class="cx"> def defaultAccessControlList(self):
</span><span class="cx"> myPrincipal = self.principalForRecord()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavscheduling_storecaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -422,8 +422,12 @@
</span><span class="cx"> authz = (yield request.locateResource(principalURL))
</span><span class="cx"> self._associatedTransaction._authz_uid = authz.record.guid
</span><span class="cx">
</span><ins>+ # Log extended item
+ if not hasattr(request, "extendedLogItems"):
+ request.extendedLogItems = {}
+
</ins><span class="cx"> # This is a local CALDAV scheduling operation.
</span><del>- scheduler = CalDAVScheduler(self._associatedTransaction, self.parent._newStoreHome.uid())
</del><ins>+ scheduler = CalDAVScheduler(self._associatedTransaction, self.parent._newStoreHome.uid(), logItems=request.extendedLogItems)
</ins><span class="cx">
</span><span class="cx"> # Do the POST processing treating
</span><span class="cx"> result = (yield scheduler.doSchedulingViaPOST(originator, recipients, calendar))
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavbasedatastoreutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -92,6 +92,12 @@
</span><span class="cx"> return "objectWithName:%s:%s" % (homeResourceID, name)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ # Home child objects by id
+
+ def keyForObjectWithResourceID(self, homeResourceID, resourceID):
+ return "objectWithName:%s:%s" % (homeResourceID, resourceID)
+
+
</ins><span class="cx"> # Home metadata (Created/Modified)
</span><span class="cx">
</span><span class="cx"> def keyForHomeMetaData(self, homeResourceID):
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastorefilepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -128,7 +128,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def hasCalendarResourceUIDSomewhereElse(self, uid, ok_object, type):
</span><span class="cx">
</span><del>- objectResources = (yield self.objectResourcesWithUID(uid, ("inbox",)))
</del><ins>+ objectResources = (yield self.getCalendarResourcesForUID(uid))
</ins><span class="cx"> for objectResource in objectResources:
</span><span class="cx"> if ok_object and objectResource._path == ok_object._path:
</span><span class="cx"> continue
</span><span class="lines">@@ -140,14 +140,9 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
</del><ins>+ def getCalendarResourcesForUID(self, uid):
</ins><span class="cx">
</span><del>- results = []
- objectResources = (yield self.objectResourcesWithUID(uid, ("inbox",)))
- for objectResource in objectResources:
- if allow_shared or objectResource._parentCollection.owned():
- results.append(objectResource)
-
</del><ins>+ results = (yield self.objectResourcesWithUID(uid, ("inbox",), allowShared=False))
</ins><span class="cx"> returnValue(results)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -112,8 +112,8 @@
</span><span class="cx"> return self._calendarHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object, type)
</span><span class="cx">
</span><span class="cx">
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
- return self._calendarHome.getCalendarResourcesForUID(uid, allow_shared)
</del><ins>+ def getCalendarResourcesForUID(self, uid):
+ return self._calendarHome.getCalendarResourcesForUID(uid)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingimplicitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -56,10 +56,10 @@
</span><span class="cx"> STATUS_ORPHANED_CANCELLED_EVENT = 1
</span><span class="cx"> STATUS_ORPHANED_EVENT = 2
</span><span class="cx">
</span><del>- def __init__(self):
</del><ins>+ def __init__(self, logItems=None):
</ins><span class="cx">
</span><span class="cx"> self.return_status = ImplicitScheduler.STATUS_OK
</span><del>- self.logItems = {}
</del><ins>+ self.logItems = logItems
</ins><span class="cx"> self.allowed_to_schedule = True
</span><span class="cx"> self.suppress_refresh = False
</span><span class="cx">
</span><span class="lines">@@ -383,7 +383,7 @@
</span><span class="cx"> if self.txn.doing_attendee_refresh == 0:
</span><span class="cx"> delattr(self.txn, "doing_attendee_refresh")
</span><span class="cx">
</span><del>- if refreshCount:
</del><ins>+ if refreshCount and self.logItems is not None:
</ins><span class="cx"> self.logItems["itip.refreshes"] = refreshCount
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -925,7 +925,8 @@
</span><span class="cx"> if self.action in ("create", "modify",):
</span><span class="cx"> total += (yield self.processRequests())
</span><span class="cx">
</span><del>- self.logItems["itip.requests"] = total
</del><ins>+ if self.logItems is not None:
+ self.logItems["itip.requests"] = total
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1304,7 +1305,8 @@
</span><span class="cx"> # First make sure we are allowed to schedule
</span><span class="cx"> self.testSchedulingAllowed()
</span><span class="cx">
</span><del>- self.logItems["itip.reply"] = "reply"
</del><ins>+ if self.logItems is not None:
+ self.logItems["itip.reply"] = "reply"
</ins><span class="cx">
</span><span class="cx"> itipmsg = iTipGenerator.generateAttendeeReply(self.calendar, self.attendee, changedRids=changedRids)
</span><span class="cx">
</span><span class="lines">@@ -1317,7 +1319,8 @@
</span><span class="cx"> # First make sure we are allowed to schedule
</span><span class="cx"> self.testSchedulingAllowed()
</span><span class="cx">
</span><del>- self.logItems["itip.reply"] = "cancel"
</del><ins>+ if self.logItems is not None:
+ self.logItems["itip.reply"] = "cancel"
</ins><span class="cx">
</span><span class="cx"> itipmsg = iTipGenerator.generateAttendeeReply(self.calendar, self.attendee, force_decline=True)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingutilspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -21,7 +21,7 @@
</span><span class="cx"> log = Logger()
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>-def getCalendarObjectForRecord(txn, record, uid, allow_shared=False):
</del><ins>+def getCalendarObjectForRecord(txn, record, uid):
</ins><span class="cx"> """
</span><span class="cx"> Get a copy of the event for a calendar user identified by a directory record.
</span><span class="cx">
</span><span class="lines">@@ -34,7 +34,7 @@
</span><span class="cx"> calendar_home = yield txn.calendarHomeWithUID(record.uid)
</span><span class="cx">
</span><span class="cx"> # Get matching newstore objects
</span><del>- objectResources = (yield calendar_home.getCalendarResourcesForUID(uid, allow_shared))
</del><ins>+ objectResources = (yield calendar_home.getCalendarResourcesForUID(uid))
</ins><span class="cx">
</span><span class="cx"> if len(objectResources) > 1:
</span><span class="cx"> # Delete all but the first one
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -527,9 +527,7 @@
</span><span class="cx"> # refer to calendar *object* UIDs, since calendar *resources* are an
</span><span class="cx"> # HTTP protocol layer thing, not a data store thing. (See also
</span><span class="cx"> # objectResourcesWithUID.)
</span><del>- objectResources = (
- yield self.objectResourcesWithUID(uid, ["inbox"], False)
- )
</del><ins>+ objectResources = (yield self.getCalendarResourcesForUID(uid))
</ins><span class="cx"> for objectResource in objectResources:
</span><span class="cx"> if ok_object and objectResource._resourceID == ok_object._resourceID:
</span><span class="cx"> continue
</span><span class="lines">@@ -541,15 +539,22 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
</del><ins>+ def getCalendarResourcesForUID(self, uid):
+ """
+ Find all calendar object resources in the calendar home that are not in the "inbox" collection
+ and not in shared collections.
+ Cache the result of this query as it can happen multiple times during scheduling under slightly
+ different circumstances.
</ins><span class="cx">
</span><del>- results = []
- objectResources = (yield self.objectResourcesWithUID(uid, ["inbox"]))
- for objectResource in objectResources:
- if allow_shared or objectResource._parentCollection.owned():
- results.append(objectResource)
</del><ins>+ @param uid: the UID of the calendar object resources to find
+ @type uid: C{str}
+ """
</ins><span class="cx">
</span><del>- returnValue(results)
</del><ins>+ if not hasattr(self, "_cachedCalendarResourcesForUID"):
+ self._cachedCalendarResourcesForUID = {}
+ if uid not in self._cachedCalendarResourcesForUID:
+ self._cachedCalendarResourcesForUID[uid] = (yield self.objectResourcesWithUID(uid, ["inbox"], allowShared=False))
+ returnValue(self._cachedCalendarResourcesForUID[uid])
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1953,7 +1958,7 @@
</span><span class="cx"> user_uuid = self._parentCollection.viewerHome().uid()
</span><span class="cx"> component = PerUserDataFilter(user_uuid).filter(component.duplicate())
</span><span class="cx">
</span><del>- scheduler = ImplicitScheduler()
</del><ins>+ scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx">
</span><span class="cx"> # PUT
</span><span class="cx"> do_implicit_action, is_scheduling_resource = (yield scheduler.testImplicitSchedulingPUT(
</span><span class="lines">@@ -2610,7 +2615,7 @@
</span><span class="cx"> if not isinbox and internal_state == ComponentRemoveState.NORMAL:
</span><span class="cx"> # Get data we need for implicit scheduling
</span><span class="cx"> calendar = (yield self.componentForUser())
</span><del>- scheduler = ImplicitScheduler()
</del><ins>+ scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx"> do_implicit_action, _ignore = (yield scheduler.testImplicitSchedulingDELETE(
</span><span class="cx"> self.calendar(),
</span><span class="cx"> self,
</span><span class="lines">@@ -2929,7 +2934,7 @@
</span><span class="cx">
</span><span class="cx"> # Only allow organizers to manipulate managed attachments for now
</span><span class="cx"> calendar = (yield self.componentForUser())
</span><del>- scheduler = ImplicitScheduler()
</del><ins>+ scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx"> is_attendee = (yield scheduler.testAttendeeEvent(self.calendar(), self, calendar,))
</span><span class="cx"> if is_attendee:
</span><span class="cx"> raise InvalidAttachmentOperation("Attendees are not allowed to manipulate managed attachments")
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -352,14 +352,18 @@
</span><span class="cx"> Print a report of all the SQL statements executed to date.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ total_statements = len(self.statements)
+ total_rows = sum([statement[1] for statement in self.statements])
+ total_time = sum([statement[2] for statement in self.statements]) * 1000.0
+
</ins><span class="cx"> toFile = StringIO()
</span><span class="cx"> toFile.write("*** SQL Stats ***\n")
</span><span class="cx"> toFile.write("\n")
</span><span class="cx"> toFile.write("Label: %s\n" % (self.label,))
</span><span class="cx"> toFile.write("Unique statements: %d\n" % (len(set([statement[0] for statement in self.statements]),),))
</span><del>- toFile.write("Total statements: %d\n" % (len(self.statements),))
- toFile.write("Total rows: %d\n" % (sum([statement[1] for statement in self.statements]),))
- toFile.write("Total time (ms): %.3f\n" % (sum([statement[2] for statement in self.statements]) * 1000.0,))
</del><ins>+ toFile.write("Total statements: %d\n" % (total_statements,))
+ toFile.write("Total rows: %d\n" % (total_rows,))
+ toFile.write("Total time (ms): %.3f\n" % (total_time,))
</ins><span class="cx"> for sql, rows, t in self.statements:
</span><span class="cx"> toFile.write("\n")
</span><span class="cx"> toFile.write("SQL: %s\n" % (sql,))
</span><span class="lines">@@ -372,8 +376,10 @@
</span><span class="cx"> else:
</span><span class="cx"> log.error(toFile.getvalue())
</span><span class="cx">
</span><ins>+ return (total_statements, total_rows, total_time,)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class CommonStoreTransactionMonitor(object):
</span><span class="cx"> """
</span><span class="cx"> Object that monitors the state of a transaction over time and logs or times out
</span><span class="lines">@@ -483,7 +489,9 @@
</span><span class="cx"> self.iudCount = 0
</span><span class="cx"> self.currentStatement = None
</span><span class="cx">
</span><ins>+ self.logItems = {}
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def enqueue(self, workItem, **kw):
</span><span class="cx"> """
</span><span class="cx"> Enqueue a L{twext.enterprise.queue.WorkItem} for later execution.
</span><span class="lines">@@ -1032,7 +1040,7 @@
</span><span class="cx">
</span><span class="cx"> # Do stats logging as a postCommit because there might be some pending preCommit SQL we want to log
</span><span class="cx"> if self._stats:
</span><del>- self.postCommit(self._stats.printReport)
</del><ins>+ self.postCommit(self.statsReport)
</ins><span class="cx"> return self._sqlTxn.commit()
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -1043,6 +1051,16 @@
</span><span class="cx"> return self._sqlTxn.abort()
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def statsReport(self):
+ """
+ Print the stats report and record log items
+ """
+ sql_statements, sql_rows, sql_time = self._stats.printReport()
+ self.logItems["sql-s"] = str(sql_statements)
+ self.logItems["sql-r"] = str(sql_rows)
+ self.logItems["sql-t"] = "%.1f" % (sql_time,)
+
+
</ins><span class="cx"> def _oldEventsBase(self, limit):
</span><span class="cx"> ch = schema.CALENDAR_HOME
</span><span class="cx"> co = schema.CALENDAR_OBJECT
</span><span class="lines">@@ -2329,16 +2347,20 @@
</span><span class="cx"> raise NotImplementedError()
</span><span class="cx">
</span><span class="cx">
</span><del>- @classproperty
- def _objectNamesSinceRevisionQuery(cls): #@NoSelf
</del><ins>+ @classmethod
+ def _objectNamesSinceRevisionQuery(cls, deleted=True): #@NoSelf
</ins><span class="cx"> """
</span><span class="cx"> DAL query for (resource, deleted-flag)
</span><span class="cx"> """
</span><span class="cx"> rev = cls._revisionsSchema
</span><del>- return Select([rev.RESOURCE_NAME, rev.DELETED],
- From=rev,
- Where=(rev.REVISION > Parameter("revision")).And(
- rev.RESOURCE_ID == Parameter("resourceID")))
</del><ins>+ where = (rev.REVISION > Parameter("revision")).And(rev.RESOURCE_ID == Parameter("resourceID"))
+ if not deleted:
+ where = where.And(rev.DELETED == False)
+ return Select(
+ [rev.RESOURCE_NAME, rev.DELETED],
+ From=rev,
+ Where=where,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def resourceNamesSinceToken(self, token):
</span><span class="lines">@@ -2363,10 +2385,10 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> results = [
</span><del>- (name if name else "", deleted)
- for name, deleted in
- (yield self._objectNamesSinceRevisionQuery.on(
- self._txn, revision=revision, resourceID=self._resourceID))
</del><ins>+ (name if name else "", deleted) for name, deleted in
+ (yield self._objectNamesSinceRevisionQuery(deleted=(revision != 0)).on(
+ self._txn, revision=revision, resourceID=self._resourceID)
+ )
</ins><span class="cx"> ]
</span><span class="cx"> results.sort(key=lambda x: x[1])
</span><span class="cx">
</span><span class="lines">@@ -3024,7 +3046,9 @@
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(shareeView._home._resourceID, shareeView._name)
</span><del>- queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+ cacheKey = queryCacher.keyForObjectWithResourceID(shareeView._home._resourceID, shareeView._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</ins><span class="cx">
</span><span class="cx"> shareeView._name = sharedname[0][0]
</span><span class="cx">
</span><span class="lines">@@ -3082,7 +3106,9 @@
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(shareeHome._resourceID, shareeChild._name)
</span><del>- queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+ cacheKey = queryCacher.keyForObjectWithResourceID(shareeHome._resourceID, shareeChild._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</ins><span class="cx"> else:
</span><span class="cx"> deletedBindName = None
</span><span class="cx">
</span><span class="lines">@@ -3347,10 +3373,9 @@
</span><span class="cx"> def invalidateQueryCache(self):
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span><span class="cx"> if queryCacher is not None:
</span><del>- cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID)
- yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
- cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name)
- yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+ yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForHomeChildMetaData(self._resourceID))
+ yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForObjectWithName(self._home._resourceID, self._name))
+ yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -3527,6 +3552,7 @@
</span><span class="cx"> if rows and queryCacher:
</span><span class="cx"> # Cache the result
</span><span class="cx"> queryCacher.setAfterCommit(home._txn, cacheKey, rows)
</span><ins>+ queryCacher.setAfterCommit(home._txn, queryCacher.keyForObjectWithResourceID(home._resourceID, rows[0][2]), rows)
</ins><span class="cx">
</span><span class="cx"> if not rows:
</span><span class="cx"> returnValue(None)
</span><span class="lines">@@ -3567,8 +3593,24 @@
</span><span class="cx"> @return: an L{CommonHomeChild} or C{None} if no such child
</span><span class="cx"> exists.
</span><span class="cx"> """
</span><del>- rows = yield cls._bindForResourceIDAndHomeID.on(
- home._txn, resourceID=resourceID, homeID=home._resourceID)
</del><ins>+
+ rows = None
+ queryCacher = home._txn._queryCacher
+
+ if queryCacher:
+ # Retrieve data from cache
+ cacheKey = queryCacher.keyForObjectWithResourceID(home._resourceID, resourceID)
+ rows = yield queryCacher.get(cacheKey)
+
+ if rows is None:
+ # No cached copy
+ rows = yield cls._bindForResourceIDAndHomeID.on(home._txn, resourceID=resourceID, homeID=home._resourceID)
+
+ if rows and queryCacher:
+ # Cache the result (under both the ID and name values)
+ queryCacher.setAfterCommit(home._txn, cacheKey, rows)
+ queryCacher.setAfterCommit(home._txn, queryCacher.keyForObjectWithName(home._resourceID, rows[0][3]), rows)
+
</ins><span class="cx"> if not rows:
</span><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="lines">@@ -3749,6 +3791,8 @@
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, oldName)
</span><span class="cx"> yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</span><ins>+ cacheKey = queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</ins><span class="cx">
</span><span class="cx"> yield self._renameQuery.on(self._txn, name=name,
</span><span class="cx"> resourceID=self._resourceID,
</span><span class="lines">@@ -3782,6 +3826,8 @@
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name)
</span><span class="cx"> yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</span><ins>+ cacheKey = queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</ins><span class="cx">
</span><span class="cx"> yield self._deletedSyncToken()
</span><span class="cx"> yield self._deleteQuery.on(self._txn, NoSuchHomeChildError,
</span><span class="lines">@@ -4498,7 +4544,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def create(cls, parent, name, component, options=None):
</span><span class="cx">
</span><del>- child = (yield cls.objectWithName(parent, name, None))
</del><ins>+ child = (yield parent.objectResourceWithName(name))
</ins><span class="cx"> if child:
</span><span class="cx"> raise ObjectResourceNameAlreadyExistsError(name)
</span><span class="cx">
</span></span></pre>
</div>
</div>
</body>
</html>