<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[11826] CalendarServer/branches/users/cdaboo/performance-tweaks</title>
</head>
<body>

<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt;  }
#msg dl a { font-weight: bold}
#msg dl a:link    { color:#fc3; }
#msg dl a:active  { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff  {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/11826">11826</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2013-10-18 12:38:34 -0700 (Fri, 18 Oct 2013)</dd>
</dl>

<h3>Log Message</h3>
<pre>Various performance tweaks and fixes to extended log items.</pre>

<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakscalendarserveraccesslogpy">CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavresourcepy">CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavscheduling_storecaldavresourcepy">CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavbasedatastoreutilpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastorefilepy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulepy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingimplicitpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingutilspy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdabooperformancetweakstxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py</a></li>
</ul>

</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdabooperformancetweakscalendarserveraccesslogpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/calendarserver/accesslog.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -173,7 +173,7 @@
</span><span class="cx">                     formatArgs[&quot;t&quot;] = (nowtime - request.timeStamps[0][1]) * 1000
</span><span class="cx"> 
</span><span class="cx">                 if hasattr(request, &quot;extendedLogItems&quot;):
</span><del>-                    for k, v in request.extendedLogItems.iteritems():
</del><ins>+                    for k, v in sorted(request.extendedLogItems.iteritems(), key=lambda x: x[0]):
</ins><span class="cx">                         k = str(k).replace('&quot;', &quot;%22&quot;)
</span><span class="cx">                         v = str(v).replace('&quot;', &quot;%22&quot;)
</span><span class="cx">                         if &quot; &quot; in v:
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/resource.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -333,6 +333,12 @@
</span><span class="cx">             else:
</span><span class="cx">                 yield transaction.commit()
</span><span class="cx"> 
</span><ins>+                # Log extended item
+                if transaction.logItems:
+                    if not hasattr(request, &quot;extendedLogItems&quot;):
+                        request.extendedLogItems = {}
+                    request.extendedLogItems.update(transaction.logItems)
+
</ins><span class="cx">                 # May need to reset the last-modified header in the response as txn.commit() can change it due to pre-commit hooks
</span><span class="cx">                 if response.headers.hasHeader(&quot;last-modified&quot;):
</span><span class="cx">                     response.headers.setHeader(&quot;last-modified&quot;, self.lastModified())
</span><span class="lines">@@ -2551,15 +2557,6 @@
</span><span class="cx">         return self._newStoreHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object._newStoreObject, mode)
</span><span class="cx"> 
</span><span class="cx"> 
</span><del>-    def getCalendarResourcesForUID(self, uid, allow_shared=False):
-        &quot;&quot;&quot;
-        Return all child object resources with the specified UID.
-
-        Pass through direct to store.
-        &quot;&quot;&quot;
-        return self._newStoreHome.getCalendarResourcesForUID(uid, allow_shared)
-
-
</del><span class="cx">     def defaultAccessControlList(self):
</span><span class="cx">         myPrincipal = self.principalForRecord()
</span><span class="cx"> 
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstwistedcaldavscheduling_storecaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/twistedcaldav/scheduling_store/caldav/resource.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -422,8 +422,12 @@
</span><span class="cx">                 authz = (yield request.locateResource(principalURL))
</span><span class="cx">                 self._associatedTransaction._authz_uid = authz.record.guid
</span><span class="cx"> 
</span><ins>+        # Log extended item
+        if not hasattr(request, &quot;extendedLogItems&quot;):
+            request.extendedLogItems = {}
+
</ins><span class="cx">         # This is a local CALDAV scheduling operation.
</span><del>-        scheduler = CalDAVScheduler(self._associatedTransaction, self.parent._newStoreHome.uid())
</del><ins>+        scheduler = CalDAVScheduler(self._associatedTransaction, self.parent._newStoreHome.uid(), logItems=request.extendedLogItems)
</ins><span class="cx"> 
</span><span class="cx">         # Do the POST processing treating
</span><span class="cx">         result = (yield scheduler.doSchedulingViaPOST(originator, recipients, calendar))
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavbasedatastoreutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/base/datastore/util.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -92,6 +92,12 @@
</span><span class="cx">         return &quot;objectWithName:%s:%s&quot; % (homeResourceID, name)
</span><span class="cx"> 
</span><span class="cx"> 
</span><ins>+    # Home child objects by id
+
+    def keyForObjectWithResourceID(self, homeResourceID, resourceID):
+        return &quot;objectWithName:%s:%s&quot; % (homeResourceID, resourceID)
+
+
</ins><span class="cx">     # Home metadata (Created/Modified)
</span><span class="cx"> 
</span><span class="cx">     def keyForHomeMetaData(self, homeResourceID):
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastorefilepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/file.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -128,7 +128,7 @@
</span><span class="cx">     @inlineCallbacks
</span><span class="cx">     def hasCalendarResourceUIDSomewhereElse(self, uid, ok_object, type):
</span><span class="cx"> 
</span><del>-        objectResources = (yield self.objectResourcesWithUID(uid, (&quot;inbox&quot;,)))
</del><ins>+        objectResources = (yield self.getCalendarResourcesForUID(uid))
</ins><span class="cx">         for objectResource in objectResources:
</span><span class="cx">             if ok_object and objectResource._path == ok_object._path:
</span><span class="cx">                 continue
</span><span class="lines">@@ -140,14 +140,9 @@
</span><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx">     @inlineCallbacks
</span><del>-    def getCalendarResourcesForUID(self, uid, allow_shared=False):
</del><ins>+    def getCalendarResourcesForUID(self, uid):
</ins><span class="cx"> 
</span><del>-        results = []
-        objectResources = (yield self.objectResourcesWithUID(uid, (&quot;inbox&quot;,)))
-        for objectResource in objectResources:
-            if allow_shared or objectResource._parentCollection.owned():
-                results.append(objectResource)
-
</del><ins>+        results = (yield self.objectResourcesWithUID(uid, (&quot;inbox&quot;,), allowShared=False))
</ins><span class="cx">         returnValue(results)
</span><span class="cx"> 
</span><span class="cx"> 
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/schedule.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -112,8 +112,8 @@
</span><span class="cx">         return self._calendarHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object, type)
</span><span class="cx"> 
</span><span class="cx"> 
</span><del>-    def getCalendarResourcesForUID(self, uid, allow_shared=False):
-        return self._calendarHome.getCalendarResourcesForUID(uid, allow_shared)
</del><ins>+    def getCalendarResourcesForUID(self, uid):
+        return self._calendarHome.getCalendarResourcesForUID(uid)
</ins><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx"> 
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingimplicitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/implicit.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -56,10 +56,10 @@
</span><span class="cx">     STATUS_ORPHANED_CANCELLED_EVENT = 1
</span><span class="cx">     STATUS_ORPHANED_EVENT = 2
</span><span class="cx"> 
</span><del>-    def __init__(self):
</del><ins>+    def __init__(self, logItems=None):
</ins><span class="cx"> 
</span><span class="cx">         self.return_status = ImplicitScheduler.STATUS_OK
</span><del>-        self.logItems = {}
</del><ins>+        self.logItems = logItems
</ins><span class="cx">         self.allowed_to_schedule = True
</span><span class="cx">         self.suppress_refresh = False
</span><span class="cx"> 
</span><span class="lines">@@ -383,7 +383,7 @@
</span><span class="cx">             if self.txn.doing_attendee_refresh == 0:
</span><span class="cx">                 delattr(self.txn, &quot;doing_attendee_refresh&quot;)
</span><span class="cx"> 
</span><del>-        if refreshCount:
</del><ins>+        if refreshCount and self.logItems is not None:
</ins><span class="cx">             self.logItems[&quot;itip.refreshes&quot;] = refreshCount
</span><span class="cx"> 
</span><span class="cx"> 
</span><span class="lines">@@ -925,7 +925,8 @@
</span><span class="cx">         if self.action in (&quot;create&quot;, &quot;modify&quot;,):
</span><span class="cx">             total += (yield self.processRequests())
</span><span class="cx"> 
</span><del>-        self.logItems[&quot;itip.requests&quot;] = total
</del><ins>+        if self.logItems is not None:
+            self.logItems[&quot;itip.requests&quot;] = total
</ins><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx">     @inlineCallbacks
</span><span class="lines">@@ -1304,7 +1305,8 @@
</span><span class="cx">         # First make sure we are allowed to schedule
</span><span class="cx">         self.testSchedulingAllowed()
</span><span class="cx"> 
</span><del>-        self.logItems[&quot;itip.reply&quot;] = &quot;reply&quot;
</del><ins>+        if self.logItems is not None:
+            self.logItems[&quot;itip.reply&quot;] = &quot;reply&quot;
</ins><span class="cx"> 
</span><span class="cx">         itipmsg = iTipGenerator.generateAttendeeReply(self.calendar, self.attendee, changedRids=changedRids)
</span><span class="cx"> 
</span><span class="lines">@@ -1317,7 +1319,8 @@
</span><span class="cx">         # First make sure we are allowed to schedule
</span><span class="cx">         self.testSchedulingAllowed()
</span><span class="cx"> 
</span><del>-        self.logItems[&quot;itip.reply&quot;] = &quot;cancel&quot;
</del><ins>+        if self.logItems is not None:
+            self.logItems[&quot;itip.reply&quot;] = &quot;cancel&quot;
</ins><span class="cx"> 
</span><span class="cx">         itipmsg = iTipGenerator.generateAttendeeReply(self.calendar, self.attendee, force_decline=True)
</span><span class="cx"> 
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoreschedulingutilspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/scheduling/utils.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -21,7 +21,7 @@
</span><span class="cx"> log = Logger()
</span><span class="cx"> 
</span><span class="cx"> @inlineCallbacks
</span><del>-def getCalendarObjectForRecord(txn, record, uid, allow_shared=False):
</del><ins>+def getCalendarObjectForRecord(txn, record, uid):
</ins><span class="cx">     &quot;&quot;&quot;
</span><span class="cx">     Get a copy of the event for a calendar user identified by a directory record.
</span><span class="cx"> 
</span><span class="lines">@@ -34,7 +34,7 @@
</span><span class="cx">         calendar_home = yield txn.calendarHomeWithUID(record.uid)
</span><span class="cx"> 
</span><span class="cx">         # Get matching newstore objects
</span><del>-        objectResources = (yield calendar_home.getCalendarResourcesForUID(uid, allow_shared))
</del><ins>+        objectResources = (yield calendar_home.getCalendarResourcesForUID(uid))
</ins><span class="cx"> 
</span><span class="cx">         if len(objectResources) &gt; 1:
</span><span class="cx">             # Delete all but the first one
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/caldav/datastore/sql.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -527,9 +527,7 @@
</span><span class="cx">         # refer to calendar *object* UIDs, since calendar *resources* are an
</span><span class="cx">         # HTTP protocol layer thing, not a data store thing.  (See also
</span><span class="cx">         # objectResourcesWithUID.)
</span><del>-        objectResources = (
-            yield self.objectResourcesWithUID(uid, [&quot;inbox&quot;], False)
-        )
</del><ins>+        objectResources = (yield self.getCalendarResourcesForUID(uid))
</ins><span class="cx">         for objectResource in objectResources:
</span><span class="cx">             if ok_object and objectResource._resourceID == ok_object._resourceID:
</span><span class="cx">                 continue
</span><span class="lines">@@ -541,15 +539,22 @@
</span><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx">     @inlineCallbacks
</span><del>-    def getCalendarResourcesForUID(self, uid, allow_shared=False):
</del><ins>+    def getCalendarResourcesForUID(self, uid):
+        &quot;&quot;&quot;
+        Find all calendar object resources in the calendar home that are not in the &quot;inbox&quot; collection
+        and not in shared collections.
+        Cache the result of this query as it can happen multiple times during scheduling under slightly
+        different circumstances.
</ins><span class="cx"> 
</span><del>-        results = []
-        objectResources = (yield self.objectResourcesWithUID(uid, [&quot;inbox&quot;]))
-        for objectResource in objectResources:
-            if allow_shared or objectResource._parentCollection.owned():
-                results.append(objectResource)
</del><ins>+        @param uid: the UID of the calendar object resources to find
+        @type uid: C{str}
+        &quot;&quot;&quot;
</ins><span class="cx"> 
</span><del>-        returnValue(results)
</del><ins>+        if not hasattr(self, &quot;_cachedCalendarResourcesForUID&quot;):
+            self._cachedCalendarResourcesForUID = {}
+        if uid not in self._cachedCalendarResourcesForUID:
+            self._cachedCalendarResourcesForUID[uid] = (yield self.objectResourcesWithUID(uid, [&quot;inbox&quot;], allowShared=False))
+        returnValue(self._cachedCalendarResourcesForUID[uid])
</ins><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx">     @inlineCallbacks
</span><span class="lines">@@ -1953,7 +1958,7 @@
</span><span class="cx">                 user_uuid = self._parentCollection.viewerHome().uid()
</span><span class="cx">                 component = PerUserDataFilter(user_uuid).filter(component.duplicate())
</span><span class="cx"> 
</span><del>-            scheduler = ImplicitScheduler()
</del><ins>+            scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx"> 
</span><span class="cx">             # PUT
</span><span class="cx">             do_implicit_action, is_scheduling_resource = (yield scheduler.testImplicitSchedulingPUT(
</span><span class="lines">@@ -2610,7 +2615,7 @@
</span><span class="cx">         if not isinbox and internal_state == ComponentRemoveState.NORMAL:
</span><span class="cx">             # Get data we need for implicit scheduling
</span><span class="cx">             calendar = (yield self.componentForUser())
</span><del>-            scheduler = ImplicitScheduler()
</del><ins>+            scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx">             do_implicit_action, _ignore = (yield scheduler.testImplicitSchedulingDELETE(
</span><span class="cx">                 self.calendar(),
</span><span class="cx">                 self,
</span><span class="lines">@@ -2929,7 +2934,7 @@
</span><span class="cx"> 
</span><span class="cx">         # Only allow organizers to manipulate managed attachments for now
</span><span class="cx">         calendar = (yield self.componentForUser())
</span><del>-        scheduler = ImplicitScheduler()
</del><ins>+        scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx">         is_attendee = (yield scheduler.testAttendeeEvent(self.calendar(), self, calendar,))
</span><span class="cx">         if is_attendee:
</span><span class="cx">             raise InvalidAttachmentOperation(&quot;Attendees are not allowed to manipulate managed attachments&quot;)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdabooperformancetweakstxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py (11825 => 11826)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py        2013-10-18 19:37:48 UTC (rev 11825)
+++ CalendarServer/branches/users/cdaboo/performance-tweaks/txdav/common/datastore/sql.py        2013-10-18 19:38:34 UTC (rev 11826)
</span><span class="lines">@@ -352,14 +352,18 @@
</span><span class="cx">         Print a report of all the SQL statements executed to date.
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="cx"> 
</span><ins>+        total_statements = len(self.statements)
+        total_rows = sum([statement[1] for statement in self.statements])
+        total_time = sum([statement[2] for statement in self.statements]) * 1000.0
+
</ins><span class="cx">         toFile = StringIO()
</span><span class="cx">         toFile.write(&quot;*** SQL Stats ***\n&quot;)
</span><span class="cx">         toFile.write(&quot;\n&quot;)
</span><span class="cx">         toFile.write(&quot;Label: %s\n&quot; % (self.label,))
</span><span class="cx">         toFile.write(&quot;Unique statements: %d\n&quot; % (len(set([statement[0] for statement in self.statements]),),))
</span><del>-        toFile.write(&quot;Total statements: %d\n&quot; % (len(self.statements),))
-        toFile.write(&quot;Total rows: %d\n&quot; % (sum([statement[1] for statement in self.statements]),))
-        toFile.write(&quot;Total time (ms): %.3f\n&quot; % (sum([statement[2] for statement in self.statements]) * 1000.0,))
</del><ins>+        toFile.write(&quot;Total statements: %d\n&quot; % (total_statements,))
+        toFile.write(&quot;Total rows: %d\n&quot; % (total_rows,))
+        toFile.write(&quot;Total time (ms): %.3f\n&quot; % (total_time,))
</ins><span class="cx">         for sql, rows, t in self.statements:
</span><span class="cx">             toFile.write(&quot;\n&quot;)
</span><span class="cx">             toFile.write(&quot;SQL: %s\n&quot; % (sql,))
</span><span class="lines">@@ -372,8 +376,10 @@
</span><span class="cx">         else:
</span><span class="cx">             log.error(toFile.getvalue())
</span><span class="cx"> 
</span><ins>+        return (total_statements, total_rows, total_time,)
</ins><span class="cx"> 
</span><span class="cx"> 
</span><ins>+
</ins><span class="cx"> class CommonStoreTransactionMonitor(object):
</span><span class="cx">     &quot;&quot;&quot;
</span><span class="cx">     Object that monitors the state of a transaction over time and logs or times out
</span><span class="lines">@@ -483,7 +489,9 @@
</span><span class="cx">         self.iudCount = 0
</span><span class="cx">         self.currentStatement = None
</span><span class="cx"> 
</span><ins>+        self.logItems = {}
</ins><span class="cx"> 
</span><ins>+
</ins><span class="cx">     def enqueue(self, workItem, **kw):
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="cx">         Enqueue a L{twext.enterprise.queue.WorkItem} for later execution.
</span><span class="lines">@@ -1032,7 +1040,7 @@
</span><span class="cx"> 
</span><span class="cx">         # Do stats logging as a postCommit because there might be some pending preCommit SQL we want to log
</span><span class="cx">         if self._stats:
</span><del>-            self.postCommit(self._stats.printReport)
</del><ins>+            self.postCommit(self.statsReport)
</ins><span class="cx">         return self._sqlTxn.commit()
</span><span class="cx"> 
</span><span class="cx"> 
</span><span class="lines">@@ -1043,6 +1051,16 @@
</span><span class="cx">         return self._sqlTxn.abort()
</span><span class="cx"> 
</span><span class="cx"> 
</span><ins>+    def statsReport(self):
+        &quot;&quot;&quot;
+        Print the stats report and record log items
+        &quot;&quot;&quot;
+        sql_statements, sql_rows, sql_time = self._stats.printReport()
+        self.logItems[&quot;sql-s&quot;] = str(sql_statements)
+        self.logItems[&quot;sql-r&quot;] = str(sql_rows)
+        self.logItems[&quot;sql-t&quot;] = &quot;%.1f&quot; % (sql_time,)
+
+
</ins><span class="cx">     def _oldEventsBase(self, limit):
</span><span class="cx">         ch = schema.CALENDAR_HOME
</span><span class="cx">         co = schema.CALENDAR_OBJECT
</span><span class="lines">@@ -2329,16 +2347,20 @@
</span><span class="cx">         raise NotImplementedError()
</span><span class="cx"> 
</span><span class="cx"> 
</span><del>-    @classproperty
-    def _objectNamesSinceRevisionQuery(cls): #@NoSelf
</del><ins>+    @classmethod
+    def _objectNamesSinceRevisionQuery(cls, deleted=True): #@NoSelf
</ins><span class="cx">         &quot;&quot;&quot;
</span><span class="cx">         DAL query for (resource, deleted-flag)
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="cx">         rev = cls._revisionsSchema
</span><del>-        return Select([rev.RESOURCE_NAME, rev.DELETED],
-                      From=rev,
-                      Where=(rev.REVISION &gt; Parameter(&quot;revision&quot;)).And(
-                          rev.RESOURCE_ID == Parameter(&quot;resourceID&quot;)))
</del><ins>+        where = (rev.REVISION &gt; Parameter(&quot;revision&quot;)).And(rev.RESOURCE_ID == Parameter(&quot;resourceID&quot;))
+        if not deleted:
+            where = where.And(rev.DELETED == False)
+        return Select(
+            [rev.RESOURCE_NAME, rev.DELETED],
+            From=rev,
+            Where=where,
+        )
</ins><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx">     def resourceNamesSinceToken(self, token):
</span><span class="lines">@@ -2363,10 +2385,10 @@
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="cx"> 
</span><span class="cx">         results = [
</span><del>-            (name if name else &quot;&quot;, deleted)
-            for name, deleted in
-            (yield self._objectNamesSinceRevisionQuery.on(
-                self._txn, revision=revision, resourceID=self._resourceID))
</del><ins>+            (name if name else &quot;&quot;, deleted) for name, deleted in
+                (yield self._objectNamesSinceRevisionQuery(deleted=(revision != 0)).on(
+                    self._txn, revision=revision, resourceID=self._resourceID)
+                )
</ins><span class="cx">         ]
</span><span class="cx">         results.sort(key=lambda x: x[1])
</span><span class="cx"> 
</span><span class="lines">@@ -3024,7 +3046,9 @@
</span><span class="cx">             queryCacher = self._txn._queryCacher
</span><span class="cx">             if queryCacher:
</span><span class="cx">                 cacheKey = queryCacher.keyForObjectWithName(shareeView._home._resourceID, shareeView._name)
</span><del>-                queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+                yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+                cacheKey = queryCacher.keyForObjectWithResourceID(shareeView._home._resourceID, shareeView._resourceID)
+                yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</ins><span class="cx"> 
</span><span class="cx">             shareeView._name = sharedname[0][0]
</span><span class="cx"> 
</span><span class="lines">@@ -3082,7 +3106,9 @@
</span><span class="cx">             queryCacher = self._txn._queryCacher
</span><span class="cx">             if queryCacher:
</span><span class="cx">                 cacheKey = queryCacher.keyForObjectWithName(shareeHome._resourceID, shareeChild._name)
</span><del>-                queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+                yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+                cacheKey = queryCacher.keyForObjectWithResourceID(shareeHome._resourceID, shareeChild._resourceID)
+                yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</ins><span class="cx">         else:
</span><span class="cx">             deletedBindName = None
</span><span class="cx"> 
</span><span class="lines">@@ -3347,10 +3373,9 @@
</span><span class="cx">     def invalidateQueryCache(self):
</span><span class="cx">         queryCacher = self._txn._queryCacher
</span><span class="cx">         if queryCacher is not None:
</span><del>-            cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID)
-            yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
-            cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name)
-            yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+            yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForHomeChildMetaData(self._resourceID))
+            yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForObjectWithName(self._home._resourceID, self._name))
+            yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID))
</ins><span class="cx"> 
</span><span class="cx"> 
</span><span class="cx"> 
</span><span class="lines">@@ -3527,6 +3552,7 @@
</span><span class="cx">             if rows and queryCacher:
</span><span class="cx">                 # Cache the result
</span><span class="cx">                 queryCacher.setAfterCommit(home._txn, cacheKey, rows)
</span><ins>+                queryCacher.setAfterCommit(home._txn, queryCacher.keyForObjectWithResourceID(home._resourceID, rows[0][2]), rows)
</ins><span class="cx"> 
</span><span class="cx">         if not rows:
</span><span class="cx">             returnValue(None)
</span><span class="lines">@@ -3567,8 +3593,24 @@
</span><span class="cx">         @return: an L{CommonHomeChild} or C{None} if no such child
</span><span class="cx">             exists.
</span><span class="cx">         &quot;&quot;&quot;
</span><del>-        rows = yield cls._bindForResourceIDAndHomeID.on(
-            home._txn, resourceID=resourceID, homeID=home._resourceID)
</del><ins>+
+        rows = None
+        queryCacher = home._txn._queryCacher
+
+        if queryCacher:
+            # Retrieve data from cache
+            cacheKey = queryCacher.keyForObjectWithResourceID(home._resourceID, resourceID)
+            rows = yield queryCacher.get(cacheKey)
+
+        if rows is None:
+            # No cached copy
+            rows = yield cls._bindForResourceIDAndHomeID.on(home._txn, resourceID=resourceID, homeID=home._resourceID)
+
+            if rows and queryCacher:
+                # Cache the result (under both the ID and name values)
+                queryCacher.setAfterCommit(home._txn, cacheKey, rows)
+                queryCacher.setAfterCommit(home._txn, queryCacher.keyForObjectWithName(home._resourceID, rows[0][3]), rows)
+
</ins><span class="cx">         if not rows:
</span><span class="cx">             returnValue(None)
</span><span class="cx"> 
</span><span class="lines">@@ -3749,6 +3791,8 @@
</span><span class="cx">         if queryCacher:
</span><span class="cx">             cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, oldName)
</span><span class="cx">             yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</span><ins>+            cacheKey = queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID)
+            yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</ins><span class="cx"> 
</span><span class="cx">         yield self._renameQuery.on(self._txn, name=name,
</span><span class="cx">                                    resourceID=self._resourceID,
</span><span class="lines">@@ -3782,6 +3826,8 @@
</span><span class="cx">         if queryCacher:
</span><span class="cx">             cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name)
</span><span class="cx">             yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</span><ins>+            cacheKey = queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID)
+            yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</ins><span class="cx"> 
</span><span class="cx">         yield self._deletedSyncToken()
</span><span class="cx">         yield self._deleteQuery.on(self._txn, NoSuchHomeChildError,
</span><span class="lines">@@ -4498,7 +4544,7 @@
</span><span class="cx">     @inlineCallbacks
</span><span class="cx">     def create(cls, parent, name, component, options=None):
</span><span class="cx"> 
</span><del>-        child = (yield cls.objectWithName(parent, name, None))
</del><ins>+        child = (yield parent.objectResourceWithName(name))
</ins><span class="cx">         if child:
</span><span class="cx">             raise ObjectResourceNameAlreadyExistsError(name)
</span><span class="cx"> 
</span></span></pre>
</div>
</div>

</body>
</html>