<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[13715] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/13715">13715</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2014-07-02 13:07:16 -0700 (Wed, 02 Jul 2014)</dd>
</dl>
<h3>Log Message</h3>
<pre>Fix issues related to cross-pod sharing sync and property handling.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkrequirementsdevtxt">CalendarServer/trunk/requirements-dev.txt</a></li>
<li><a href="#CalendarServertrunktxdavcaldavdatastoresqlpy">CalendarServer/trunk/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcarddavdatastoresqlpy">CalendarServer/trunk/txdav/carddav/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresqlpy">CalendarServer/trunk/txdav/common/datastore/sql.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkrequirementsdevtxt"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/requirements-dev.txt (13714 => 13715)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/requirements-dev.txt 2014-07-02 19:59:58 UTC (rev 13714)
+++ CalendarServer/trunk/requirements-dev.txt 2014-07-02 20:07:16 UTC (rev 13715)
</span><span class="lines">@@ -7,4 +7,4 @@
</span><span class="cx"> mockldap
</span><span class="cx"> q
</span><span class="cx"> --editable svn+http://svn.calendarserver.org/repository/calendarserver/CalDAVClientLibrary/trunk@13420#egg=CalDAVClientLibrary
</span><del>---editable svn+http://svn.calendarserver.org/repository/calendarserver/CalDAVTester/trunk@13708#egg=CalDAVTester
</del><ins>+--editable svn+http://svn.calendarserver.org/repository/calendarserver/CalDAVTester/trunk@13714#egg=CalDAVTester
</ins></span></pre></div>
<a id="CalendarServertrunktxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py (13714 => 13715)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2014-07-02 19:59:58 UTC (rev 13714)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2014-07-02 20:07:16 UTC (rev 13715)
</span><span class="lines">@@ -1366,13 +1366,13 @@
</span><span class="cx"> # Initialize these for all shares
</span><span class="cx"> for ename in self._shadowProperties:
</span><span class="cx"> if ename not in self.properties() and ename.toString() in props:
</span><del>- self.properties()[ename] = WebDAVDocument.fromString(props[ename]).root_element
</del><ins>+ self.properties()[ename] = WebDAVDocument.fromString(props[ename.toString()]).root_element
</ins><span class="cx">
</span><span class="cx"> # Only initialize these for direct shares
</span><span class="cx"> if self.direct():
</span><span class="cx"> for ename in (PropertyName.fromElement(element.DisplayName),):
</span><span class="cx"> if ename not in self.properties() and ename.toString() in props:
</span><del>- self.properties()[ename] = WebDAVDocument.fromString(props[ename]).root_element
</del><ins>+ self.properties()[ename] = WebDAVDocument.fromString(props[ename.toString()]).root_element
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> # FIXME: this is DAV-ish. Data store calendar objects don't have
</span></span></pre></div>
<a id="CalendarServertrunktxdavcarddavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py (13714 => 13715)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2014-07-02 19:59:58 UTC (rev 13714)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2014-07-02 20:07:16 UTC (rev 13715)
</span><span class="lines">@@ -839,13 +839,13 @@
</span><span class="cx"> # Initialize these for all shares
</span><span class="cx"> for ename in self._shadowProperties:
</span><span class="cx"> if ename not in self.properties() and ename.toString() in props:
</span><del>- self.properties()[ename] = WebDAVDocument.fromString(props[ename]).root_element
</del><ins>+ self.properties()[ename] = WebDAVDocument.fromString(props[ename.toString()]).root_element
</ins><span class="cx">
</span><span class="cx"> # Only initialize these for direct shares
</span><span class="cx"> if self.direct():
</span><span class="cx"> for ename in (PropertyName.fromElement(element.DisplayName),):
</span><span class="cx"> if ename not in self.properties() and ename.toString() in props:
</span><del>- self.properties()[ename] = WebDAVDocument.fromString(props[ename]).root_element
</del><ins>+ self.properties()[ename] = WebDAVDocument.fromString(props[ename.toString()]).root_element
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def contentType(self):
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql.py (13714 => 13715)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql.py 2014-07-02 19:59:58 UTC (rev 13714)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2014-07-02 20:07:16 UTC (rev 13715)
</span><span class="lines">@@ -3482,6 +3482,11 @@
</span><span class="cx"> We do the same SQL query for both depth "1" and "infinity", but filter the results for
</span><span class="cx"> "1" to only account for a collection change.
</span><span class="cx">
</span><ins>+ Now that we are truncating the revision table, we need to handle the full sync (revision == 0)
+ case a little differently as the revision table will not contain data for resources that exist,
+ but were last modified before the revision cut-off. Instead for revision == 0 we need to list
+ all existing child resources.
+
</ins><span class="cx"> We need to handle shared collection a little differently from owned ones. When a shared collection
</span><span class="cx"> is bound into a home we record a revision for it using the sharee home id and sharee collection name.
</span><span class="cx"> That revision is the "starting point" for changes: so if sync occurs with a revision earlier than
</span><span class="lines">@@ -3501,61 +3506,96 @@
</span><span class="cx"> @type depth: C{str}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- changed = set()
- deleted = set()
- invalid = set()
</del><span class="cx"> if revision:
</span><span class="cx"> minValidRevision = yield self._txn.calendarserverValue("MIN-VALID-REVISION")
</span><span class="cx"> if revision < int(minValidRevision):
</span><span class="cx"> raise SyncTokenValidException
</span><ins>+ else:
+ results = yield self.resourceNamesSinceRevisionZero(depth)
+ returnValue(results)
</ins><span class="cx">
</span><del>- results = [
- (
- path if path else (collection if collection else ""),
- name if name else "",
- wasdeleted
- )
- for path, collection, name, wasdeleted in
- (yield self.doChangesQuery(revision))
- ]
</del><ins>+ # Use revision table to find changes since the last revision - this will not include
+ # changes to child resources of shared collections - those we will get later
+ results = [
+ (
+ path if path else (collection if collection else ""),
+ name if name else "",
+ wasdeleted
+ )
+ for path, collection, name, wasdeleted in
+ (yield self.doChangesQuery(revision))
+ ]
</ins><span class="cx">
</span><del>- deleted_collections = set()
- for path, name, wasdeleted in results:
- if wasdeleted:
- if name:
- # Resource deleted - for depth "1" report collection as changed,
- # otherwise report resource as deleted
- if depth == "1":
- changed.add("%s/" % (path,))
- else:
- deleted.add("%s/%s" % (path, name,))
</del><ins>+ changed = set()
+ deleted = set()
+ invalid = set()
+ deleted_collections = set()
+ for path, name, wasdeleted in results:
+ if wasdeleted:
+ if name:
+ # Resource deleted - for depth "1" report collection as changed,
+ # otherwise report resource as deleted
+ if depth == "1":
+ changed.add("%s/" % (path,))
</ins><span class="cx"> else:
</span><del>- # Collection was deleted
- deleted.add("%s/" % (path,))
- deleted_collections.add(path)
</del><ins>+ deleted.add("%s/%s" % (path, name,))
+ else:
+ # Collection was deleted
+ deleted.add("%s/" % (path,))
+ deleted_collections.add(path)
</ins><span class="cx">
</span><del>- if path not in deleted_collections:
- # Always report collection as changed
- changed.add("%s/" % (path,))
- if name:
- # Resource changed - for depth "infinity" report resource as changed
- if depth != "1":
- changed.add("%s/%s" % (path, name,))
</del><ins>+ if path not in deleted_collections:
+ # Always report collection as changed
+ changed.add("%s/" % (path,))
</ins><span class="cx">
</span><del>- # Now deal with shared collections (and owned if revision == 0)
</del><ins>+ # Resource changed - for depth "infinity" report resource as changed
+ if name and depth != "1":
+ changed.add("%s/%s" % (path, name,))
+
+ # Now deal with existing shared collections
+ # TODO: think about whether this can be done in one query rather than looping over each share
</ins><span class="cx"> for share in (yield self.children()):
</span><del>- if share.owned():
- if not revision:
- path = share.name()
- # Always report collection as changed
- changed.add("%s/" % (path,))
</del><ins>+ if not share.owned():
+ sharedChanged, sharedDeleted, sharedInvalid = yield share.sharedChildResourceNamesSinceRevision(revision, depth)
+ changed |= sharedChanged
+ changed -= sharedInvalid
+ deleted |= sharedDeleted
+ deleted -= sharedInvalid
+ invalid |= sharedInvalid
</ins><span class="cx">
</span><del>- # Resource changed - for depth "infinity" report resource as changed
- if depth != "1":
- for name in (yield share.listObjectResources()):
- changed.add("%s/%s" % (path, name,))
</del><ins>+ changed = sorted(changed)
+ deleted = sorted(deleted)
+ invalid = sorted(invalid)
+ returnValue((changed, deleted, invalid,))
+
+
+
+
+ @inlineCallbacks
+ def resourceNamesSinceRevisionZero(self, depth):
+ """
+ Revision == 0 specialization of L{resourceNamesSinceRevision} .
+
+ @param depth: depth for determine what changed
+ @type depth: C{str}
+ """
+
+ # Scan each child
+ changed = set()
+ deleted = set()
+ invalid = set()
+ for child in (yield self.children()):
+ if child.owned():
+ path = child.name()
+ # Always report collection as changed
+ changed.add("%s/" % (path,))
+
+ # Resource changed - for depth "infinity" report resource as changed
+ if depth != "1":
+ for name in (yield child.listObjectResources()):
+ changed.add("%s/%s" % (path, name,))
</ins><span class="cx"> else:
</span><del>- sharedChanged, sharedDeleted, sharedInvalid = yield share.sharedChildResourceNamesSinceRevision(revision, depth)
</del><ins>+ sharedChanged, sharedDeleted, sharedInvalid = yield child.sharedChildResourceNamesSinceRevisionZero(depth)
</ins><span class="cx"> changed |= sharedChanged
</span><span class="cx"> changed -= sharedInvalid
</span><span class="cx"> deleted |= sharedDeleted
</span><span class="lines">@@ -6203,57 +6243,83 @@
</span><span class="cx"> else:
</span><span class="cx"> invalid.add(self.name() + "/")
</span><span class="cx"> else:
</span><ins>+ # If revision is prior to when the share was created, then treat as a full sync of the share
</ins><span class="cx"> if revision != 0 and revision < self._bindRevision:
</span><span class="cx"> if depth != "1":
</span><ins>+ # This should never happen unless the client the share existed, was removed and then
+ # re-added and the client has a token from before the remove. In that case the token is no
+ # longer valid - a full sync has to be done.
</ins><span class="cx"> raise SyncTokenValidException
</span><span class="cx"> else:
</span><del>- revision = 0
</del><ins>+ results = yield self.sharedChildResourceNamesSinceRevisionZero(depth)
+ returnValue(results)
</ins><span class="cx">
</span><del>- if revision:
- rev = self._revisionsSchema
- results = [
- (
- self.name(),
- name if name else "",
- wasdeleted
- )
- for name, wasdeleted in
- (yield Select(
- [rev.RESOURCE_NAME, rev.DELETED],
- From=rev,
- Where=(rev.REVISION > revision).And(
- rev.RESOURCE_ID == self._resourceID)
- ).on(self._txn))
- if name
- ]
</del><ins>+ rev = self._revisionsSchema
+ results = [
+ (
+ self.name(),
+ name if name else "",
+ wasdeleted
+ )
+ for name, wasdeleted in
+ (yield Select(
+ [rev.RESOURCE_NAME, rev.DELETED],
+ From=rev,
+ Where=(rev.REVISION > revision).And(
+ rev.RESOURCE_ID == self._resourceID)
+ ).on(self._txn))
+ if name
+ ]
</ins><span class="cx">
</span><del>- for path, name, wasdeleted in results:
- if wasdeleted:
- if depth == "1":
- changed.add("%s/" % (path,))
- else:
- deleted.add("%s/%s" % (path, name,))
</del><ins>+ for path, name, wasdeleted in results:
+ if wasdeleted:
+ if depth == "1":
+ changed.add("%s/" % (path,))
+ else:
+ deleted.add("%s/%s" % (path, name,))
</ins><span class="cx">
</span><del>- # Always report collection as changed
- changed.add("%s/" % (path,))
-
- # Resource changed - for depth "infinity" report resource as changed
- if depth != "1":
- changed.add("%s/%s" % (path, name,))
- else:
- path = self.name()
</del><span class="cx"> # Always report collection as changed
</span><span class="cx"> changed.add("%s/" % (path,))
</span><span class="cx">
</span><span class="cx"> # Resource changed - for depth "infinity" report resource as changed
</span><del>- if depth != "1":
- for name in (yield self.listObjectResources()):
- changed.add("%s/%s" % (path, name,))
</del><ins>+ if name and depth != "1":
+ changed.add("%s/%s" % (path, name,))
</ins><span class="cx">
</span><span class="cx"> returnValue((changed, deleted, invalid,))
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def sharedChildResourceNamesSinceRevisionZero(self, depth):
+ """
+ Revision == 0 specialization of L{sharedChildResourceNamesSinceRevision}. We report on all
+ existing resources -= this collection and children (if depth == infinite).
+
+ @param depth: depth for determine what changed
+ @type depth: C{str}
+ """
+ changed = set()
+ deleted = set()
+ invalid = set()
+ path = self.name()
+ if self.external():
+ if depth == "1":
+ changed.add("{}/".format(path))
+ else:
+ invalid.add("{}/".format(path))
+ else:
+ path = self.name()
+ # Always report collection as changed
+ changed.add(path + "/")
+
+ # Resource changed - for depth "infinity" report resource as changed
+ if depth != "1":
+ for name in (yield self.listObjectResources()):
+ changed.add("%s/%s" % (path, name,))
+
+ returnValue((changed, deleted, invalid,))
+
+
+ @inlineCallbacks
</ins><span class="cx"> def _loadPropertyStore(self, props=None):
</span><span class="cx"> if props is None:
</span><span class="cx"> # Use any authz uid in place of the viewer uid so delegates have their own
</span></span></pre>
</div>
</div>
</body>
</html>