<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14925] CalendarServer/trunk/txdav</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14925">14925</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-06-26 14:21:39 -0700 (Fri, 26 Jun 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Fix revision clean-up and revision MODIFIED updating.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunktxdavcaldavdatastoresqlpy">CalendarServer/trunk/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcaldavdatastoretesttest_sqlpy">CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcarddavdatastoresqlpy">CalendarServer/trunk/txdav/carddav/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcarddavdatastoretesttest_sqlpy">CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresqlpy">CalendarServer/trunk/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_sharingpy">CalendarServer/trunk/txdav/common/datastore/sql_sharing.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_utilpy">CalendarServer/trunk/txdav/common/datastore/sql_util.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoreworkrevision_cleanuppy">CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoreworktesttest_revision_cleanuppy">CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunktxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -2568,11 +2568,8 @@
</span><span class="cx"> }
</span><span class="cx"> accesstype_to_accessMode = dict([(v, k) for k, v in accessMode_to_type.items()])
</span><span class="cx">
</span><del>-def _pathToName(path):
- return path.rsplit(".", 1)[0]
</del><span class="cx">
</span><span class="cx">
</span><del>-
</del><span class="cx"> class CalendarObject(CommonObjectResource, CalendarObjectBase):
</span><span class="cx"> implements(ICalendarObject)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavcaldavdatastoretesttest_sqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -58,10 +58,10 @@
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema, _BIND_MODE_DIRECT, \
</span><span class="cx"> _BIND_STATUS_ACCEPTED, _TRANSP_OPAQUE
</span><span class="cx"> from txdav.caldav.datastore.test.common import CommonTests as CalendarCommonTests, \
</span><del>- test_event_text
</del><ins>+ test_event_text, cal1Root
</ins><span class="cx"> from txdav.caldav.datastore.test.test_file import setUpCalendarStore
</span><span class="cx"> from txdav.common.datastore.test.util import populateCalendarsFrom, \
</span><del>- CommonCommonTests
</del><ins>+ CommonCommonTests, updateToCurrentYear
</ins><span class="cx"> from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
</span><span class="cx"> from txdav.caldav.icalendarstore import ComponentUpdateState, InvalidDefaultCalendar, \
</span><span class="cx"> InvalidSplit, UnknownTimezone
</span><span class="lines">@@ -1913,117 +1913,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_calendarRevisionChangeConcurrency(self):
- """
- Test that two concurrent attempts to add resources in two separate
- calendar homes does not deadlock on the revision table update.
- """
-
- calendarStore = self._sqlCalendarStore
-
- # Make sure homes are provisioned
- txn = self.transactionUnderTest()
- home_uid1 = yield txn.homeWithUID(ECALENDARTYPE, "user01", create=True)
- home_uid2 = yield txn.homeWithUID(ECALENDARTYPE, "user02", create=True)
- self.assertNotEqual(home_uid1, None)
- self.assertNotEqual(home_uid2, None)
- yield self.commit()
-
- # Create first events in different calendar homes
- txn1 = calendarStore.newTransaction()
- txn2 = calendarStore.newTransaction()
-
- calendar_uid1_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user01")
- calendar_uid2_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user02")
-
- data = """BEGIN:VCALENDAR
-VERSION:2.0
-CALSCALE:GREGORIAN
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:data%(ctr)s
-DTSTART:20130102T140000Z
-DURATION:PT1H
-CREATED:20060102T190000Z
-DTSTAMP:20051222T210507Z
-SUMMARY:data%(ctr)s
-END:VEVENT
-END:VCALENDAR
-"""
-
- component = Component.fromString(data % {"ctr": 1})
- yield calendar_uid1_in_txn1.createCalendarObjectWithName("data1.ics", component)
-
- component = Component.fromString(data % {"ctr": 2})
- yield calendar_uid2_in_txn2.createCalendarObjectWithName("data2.ics", component)
-
- # Setup deferreds to run concurrently and create second events in the calendar homes
- # previously used by the other transaction - this could create the deadlock.
- @inlineCallbacks
- def _defer_uid3():
- calendar_uid1_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user01")
- component = Component.fromString(data % {"ctr": 3})
- yield calendar_uid1_in_txn2.createCalendarObjectWithName("data3.ics", component)
- yield txn2.commit()
- d1 = _defer_uid3()
-
- @inlineCallbacks
- def _defer_uid4():
- calendar_uid2_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user02")
- component = Component.fromString(data % {"ctr": 4})
- yield calendar_uid2_in_txn1.createCalendarObjectWithName("data4.ics", component)
- yield txn1.commit()
- d2 = _defer_uid4()
-
- # Now do the concurrent provision attempt
- yield DeferredList([d1, d2])
-
- # Verify we did not have a deadlock and all resources have been created.
- caldata1 = yield self.calendarObjectUnderTest(name="data1.ics", calendar_name="calendar", home="user01")
- caldata2 = yield self.calendarObjectUnderTest(name="data2.ics", calendar_name="calendar", home="user02")
- caldata3 = yield self.calendarObjectUnderTest(name="data3.ics", calendar_name="calendar", home="user01")
- caldata4 = yield self.calendarObjectUnderTest(name="data4.ics", calendar_name="calendar", home="user02")
- self.assertNotEqual(caldata1, None)
- self.assertNotEqual(caldata2, None)
- self.assertNotEqual(caldata3, None)
- self.assertNotEqual(caldata4, None)
-
-
- @inlineCallbacks
- def test_calendarMissingRevision(self):
- """
- Test that two concurrent attempts to add resources in two separate
- calendar homes does not deadlock on the revision table update.
- """
-
- # Get details
- home = yield self.homeUnderTest(name="user01", create=True)
- self.assertNotEqual(home, None)
- calendar = yield home.childWithName("calendar")
- self.assertNotEqual(calendar, None)
-
- rev = calendar._revisionsSchema
- yield Delete(
- From=rev,
- Where=(
- rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
- rev.COLLECTION_NAME == Parameter("collectionName")
- )
- ).on(self.transactionUnderTest(), homeID=home.id(), collectionName="calendar")
-
- yield self.commit()
-
- home = yield self.homeUnderTest(name="user01")
- children = yield home.loadChildren()
- self.assertEqual(len(children), 3)
- yield self.commit()
-
- calendar = yield self.calendarUnderTest(home="user01", name="calendar")
- token = yield calendar.syncToken()
- self.assertTrue(token is not None)
-
-
- @inlineCallbacks
</del><span class="cx"> def test_inboxTransp(self):
</span><span class="cx"> """
</span><span class="cx"> Make sure inbox is always transparent no matter what is stored in the DB.
</span><span class="lines">@@ -2319,7 +2208,164 @@
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="cx">
</span><ins>+
+class SyncTests(CommonCommonTests, unittest.TestCase):
+ """
+ Revision table/sync report tests.
+ """
+
</ins><span class="cx"> @inlineCallbacks
</span><ins>+ def setUp(self):
+ yield super(SyncTests, self).setUp()
+ yield self.buildStoreAndDirectory()
+ yield self.populate()
+
+
+ requirements = {
+ "user01": {
+ "calendar": {
+ "1.ics": (cal1Root.child("1.ics").getContent(), CalendarCommonTests.metadata1),
+ "2.ics": (cal1Root.child("2.ics").getContent(), CalendarCommonTests.metadata2),
+ "3.ics": (cal1Root.child("3.ics").getContent(), CalendarCommonTests.metadata3),
+ "4.ics": (cal1Root.child("4.ics").getContent(), CalendarCommonTests.metadata4),
+ "5.ics": (cal1Root.child("5.ics").getContent(), CalendarCommonTests.metadata5),
+ },
+ },
+ }
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateCalendarsFrom(self.requirements, self.storeUnderTest())
+ self.notifierFactory.reset()
+
+
+ def token2revision(self, token):
+ """
+ FIXME: the API names for L{syncToken}() and L{resourceNamesSinceToken}()
+ are slightly inaccurate; one doesn't produce input for the other.
+ Actually it should be resource names since I{revision} and you need to
+ understand the structure of the tokens to extract the revision. Right
+ now that logic lives in the protocol layer, so this testing method
+ replicates it.
+ """
+ _ignore_uuid, rev = token.split("_", 1)
+ rev = int(rev)
+ return rev
+
+
+ @inlineCallbacks
+ def test_calendarRevisionChangeConcurrency(self):
+ """
+ Test that two concurrent attempts to add resources in two separate
+ calendar homes does not deadlock on the revision table update.
+ """
+
+ calendarStore = self._sqlCalendarStore
+
+ # Make sure homes are provisioned
+ txn = self.transactionUnderTest()
+ home_uid1 = yield txn.homeWithUID(ECALENDARTYPE, "user01", create=True)
+ home_uid2 = yield txn.homeWithUID(ECALENDARTYPE, "user02", create=True)
+ self.assertNotEqual(home_uid1, None)
+ self.assertNotEqual(home_uid2, None)
+ yield self.commit()
+
+ # Create first events in different calendar homes
+ txn1 = calendarStore.newTransaction()
+ txn2 = calendarStore.newTransaction()
+
+ calendar_uid1_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user01")
+ calendar_uid2_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user02")
+
+ data = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:data%(ctr)s
+DTSTART:20130102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data%(ctr)s
+END:VEVENT
+END:VCALENDAR
+"""
+
+ component = Component.fromString(data % {"ctr": 1})
+ yield calendar_uid1_in_txn1.createCalendarObjectWithName("data1.ics", component)
+
+ component = Component.fromString(data % {"ctr": 2})
+ yield calendar_uid2_in_txn2.createCalendarObjectWithName("data2.ics", component)
+
+ # Setup deferreds to run concurrently and create second events in the calendar homes
+ # previously used by the other transaction - this could create the deadlock.
+ @inlineCallbacks
+ def _defer_uid3():
+ calendar_uid1_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user01")
+ component = Component.fromString(data % {"ctr": 3})
+ yield calendar_uid1_in_txn2.createCalendarObjectWithName("data3.ics", component)
+ yield txn2.commit()
+ d1 = _defer_uid3()
+
+ @inlineCallbacks
+ def _defer_uid4():
+ calendar_uid2_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user02")
+ component = Component.fromString(data % {"ctr": 4})
+ yield calendar_uid2_in_txn1.createCalendarObjectWithName("data4.ics", component)
+ yield txn1.commit()
+ d2 = _defer_uid4()
+
+ # Now do the concurrent provision attempt
+ yield DeferredList([d1, d2])
+
+ # Verify we did not have a deadlock and all resources have been created.
+ caldata1 = yield self.calendarObjectUnderTest(name="data1.ics", calendar_name="calendar", home="user01")
+ caldata2 = yield self.calendarObjectUnderTest(name="data2.ics", calendar_name="calendar", home="user02")
+ caldata3 = yield self.calendarObjectUnderTest(name="data3.ics", calendar_name="calendar", home="user01")
+ caldata4 = yield self.calendarObjectUnderTest(name="data4.ics", calendar_name="calendar", home="user02")
+ self.assertNotEqual(caldata1, None)
+ self.assertNotEqual(caldata2, None)
+ self.assertNotEqual(caldata3, None)
+ self.assertNotEqual(caldata4, None)
+
+
+ @inlineCallbacks
+ def test_calendarMissingRevision(self):
+ """
+ Test that two concurrent attempts to add resources in two separate
+ calendar homes does not deadlock on the revision table update.
+ """
+
+ # Get details
+ home = yield self.homeUnderTest(name="user02", create=True)
+ self.assertNotEqual(home, None)
+ calendar = yield home.childWithName("calendar")
+ self.assertNotEqual(calendar, None)
+
+ rev = calendar._revisionsSchema
+ yield Delete(
+ From=rev,
+ Where=(
+ rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
+ rev.COLLECTION_NAME == Parameter("collectionName")
+ )
+ ).on(self.transactionUnderTest(), homeID=home.id(), collectionName="calendar")
+
+ yield self.commit()
+
+ home = yield self.homeUnderTest(name="user02")
+ children = yield home.loadChildren()
+ self.assertEqual(len(children), 3)
+ yield self.commit()
+
+ calendar = yield self.calendarUnderTest(home="user02", name="calendar")
+ token = yield calendar.syncToken()
+ self.assertTrue(token is not None)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def test_removeAfterRevisionCleanup(self):
</span><span class="cx"> """
</span><span class="cx"> Make sure L{Calendar}'s can be renamed after revision cleanup
</span><span class="lines">@@ -2350,6 +2396,58 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def test_revisionModified(self):
+ """
+ Make sure the revision table MODIFIED value changes for an update or delete
+ """
+
+ @inlineCallbacks
+ def _getModified():
+ home = yield self.homeUnderTest(name="user01")
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ rev = calendar._revisionsSchema
+ modified = yield Select(
+ [rev.MODIFIED, ],
+ From=rev,
+ Where=(
+ rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
+ rev.CALENDAR_RESOURCE_ID == Parameter("collectionID")).And(
+ rev.RESOURCE_NAME == Parameter("resourceName")
+ )
+ ).on(
+ home._txn,
+ homeID=home.id(),
+ collectionID=calendar.id(),
+ resourceName="1.ics",
+ )
+ yield self.commit()
+ returnValue(modified[0][0])
+
+ # Get current modified
+ old_modified = yield _getModified()
+ self.assertNotEqual(old_modified, None)
+
+ # Update resource
+ cobj = yield self.calendarObjectUnderTest(home="user01", calendar_name="calendar", name="1.ics")
+ yield cobj.setComponent(Component.fromString(updateToCurrentYear(cal1Root.child("1.ics").getContent())))
+ yield self.commit()
+
+ # Modified changed
+ update_modified = yield _getModified()
+ self.assertGreater(update_modified, old_modified)
+
+ # Delete resource
+ cobj = yield self.calendarObjectUnderTest(home="user01", calendar_name="calendar", name="1.ics")
+ yield cobj.remove()
+ yield self.commit()
+
+ # Modified changed
+ delete_modified = yield _getModified()
+ self.assertGreater(delete_modified, old_modified)
+ self.assertGreater(delete_modified, update_modified)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def test_homeSyncTokenWithTrash_Visible(self):
</span><span class="cx"> """
</span><span class="cx"> L{ICalendarHome.resourceNamesSinceToken} will return the names of
</span><span class="lines">@@ -2360,8 +2458,8 @@
</span><span class="cx"> self.patch(config, "EnableTrashCollection", True)
</span><span class="cx"> self.patch(config, "ExposeTrashCollection", True)
</span><span class="cx">
</span><del>- home = yield self.homeUnderTest()
- cal = yield self.calendarUnderTest()
</del><ins>+ home = yield self.homeUnderTest(name="user01")
+ cal = yield self.calendarUnderTest(home="user01", name="calendar")
</ins><span class="cx"> st = yield home.syncToken()
</span><span class="cx"> yield cal.createCalendarObjectWithName("new.ics", Component.fromString(
</span><span class="cx"> test_event_text
</span><span class="lines">@@ -2373,12 +2471,12 @@
</span><span class="cx"> st2 = yield home.syncToken()
</span><span class="cx"> self.failIfEquals(st, st2)
</span><span class="cx">
</span><del>- home = yield self.homeUnderTest()
</del><ins>+ home = yield self.homeUnderTest(name="user01")
</ins><span class="cx">
</span><span class="cx"> expected = [
</span><del>- "calendar_1/",
- "calendar_1/new.ics",
- "calendar_1/2.ics",
</del><ins>+ "calendar/",
+ "calendar/new.ics",
+ "calendar/2.ics",
</ins><span class="cx"> "other-calendar/"
</span><span class="cx"> ]
</span><span class="cx">
</span><span class="lines">@@ -2394,7 +2492,7 @@
</span><span class="cx"> self.token2revision(st), "infinity")
</span><span class="cx">
</span><span class="cx"> self.assertEquals(set(changed), set(expected))
</span><del>- self.assertEquals(set(deleted), set(["calendar_1/2.ics"]))
</del><ins>+ self.assertEquals(set(deleted), set(["calendar/2.ics"]))
</ins><span class="cx"> self.assertEquals(invalid, [])
</span><span class="cx">
</span><span class="cx"> changed, deleted, invalid = yield home.resourceNamesSinceToken(
</span><span class="lines">@@ -2414,8 +2512,8 @@
</span><span class="cx">
</span><span class="cx"> self.patch(config, "EnableTrashCollection", True)
</span><span class="cx">
</span><del>- home = yield self.homeUnderTest()
- cal = yield self.calendarUnderTest()
</del><ins>+ home = yield self.homeUnderTest(name="user01")
+ cal = yield self.calendarUnderTest(home="user01", name="calendar")
</ins><span class="cx"> st = yield home.syncToken()
</span><span class="cx"> yield cal.createCalendarObjectWithName("new.ics", Component.fromString(
</span><span class="cx"> test_event_text
</span><span class="lines">@@ -2427,12 +2525,12 @@
</span><span class="cx"> st2 = yield home.syncToken()
</span><span class="cx"> self.failIfEquals(st, st2)
</span><span class="cx">
</span><del>- home = yield self.homeUnderTest()
</del><ins>+ home = yield self.homeUnderTest(name="user01")
</ins><span class="cx">
</span><span class="cx"> expected = [
</span><del>- "calendar_1/",
- "calendar_1/new.ics",
- "calendar_1/2.ics",
</del><ins>+ "calendar/",
+ "calendar/new.ics",
+ "calendar/2.ics",
</ins><span class="cx"> "other-calendar/"
</span><span class="cx"> ]
</span><span class="cx">
</span><span class="lines">@@ -2440,7 +2538,7 @@
</span><span class="cx"> self.token2revision(st), "infinity")
</span><span class="cx">
</span><span class="cx"> self.assertEquals(set(changed), set(expected))
</span><del>- self.assertEquals(set(deleted), set(["calendar_1/2.ics"]))
</del><ins>+ self.assertEquals(set(deleted), set(["calendar/2.ics"]))
</ins><span class="cx"> self.assertEquals(invalid, [])
</span><span class="cx">
</span><span class="cx"> changed, deleted, invalid = yield home.resourceNamesSinceToken(
</span></span></pre></div>
<a id="CalendarServertrunktxdavcarddavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -570,7 +570,8 @@
</span><span class="cx"> {
</span><span class="cx"> rev.REVISION: schema.REVISION_SEQ,
</span><span class="cx"> rev.OBJECT_RESOURCE_ID: Parameter("id"),
</span><del>- rev.DELETED: True
</del><ins>+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
</ins><span class="cx"> },
</span><span class="cx"> Where=(
</span><span class="cx"> rev.RESOURCE_ID == Parameter("resourceID")).And(
</span></span></pre></div>
<a id="CalendarServertrunktxdavcarddavdatastoretesttest_sqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -28,20 +28,21 @@
</span><span class="cx"> from twisted.trial import unittest
</span><span class="cx">
</span><span class="cx"> from twistedcaldav import carddavxml
</span><del>-from twistedcaldav.vcard import Component as VCard
</del><ins>+from twistedcaldav.vcard import Component as VCard, Component
</ins><span class="cx"> from twistedcaldav.vcard import Component as VComponent
</span><span class="cx">
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="cx">
</span><span class="cx"> from txdav.carddav.datastore.test.common import CommonTests as AddressBookCommonTests, \
</span><del>- vcard4_text
</del><ins>+ vcard4_text, adbk1Root
</ins><span class="cx"> from txdav.carddav.datastore.test.test_file import setUpAddressBookStore
</span><span class="cx"> from txdav.carddav.datastore.util import _migrateAddressbook, migrateHome
</span><span class="cx">
</span><span class="cx"> from txdav.common.icommondatastore import NoSuchObjectResourceError
</span><span class="cx"> from txdav.common.datastore.sql import EADDRESSBOOKTYPE, CommonObjectResource
</span><span class="cx"> from txdav.common.datastore.sql_tables import _ABO_KIND_PERSON, _ABO_KIND_GROUP, schema
</span><del>-from txdav.common.datastore.test.util import cleanStore
</del><ins>+from txdav.common.datastore.test.util import cleanStore, CommonCommonTests, \
+ populateAddressBooksFrom
</ins><span class="cx"> from txdav.carddav.datastore.sql import AddressBook
</span><span class="cx">
</span><span class="cx"> from txdav.xml.rfc2518 import GETContentLanguage, ResourceType
</span><span class="lines">@@ -919,7 +920,38 @@
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="cx">
</span><ins>+
+class SyncTests(CommonCommonTests, unittest.TestCase):
+ """
+ Revision table/sync report tests.
+ """
+
</ins><span class="cx"> @inlineCallbacks
</span><ins>+ def setUp(self):
+ yield super(SyncTests, self).setUp()
+ yield self.buildStoreAndDirectory()
+ yield self.populate()
+
+
+ requirements = {
+ "user01": {
+ "addressbook": {
+ "1.vcf": adbk1Root.child("1.vcf").getContent(),
+ "2.vcf": adbk1Root.child("2.vcf").getContent(),
+ "3.vcf": adbk1Root.child("3.vcf").getContent(),
+ },
+ "not_a_addressbook": None
+ },
+ }
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateAddressBooksFrom(self.requirements, self.storeUnderTest())
+ self.notifierFactory.reset()
+
+
+ @inlineCallbacks
</ins><span class="cx"> def test_updateAfterRevisionCleanup(self):
</span><span class="cx"> """
</span><span class="cx"> Make sure L{AddressBookObject}'s can be updated or removed after revision cleanup
</span><span class="lines">@@ -957,26 +989,26 @@
</span><span class="cx"> END:VCARD
</span><span class="cx"> """
</span><span class="cx">
</span><del>- yield self.homeUnderTest()
- adbk = yield self.addressbookUnderTest(name="addressbook")
</del><ins>+ yield self.addressbookHomeUnderTest(name="user01")
+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
</ins><span class="cx"> yield adbk.createAddressBookObjectWithName("person.vcf", VCard.fromString(person))
</span><span class="cx"> yield adbk.createAddressBookObjectWithName("group.vcf", VCard.fromString(group))
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="cx"> # Remove the revision
</span><del>- adbk = yield self.addressbookUnderTest(name="addressbook")
</del><ins>+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
</ins><span class="cx"> yield adbk.syncToken()
</span><span class="cx"> yield self.transactionUnderTest().deleteRevisionsBefore(adbk._syncTokenRevision + 1)
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="cx"> # Update the object
</span><del>- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
</del><ins>+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
</ins><span class="cx"> yield obj.setComponent(VCard.fromString(group_update))
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
</del><ins>+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
</ins><span class="cx"> self.assertTrue(obj is not None)
</span><del>- obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook")
</del><ins>+ obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook", home="user01")
</ins><span class="cx"> self.assertTrue(obj is not None)
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="lines">@@ -1010,26 +1042,76 @@
</span><span class="cx"> END:VCARD
</span><span class="cx"> """
</span><span class="cx">
</span><del>- yield self.homeUnderTest()
- adbk = yield self.addressbookUnderTest(name="addressbook")
</del><ins>+ yield self.addressbookHomeUnderTest(name="user01")
+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
</ins><span class="cx"> yield adbk.createAddressBookObjectWithName("person.vcf", VCard.fromString(person))
</span><span class="cx"> yield adbk.createAddressBookObjectWithName("group.vcf", VCard.fromString(group))
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="cx"> # Remove the revision
</span><del>- adbk = yield self.addressbookUnderTest(name="addressbook")
</del><ins>+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
</ins><span class="cx"> yield adbk.syncToken()
</span><span class="cx"> yield self.transactionUnderTest().deleteRevisionsBefore(adbk._syncTokenRevision + 1)
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="cx"> # Remove the object
</span><del>- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
</del><ins>+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
</ins><span class="cx"> self.assertTrue(obj is not None)
</span><span class="cx"> yield obj.remove()
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
</del><ins>+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
</ins><span class="cx"> self.assertTrue(obj is None)
</span><del>- obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook")
</del><ins>+ obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook", home="user01")
</ins><span class="cx"> self.assertTrue(obj is not None)
</span><span class="cx"> yield self.commit()
</span><ins>+
+
+ @inlineCallbacks
+ def test_revisionModified(self):
+ """
+ Make sure the revision table MODIFIED value changes for an update or delete
+ """
+
+ @inlineCallbacks
+ def _getModified():
+ home = yield self.addressbookHomeUnderTest(name="user01")
+ addressbook = yield self.addressbookUnderTest(home="user01", name="addressbook")
+ rev = addressbook._revisionsSchema
+ modified = yield Select(
+ [rev.MODIFIED, ],
+ From=rev,
+ Where=(
+ rev.ADDRESSBOOK_HOME_RESOURCE_ID == Parameter("homeID")).And(
+ rev.RESOURCE_NAME == Parameter("resourceName")
+ )
+ ).on(
+ home._txn,
+ homeID=home.id(),
+ resourceName="1.vcf",
+ )
+ yield self.commit()
+ returnValue(modified[0][0])
+
+ # Get current modified
+ old_modified = yield _getModified()
+ self.assertNotEqual(old_modified, None)
+
+ # Update resource
+ aobj = yield self.addressbookObjectUnderTest(home="user01", addressbook_name="addressbook", name="1.vcf")
+ yield aobj.setComponent(Component.fromString(adbk1Root.child("1.vcf").getContent()))
+ yield self.commit()
+
+ # Modified changed
+ update_modified = yield _getModified()
+ self.assertGreater(update_modified, old_modified)
+
+ # Delete resource
+ aobj = yield self.addressbookObjectUnderTest(home="user01", addressbook_name="addressbook", name="1.vcf")
+ yield aobj.remove()
+ yield self.commit()
+
+ # Modified changed
+ delete_modified = yield _getModified()
+ self.assertGreater(delete_modified, old_modified)
+ self.assertGreater(delete_modified, update_modified)
</ins></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -3409,8 +3409,7 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> # Get revisions
</span><del>- revisions = (yield cls._revisionsForResourceIDs(childResourceIDs).on(home._txn, resourceIDs=childResourceIDs))
- revisions = dict(revisions)
</del><ins>+ revisions = yield cls.childSyncTokenRevisions(home, childResourceIDs)
</ins><span class="cx">
</span><span class="cx"> # Create the actual objects merging in properties
</span><span class="cx"> for dataRow in dataRows:
</span><span class="lines">@@ -3421,7 +3420,7 @@
</span><span class="cx"> propstore = propertyStores.get(resourceID, None)
</span><span class="cx">
</span><span class="cx"> child = yield cls.makeClass(home, bindData, additionalBindData, metadataData, propstore)
</span><del>- child._syncTokenRevision = revisions.get(resourceID, 0)
</del><ins>+ child._syncTokenRevision = revisions.get(resourceID, None)
</ins><span class="cx"> results.append(child)
</span><span class="cx">
</span><span class="cx"> returnValue(results)
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_sharing.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_sharing.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/sql_sharing.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -19,7 +19,7 @@
</span><span class="cx"> from pycalendar.datetime import DateTime
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.syntax import Insert, Parameter, Update, Delete, \
</span><del>- Select, Max
</del><ins>+ Select
</ins><span class="cx"> from twext.python.clsprop import classproperty
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx">
</span><span class="lines">@@ -1451,18 +1451,6 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><del>- @classmethod
- def _revisionsForResourceIDs(cls, resourceIDs):
- rev = cls._revisionsSchema
- return Select(
- [rev.RESOURCE_ID, Max(rev.REVISION)],
- From=rev,
- Where=rev.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs))).And(
- (rev.RESOURCE_NAME != None).Or(rev.DELETED == False)),
- GroupBy=rev.RESOURCE_ID
- )
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def invalidateQueryCache(self):
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_utilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_util.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_util.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/sql_util.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -16,7 +16,7 @@
</span><span class="cx"> ##
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.syntax import Max, Select, Parameter, Delete, Insert, \
</span><del>- Update, ColumnSyntax, TableSyntax, Upper
</del><ins>+ Update, ColumnSyntax, TableSyntax, Upper, utcNowSQL
</ins><span class="cx"> from twext.python.clsprop import classproperty
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twisted.internet.defer import succeed, inlineCallbacks, returnValue
</span><span class="lines">@@ -66,6 +66,18 @@
</span><span class="cx"> Where=rev.RESOURCE_ID == Parameter("resourceID"))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @classmethod
+ def _revisionsForResourceIDs(cls, resourceIDs):
+ rev = cls._revisionsSchema
+ return Select(
+ [rev.RESOURCE_ID, Max(rev.REVISION)],
+ From=rev,
+ Where=rev.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs))).And(
+ (rev.RESOURCE_NAME != None).Or(rev.DELETED == False)),
+ GroupBy=rev.RESOURCE_ID
+ )
+
+
</ins><span class="cx"> def revisionFromToken(self, token):
</span><span class="cx"> if token is None:
</span><span class="cx"> return 0
</span><span class="lines">@@ -91,6 +103,21 @@
</span><span class="cx"> returnValue(revision)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @classmethod
+ @inlineCallbacks
+ def childSyncTokenRevisions(cls, home, childResourceIDs):
+ rows = (yield cls._revisionsForResourceIDs(childResourceIDs).on(home._txn, resourceIDs=childResourceIDs))
+ revisions = dict(rows)
+
+ # Add in any that were missing - this assumes that childResourceIDs were all valid to begin with
+ missingIDs = set(childResourceIDs) - set(revisions.keys())
+ if missingIDs:
+ min_revision = int((yield home._txn.calendarserverValue("MIN-VALID-REVISION")))
+ for resourceID in missingIDs:
+ revisions[resourceID] = min_revision
+ returnValue(revisions)
+
+
</ins><span class="cx"> def objectResourcesSinceToken(self, token):
</span><span class="cx"> raise NotImplementedError()
</span><span class="cx">
</span><span class="lines">@@ -206,7 +233,8 @@
</span><span class="cx"> return Update(
</span><span class="cx"> {
</span><span class="cx"> rev.REVISION: schema.REVISION_SEQ,
</span><del>- rev.COLLECTION_NAME: Parameter("name")
</del><ins>+ rev.COLLECTION_NAME: Parameter("name"),
+ rev.MODIFIED: utcNowSQL,
</ins><span class="cx"> },
</span><span class="cx"> Where=(rev.RESOURCE_ID == Parameter("resourceID")).And
</span><span class="cx"> (rev.RESOURCE_NAME == None),
</span><span class="lines">@@ -233,7 +261,10 @@
</span><span class="cx"> """
</span><span class="cx"> rev = cls._revisionsSchema
</span><span class="cx"> return Update(
</span><del>- {rev.REVISION: schema.REVISION_SEQ, },
</del><ins>+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.MODIFIED: utcNowSQL,
+ },
</ins><span class="cx"> Where=(rev.RESOURCE_ID == Parameter("resourceID")).And
</span><span class="cx"> (rev.RESOURCE_NAME == None)
</span><span class="cx"> )
</span><span class="lines">@@ -276,7 +307,8 @@
</span><span class="cx"> {
</span><span class="cx"> rev.RESOURCE_ID: None,
</span><span class="cx"> rev.REVISION: schema.REVISION_SEQ,
</span><del>- rev.DELETED: True
</del><ins>+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
</ins><span class="cx"> },
</span><span class="cx"> Where=(rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
</span><span class="cx"> rev.RESOURCE_ID == Parameter("resourceID")).And(
</span><span class="lines">@@ -294,7 +326,8 @@
</span><span class="cx"> {
</span><span class="cx"> rev.RESOURCE_ID: None,
</span><span class="cx"> rev.REVISION: schema.REVISION_SEQ,
</span><del>- rev.DELETED: True
</del><ins>+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
</ins><span class="cx"> },
</span><span class="cx"> Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
</span><span class="cx"> rev.RESOURCE_NAME == None),
</span><span class="lines">@@ -346,7 +379,11 @@
</span><span class="cx"> def _deleteBumpTokenQuery(cls):
</span><span class="cx"> rev = cls._revisionsSchema
</span><span class="cx"> return Update(
</span><del>- {rev.REVISION: schema.REVISION_SEQ, rev.DELETED: True},
</del><ins>+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
+ },
</ins><span class="cx"> Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
</span><span class="cx"> rev.RESOURCE_NAME == Parameter("name")),
</span><span class="cx"> Return=rev.REVISION
</span><span class="lines">@@ -357,7 +394,10 @@
</span><span class="cx"> def _updateBumpTokenQuery(cls):
</span><span class="cx"> rev = cls._revisionsSchema
</span><span class="cx"> return Update(
</span><del>- {rev.REVISION: schema.REVISION_SEQ},
</del><ins>+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.MODIFIED: utcNowSQL,
+ },
</ins><span class="cx"> Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
</span><span class="cx"> rev.RESOURCE_NAME == Parameter("name")),
</span><span class="cx"> Return=rev.REVISION
</span><span class="lines">@@ -379,7 +419,11 @@
</span><span class="cx"> def _updatePreviouslyNamedQuery(cls):
</span><span class="cx"> rev = cls._revisionsSchema
</span><span class="cx"> return Update(
</span><del>- {rev.REVISION: schema.REVISION_SEQ, rev.DELETED: False},
</del><ins>+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.DELETED: False,
+ rev.MODIFIED: utcNowSQL,
+ },
</ins><span class="cx"> Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
</span><span class="cx"> rev.RESOURCE_NAME == Parameter("name")),
</span><span class="cx"> Return=rev.REVISION
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoreworkrevision_cleanuppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -53,6 +53,10 @@
</span><span class="cx"> return float(config.RevisionCleanup.CleanupPeriodDays) * 24 * 60 * 60
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def dateCutoff(self):
+ return datetime.datetime.utcnow() - datetime.timedelta(days=float(config.RevisionCleanup.SyncTokenLifetimeDays))
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def doWork(self):
</span><span class="cx">
</span><span class="lines">@@ -60,10 +64,7 @@
</span><span class="cx"> minValidRevision = int((yield self.transaction.calendarserverValue("MIN-VALID-REVISION")))
</span><span class="cx">
</span><span class="cx"> # get max revision on table rows before dateLimit
</span><del>- dateLimit = (
- datetime.datetime.utcnow() -
- datetime.timedelta(days=float(config.RevisionCleanup.SyncTokenLifetimeDays))
- )
</del><ins>+ dateLimit = self.dateCutoff()
</ins><span class="cx"> maxRevOlderThanDate = 0
</span><span class="cx">
</span><span class="cx"> # TODO: Use one Select statement
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoreworktesttest_revision_cleanuppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py (14924 => 14925)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py 2015-06-26 21:21:39 UTC (rev 14925)
</span><span class="lines">@@ -23,12 +23,14 @@
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx"> from twisted.trial.unittest import TestCase
</span><span class="cx"> from twistedcaldav.config import config
</span><ins>+from twistedcaldav.ical import Component
</ins><span class="cx"> from twistedcaldav.vcard import Component as VCard
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema, _BIND_MODE_READ
</span><span class="cx"> from txdav.common.datastore.test.util import CommonCommonTests, populateCalendarsFrom
</span><span class="cx"> from txdav.common.datastore.work.revision_cleanup import FindMinValidRevisionWork, RevisionCleanupWork
</span><span class="cx"> from txdav.common.icommondatastore import SyncTokenValidException
</span><span class="cx"> import datetime
</span><ins>+import time
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -88,6 +90,21 @@
</span><span class="cx"> END:VCALENDAR
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ cal1_mod = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid1
+DTSTART:20131122T140000
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:event 1.1
+END:VEVENT
+END:VCALENDAR
+"""
+
</ins><span class="cx"> cal2 = """BEGIN:VCALENDAR
</span><span class="cx"> VERSION:2.0
</span><span class="cx"> CALSCALE:GREGORIAN
</span><span class="lines">@@ -242,6 +259,10 @@
</span><span class="cx"> Verify that all extra calendar object revisions are deleted by FindMinValidRevisionWork and RevisionCleanupWork
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ # get home sync token
+ home = yield self.homeUnderTest(name="user01")
+ hometoken = yield home.syncToken()
+
</ins><span class="cx"> # get sync token
</span><span class="cx"> calendar = yield self.calendarUnderTest(home="user01", name="calendar")
</span><span class="cx"> token = yield calendar.syncToken()
</span><span class="lines">@@ -267,7 +288,7 @@
</span><span class="cx">
</span><span class="cx"> # Get the minimum valid revision and check it
</span><span class="cx"> minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
</span><del>- self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
</del><ins>+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]) + 1)
</ins><span class="cx">
</span><span class="cx"> # do RevisionCleanupWork
</span><span class="cx"> yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
</span><span class="lines">@@ -280,14 +301,111 @@
</span><span class="cx"> [rev.REVISION],
</span><span class="cx"> From=rev,
</span><span class="cx"> ).on(self.transactionUnderTest())
</span><del>- self.assertEqual(len(revisionRows), 1) # deleteRevisionsBefore() leaves 1 revision behind
</del><ins>+ self.assertEqual(len(revisionRows), 0)
</ins><span class="cx">
</span><span class="cx"> # old sync token fails
</span><span class="cx"> calendar = yield self.calendarUnderTest(home="user01", name="calendar")
</span><span class="cx"> yield self.failUnlessFailure(calendar.resourceNamesSinceToken(token), SyncTokenValidException)
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><ins>+ # old sync token fails
+ home = yield self.homeUnderTest(name="user01")
+ yield self.failUnlessFailure(home.resourceNamesSinceToken(hometoken, 1), SyncTokenValidException)
+ yield self.commit()
</ins><span class="cx">
</span><ins>+ # calendar sync token changed
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ newtoken = yield calendar.syncToken()
+ self.assertGreater(newtoken, token)
+ yield self.commit()
+
+ # home sync token changed
+ home = yield self.homeUnderTest(name="user01")
+ newhometoken = yield home.syncToken()
+ self.assertGreater(newhometoken, hometoken)
+ yield self.commit()
+
+ # Depth:1 tokens match
+ home = yield self.homeUnderTest(name="user01")
+ yield home.loadChildren()
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ newtoken1 = yield calendar.syncToken()
+ self.assertEqual(newtoken1, newtoken)
+ yield self.commit()
+
+
</ins><span class="cx"> @inlineCallbacks
</span><ins>+ def test_calendarObjectRevisions_Modified(self):
+ """
+ Verify that a calendar object created before the revision cut-off, but modified after it is correctly reported as changed
+ after revision clean-up
+ """
+
+ # Need to add one non-event change that creates a revision after the last event change revisions in order
+ # for the logic in this test to work correctly
+ home = yield self.homeUnderTest(name="user01")
+ yield home.createCalendarWithName("_ignore_me")
+ yield self.commit()
+
+ # get initial sync token
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ initial_token = yield calendar.syncToken()
+ yield self.commit()
+
+ # Pause to give some space in the modified time
+ time.sleep(1)
+ modified = datetime.datetime.utcnow()
+ time.sleep(1)
+
+ # Patch the work item to use the modified cut-off we need
+ def _dateCutoff(self):
+ return modified
+ self.patch(FindMinValidRevisionWork, "dateCutoff", _dateCutoff)
+
+ # Make a change to get a pre-update token
+ cal2Object = yield self.calendarObjectUnderTest(self.transactionUnderTest(), name="cal2.ics", calendar_name="calendar", home="user01")
+ yield cal2Object.remove()
+ yield self.commit()
+
+ # get changed sync token
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ pre_update_token = yield calendar.syncToken()
+ yield self.commit()
+
+ # make changes
+ cal1Object = yield self.calendarObjectUnderTest(self.transactionUnderTest(), name="cal1.ics", calendar_name="calendar", home="user01")
+ yield cal1Object.setComponent(Component.fromString(self.cal1_mod))
+ yield self.commit()
+
+ # get changed sync token
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ update_token = yield calendar.syncToken()
+ yield self.commit()
+
+ # do FindMinValidRevisionWork and RevisionCleanupWork
+ yield FindMinValidRevisionWork.reschedule(self.transactionUnderTest(), 0)
+ yield self.commit()
+ yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60)
+
+ # initial sync token fails
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ yield self.failUnlessFailure(calendar.resourceNamesSinceToken(initial_token), SyncTokenValidException)
+ yield self.commit()
+
+ # Pre-update sync token returns one item
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ names = yield calendar.resourceNamesSinceToken(pre_update_token)
+ self.assertEqual(names, (['cal1.ics'], [], []))
+ yield self.commit()
+
+ # Post-update sync token returns one item
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ names = yield calendar.resourceNamesSinceToken(update_token)
+ self.assertEqual(names, ([], [], []))
+ yield self.commit()
+
+
+ @inlineCallbacks
</ins><span class="cx"> def test_notificationObjectRevisions(self):
</span><span class="cx"> """
</span><span class="cx"> Verify that all extra notification object revisions are deleted by FindMinValidRevisionWork and RevisionCleanupWork
</span><span class="lines">@@ -315,7 +433,7 @@
</span><span class="cx">
</span><span class="cx"> # Get the minimum valid revision and check it
</span><span class="cx"> minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
</span><del>- self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
</del><ins>+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]) + 1)
</ins><span class="cx">
</span><span class="cx"> # do RevisionCleanupWork
</span><span class="cx"> yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
</span><span class="lines">@@ -328,7 +446,7 @@
</span><span class="cx"> [rev.REVISION],
</span><span class="cx"> From=rev,
</span><span class="cx"> ).on(self.transactionUnderTest())
</span><del>- self.assertEqual(len(revisionRows), 1) # deleteRevisionsBefore() leaves 1 revision behind
</del><ins>+ self.assertEqual(len(revisionRows), 0)
</ins><span class="cx">
</span><span class="cx"> # old sync token fails
</span><span class="cx"> home = yield self.homeUnderTest(name="user01")
</span><span class="lines">@@ -367,7 +485,7 @@
</span><span class="cx">
</span><span class="cx"> # Get the minimum valid revision and check it
</span><span class="cx"> minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
</span><del>- self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
</del><ins>+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]) + 1)
</ins><span class="cx">
</span><span class="cx"> # do RevisionCleanupWork
</span><span class="cx"> yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
</span><span class="lines">@@ -380,7 +498,7 @@
</span><span class="cx"> [rev.REVISION],
</span><span class="cx"> From=rev,
</span><span class="cx"> ).on(self.transactionUnderTest())
</span><del>- self.assertEqual(len(revisionRows), 1) # deleteRevisionsBefore() leaves 1 revision behind
</del><ins>+ self.assertEqual(len(revisionRows), 0)
</ins><span class="cx">
</span><span class="cx"> # old sync token fails
</span><span class="cx"> addressbook = yield self.addressbookUnderTest(home="user01", name="addressbook")
</span><span class="lines">@@ -440,7 +558,7 @@
</span><span class="cx">
</span><span class="cx"> # Get the minimum valid revision and check it
</span><span class="cx"> minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
</span><del>- self.assertEqual(int(minValidRevision), max([row[3] for row in group1Rows + group2Rows]))
</del><ins>+ self.assertEqual(int(minValidRevision), max([row[3] for row in group1Rows + group2Rows]) + 1)
</ins><span class="cx">
</span><span class="cx"> # do RevisionCleanupWork
</span><span class="cx"> yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
</span></span></pre>
</div>
</div>
</body>
</html>