<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14507] CalendarServer/branches/users/cdaboo/pod2pod-migration</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14507">14507</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-03-04 18:41:22 -0800 (Wed, 04 Mar 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Implement a test that covers the complete migration cycle.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationcalendarserverpushtesttest_notifierpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationcalendarservertoolspurgepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtwistedcaldavresourcepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoreschedulingischeduledeliverypy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretestcommonpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretesttest_sqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretesttest_sql_sharingpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sql_sharingpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorefilepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtestaccountsgroupAccountsxml">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_notificationpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_sharingpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_utilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoretestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoreupgradesqlupgradestesttest_notification_upgrade_from_0_to_1py">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavwhotesttest_group_shareespy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_migrationpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationcalendarserverpushtesttest_notifierpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -297,7 +297,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def test_notificationNotifier(self):
</span><span class="cx">
</span><del>- notifications = yield self.transactionUnderTest().notificationsWithUID("user01")
</del><ins>+ notifications = yield self.transactionUnderTest().notificationsWithUID("user01", create=True)
</ins><span class="cx"> yield notifications.notifyChanged(category=ChangeCategory.default)
</span><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationcalendarservertoolspurgepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -992,7 +992,7 @@
</span><span class="cx">
</span><span class="cx"> if not self.dryrun:
</span><span class="cx"> yield storeCalHome.removeUnacceptedShares()
</span><del>- notificationHome = yield txn.notificationsWithUID(storeCalHome.uid(), create=False)
</del><ins>+ notificationHome = yield txn.notificationsWithUID(storeCalHome.uid())
</ins><span class="cx"> if notificationHome is not None:
</span><span class="cx"> yield notificationHome.remove()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtwistedcaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -2139,7 +2139,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def createNotificationsCollection(self):
</span><span class="cx"> txn = self._associatedTransaction
</span><del>- notifications = yield txn.notificationsWithUID(self._newStoreHome.uid())
</del><ins>+ notifications = yield txn.notificationsWithUID(self._newStoreHome.uid(), create=True)
</ins><span class="cx">
</span><span class="cx"> from twistedcaldav.storebridge import StoreNotificationCollectionResource
</span><span class="cx"> similar = StoreNotificationCollectionResource(
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoreschedulingischeduledeliverypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -448,15 +448,6 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _processRequest(self, ssl, host, port, path):
</span><del>- from twisted.internet import reactor
- f = Factory()
- f.protocol = HTTPClientProtocol
- if ssl:
- ep = GAIEndpoint(reactor, host, port, _configuredClientContextFactory())
- else:
- ep = GAIEndpoint(reactor, host, port)
- proto = (yield ep.connect(f))
-
</del><span class="cx"> if not self.server.podding() and config.Scheduling.iSchedule.DKIM.Enabled:
</span><span class="cx"> domain, selector, key_file, algorithm, useDNSKey, useHTTPKey, usePrivateExchangeKey, expire = DKIMUtils.getConfiguration(config)
</span><span class="cx"> request = DKIMRequest(
</span><span class="lines">@@ -481,6 +472,21 @@
</span><span class="cx"> if accountingEnabledForCategory("iSchedule"):
</span><span class="cx"> self.loggedRequest = yield self.logRequest(request)
</span><span class="cx">
</span><ins>+ response = yield self._submitRequest(ssl, host, port, request)
+ returnValue(response)
+
+
+ @inlineCallbacks
+ def _submitRequest(self, ssl, host, port, request):
+ from twisted.internet import reactor
+ f = Factory()
+ f.protocol = HTTPClientProtocol
+ if ssl:
+ ep = GAIEndpoint(reactor, host, port, _configuredClientContextFactory())
+ else:
+ ep = GAIEndpoint(reactor, host, port)
+ proto = (yield ep.connect(f))
+
</ins><span class="cx"> response = (yield proto.submitRequest(request))
</span><span class="cx">
</span><span class="cx"> returnValue(response)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretestcommonpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -375,7 +375,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def notificationUnderTest(self):
</span><span class="cx"> txn = self.transactionUnderTest()
</span><del>- notifications = yield txn.notificationsWithUID("home1")
</del><ins>+ notifications = yield txn.notificationsWithUID("home1", create=True)
</ins><span class="cx"> yield notifications.writeNotificationObject(
</span><span class="cx"> "abc",
</span><span class="cx"> json.loads("{\"notification-type\":\"invite-notification\"}"),
</span><span class="lines">@@ -402,7 +402,7 @@
</span><span class="cx"> objects changed or deleted since
</span><span class="cx"> """
</span><span class="cx"> txn = self.transactionUnderTest()
</span><del>- coll = yield txn.notificationsWithUID("home1")
</del><ins>+ coll = yield txn.notificationsWithUID("home1", create=True)
</ins><span class="cx"> yield coll.writeNotificationObject(
</span><span class="cx"> "1",
</span><span class="cx"> json.loads("{\"notification-type\":\"invite-notification\"}"),
</span><span class="lines">@@ -435,7 +435,7 @@
</span><span class="cx"> overwrite the notification object.
</span><span class="cx"> """
</span><span class="cx"> notifications = yield self.transactionUnderTest().notificationsWithUID(
</span><del>- "home1"
</del><ins>+ "home1", create=True
</ins><span class="cx"> )
</span><span class="cx"> yield notifications.writeNotificationObject(
</span><span class="cx"> "abc",
</span><span class="lines">@@ -462,7 +462,7 @@
</span><span class="cx"> """
</span><span class="cx"> # Prime the home collection first
</span><span class="cx"> yield self.transactionUnderTest().notificationsWithUID(
</span><del>- "home1"
</del><ins>+ "home1", create=True
</ins><span class="cx"> )
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><span class="lines">@@ -512,7 +512,7 @@
</span><span class="cx"> overwrite the notification object.
</span><span class="cx"> """
</span><span class="cx"> notifications = yield self.transactionUnderTest().notificationsWithUID(
</span><del>- "home1"
</del><ins>+ "home1", create=True
</ins><span class="cx"> )
</span><span class="cx"> yield notifications.writeNotificationObject(
</span><span class="cx"> "abc",
</span><span class="lines">@@ -555,7 +555,7 @@
</span><span class="cx"> L{INotificationCollection} that the object was retrieved from.
</span><span class="cx"> """
</span><span class="cx"> txn = self.transactionUnderTest()
</span><del>- collection = yield txn.notificationsWithUID("home1")
</del><ins>+ collection = yield txn.notificationsWithUID("home1", create=True)
</ins><span class="cx"> notification = yield self.notificationUnderTest()
</span><span class="cx"> self.assertIdentical(collection, notification.notificationCollection())
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretesttest_sqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -767,13 +767,13 @@
</span><span class="cx"> txn2 = calendarStore.newTransaction()
</span><span class="cx">
</span><span class="cx"> notification_uid1_1 = yield txn1.notificationsWithUID(
</span><del>- "uid1",
</del><ins>+ "uid1", create=True
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _defer_notification_uid1_2():
</span><span class="cx"> notification_uid1_2 = yield txn2.notificationsWithUID(
</span><del>- "uid1",
</del><ins>+ "uid1", create=True
</ins><span class="cx"> )
</span><span class="cx"> yield txn2.commit()
</span><span class="cx"> returnValue(notification_uid1_2)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretesttest_sql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -464,7 +464,7 @@
</span><span class="cx"> shared = yield self.calendarUnderTest(home="user02", name=sharedName)
</span><span class="cx"> self.assertTrue(shared is not None)
</span><span class="cx">
</span><del>- notifyHome = yield self.transactionUnderTest().notificationsWithUID("user02")
</del><ins>+ notifyHome = yield self.transactionUnderTest().notificationsWithUID("user02", create=True)
</ins><span class="cx"> notifications = yield notifyHome.listNotificationObjects()
</span><span class="cx"> self.assertEqual(len(notifications), 0)
</span><span class="cx">
</span><span class="lines">@@ -654,7 +654,7 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _check_notifications(self, uid, items):
</span><del>- notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid)
</del><ins>+ notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid, create=True)
</ins><span class="cx"> notifications = yield notifyHome.listNotificationObjects()
</span><span class="cx"> self.assertEqual(set(notifications), set(items))
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -364,13 +364,13 @@
</span><span class="cx"> txn2 = addressbookStore.newTransaction()
</span><span class="cx">
</span><span class="cx"> notification_uid1_1 = yield txn1.notificationsWithUID(
</span><del>- "uid1",
</del><ins>+ "uid1", create=True,
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _defer_notification_uid1_2():
</span><span class="cx"> notification_uid1_2 = yield txn2.notificationsWithUID(
</span><del>- "uid1",
</del><ins>+ "uid1", create=True,
</ins><span class="cx"> )
</span><span class="cx"> yield txn2.commit()
</span><span class="cx"> returnValue(notification_uid1_2)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -198,7 +198,7 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _check_notifications(self, home, items):
</span><del>- notifyHome = yield self.transactionUnderTest().notificationsWithUID(home)
</del><ins>+ notifyHome = yield self.transactionUnderTest().notificationsWithUID(home, create=True)
</ins><span class="cx"> notifications = yield notifyHome.listNotificationObjects()
</span><span class="cx"> self.assertEqual(set(notifications), set(items))
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorefilepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -376,7 +376,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @memoizedKey("uid", "_notificationHomes", deferredResult=False)
</span><del>- def notificationsWithUID(self, uid, home=None):
</del><ins>+ def notificationsWithUID(self, uid, home=None, create=False):
</ins><span class="cx">
</span><span class="cx"> if home is None:
</span><span class="cx"> home = self.homeWithUID(self._notificationHomeType, uid, create=True)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -85,17 +85,22 @@
</span><span class="cx">
</span><span class="cx"> BATCH_SIZE = 50
</span><span class="cx">
</span><del>- def __init__(self, store, diruid):
</del><ins>+ def __init__(self, store, diruid, final=False):
</ins><span class="cx"> """
</span><span class="cx"> @param store: the data store
</span><span class="cx"> @type store: L{CommonDataStore}
</span><span class="cx"> @param diruid: directory uid of the user whose home is to be sync'd
</span><span class="cx"> @type diruid: L{str}
</span><ins>+ @param final: indicates whether this is in the final sync stage with the remote home
+ already disabled
+ @type final: L{bool}
</ins><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> self.store = store
</span><span class="cx"> self.diruid = diruid
</span><del>- self.disabledRemote = False
</del><ins>+ self.disabledRemote = final
+ self.record = None
+ self.homeId = None
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def label(self, detail):
</span><span class="lines">@@ -133,7 +138,7 @@
</span><span class="cx"> # Step 6 - enable new home
</span><span class="cx"> yield self.enableLocalHome()
</span><span class="cx">
</span><del>- # Step 7 - remote remote home
</del><ins>+ # Step 7 - remove remote home
</ins><span class="cx"> yield self.removeRemoteHome()
</span><span class="cx">
</span><span class="cx"> # Step 8 - say phew! TODO: Actually alert everyone else
</span><span class="lines">@@ -168,6 +173,9 @@
</span><span class="cx"> rows, recalculate quota etc.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
+
</ins><span class="cx"> # Link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
</span><span class="cx"> yield self.linkAttachments()
</span><span class="cx">
</span><span class="lines">@@ -198,6 +206,9 @@
</span><span class="cx"> Mark the remote home as disabled.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
+
</ins><span class="cx"> # Calendar home
</span><span class="cx"> remote_home = yield self._remoteHome(txn)
</span><span class="cx"> yield remote_home.setStatus(_HOME_STATUS_DISABLED)
</span><span class="lines">@@ -216,13 +227,16 @@
</span><span class="cx"> Mark the local home as enabled and remove any previously existing external home.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
+
</ins><span class="cx"> # Disable any local external homes
</span><span class="cx"> oldhome = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_EXTERNAL)
</span><span class="cx"> if oldhome is not None:
</span><del>- yield oldhome.setStatus(_HOME_STATUS_DISABLED)
</del><ins>+ yield oldhome.setLocalStatus(_HOME_STATUS_DISABLED)
</ins><span class="cx"> oldnotifications = yield txn.notificationsWithUID(self.diruid, status=_HOME_STATUS_EXTERNAL)
</span><span class="cx"> if oldnotifications:
</span><del>- yield oldnotifications.setStatus(_HOME_STATUS_DISABLED)
</del><ins>+ yield oldnotifications.setLocalStatus(_HOME_STATUS_DISABLED)
</ins><span class="cx">
</span><span class="cx"> # Enable the migrating ones
</span><span class="cx"> newhome = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING)
</span><span class="lines">@@ -244,7 +258,8 @@
</span><span class="cx">
</span><span class="cx"> # TODO: implement API on CommonHome to purge the old data without
</span><span class="cx"> # any side-effects (scheduling, sharing etc).
</span><del>- pass
</del><ins>+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -253,11 +268,12 @@
</span><span class="cx"> Initiate a sync of the home.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- self.record = yield self.store.directoryService().recordWithUID(self.diruid)
</del><span class="cx"> if self.record is None:
</span><del>- raise DirectoryRecordNotFoundError("Cross-pod Migration Sync missing directory record for {}".format(self.diruid))
- if self.record.thisServer():
- raise ValueError("Cross-pod Migration Sync cannot sync with user already on this server: {}".format(self.diruid))
</del><ins>+ self.record = yield self.store.directoryService().recordWithUID(self.diruid)
+ if self.record is None:
+ raise DirectoryRecordNotFoundError("Cross-pod Migration Sync missing directory record for {}".format(self.diruid))
+ if self.record.thisServer():
+ raise ValueError("Cross-pod Migration Sync cannot sync with user already on this server: {}".format(self.diruid))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inTransactionWrapper
</span><span class="lines">@@ -267,10 +283,14 @@
</span><span class="cx"> Make sure the inactive home to migrate into is present on this pod.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- home = yield self._localHome(txn)
- if home is None:
- home = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
- self.homeId = home.id()
</del><ins>+ if self.homeId is None:
+ home = yield self._localHome(txn)
+ if home is None:
+ if self.disabledRemote:
+ self.homeId = None
+ else:
+ home = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
+ self.homeId = home.id() if home is not None else None
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inTransactionWrapper
</span><span class="lines">@@ -1004,6 +1024,8 @@
</span><span class="cx"> len_records = 0
</span><span class="cx"> for calendar in calendars.values():
</span><span class="cx"> records, bindUID = yield self.sharedByCollectionRecords(calendar.remoteResourceID, calendar.localResourceID)
</span><ins>+ if not records:
+ continue
</ins><span class="cx"> records = records.items()
</span><span class="cx">
</span><span class="cx"> # Batch setting resources for the local home
</span><span class="lines">@@ -1039,7 +1061,7 @@
</span><span class="cx"> calendarHomeResourceID=self.homeId,
</span><span class="cx"> calendarResourceID=local_id,
</span><span class="cx"> )
</span><del>- if not local_records[0].bindUID:
</del><ins>+ if records and not local_records[0].bindUID:
</ins><span class="cx"> yield local_records[0].update(bindUID=str(uuid4()))
</span><span class="cx">
</span><span class="cx"> returnValue((records, local_records[0].bindUID,))
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtestaccountsgroupAccountsxml"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -106,6 +106,7 @@
</span><span class="cx"> <full-name>Group 01</full-name>
</span><span class="cx"> <email>group01@example.com</email>
</span><span class="cx"> <member-uid>user01</member-uid>
</span><ins>+ <member-uid>puser01</member-uid>
</ins><span class="cx"> </record>
</span><span class="cx"> <record type="group">
</span><span class="cx"> <short-name>group02</short-name>
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -744,6 +744,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><ins>+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</ins><span class="cx"> calendar0 = yield home0.childWithName("calendar")
</span><span class="cx"> object0_1 = yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
</span><span class="cx"> object0_2 = yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
</span><span class="lines">@@ -854,6 +855,7 @@
</span><span class="cx">
</span><span class="cx"> # Create remote home
</span><span class="cx"> yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><ins>+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</ins><span class="cx"> yield self.commitTransaction(0)
</span><span class="cx">
</span><span class="cx"> # Add some delegates
</span><span class="lines">@@ -941,7 +943,7 @@
</span><span class="cx">
</span><span class="cx"> # Create remote home - and add some fake notifications
</span><span class="cx"> yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><del>- notifications = yield self.theTransactionUnderTest(0).notificationsWithUID("user01")
</del><ins>+ notifications = yield self.theTransactionUnderTest(0).notificationsWithUID("user01", create=True)
</ins><span class="cx"> uid1 = str(uuid4())
</span><span class="cx"> obj1 = yield notifications.writeNotificationObject(uid1, "type1", "data1")
</span><span class="cx"> id1 = obj1.id()
</span><span class="lines">@@ -1047,6 +1049,7 @@
</span><span class="cx">
</span><span class="cx"> # Create home
</span><span class="cx"> yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><ins>+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</ins><span class="cx"> yield self.commitTransaction(0)
</span><span class="cx">
</span><span class="cx"> # Shared by migrating user
</span><span class="lines">@@ -1119,6 +1122,7 @@
</span><span class="cx">
</span><span class="cx"> # Create home
</span><span class="cx"> yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><ins>+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</ins><span class="cx"> yield self.commitTransaction(0)
</span><span class="cx">
</span><span class="cx"> # Shared by migrating user
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_migrationpy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py (0 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -0,0 +1,691 @@
</span><ins>+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from pycalendar.datetime import DateTime
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python.filepath import FilePath
+from twistedcaldav.config import config
+from twistedcaldav.ical import Component
+from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
+from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
+from txdav.common.datastore.sql_tables import _BIND_MODE_READ, \
+ _HOME_STATUS_DISABLED, _HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL, \
+ _HOME_STATUS_MIGRATING
+from txdav.common.datastore.test.util import populateCalendarsFrom
+from txdav.who.delegates import Delegates
+from txweb2.http_headers import MimeType
+from txweb2.stream import MemoryStream
+from txdav.caldav.datastore.scheduling.ischedule.delivery import IScheduleRequest
+from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
+from txweb2.dav.test.util import SimpleRequest
+from txdav.caldav.datastore.test.common import CaptureProtocol
+
+
+class TestCompleteMigrationCycle(MultiStoreConduitTest):
+ """
+ Test that a full migration cycle using L{CrossPodHomeSync} works.
+ """
+
+ def __init__(self, methodName='runTest'):
+ super(TestCompleteMigrationCycle, self).__init__(methodName)
+ self.stash = {}
+
+
+ @inlineCallbacks
+ def setUp(self):
+ @inlineCallbacks
+ def _fakeSubmitRequest(iself, ssl, host, port, request):
+ pod = (port - 8008) / 100
+ inbox = IScheduleInboxResource(self.site.resource, self.theStoreUnderTest(pod), podding=True)
+ response = yield inbox.http_POST(SimpleRequest(
+ self.site,
+ "POST",
+ "http://{host}:{port}/podding".format(host=host, port=port),
+ request.headers,
+ request.stream.mem,
+ ))
+ returnValue(response)
+
+
+ self.patch(IScheduleRequest, "_submitRequest", _fakeSubmitRequest)
+ self.accounts = FilePath(__file__).sibling("accounts").child("groupAccounts.xml")
+ self.augments = FilePath(__file__).sibling("accounts").child("augments.xml")
+ yield super(TestCompleteMigrationCycle, self).setUp()
+ yield self.populate()
+
+
+ def configure(self):
+ super(TestCompleteMigrationCycle, self).configure()
+ config.GroupAttendees.Enabled = True
+ config.GroupAttendees.ReconciliationDelaySeconds = 0
+ config.GroupAttendees.AutoUpdateSecondsFromNow = 0
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateCalendarsFrom(self.requirements0, self.theStoreUnderTest(0))
+ yield populateCalendarsFrom(self.requirements1, self.theStoreUnderTest(1))
+
+ requirements0 = {
+ "user01" : None,
+ "user02" : None,
+ "user03" : None,
+ "user04" : None,
+ "user05" : None,
+ "user06" : None,
+ "user07" : None,
+ "user08" : None,
+ "user09" : None,
+ "user10" : None,
+ }
+
+ requirements1 = {
+ "puser01" : None,
+ "puser02" : None,
+ "puser03" : None,
+ "puser04" : None,
+ "puser05" : None,
+ "puser06" : None,
+ "puser07" : None,
+ "puser08" : None,
+ "puser09" : None,
+ "puser10" : None,
+ }
+
+
+ @inlineCallbacks
+ def _createShare(self, shareFrom, shareTo, accept=True):
+ # Invite
+ txnindex = 1 if shareFrom[0] == "p" else 0
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(txnindex), name=shareFrom, create=True)
+ calendar = yield home.childWithName("calendar")
+ shareeView = yield calendar.inviteUIDToShare(shareTo, _BIND_MODE_READ, "summary")
+ yield self.commitTransaction(txnindex)
+
+ # Accept
+ if accept:
+ inviteUID = shareeView.shareUID()
+ txnindex = 1 if shareTo[0] == "p" else 0
+ shareeHome = yield self.homeUnderTest(txn=self.theTransactionUnderTest(txnindex), name=shareTo)
+ shareeView = yield shareeHome.acceptShare(inviteUID)
+ sharedName = shareeView.name()
+ yield self.commitTransaction(txnindex)
+ else:
+ sharedName = None
+
+ returnValue(sharedName)
+
+
+ def attachmentToString(self, attachment):
+ """
+ Convenience to convert an L{IAttachment} to a string.
+
+ @param attachment: an L{IAttachment} provider to convert into a string.
+
+ @return: a L{Deferred} that fires with the contents of the attachment.
+
+ @rtype: L{Deferred} firing C{bytes}
+ """
+ capture = CaptureProtocol()
+ attachment.retrieve(capture)
+ return capture.deferred
+
+
+ now = {
+ "now": DateTime.getToday().getYear(),
+ "now1": DateTime.getToday().getYear() + 1,
+ }
+
+ data01_1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_1
+DTSTART:{now1:04d}0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:data01_1
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data01_1_changed = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_1
+DTSTART:{now1:04d}0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:data01_1_changed
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data01_2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_2
+DTSTART:{now1:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_2
+ORGANIZER:mailto:user01@example.com
+ATTENDEE:mailto:user01@example.com
+ATTENDEE:mailto:user02@example.com
+ATTENDEE:mailto:puser02@example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data01_3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_3
+DTSTART:{now1:04d}0102T180000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_3
+ORGANIZER:mailto:user01@example.com
+ATTENDEE:mailto:user01@example.com
+ATTENDEE:mailto:group02@example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data02_1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data02_1
+DTSTART:{now1:04d}0103T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:data02_1
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data02_2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data02_2
+DTSTART:{now1:04d}0103T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data02_2
+ORGANIZER:mailto:user02@example.com
+ATTENDEE:mailto:user02@example.com
+ATTENDEE:mailto:user01@example.com
+ATTENDEE:mailto:puser02@example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data02_3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data02_3
+DTSTART:{now1:04d}0103T180000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data02_3
+ORGANIZER:mailto:user02@example.com
+ATTENDEE:mailto:user02@example.com
+ATTENDEE:mailto:group01@example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ datap02_1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_datap02_1
+DTSTART:{now1:04d}0103T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:datap02_1
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ datap02_2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_datap02_2
+DTSTART:{now1:04d}0103T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:datap02_2
+ORGANIZER:mailto:puser02@example.com
+ATTENDEE:mailto:puser02@example.com
+ATTENDEE:mailto:user01@example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ datap02_3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_datap02_3
+DTSTART:{now1:04d}0103T180000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:datap02_3
+ORGANIZER:mailto:puser02@example.com
+ATTENDEE:mailto:puser02@example.com
+ATTENDEE:mailto:group01@example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+
+ @inlineCallbacks
+ def preCheck(self):
+ """
+ Checks prior to starting any tests
+ """
+
+ for i in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(i)
+ record = yield txn.directoryService().recordWithUID(u"user01")
+ self.assertEqual(record.serviceNodeUID, "A")
+ self.assertEqual(record.thisServer(), i == 0)
+ record = yield txn.directoryService().recordWithUID(u"user02")
+ self.assertEqual(record.serviceNodeUID, "A")
+ self.assertEqual(record.thisServer(), i == 0)
+ record = yield txn.directoryService().recordWithUID(u"puser02")
+ self.assertEqual(record.serviceNodeUID, "B")
+ self.assertEqual(record.thisServer(), i == 1)
+ yield self.commitTransaction(i)
+
+
+ @inlineCallbacks
+ def initialState(self):
+ """
+ Setup the server with an initial set of data
+
+ user01 - migrating user
+ user02 - has a calendar shared with user01
+ user03 - shared to by user01
+
+ puser01 - user on other pod
+ puser02 - has a calendar shared with user01
+ puser03 - shared to by user01
+ """
+
+ # Data for user01
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("01_1.ics", Component.fromString(self.data01_1))
+ yield calendar.createCalendarObjectWithName("01_2.ics", Component.fromString(self.data01_2))
+ obj3 = yield calendar.createCalendarObjectWithName("01_3.ics", Component.fromString(self.data01_3))
+ attachment, _ignore_location = yield obj3.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1."))
+ self.stash["user01_attachment_id"] = attachment.id()
+ self.stash["user01_attachment_md5"] = attachment.md5()
+ self.stash["user01_attachment_mid"] = attachment.managedID()
+ yield self.commitTransaction(0)
+
+ # Data for user02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user02", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("02_1.ics", Component.fromString(self.data02_1))
+ yield calendar.createCalendarObjectWithName("02_2.ics", Component.fromString(self.data02_2))
+ yield calendar.createCalendarObjectWithName("02_3.ics", Component.fromString(self.data02_3))
+ yield self.commitTransaction(0)
+
+ # Data for puser02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("p02_1.ics", Component.fromString(self.datap02_1))
+ yield calendar.createCalendarObjectWithName("p02_2.ics", Component.fromString(self.datap02_2))
+ yield calendar.createCalendarObjectWithName("p02_3.ics", Component.fromString(self.datap02_3))
+ yield self.commitTransaction(1)
+
+ # Share calendars
+ self.stash["sharename_user01_to_user03"] = yield self._createShare("user01", "user03")
+ self.stash["sharename_user01_to_puser03"] = yield self._createShare("user01", "puser03")
+ self.stash["sharename_user02_to_user01"] = yield self._createShare("user02", "user01")
+ self.stash["sharename_puser02_to_user01"] = yield self._createShare("puser02", "user01")
+
+ # Add some delegates
+ txn = self.theTransactionUnderTest(0)
+ record01 = yield txn.directoryService().recordWithUID(u"user01")
+ record02 = yield txn.directoryService().recordWithUID(u"user02")
+ record03 = yield txn.directoryService().recordWithUID(u"user03")
+ precord01 = yield txn.directoryService().recordWithUID(u"puser01")
+
+ group02 = yield txn.directoryService().recordWithUID(u"group02")
+ group03 = yield txn.directoryService().recordWithUID(u"group03")
+
+ # Add user02 and user03 as individual delegates
+ yield Delegates.addDelegate(txn, record01, record02, True)
+ yield Delegates.addDelegate(txn, record01, record03, False)
+ yield Delegates.addDelegate(txn, record01, precord01, False)
+
+ # Add group delegates
+ yield Delegates.addDelegate(txn, record01, group02, True)
+ yield Delegates.addDelegate(txn, record01, group03, False)
+
+ # Add external delegates
+ yield txn.assignExternalDelegates(u"user01", None, None, u"external1", u"external2")
+
+ yield self.commitTransaction(0)
+
+ yield self.waitAllEmpty()
+
+
+ @inlineCallbacks
+ def secondState(self):
+ """
+ Setup the server with data changes appearing after the first sync
+ """
+ txn = self.theTransactionUnderTest(0)
+ obj = yield self.calendarObjectUnderTest(txn, name="01_1.ics", calendar_name="calendar", home="user01")
+ yield obj.setComponent(self.data01_1_changed)
+
+ obj = yield self.calendarObjectUnderTest(txn, name="02_2.ics", calendar_name="calendar", home="user02")
+ attachment, _ignore_location = yield obj.addAttachment(None, MimeType.fromString("text/plain"), "test_02.txt", MemoryStream("Here is some text #02."))
+ self.stash["user02_attachment_id"] = attachment.id()
+ self.stash["user02_attachment_md5"] = attachment.md5()
+ self.stash["user02_attachment_mid"] = attachment.managedID()
+
+ yield self.commitTransaction(0)
+
+ yield self.waitAllEmpty()
+
+
+ @inlineCallbacks
+ def finalState(self):
+ """
+ Setup the server with data changes appearing before the final sync
+ """
+ txn = self.theTransactionUnderTest(1)
+ obj = yield self.calendarObjectUnderTest(txn, name="p02_2.ics", calendar_name="calendar", home="puser02")
+ attachment, _ignore_location = yield obj.addAttachment(None, MimeType.fromString("text/plain"), "test_p02.txt", MemoryStream("Here is some text #p02."))
+ self.stash["puser02_attachment_id"] = attachment.id()
+ self.stash["puser02_attachment_mid"] = attachment.managedID()
+ self.stash["puser02_attachment_md5"] = attachment.md5()
+
+ yield self.commitTransaction(1)
+
+ yield self.waitAllEmpty()
+
+
+ @inlineCallbacks
+ def switchAccounts(self):
+ """
+ Switch the migrated user accounts to point to the new pod
+ """
+
+ for i in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(i)
+ record = yield txn.directoryService().recordWithUID(u"user01")
+ yield self.changeRecord(record, txn.directoryService().fieldName.serviceNodeUID, u"B", directory=txn.directoryService())
+ yield self.commitTransaction(i)
+
+ for i in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(i)
+ record = yield txn.directoryService().recordWithUID(u"user01")
+ self.assertEqual(record.serviceNodeUID, "B")
+ self.assertEqual(record.thisServer(), i == 1)
+ record = yield txn.directoryService().recordWithUID(u"user02")
+ self.assertEqual(record.serviceNodeUID, "A")
+ self.assertEqual(record.thisServer(), i == 0)
+ record = yield txn.directoryService().recordWithUID(u"puser02")
+ self.assertEqual(record.serviceNodeUID, "B")
+ self.assertEqual(record.thisServer(), i == 1)
+ yield self.commitTransaction(i)
+
+
+ @inlineCallbacks
+ def postCheck(self):
+ """
+ Checks after migration is done
+ """
+
+ # Check that the home has been moved
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01")
+ self.assertTrue(home.external())
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(home is None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_MIGRATING)
+ self.assertTrue(home is None)
+ yield self.commitTransaction(0)
+
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01")
+ self.assertTrue(home.normal())
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(home is None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
+ self.assertTrue(home is None)
+ yield self.commitTransaction(1)
+
+ # Check that the notifications have been moved
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(notifications is None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(notifications is None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(notifications is not None)
+ yield self.commitTransaction(0)
+
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(notifications is not None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(notifications is None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(notifications is not None)
+ yield self.commitTransaction(1)
+
+ # New pod data
+ homes = {}
+ homes["user01"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01")
+ homes["user02"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user02")
+ self.assertTrue(homes["user02"].external())
+ homes["user03"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user03")
+ self.assertTrue(homes["user03"].external())
+ homes["puser01"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="puser01")
+ self.assertTrue(homes["puser01"].normal())
+ homes["puser02"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="puser02")
+ self.assertTrue(homes["puser02"].normal())
+ homes["puser03"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="puser03")
+ self.assertTrue(homes["puser03"].normal())
+
+ # Check calendar data on new pod
+ calendars = yield homes["user01"].loadChildren()
+ calnames = dict([(calendar.name(), calendar) for calendar in calendars])
+ self.assertEqual(
+ set(calnames.keys()),
+ set(("calendar", "tasks", "inbox", self.stash["sharename_user02_to_user01"], self.stash["sharename_puser02_to_user01"],))
+ )
+
+ # Check shared-by user01 on new pod
+ shared = calnames["calendar"]
+ invitations = yield shared.sharingInvites()
+ by_sharee = dict([(invitation.shareeUID, invitation) for invitation in invitations])
+ self.assertEqual(len(invitations), 2)
+ self.assertEqual(set(by_sharee.keys()), set(("user03", "puser03",)))
+ self.assertEqual(by_sharee["user03"].shareeHomeID, homes["user03"].id())
+ self.assertEqual(by_sharee["puser03"].shareeHomeID, homes["puser03"].id())
+
+ # Check shared-to user01 on new pod
+ shared = calnames[self.stash["sharename_user02_to_user01"]]
+ self.assertEqual(shared.ownerHome().uid(), "user02")
+ self.assertEqual(shared.ownerHome().id(), homes["user02"].id())
+
+ shared = calnames[self.stash["sharename_puser02_to_user01"]]
+ self.assertEqual(shared.ownerHome().uid(), "puser02")
+ self.assertEqual(shared.ownerHome().id(), homes["puser02"].id())
+
+ shared = yield homes["puser02"].calendarWithName("calendar")
+ invitations = yield shared.sharingInvites()
+ self.assertEqual(len(invitations), 1)
+ self.assertEqual(invitations[0].shareeHomeID, homes["user01"].id())
+
+ yield self.commitTransaction(1)
+
+ # Old pod data
+ homes = {}
+ homes["user01"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01")
+ homes["user02"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user02")
+ self.assertTrue(homes["user02"].normal())
+ homes["user03"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user03")
+ self.assertTrue(homes["user03"].normal())
+ homes["puser01"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="puser01")
+ self.assertTrue(homes["puser01"] is None)
+ homes["puser02"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="puser02")
+ self.assertTrue(homes["puser02"].external())
+ homes["puser03"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="puser03")
+ self.assertTrue(homes["puser03"].external())
+
+ # Check shared-by user01 on old pod
+ shared = yield homes["user03"].calendarWithName(self.stash["sharename_user01_to_user03"])
+ self.assertEqual(shared.ownerHome().uid(), "user01")
+ self.assertEqual(shared.ownerHome().id(), homes["user01"].id())
+
+ # Check shared-to user01 on old pod
+ shared = yield homes["user02"].calendarWithName("calendar")
+ invitations = yield shared.sharingInvites()
+ self.assertEqual(len(invitations), 1)
+ self.assertEqual(invitations[0].shareeHomeID, homes["user01"].id())
+
+ yield self.commitTransaction(0)
+
+ # Delegates on each pod
+ for pod in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(pod)
+ records = {}
+ for ctr in range(10):
+ uid = u"user{:02d}".format(ctr + 1)
+ records[uid] = yield txn.directoryService().recordWithUID(uid)
+ for ctr in range(10):
+ uid = u"puser{:02d}".format(ctr + 1)
+ records[uid] = yield txn.directoryService().recordWithUID(uid)
+ for ctr in range(10):
+ uid = u"group{:02d}".format(ctr + 1)
+ records[uid] = yield txn.directoryService().recordWithUID(uid)
+
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], True, False)
+ self.assertTrue(records["user02"] in delegates)
+ self.assertTrue(records["group02"] in delegates)
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], True, True)
+ self.assertTrue(records["user02"] in delegates)
+ self.assertTrue(records["user06"] in delegates)
+ self.assertTrue(records["user07"] in delegates)
+ self.assertTrue(records["user08"] in delegates)
+
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], False, False)
+ self.assertTrue(records["user03"] in delegates)
+ self.assertTrue(records["group03"] in delegates)
+ self.assertTrue(records["puser01"] in delegates)
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], False, True)
+ self.assertTrue(records["user03"] in delegates)
+ self.assertTrue(records["user07"] in delegates)
+ self.assertTrue(records["user08"] in delegates)
+ self.assertTrue(records["user09"] in delegates)
+ self.assertTrue(records["puser01"] in delegates)
+
+ # Attachments
+ obj = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), name="01_3.ics", calendar_name="calendar", home="user01")
+ attachment = yield obj.attachmentWithManagedID(self.stash["user01_attachment_mid"])
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), self.stash["user01_attachment_md5"])
+ data = yield self.attachmentToString(attachment)
+ self.assertEqual(data, "Here is some text #1.")
+
+
+ @inlineCallbacks
+ def test_migration(self):
+ """
+ Full migration cycle.
+ """
+
+ yield self.preCheck()
+
+ # Step 1. Live full sync
+ yield self.initialState()
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.sync()
+
+ # Step 2. Live incremental sync
+ yield self.secondState()
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.sync()
+
+ # Step 3. Disable home after final changes
+ yield self.finalState()
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.disableRemoteHome()
+
+ # Step 4. Final incremental sync
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.sync()
+
+ # Step 5. Final reconcile sync
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.finalSync()
+
+ # Step 6. Enable new home
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.enableLocalHome()
+
+ # Step 7. Remove old home
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.removeRemoteHome()
+
+ yield self.switchAccounts()
+
+ yield self.postCheck()
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -14,6 +14,7 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><ins>+from twisted.internet import reactor
</ins><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx"> from twisted.internet.protocol import Protocol
</span><span class="cx">
</span><span class="lines">@@ -34,6 +35,7 @@
</span><span class="cx"> from txweb2.stream import ProducerStream
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.ienterprise import AlreadyFinishedError
</span><ins>+from twext.enterprise.jobqueue import JobItem
</ins><span class="cx">
</span><span class="cx"> import json
</span><span class="cx">
</span><span class="lines">@@ -225,6 +227,12 @@
</span><span class="cx"> self.activeTransactions[count] = None
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def waitAllEmpty(self):
+ for i in range(self.numberOfStores):
+ yield JobItem.waitEmpty(self.theStoreUnderTest(i).newTransaction, reactor, 60.0)
+
+
</ins><span class="cx"> def makeConduit(self, store):
</span><span class="cx"> conduit = PoddingConduit(store)
</span><span class="cx"> conduit.conduitRequestClass = FakeConduitRequest
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -579,7 +579,10 @@
</span><span class="cx"> "byID": collections.defaultdict(dict),
</span><span class="cx"> },
</span><span class="cx"> }
</span><del>- self._notificationHomes = {}
</del><ins>+ self._notificationHomes = {
+ "byUID": collections.defaultdict(dict),
+ "byID": collections.defaultdict(dict),
+ }
</ins><span class="cx"> self._notifierFactories = notifierFactories
</span><span class="cx"> self._notifiedAlready = set()
</span><span class="cx"> self._bumpedRevisionAlready = set()
</span><span class="lines">@@ -753,7 +756,7 @@
</span><span class="cx"> result = yield self._homeClass[storeType].homeWithResourceID(self, rid)
</span><span class="cx"> if result:
</span><span class="cx"> self._determineMemo(storeType, "byID", None)[rid] = result
</span><del>- self._determineMemo(storeType, "byUID", result._status)[result.uid()] = result
</del><ins>+ self._determineMemo(storeType, "byUID", result.status())[result.uid()] = result
</ins><span class="cx"> returnValue(result)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -765,22 +768,36 @@
</span><span class="cx"> return self.homeWithResourceID(EADDRESSBOOKTYPE, rid)
</span><span class="cx">
</span><span class="cx">
</span><del>- @memoizedKey("uid", "_notificationHomes")
- def notificationsWithUID(self, uid, status=None, create=True):
</del><ins>+ @inlineCallbacks
+ def notificationsWithUID(self, uid, status=None, create=False):
</ins><span class="cx"> """
</span><span class="cx"> Implement notificationsWithUID.
</span><span class="cx"> """
</span><del>- return NotificationCollection.notificationsWithUID(self, uid, create=create)
</del><span class="cx">
</span><ins>+ result = self._notificationHomes["byUID"][status].get(uid)
+ if result is None:
+ result = yield NotificationCollection.notificationsWithUID(self, uid, status=status, create=create)
+ if result:
+ self._notificationHomes["byUID"][status][uid] = result
+ self._notificationHomes["byID"][None][result.id()] = result
+ returnValue(result)
</ins><span class="cx">
</span><del>- @memoizedKey("rid", "_notificationHomes")
</del><ins>+
+ @inlineCallbacks
</ins><span class="cx"> def notificationsWithResourceID(self, rid):
</span><span class="cx"> """
</span><span class="cx"> Implement notificationsWithResourceID.
</span><span class="cx"> """
</span><del>- return NotificationCollection.notificationsWithResourceID(self, rid)
</del><span class="cx">
</span><ins>+ result = self._notificationHomes["byID"][None].get(rid)
+ if result is None:
+ result = yield NotificationCollection.notificationsWithResourceID(self, rid)
+ if result:
+ self._notificationHomes["byID"][None][rid] = result
+ self._notificationHomes["byUID"][result.status()][result.uid()] = result
+ returnValue(result)
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def preCommit(self, operation):
</span><span class="cx"> """
</span><span class="cx"> Run things before C{commit}. (Note: only provided by SQL
</span><span class="lines">@@ -1968,6 +1985,10 @@
</span><span class="cx"> return self._authzUID
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def status(self):
+ return self._status
+
+
</ins><span class="cx"> def normal(self):
</span><span class="cx"> """
</span><span class="cx"> Is this an normal (internal) home.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -89,6 +89,16 @@
</span><span class="cx"> return self._txn.store().conduit.send_home_set_status(self, newStatus)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def setLocalStatus(self, newStatus):
+ """
+ Set the status on the object in the local store not the remote one.
+
+ @param newStatus: the new status to set
+ @type newStatus: L{int}
+ """
+ return super(CommonHomeExternal, self).setStatus(newStatus)
+
+
</ins><span class="cx"> def external(self):
</span><span class="cx"> """
</span><span class="cx"> Is this an external home.
</span><span class="lines">@@ -517,4 +527,14 @@
</span><span class="cx"> def setStatus(self, newStatus):
</span><span class="cx"> return self._txn.store().conduit.send_notification_set_status(self, newStatus)
</span><span class="cx">
</span><ins>+
+ def setLocalStatus(self, newStatus):
+ """
+ Set the status on the object in the local store not the remote one.
+
+ @param newStatus: the new status to set
+ @type newStatus: L{int}
+ """
+ return super(NotificationCollectionExternal, self).setStatus(newStatus)
+
</ins><span class="cx"> NotificationCollection._externalClass = NotificationCollectionExternal
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_notificationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -147,8 +147,8 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><del>- def notificationsWithUID(cls, txn, uid, status=None, create=True):
- return cls.notificationsWith(txn, None, uid, status, create=create)
</del><ins>+ def notificationsWithUID(cls, txn, uid, status=None, create=False):
+ return cls.notificationsWith(txn, None, uid, status=status, create=create)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="lines">@@ -158,7 +158,7 @@
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="cx"> @inlineCallbacks
</span><del>- def notificationsWith(cls, txn, rid, uid, status=None, create=True):
</del><ins>+ def notificationsWith(cls, txn, rid, uid, status=None, create=False):
</ins><span class="cx"> """
</span><span class="cx"> @param uid: I'm going to assume uid is utf-8 encoded bytes
</span><span class="cx"> """
</span><span class="lines">@@ -318,6 +318,10 @@
</span><span class="cx"> return self._ownerUID
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def status(self):
+ return self._status
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def setStatus(self, newStatus):
</span><span class="cx"> """
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -601,7 +601,7 @@
</span><span class="cx"> notificationdata["supported-components"] = self.getSupportedComponents()
</span><span class="cx">
</span><span class="cx"> # Add to sharee's collection
</span><del>- notifications = yield self._txn.notificationsWithUID(shareeView.viewerHome().uid())
</del><ins>+ notifications = yield self._txn.notificationsWithUID(shareeView.viewerHome().uid(), create=True)
</ins><span class="cx"> yield notifications.writeNotificationObject(shareeView.shareUID(), notificationtype, notificationdata)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -632,7 +632,7 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> # Add to owner notification collection
</span><del>- notifications = yield self._txn.notificationsWithUID(self.ownerHome().uid())
</del><ins>+ notifications = yield self._txn.notificationsWithUID(self.ownerHome().uid(), create=True)
</ins><span class="cx"> yield notifications.writeNotificationObject(notificationUID, notificationtype, notificationdata)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_utilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -691,7 +691,7 @@
</span><span class="cx"> L{NotificationHome} when it has been retrieved.
</span><span class="cx"> """
</span><span class="cx"> if homeType == ENOTIFICATIONTYPE:
</span><del>- return txn.notificationsWithUID(uid, create=False)
</del><ins>+ return txn.notificationsWithUID(uid)
</ins><span class="cx"> else:
</span><span class="cx"> return txn.homeWithUID(homeType, uid)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoretestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -939,6 +939,13 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def notificationCollectionUnderTest(self, txn=None, name="home1", status=None, create=False):
+ if txn is None:
+ txn = self.transactionUnderTest()
+ returnValue((yield txn.notificationsWithUID(name, status=status, create=create)))
+
+
+ @inlineCallbacks
</ins><span class="cx"> def userRecordWithShortName(self, shortname):
</span><span class="cx"> record = yield self.directory.recordWithShortName(self.directory.recordType.user, shortname)
</span><span class="cx"> returnValue(record)
</span><span class="lines">@@ -962,11 +969,13 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def changeRecord(self, record, fieldname, value):
</del><ins>+ def changeRecord(self, record, fieldname, value, directory=None):
+ if directory is None:
+ directory = self.directory
</ins><span class="cx"> fields = record.fields.copy()
</span><span class="cx"> fields[fieldname] = value
</span><del>- updatedRecord = DirectoryRecord(self.directory, fields)
- yield self.directory.updateRecords((updatedRecord,))
</del><ins>+ updatedRecord = DirectoryRecord(directory, fields)
+ yield directory.updateRecords((updatedRecord,))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoreupgradesqlupgradestesttest_notification_upgrade_from_0_to_1py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -169,7 +169,7 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> for uid, notificationtype, _ignore_jtype, notificationdata, _ignore_jdata in data:
</span><del>- notifications = yield self.transactionUnderTest().notificationsWithUID("user01")
</del><ins>+ notifications = yield self.transactionUnderTest().notificationsWithUID("user01", create=True)
</ins><span class="cx"> yield notifications.writeNotificationObject(uid, notificationtype, notificationdata)
</span><span class="cx">
</span><span class="cx"> # Force data version to previous
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavwhotesttest_group_shareespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py (14506 => 14507)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py 2015-03-05 02:41:22 UTC (rev 14507)
</span><span class="lines">@@ -84,7 +84,7 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _check_notifications(self, uid, items):
</span><del>- notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid)
</del><ins>+ notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid, create=True)
</ins><span class="cx"> notifications = yield notifyHome.listNotificationObjects()
</span><span class="cx"> self.assertEqual(set(notifications), set(items))
</span><span class="cx">
</span></span></pre>
</div>
</div>
</body>
</html>