<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14490] CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14490">14490</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-02-28 14:26:20 -0800 (Sat, 28 Feb 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Checkpoint: final sync steps.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_notificationpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -25,8 +25,8 @@
</span><span class="cx"> from txdav.caldav.datastore.sql import ManagedAttachment, CalendarBindRecord
</span><span class="cx"> from txdav.common.datastore.sql_external import NotificationCollectionExternal
</span><span class="cx"> from txdav.common.datastore.sql_notification import NotificationCollection
</span><del>-from txdav.common.datastore.sql_tables import _HOME_STATUS_EXTERNAL, \
- _HOME_STATUS_MIGRATING
</del><ins>+from txdav.common.datastore.sql_tables import _HOME_STATUS_MIGRATING, _HOME_STATUS_DISABLED, \
+ _HOME_STATUS_EXTERNAL, _HOME_STATUS_NORMAL
</ins><span class="cx"> from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
</span><span class="cx">
</span><span class="cx"> from uuid import uuid4
</span><span class="lines">@@ -95,6 +95,7 @@
</span><span class="cx">
</span><span class="cx"> self.store = store
</span><span class="cx"> self.diruid = diruid
</span><ins>+ self.disabledRemote = False
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def label(self, detail):
</span><span class="lines">@@ -190,25 +191,48 @@
</span><span class="cx"> pass
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inTransactionWrapper
</ins><span class="cx"> @inlineCallbacks
</span><del>- def disableRemoteHome(self):
</del><ins>+ def disableRemoteHome(self, txn):
</ins><span class="cx"> """
</span><span class="cx"> Mark the remote home as disabled.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- # TODO: implement API on CommonHome to rename the ownerUID column and
- # change the status column.
- pass
</del><ins>+ # Calendar home
+ remote_home = yield self._remoteHome(txn)
+ yield remote_home.setStatus(_HOME_STATUS_DISABLED)
</ins><span class="cx">
</span><ins>+ # Notification home
+ notifications = yield self._remoteNotificationsHome(txn)
+ yield notifications.setStatus(_HOME_STATUS_DISABLED)
</ins><span class="cx">
</span><ins>+ self.disabledRemote = True
+
+
+ @inTransactionWrapper
</ins><span class="cx"> @inlineCallbacks
</span><del>- def enableLocalHome(self):
</del><ins>+ def enableLocalHome(self, txn):
</ins><span class="cx"> """
</span><del>- Mark the local home as enabled.
</del><ins>+ Mark the local home as enabled and remove any previously existing external home.
</ins><span class="cx"> """
</span><span class="cx">
</span><del>- # TODO: implement API on CommonHome to rename the ownerUID column and
- # change the status column. Also adjust NotificationCollection.
</del><ins>+ # Disable any local external homes
+ oldhome = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_EXTERNAL)
+ if oldhome is not None:
+ yield oldhome.setStatus(_HOME_STATUS_DISABLED)
+ oldnotifications = yield txn.notificationsWithUID(self.diruid, status=_HOME_STATUS_EXTERNAL)
+ if oldnotifications:
+ yield oldnotifications.setStatus(_HOME_STATUS_DISABLED)
+
+ # Enable the migrating ones
+ newhome = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING)
+ if newhome is not None:
+ yield newhome.setStatus(_HOME_STATUS_NORMAL)
+ newnotifications = yield txn.notificationsWithUID(self.diruid, status=_HOME_STATUS_MIGRATING)
+ if newnotifications:
+ yield newnotifications.setStatus(_HOME_STATUS_NORMAL)
+
+ # TODO: purge the old ones
</ins><span class="cx"> pass
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -256,7 +280,7 @@
</span><span class="cx"> Make sure the home meta-data (alarms, default calendars) is properly sync'd
</span><span class="cx"> """
</span><span class="cx">
</span><del>- remote_home = yield self._remoteHome(txn=txn)
</del><ins>+ remote_home = yield self._remoteHome(txn)
</ins><span class="cx"> yield remote_home.readMetaData()
</span><span class="cx">
</span><span class="cx"> calendars = yield CalendarMigrationRecord.querysimple(txn, calendarHomeResourceID=self.homeId)
</span><span class="lines">@@ -273,11 +297,25 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.sql_external import CalendarHomeExternal
</span><del>- resourceID = yield txn.store().conduit.send_home_resource_id(txn, self.record)
</del><ins>+ resourceID = yield txn.store().conduit.send_home_resource_id(txn, self.record, migrating=True)
</ins><span class="cx"> home = CalendarHomeExternal.makeSyntheticExternalHome(txn, self.record.uid, resourceID) if resourceID is not None else None
</span><ins>+ if self.disabledRemote:
+ home._migratingHome = True
</ins><span class="cx"> returnValue(home)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _remoteNotificationsHome(self, txn):
+ """
+ Create a synthetic external home object that maps to the actual remote home.
+ """
+
+ notifications = yield NotificationCollectionExternal.notificationsWithUID(txn, self.diruid, create=True)
+ if self.disabledRemote:
+ notifications._migratingHome = True
+ returnValue(notifications)
+
+
</ins><span class="cx"> def _localHome(self, txn):
</span><span class="cx"> """
</span><span class="cx"> Get the home on this pod that will have data migrated to it.
</span><span class="lines">@@ -917,7 +955,7 @@
</span><span class="cx"> Get all the existing L{NotificationObjectRecord}'s from the remote store.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- notifications = yield NotificationCollectionExternal.notificationsWithUID(txn, self.diruid, True)
</del><ins>+ notifications = yield self._remoteNotificationsHome(txn)
</ins><span class="cx"> records = yield notifications.notificationObjectRecords()
</span><span class="cx"> for record in records:
</span><span class="cx"> # This needs to be reset when added to the local store
</span><span class="lines">@@ -936,7 +974,7 @@
</span><span class="cx"> Create L{NotificationObjectRecord} records in the local store.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- notifications = yield NotificationCollection.notificationsWithUID(txn, self.diruid, True, _HOME_STATUS_EXTERNAL)
</del><ins>+ notifications = yield NotificationCollection.notificationsWithUID(txn, self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
</ins><span class="cx"> for record in records:
</span><span class="cx"> # Do this via the "write" API so that sync revisions are updated properly, rather than just
</span><span class="cx"> # inserting the records directly.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -806,7 +806,8 @@
</span><span class="cx"> self.assertEqual(changed, set(((yield _mapLocalIDToRemote(id0_1)), (yield _mapLocalIDToRemote(id0_2)),)))
</span><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><del>- # Link attachments
</del><ins>+ # Link attachments (after home is disabled)
+ yield syncer.disableRemoteHome()
</ins><span class="cx"> len_links = yield syncer.linkAttachments()
</span><span class="cx"> self.assertEqual(len_links, 3)
</span><span class="cx">
</span><span class="lines">@@ -891,6 +892,7 @@
</span><span class="cx"> # Sync from remote side
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><ins>+ yield syncer.disableRemoteHome()
</ins><span class="cx"> yield syncer.delegateReconcile()
</span><span class="cx">
</span><span class="cx"> # Now have local delegates
</span><span class="lines">@@ -952,6 +954,7 @@
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><span class="cx"> yield syncer.prepareCalendarHome()
</span><ins>+ yield syncer.disableRemoteHome()
</ins><span class="cx"> changes = yield syncer.notificationsReconcile()
</span><span class="cx"> self.assertEqual(changes, 2)
</span><span class="cx">
</span><span class="lines">@@ -959,8 +962,7 @@
</span><span class="cx"> notifications = yield NotificationCollection.notificationsWithUID(
</span><span class="cx"> self.theTransactionUnderTest(1),
</span><span class="cx"> "user01",
</span><del>- True,
- _HOME_STATUS_EXTERNAL
</del><ins>+ status=_HOME_STATUS_MIGRATING,
</ins><span class="cx"> )
</span><span class="cx"> results = yield notifications.notificationObjects()
</span><span class="cx"> self.assertEqual(len(results), 2)
</span><span class="lines">@@ -1059,6 +1061,7 @@
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><span class="cx"> yield syncer.sync()
</span><ins>+ yield syncer.disableRemoteHome()
</ins><span class="cx"> changes = yield syncer.sharedByCollectionsReconcile()
</span><span class="cx"> self.assertEqual(changes, 2)
</span><span class="cx"> changes = yield syncer.sharedToCollectionsReconcile()
</span><span class="lines">@@ -1125,6 +1128,7 @@
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><span class="cx"> yield syncer.sync()
</span><ins>+ yield syncer.disableRemoteHome()
</ins><span class="cx"> changes = yield syncer.sharedByCollectionsReconcile()
</span><span class="cx"> self.assertEqual(changes, 3)
</span><span class="cx"> changes = yield syncer.sharedToCollectionsReconcile()
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -18,6 +18,7 @@
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
</span><span class="cx"> from txdav.common.datastore.podding.util import UtilityConduitMixin
</span><ins>+from txdav.common.datastore.sql_tables import _HOME_STATUS_DISABLED
</ins><span class="cx">
</span><span class="cx"> from twistedcaldav.caldavxml import TimeRange
</span><span class="cx">
</span><span class="lines">@@ -28,17 +29,20 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def send_home_resource_id(self, txn, recipient):
</del><ins>+ def send_home_resource_id(self, txn, recipient, migrating=False):
</ins><span class="cx"> """
</span><span class="cx"> Lookup the remote resourceID matching the specified directory uid.
</span><span class="cx">
</span><span class="cx"> @param ownerUID: directory record for user whose home is needed
</span><span class="cx"> @type ownerUID: L{DirectroryRecord}
</span><ins>+ @param migrating: if L{True} then also return a disbaled home
+ @type migrating: L{bool}
</ins><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> request = {
</span><span class="cx"> "action": "home-resource_id",
</span><span class="cx"> "ownerUID": recipient.uid,
</span><ins>+ "migrating": migrating,
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> response = yield self.sendRequest(txn, recipient, request)
</span><span class="lines">@@ -55,6 +59,8 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> home = yield txn.calendarHomeWithUID(request["ownerUID"])
</span><ins>+ if home is None and request["migrating"]:
+ home = yield txn.calendarHomeWithUID(request["ownerUID"], status=_HOME_STATUS_DISABLED)
</ins><span class="cx"> returnValue(home.id() if home is not None else None)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -154,8 +160,9 @@
</span><span class="cx"> # These are the actions on store objects we need to expose via the conduit api
</span><span class="cx">
</span><span class="cx"> # Calls on L{CommonHome} objects
</span><del>-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_metadata", "serialize", classMethod=False)
-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_get_all_group_attendees", "getAllGroupAttendees", classMethod=False, transform_recv_result=StoreAPIConduitMixin._to_serialize_pair_list)
</del><ins>+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_metadata", "serialize")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_set_status", "setStatus")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_get_all_group_attendees", "getAllGroupAttendees", transform_recv_result=StoreAPIConduitMixin._to_serialize_pair_list)
</ins><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "home_shared_to_records", "sharedToBindRecords", transform_recv_result=StoreAPIConduitMixin._to_serialize_dict_list_serialized_value)
</span><span class="cx">
</span><span class="cx"> # Calls on L{CommonHomeChild} objects
</span><span class="lines">@@ -185,4 +192,5 @@
</span><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_remove", "remove")
</span><span class="cx">
</span><span class="cx"> # Calls on L{NotificationCollection} objects
</span><del>-UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "notification_all_records", "notificationObjectRecords", classMethod=False, transform_recv_result=UtilityConduitMixin._to_serialize_list)
</del><ins>+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "notification_set_status", "setStatus")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "notification_all_records", "notificationObjectRecords", transform_recv_result=UtilityConduitMixin._to_serialize_list)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -83,6 +83,8 @@
</span><span class="cx"> if viewer_home:
</span><span class="cx"> result["homeType"] = viewer_home._homeType
</span><span class="cx"> result["homeUID"] = viewer_home.uid()
</span><ins>+ if getattr(viewer_home, "_migratingHome", False):
+ result["allowDisabledHome"] = True
</ins><span class="cx"> if home_child:
</span><span class="cx"> if home_child.owned():
</span><span class="cx"> result["homeChildID"] = home_child.id()
</span><span class="lines">@@ -97,6 +99,8 @@
</span><span class="cx">
</span><span class="cx"> elif notification:
</span><span class="cx"> result["notificationUID"] = notification.uid()
</span><ins>+ if getattr(notification, "_migratingHome", False):
+ result["allowDisabledHome"] = True
</ins><span class="cx"> recipient = yield self.store.directoryService().recordWithUID(notification.uid())
</span><span class="cx">
</span><span class="cx"> returnValue((txn, result, recipient.server(),))
</span><span class="lines">@@ -111,6 +115,9 @@
</span><span class="cx"> returnObject = txn
</span><span class="cx"> classObject = None
</span><span class="cx">
</span><ins>+ if "allowDisabledHome" in request:
+ txn._allowDisabled = True
+
</ins><span class="cx"> if "homeUID" in request:
</span><span class="cx"> home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
</span><span class="cx"> if home is None:
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -64,7 +64,8 @@
</span><span class="cx"> from txdav.common.datastore.sql_notification import NotificationCollection
</span><span class="cx"> from txdav.common.datastore.sql_tables import _BIND_MODE_OWN, _BIND_STATUS_ACCEPTED, \
</span><span class="cx"> _HOME_STATUS_EXTERNAL, _HOME_STATUS_NORMAL, \
</span><del>- _HOME_STATUS_PURGING, schema, splitSQLString, _HOME_STATUS_MIGRATING
</del><ins>+ _HOME_STATUS_PURGING, schema, splitSQLString, _HOME_STATUS_MIGRATING, \
+ _HOME_STATUS_DISABLED
</ins><span class="cx"> from txdav.common.datastore.sql_util import _SharedSyncLogic
</span><span class="cx"> from txdav.common.datastore.sql_sharing import SharingHomeMixIn, SharingMixIn
</span><span class="cx"> from txdav.common.icommondatastore import ConcurrentModification, \
</span><span class="lines">@@ -584,6 +585,7 @@
</span><span class="cx"> self._bumpedRevisionAlready = set()
</span><span class="cx"> self._label = label
</span><span class="cx"> self._migrating = migrating
</span><ins>+ self._allowDisabled = False
</ins><span class="cx"> self._primaryHomeType = None
</span><span class="cx"> self._disableCache = disableCache or not store.queryCachingEnabled()
</span><span class="cx"> if disableCache:
</span><span class="lines">@@ -764,11 +766,11 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @memoizedKey("uid", "_notificationHomes")
</span><del>- def notificationsWithUID(self, uid, create=True):
</del><ins>+ def notificationsWithUID(self, uid, status=None, create=True):
</ins><span class="cx"> """
</span><span class="cx"> Implement notificationsWithUID.
</span><span class="cx"> """
</span><del>- return NotificationCollection.notificationsWithUID(self, uid, create)
</del><ins>+ return NotificationCollection.notificationsWithUID(self, uid, create=create)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @memoizedKey("rid", "_notificationHomes")
</span><span class="lines">@@ -1811,6 +1813,8 @@
</span><span class="cx"> cacheKeys.append(queryCacher.keyForHomeWithUID(cls._homeType, uid, status))
</span><span class="cx"> else:
</span><span class="cx"> statusSet = (_HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL, _HOME_STATUS_PURGING)
</span><ins>+ if txn._allowDisabled:
+ statusSet += (_HOME_STATUS_DISABLED,)
</ins><span class="cx"> query = query.And(cls._homeSchema.STATUS.In(statusSet))
</span><span class="cx"> if queryCacher:
</span><span class="cx"> for item in statusSet:
</span><span class="lines">@@ -1826,7 +1830,7 @@
</span><span class="cx"> else:
</span><span class="cx"> result = None
</span><span class="cx">
</span><del>- # If nothing in thr cache, do the SQL query and cache the result
</del><ins>+ # If nothing in the cache, do the SQL query and cache the result
</ins><span class="cx"> if result is None:
</span><span class="cx"> results = yield Select(
</span><span class="cx"> cls.homeColumns(),
</span><span class="lines">@@ -1834,12 +1838,14 @@
</span><span class="cx"> Where=query,
</span><span class="cx"> ).on(txn)
</span><span class="cx">
</span><del>- # Pick the internal one
</del><span class="cx"> if len(results) > 1:
</span><del>- if results[0][cls.homeColumns().index(cls._homeSchema.STATUS)] == _HOME_STATUS_NORMAL:
- result = results[0]
- else:
- result = results[1]
</del><ins>+ # Pick the best one in order: normal, disabled and external
+ byStatus = dict([(result[cls.homeColumns().index(cls._homeSchema.STATUS)], result) for result in results])
+ result = byStatus.get(_HOME_STATUS_NORMAL)
+ if result is None:
+ result = byStatus.get(_HOME_STATUS_DISABLED)
+ if result is None:
+ result = byStatus.get(_HOME_STATUS_EXTERNAL)
</ins><span class="cx"> elif results:
</span><span class="cx"> result = results[0]
</span><span class="cx"> else:
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -85,6 +85,10 @@
</span><span class="cx"> self.deserialize(mapping)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def setStatus(self, newStatus):
+ return self._txn.store().conduit.send_home_set_status(self, newStatus)
+
+
</ins><span class="cx"> def external(self):
</span><span class="cx"> """
</span><span class="cx"> Is this an external home.
</span><span class="lines">@@ -493,11 +497,24 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><del>- def notificationsWithUID(cls, txn, uid, create):
- return super(NotificationCollectionExternal, cls).notificationsWithUID(txn, uid, create, expected_status=_HOME_STATUS_EXTERNAL)
</del><ins>+ def notificationsWithUID(cls, txn, uid, create=False):
+ return super(NotificationCollectionExternal, cls).notificationsWithUID(txn, uid, status=_HOME_STATUS_EXTERNAL, create=create)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ def initFromStore(self):
+ """
+ NoOp for an external share as there are no properties.
+ """
+ return succeed(self)
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def notificationObjectRecords(self):
</span><span class="cx"> results = yield self._txn.store().conduit.send_notification_all_records(self)
</span><span class="cx"> returnValue(map(NotificationObjectRecord.deserialize, results))
</span><ins>+
+
+ def setStatus(self, newStatus):
+ return self._txn.store().conduit.send_notification_set_status(self, newStatus)
+
+NotificationCollection._externalClass = NotificationCollectionExternal
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_notificationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py (14489 => 14490)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py 2015-02-28 16:40:03 UTC (rev 14489)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py 2015-02-28 22:26:20 UTC (rev 14490)
</span><span class="lines">@@ -27,7 +27,7 @@
</span><span class="cx"> from twistedcaldav.dateops import datetimeMktime
</span><span class="cx"> from txdav.base.propertystore.sql import PropertyStore
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema, _HOME_STATUS_NORMAL, \
</span><del>- _HOME_STATUS_EXTERNAL
</del><ins>+ _HOME_STATUS_EXTERNAL, _HOME_STATUS_DISABLED, _HOME_STATUS_MIGRATING
</ins><span class="cx"> from txdav.common.datastore.sql_util import _SharedSyncLogic
</span><span class="cx"> from txdav.common.icommondatastore import RecordNotAllowedError
</span><span class="cx"> from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
</span><span class="lines">@@ -49,20 +49,74 @@
</span><span class="cx"> implements(INotificationCollection)
</span><span class="cx">
</span><span class="cx"> compareAttributes = (
</span><del>- "_uid",
</del><ins>+ "_ownerUID",
</ins><span class="cx"> "_resourceID",
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> _revisionsSchema = schema.NOTIFICATION_OBJECT_REVISIONS
</span><span class="cx"> _homeSchema = schema.NOTIFICATION_HOME
</span><span class="cx">
</span><ins>+ _externalClass = None
</ins><span class="cx">
</span><del>- def __init__(self, txn, uid, resourceID, status):
</del><span class="cx">
</span><ins>+ @classmethod
+ def makeClass(cls, transaction, homeData):
+ """
+ Build the actual home class taking into account the possibility that we might need to
+ switch in the external version of the class.
+
+ @param transaction: transaction
+ @type transaction: L{CommonStoreTransaction}
+ @param homeData: home table column data
+ @type homeData: C{list}
+ """
+
+ status = homeData[cls.homeColumns().index(cls._homeSchema.STATUS)]
+ if status == _HOME_STATUS_EXTERNAL:
+ home = cls._externalClass(transaction, homeData)
+ else:
+ home = cls(transaction, homeData)
+ return home.initFromStore()
+
+
+ @classmethod
+ def homeColumns(cls):
+ """
+ Return a list of column names to retrieve when doing an ownerUID->home lookup.
+ """
+
+ # Common behavior is to have created and modified
+
+ return (
+ cls._homeSchema.RESOURCE_ID,
+ cls._homeSchema.OWNER_UID,
+ cls._homeSchema.STATUS,
+ )
+
+
+ @classmethod
+ def homeAttributes(cls):
+ """
+ Return a list of attributes names to map L{homeColumns} to.
+ """
+
+ # Common behavior is to have created and modified
+
+ return (
+ "_resourceID",
+ "_ownerUID",
+ "_status",
+ )
+
+
+ def __init__(self, txn, homeData):
+
</ins><span class="cx"> self._txn = txn
</span><del>- self._uid = uid
- self._resourceID = resourceID
- self._status = status
</del><ins>+
+ for attr, value in zip(self.homeAttributes(), homeData):
+ setattr(self, attr, value)
+
+ self._txn = txn
</ins><span class="cx"> self._dataVersion = None
</span><span class="cx"> self._notifications = {}
</span><span class="cx"> self._notificationNames = None
</span><span class="lines">@@ -72,25 +126,15 @@
</span><span class="cx"> # as well as the home it is in
</span><span class="cx"> self._notifiers = dict([(factory_name, factory.newNotifier(self),) for factory_name, factory in txn._notifierFactories.items()])
</span><span class="cx">
</span><del>- _resourceIDFromUIDQuery = Select(
- [_homeSchema.RESOURCE_ID, _homeSchema.STATUS],
- From=_homeSchema,
- Where=_homeSchema.OWNER_UID == Parameter("uid")
- )
</del><span class="cx">
</span><del>- _UIDFromResourceIDQuery = Select(
- [_homeSchema.OWNER_UID],
- From=_homeSchema,
- Where=_homeSchema.RESOURCE_ID == Parameter("rid")
- )
</del><ins>+ @inlineCallbacks
+ def initFromStore(self):
+ """
+ Initialize this object from the store.
+ """
</ins><span class="cx">
</span><del>- _provisionNewNotificationsQuery = Insert(
- {
- _homeSchema.OWNER_UID: Parameter("uid"),
- _homeSchema.STATUS: Parameter("status"),
- },
- Return=_homeSchema.RESOURCE_ID
- )
</del><ins>+ yield self._loadPropertyStore()
+ returnValue(self)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @property
</span><span class="lines">@@ -103,28 +147,78 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><ins>+ def notificationsWithUID(cls, txn, uid, status=None, create=True):
+ return cls.notificationsWith(txn, None, uid, status, create=create)
+
+
+ @classmethod
+ def notificationsWithResourceID(cls, txn, rid):
+ return cls.notificationsWith(txn, rid, None)
+
+
+ @classmethod
</ins><span class="cx"> @inlineCallbacks
</span><del>- def notificationsWithUID(cls, txn, uid, create, expected_status=_HOME_STATUS_NORMAL):
</del><ins>+ def notificationsWith(cls, txn, rid, uid, status=None, create=True):
</ins><span class="cx"> """
</span><span class="cx"> @param uid: I'm going to assume uid is utf-8 encoded bytes
</span><span class="cx"> """
</span><del>- rows = yield cls._resourceIDFromUIDQuery.on(txn, uid=uid)
</del><ins>+ if rid is not None:
+ query = cls._homeSchema.RESOURCE_ID == rid
+ elif uid is not None:
+ query = cls._homeSchema.OWNER_UID == uid
+ if status is not None:
+ query = query.And(cls._homeSchema.STATUS == status)
+ else:
+ statusSet = (_HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL,)
+ if txn._allowDisabled:
+ statusSet += (_HOME_STATUS_DISABLED,)
+ query = query.And(cls._homeSchema.STATUS.In(statusSet))
+ else:
+ raise AssertionError("One of rid or uid must be set")
</ins><span class="cx">
</span><del>- if rows:
- resourceID = rows[0][0]
- status = rows[0][1]
- if status != expected_status:
- raise RecordNotAllowedError("Notifications status mismatch: {} != {}".format(status, expected_status))
- created = False
- elif create:
</del><ins>+ results = yield Select(
+ cls.homeColumns(),
+ From=cls._homeSchema,
+ Where=query,
+ ).on(txn)
+
+ if len(results) > 1:
+ # Pick the best one in order: normal, disabled and external
+ byStatus = dict([(result[cls.homeColumns().index(cls._homeSchema.STATUS)], result) for result in results])
+ result = byStatus.get(_HOME_STATUS_NORMAL)
+ if result is None:
+ result = byStatus.get(_HOME_STATUS_DISABLED)
+ if result is None:
+ result = byStatus.get(_HOME_STATUS_EXTERNAL)
+ elif results:
+ result = results[0]
+ else:
+ result = None
+
+ if result:
+ # Return object that already exists in the store
+ homeObject = yield cls.makeClass(txn, result)
+ returnValue(homeObject)
+ else:
+ # Can only create when uid is specified
+ if not create or uid is None:
+ returnValue(None)
+
</ins><span class="cx"> # Determine if the user is local or external
</span><span class="cx"> record = yield txn.directoryService().recordWithUID(uid.decode("utf-8"))
</span><span class="cx"> if record is None:
</span><span class="cx"> raise DirectoryRecordNotFoundError("Cannot create home for UID since no directory record exists: {}".format(uid))
</span><span class="cx">
</span><del>- status = _HOME_STATUS_NORMAL if record.thisServer() else _HOME_STATUS_EXTERNAL
- if status != expected_status:
- raise RecordNotAllowedError("Notifications status mismatch: {} != {}".format(status, expected_status))
</del><ins>+ if status is None:
+ createStatus = _HOME_STATUS_NORMAL if record.thisServer() else _HOME_STATUS_EXTERNAL
+ elif status == _HOME_STATUS_MIGRATING:
+ if record.thisServer():
+ raise RecordNotAllowedError("Cannot migrate a user data for a user already hosted on this server")
+ createStatus = status
+ elif status in (_HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL,):
+ createStatus = status
+ else:
+ raise RecordNotAllowedError("Cannot create home with status {}: {}".format(status, uid))
</ins><span class="cx">
</span><span class="cx"> # Use savepoint so we can do a partial rollback if there is a race
</span><span class="cx"> # condition where this row has already been inserted
</span><span class="lines">@@ -132,55 +226,52 @@
</span><span class="cx"> yield savepoint.acquire(txn)
</span><span class="cx">
</span><span class="cx"> try:
</span><del>- resourceID = str((
- yield cls._provisionNewNotificationsQuery.on(txn, uid=uid, status=status)
- )[0][0])
</del><ins>+ resourceid = (yield Insert(
+ {
+ cls._homeSchema.OWNER_UID: uid,
+ cls._homeSchema.STATUS: createStatus,
+ },
+ Return=cls._homeSchema.RESOURCE_ID
+ ).on(txn))[0][0]
</ins><span class="cx"> except Exception:
</span><span class="cx"> # FIXME: Really want to trap the pg.DatabaseError but in a non-
</span><span class="cx"> # DB specific manner
</span><span class="cx"> yield savepoint.rollback(txn)
</span><span class="cx">
</span><span class="cx"> # Retry the query - row may exist now, if not re-raise
</span><del>- rows = yield cls._resourceIDFromUIDQuery.on(txn, uid=uid)
- if rows:
- resourceID = rows[0][0]
- status = rows[0][1]
- if status != expected_status:
- raise RecordNotAllowedError("Notifications status mismatch: {} != {}".format(status, expected_status))
- created = False
</del><ins>+ results = yield Select(
+ cls.homeColumns(),
+ From=cls._homeSchema,
+ Where=query,
+ ).on(txn)
+ if results:
+ homeObject = yield cls.makeClass(txn, results[0])
+ returnValue(homeObject)
</ins><span class="cx"> else:
</span><span class="cx"> raise
</span><span class="cx"> else:
</span><del>- created = True
</del><span class="cx"> yield savepoint.release(txn)
</span><del>- else:
- returnValue(None)
- collection = cls(txn, uid, resourceID, status)
- yield collection._loadPropertyStore()
- if created:
- yield collection._initSyncToken()
- yield collection.notifyChanged()
- returnValue(collection)
</del><span class="cx">
</span><ins>+ # Note that we must not cache the owner_uid->resource_id
+ # mapping in the query cacher when creating as we don't want that to appear
+ # until AFTER the commit
+ results = yield Select(
+ cls.homeColumns(),
+ From=cls._homeSchema,
+ Where=cls._homeSchema.RESOURCE_ID == resourceid,
+ ).on(txn)
+ homeObject = yield cls.makeClass(txn, results[0])
+ if homeObject.normal():
+ yield homeObject._initSyncToken()
+ yield homeObject.notifyChanged()
+ returnValue(homeObject)
</ins><span class="cx">
</span><del>- @classmethod
- @inlineCallbacks
- def notificationsWithResourceID(cls, txn, rid):
- rows = yield cls._UIDFromResourceIDQuery.on(txn, rid=rid)
</del><span class="cx">
</span><del>- if rows:
- uid = rows[0][0]
- result = (yield cls.notificationsWithUID(txn, uid, create=False))
- returnValue(result)
- else:
- returnValue(None)
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _loadPropertyStore(self):
</span><span class="cx"> self._propertyStore = yield PropertyStore.load(
</span><del>- self._uid,
- self._uid,
</del><ins>+ self._ownerUID,
+ self._ownerUID,
</ins><span class="cx"> None,
</span><span class="cx"> self._txn,
</span><span class="cx"> self._resourceID,
</span><span class="lines">@@ -224,9 +315,41 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def uid(self):
</span><del>- return self._uid
</del><ins>+ return self._ownerUID
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def setStatus(self, newStatus):
+ """
+ Mark this home as being purged.
+ """
+ # Only if different
+ if self._status != newStatus:
+ yield Update(
+ {self._homeSchema.STATUS: newStatus},
+ Where=(self._homeSchema.RESOURCE_ID == self._resourceID),
+ ).on(self._txn)
+ self._status = newStatus
+
+
+ def normal(self):
+ """
+ Is this an normal (internal) home.
+
+ @return: a L{bool}.
+ """
+ return self._status == _HOME_STATUS_NORMAL
+
+
+ def external(self):
+ """
+ Is this an external home.
+
+ @return: a L{bool}.
+ """
+ return self._status == _HOME_STATUS_EXTERNAL
+
+
</ins><span class="cx"> def owned(self):
</span><span class="cx"> return True
</span><span class="cx">
</span></span></pre>
</div>
</div>
</body>
</html>