<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14339] CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14339">14339</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-01-23 16:51:33 -0800 (Fri, 23 Jan 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Checkpoint current migration code.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavicalendarstorepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/icalendarstore.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_tablespy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li>CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/</li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigration__init__py">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/__init__.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li>CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/</li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtest__init__py">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/__init__.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -526,6 +526,48 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def copyMetadata(self, other):
+ """
+ Copy metadata from one L{CalendarObjectResource} to another. This is only
+ used during a migration step.
+ """
+ assert self._txn._migrating
+
+ # Simple attributes that can be copied over as-is
+ chm = schema.CALENDAR_HOME_METADATA
+ values = {
+ chm.ALARM_VEVENT_TIMED : other._alarm_vevent_timed,
+ chm.ALARM_VEVENT_ALLDAY : other._alarm_vevent_allday,
+ chm.ALARM_VTODO_TIMED : other._alarm_vtodo_timed,
+ chm.ALARM_VTODO_ALLDAY : other._alarm_vtodo_allday,
+ chm.AVAILABILITY : other._availability,
+ }
+
+ # Need to map the default collection references from the remote ids to
+ # the local ones using names
+ remote_calendars = yield other.loadChildren()
+ remote_calendars = dict([(calendar.id(), calendar,) for calendar in remote_calendars])
+ local_calendars = yield self.loadChildren()
+ local_calendars = dict([(calendar.name(), calendar,) for calendar in local_calendars])
+
+ for componentType in self._componentDefaultColumn.keys():
+ attr_name = self._componentDefaultAttribute[componentType]
+ remote_id = getattr(other, attr_name)
+ if remote_id is not None:
+ remote_calendar = remote_calendars.get(remote_id)
+ if remote_calendar is not None:
+ remote_id = local_calendars.get(remote_calendar.name())
+ setattr(self, attr_name, remote_id)
+ values[self._componentDefaultColumn[componentType]] = remote_id
+
+ # Update the local data
+ yield Update(
+ values,
+ Where=chm.RESOURCE_ID == self._resourceID
+ ).on(self._txn)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def hasCalendarResourceUIDSomewhereElse(self, uid, ok_object, mode):
</span><span class="cx"> """
</span><span class="cx"> Determine if this calendar home contains any calendar objects which
</span><span class="lines">@@ -3694,6 +3736,31 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def copyMetadata(self, other):
+ """
+ Copy metadata from one L{CalendarObjectResource} to another. This is only
+ used during a migration step.
+ """
+ assert self._txn._migrating
+
+ co = schema.CALENDAR_OBJECT
+ values = {
+ co.ATTACHMENTS_MODE : other._attachment,
+ co.DROPBOX_ID : other._dropboxID,
+ co.ACCESS : other._access,
+ co.SCHEDULE_OBJECT : other._schedule_object,
+ co.SCHEDULE_TAG : other._schedule_tag,
+ co.SCHEDULE_ETAGS : other._schedule_etags,
+ co.PRIVATE_COMMENTS : other._private_comments,
+ }
+
+ yield Update(
+ values,
+ Where=co.RESOURCE_ID == self._resourceID
+ ).on(self._txn)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def component(self, doUpdate=False):
</span><span class="cx"> """
</span><span class="cx"> Read calendar data and validate/fix it. Do not raise a store error here
</span><span class="lines">@@ -3859,6 +3926,15 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def purge(self):
+ """
+ Do a "silent" removal of this object resource.
+ """
+ return self._removeInternal(
+ ComponentRemoveState.NORMAL_NO_IMPLICIT
+ )
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _removeInternal(self, internal_state=ComponentRemoveState.NORMAL):
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavicalendarstorepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/icalendarstore.py (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/icalendarstore.py 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/icalendarstore.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -906,7 +906,7 @@
</span><span class="cx"> is done (more than RAW).
</span><span class="cx">
</span><span class="cx"> RAW - store the supplied data as-is without any processing or validation. This is used
</span><del>- for unit testing purposes only.
</del><ins>+ for unit testing purposes only, or during migration.
</ins><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> NORMAL = NamedConstant()
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigration__init__py"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/__init__.py (0 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/__init__.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/__init__.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -0,0 +1,15 @@
</span><ins>+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (0 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -0,0 +1,540 @@
</span><ins>+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.python.log import Logger
+
+from twisted.python.failure import Failure
+from twisted.internet.defer import returnValue, inlineCallbacks, succeed
+from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
+
+from functools import wraps
+from txdav.caldav.icalendarstore import ComponentUpdateState
+
+log = Logger()
+
+
+def inTransactionWrapper(operation):
+ """
+ This wrapper converts an instance method that takes a transaction as its
+ first parameter into one where the transaction parameter is an optional
+ keyword argument. If the keyword argument is present and not None, then
+ the instance method is called with that keyword as the first positional
+ argument (i.e., almost a NoOp). If the keyword argument is not present,
+ then a new transaction is created and the instance method called with
+ it as the first positional argument, plus the call is wrapped with
+ try/except/else to ensure proper commit and abort of the internally
+ created transaction is done.
+
+ So this wrapper allows for a method that requires a transaction to be run
+ with either an existing transaction or one created just for the purpose
+ of running it.
+
+ @param operation: a callable that takes an L{IAsyncTransaction} as its first
+ argument, and returns a value.
+ """
+
+ @wraps(operation)
+ @inlineCallbacks
+ def _inTxn(self, *args, **kwargs):
+ label = self.label(operation.__name__)
+ if "txn" in kwargs:
+ txn = kwargs["txn"]
+ del kwargs["txn"]
+ result = yield operation(self, txn, *args, **kwargs)
+ returnValue(result)
+ else:
+ txn = self.store.newTransaction(label=label)
+ try:
+ result = yield operation(self, txn, *args, **kwargs)
+ except Exception as ex:
+ f = Failure()
+ yield txn.abort()
+ log.error("{label} failed: {e}".format(label=label, e=str(ex)))
+ returnValue(f)
+ else:
+ yield txn.commit()
+ returnValue(result)
+
+ return _inTxn
+
+
+
+# Cross-pod synchronization of an entire calendar home
+class CrossPodHomeSync(object):
+
+ BATCH_SIZE = 50
+
+ def __init__(self, store, diruid):
+ """
+ @param store: the data store
+ @type store: L{CommonDataStore}
+ @param diruid: directory uid of the user whose home is to be sync'd
+ @type diruid: L{str}
+ """
+
+ self.store = store
+ self.diruid = diruid
+
+
+ def label(self, detail):
+ return "Cross-pod Migration Sync for {}: {}".format(self.diruid, detail)
+
+
+ def migratingUid(self):
+ return "Migrating-{}".format(self.diruid)
+
+
+ @inlineCallbacks
+ def migrateHere(self):
+ """
+ This is a full, serialized version of a data migration (minus any directory
+ update) that can be triggered via a command line tool. It is designed to
+ minimize down time for the migrating user.
+ """
+
+ # Step 1 - initial full sync
+ yield self.sync()
+
+ # Step 2 - increment sync (since the initial sync may take a long time
+ # to run we should do one incremental sync before bringing down the
+ # account being migrated)
+ yield self.sync()
+
+ # Step 3 - disable remote home
+ # NB Any failure from this point on will need to be caught and
+ # handled by re-enabling the old home (and fixing any sharing state
+ # that may have been changed)
+ yield self.disableRemoteHome()
+
+ # Step 4 - final incremental sync
+ yield self.sync()
+
+ # Step 5 - final overell sync of meta-data (including sharing re-linking)
+ yield self.finalSync()
+
+ # Step 6 - enable new home
+ yield self.enableLocalHome()
+
+ # Step 7 - remote remote home
+ yield self.removeRemoteHome()
+
+ # Step 8 - say phew! TODO: Actually alert everyone else
+ pass
+
+
+ @inlineCallbacks
+ def sync(self):
+ """
+ Initiate a sync of the home. This is a simple data sync that does not
+ reconcile sharing state etc. The L{finalSync} method will do a full
+ sharing reconcile as well as disable the migration source home.
+ """
+
+ yield self.loadRecord()
+ self.homeId = yield self.prepareCalendarHome()
+
+ yield self.syncCalendarList()
+
+ # TODO: sync home metadata such as alarms, default calendars, etc
+ yield self.syncCalendarHomeMetaData()
+
+ # TODO: sync attachments
+ pass
+
+ # TODO: group attendee reconcile
+ pass
+
+
+ @inlineCallbacks
+ def finalSync(self):
+ """
+ Do the final sync up of any additional data, re-link sharing bind
+ rows, recalculate quota etc.
+ """
+
+ # TODO:
+ pass
+
+
+ @inlineCallbacks
+ def disableRemoteHome(self):
+ """
+ Mark the remote home as disabled.
+ """
+
+ # TODO: implement API on CommonHome to rename the ownerUID column and
+ # change the status column.
+ pass
+
+
+ @inlineCallbacks
+ def enableLocalHome(self):
+ """
+ Mark the local home as enabled.
+ """
+
+ # TODO: implement API on CommonHome to rename the ownerUID column and
+ # change the status column.
+ pass
+
+
+ @inlineCallbacks
+ def removeRemoteHome(self):
+ """
+ Remove all the old data on the remote pod.
+ """
+
+ # TODO: implement API on CommonHome to purge the old data without
+ # any side-effects (scheduling, sharing etc).
+ pass
+
+
+ @inlineCallbacks
+ def loadRecord(self):
+ """
+ Initiate a sync of the home.
+ """
+
+ self.record = yield self.store.directoryService().recordWithUID(self.diruid)
+ if self.record is None:
+ raise DirectoryRecordNotFoundError("Cross-pod Migration Sync missing directory record for {}".format(self.diruid))
+ if self.record.thisServer():
+ raise ValueError("Cross-pod Migration Sync cannot sync with user already on this server: {}".format(self.diruid))
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def prepareCalendarHome(self, txn):
+ """
+ Make sure the inactive home to migrate into is present on this pod.
+ """
+
+ home = yield txn.calendarHomeWithUID(self.migratingUid())
+ if home is None:
+ home = yield txn.calendarHomeWithUID(self.migratingUid(), create=True, migratingUID=self.diruid)
+ returnValue(home.id())
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def syncCalendarHomeMetaData(self, txn):
+ """
+ Make sure the home meta-data (alarms, default calendars) is properly sync'd
+ """
+
+ remote_home = yield self._remoteHome(txn=txn)
+ local_home = yield txn.calendarHomeWithUID(self.migratingUid())
+ yield local_home.copyMetadata(remote_home)
+
+
+ @inlineCallbacks
+ def _remoteHome(self, txn):
+ """
+ Create a synthetic external home object that maps to the actual remote home.
+
+ @param ownerUID: directory uid of the user's home
+ @type ownerUID: L{str}
+ """
+
+ from txdav.caldav.datastore.sql_external import CalendarHomeExternal
+ resourceID = yield txn.store().conduit.send_home_resource_id(self, self.record)
+ home = CalendarHomeExternal(txn, self.record.uid, resourceID) if resourceID is not None else None
+ if home:
+ home._childClass = home._childClass._externalClass
+ returnValue(home)
+
+
+ @inlineCallbacks
+ def syncCalendarList(self):
+ """
+ Synchronize each owned calendar.
+ """
+
+ # Remote sync details
+ remote_sync_state = yield self.getCalendarSyncList()
+
+ # TODO: get local sync details from local DB
+ local_sync_state = yield self.getSyncState()
+
+ # Remove local calendars no longer on the remote side
+ yield self.purgeLocal(local_sync_state, remote_sync_state)
+
+ # Sync each calendar that matches on both sides
+ for name in remote_sync_state.keys():
+ yield self.syncCalendar(name, local_sync_state, remote_sync_state)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def getCalendarSyncList(self, txn):
+ """
+ Get the names and sync-tokens for each remote owned calendar.
+ """
+
+ # List of calendars from the remote side
+ home = yield self._remoteHome(txn)
+ if home is None:
+ returnValue(None)
+ calendars = yield home.loadChildren()
+ results = {}
+ for calendar in calendars:
+ if calendar.owned():
+ sync_token = yield calendar.syncToken()
+ results[calendar.name()] = sync_token
+
+ returnValue(results)
+
+
+ def getSyncState(self):
+ """
+ Get local synchronization state for the home being migrated.
+ """
+ return succeed({})
+
+
+ @inTransactionWrapper
+ def setSyncState(self, txn, details):
+ """
+ Get local synchronization state for the home being migrated.
+ """
+ return succeed(None)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def purgeLocal(self, txn, local_sync_state, remote_sync_state):
+ """
+ Remove (silently - i.e., no scheduling) local calendars that are no longer on the remote side.
+
+ @param txn: transaction to use
+ @type txn: L{CommonStoreTransaction}
+ @param local_sync_state: local sync state
+ @type local_sync_state: L{dict}
+ @param remote_sync_state: remote sync state
+ @type remote_sync_state: L{dict}
+ """
+ home = yield txn.calendarHomeWithUID(self.migratingUid())
+ for name in set(local_sync_state.keys()) - set(remote_sync_state.keys()):
+ calendar = yield home.childWithName(name)
+ if calendar is not None:
+ yield calendar.purge()
+ del local_sync_state[name]
+
+ yield self.setSyncState(local_sync_state, txn=txn)
+
+
+ @inlineCallbacks
+ def syncCalendar(self, name, local_sync_state, remote_sync_state):
+ """
+ Sync the contents of a calendar from the remote side. The local calendar may need to be created
+ on initial sync. Make use of sync tokens to avoid unnecessary work.
+
+ @param name: name of the calendar to sync
+ @type name: L{str}
+ @param local_sync_state: local sync state
+ @type local_sync_state: L{dict}
+ @param remote_sync_state: remote sync state
+ @type remote_sync_state: L{dict}
+ """
+
+ local_token = local_sync_state.get(name, None)
+ remote_token = remote_sync_state[name]
+ if local_token != remote_token:
+ # See if we need to create the local one first
+ if local_token is None:
+ yield self.newCalendar(name)
+
+ # TODO: sync meta-data such as alarms, supported-components, transp, etc
+ pass
+
+ # Sync object resources
+ changed, deleted = yield self.findObjectsToSync(name, local_token)
+ yield self.purgeDeletedObjectsInBatches(name, deleted)
+ yield self.updateChangedObjectsInBatches(name, changed)
+
+ local_sync_state[name] = remote_token
+ yield self.setSyncState(local_sync_state)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def newCalendar(self, txn, name):
+ """
+ Create a new local calendar to sync remote data to.
+
+ @param name: name of the calendar to create
+ @type name: L{str}
+ """
+
+ home = yield txn.calendarHomeWithUID(self.migratingUid())
+ calendar = yield home.childWithName(name)
+ if calendar is None:
+ yield home.createChildWithName(name)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def findObjectsToSync(self, txn, name, local_token):
+ """
+ Find the set of object resources that need to be sync'd from the remote
+ side and the set that need to be removed locally. Take into account the
+ possibility that this is a partial sync and removals or additions might
+ be false positives.
+
+ @param name: name of the calendar to sync
+ @type name: L{str}
+ @param local_token: sync token last used to sync the calendar
+ @type local_token: L{str}
+ """
+
+ # Remote changes
+ remote_home = yield self._remoteHome(txn)
+ remote_calendar = yield remote_home.childWithName(name)
+ if remote_calendar is None:
+ returnValue(None)
+ changed, deleted, _ignore_invalid = yield remote_calendar.resourceNamesSinceToken(local_token)
+
+ # Check whether the deleted set items
+ local_home = yield txn.calendarHomeWithUID(self.migratingUid())
+ local_calendar = yield local_home.childWithName(name)
+
+ # Check the md5's on each changed remote with the local one to filter out ones
+ # we don't actually need to sync
+ remote_changes = yield remote_calendar.objectResourcesWithNames(changed)
+ remote_changes = dict([(calendar.name(), calendar) for calendar in remote_changes])
+
+ local_changes = yield local_calendar.objectResourcesWithNames(changed)
+ local_changes = dict([(calendar.name(), calendar) for calendar in local_changes])
+
+ actual_changes = []
+ for name, calendar in remote_changes.items():
+ if name not in local_changes or remote_changes[name].md5() != local_changes[name].md5():
+ actual_changes.append(name)
+
+ returnValue((actual_changes, deleted,))
+
+
+ @inlineCallbacks
+ def purgeDeletedObjectsInBatches(self, name, deleted):
+ """
+ Purge (silently remove) the specified object resources. This needs to
+ succeed in the case where some or all resources have already been deleted.
+ Do this in batches to keep transaction times small.
+
+ @param name: name of the calendar to purge from
+ @type name: L{str}
+ @param deleted: list of names to purge
+ @type deleted: L{list} of L{str}
+ """
+
+ remaining = list(deleted)
+ while remaining:
+ yield self.purgeBatch(name, remaining[:self.BATCH_SIZE])
+ del remaining[:self.BATCH_SIZE]
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def purgeBatch(self, txn, name, purge_names):
+ """
+ Purge a bunch of object resources from the specified calendar.
+
+ @param txn: transaction to use
+ @type txn: L{CommonStoreTransaction}
+ @param name: name of calendar
+ @type name: L{str}
+ @param purge_names: object resource names to purge
+ @type purge_names: L{list} of L{str}
+ """
+
+ # Check whether the deleted set items
+ local_home = yield txn.calendarHomeWithUID(self.migratingUid())
+ local_calendar = yield local_home.childWithName(name)
+ local_objects = yield local_calendar.objectResourcesWithNames(purge_names)
+
+ for local_object in local_objects:
+ yield local_object.purge()
+
+
+ @inlineCallbacks
+ def updateChangedObjectsInBatches(self, name, changed):
+ """
+ Update the specified object resources. This needs to succeed in the
+ case where some or all resources have already been deleted.
+ Do this in batches to keep transaction times small.
+
+ @param name: name of the calendar to purge from
+ @type name: L{str}
+ @param changed: list of names to update
+ @type changed: L{list} of L{str}
+ """
+
+ remaining = list(changed)
+ while remaining:
+ yield self.updateBatch(name, remaining[:self.BATCH_SIZE])
+ del remaining[:self.BATCH_SIZE]
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def updateBatch(self, txn, name, remaining):
+ """
+ Update a bunch of object resources from the specified remote calendar.
+
+ @param txn: transaction to use
+ @type txn: L{CommonStoreTransaction}
+ @param name: name of calendar
+ @type name: L{str}
+ @param purge_names: object resource names to update
+ @type purge_names: L{list} of L{str}
+ """
+
+ # Get remote objects
+ remote_home = yield self._remoteHome(txn)
+ remote_calendar = yield remote_home.childWithName(name)
+ if remote_calendar is None:
+ returnValue(None)
+ remote_objects = yield remote_calendar.objectResourcesWithNames(remaining)
+ remote_objects = dict([(obj.name(), obj) for obj in remote_objects])
+
+ # Get local objects
+ local_home = yield txn.calendarHomeWithUID(self.migratingUid())
+ local_calendar = yield local_home.childWithName(name)
+ local_objects = yield local_calendar.objectResourcesWithNames(remaining)
+ local_objects = dict([(obj.name(), obj) for obj in local_objects])
+
+ # Sync ones that still exist - use txn._migrating together with stuffing the remote md5
+ # value onto the component being stored to ensure that the md5 value stored locally
+ # matches the remote one (which should help reduce the need for a client to resync
+ # the data when moved from one pod to the other).
+ txn._migrating = True
+ for obj_name in remote_objects.keys():
+ remote_object = remote_objects[obj_name]
+ remote_data = yield remote_object.component()
+ remote_data.md5 = remote_object.md5()
+ if obj_name in local_objects:
+ local_object = yield local_objects[obj_name]
+ yield local_object._setComponentInternal(remote_data, internal_state=ComponentUpdateState.RAW)
+ del local_objects[obj_name]
+ else:
+ local_object = yield local_calendar._createCalendarObjectWithNameInternal(obj_name, remote_data, internal_state=ComponentUpdateState.RAW)
+
+ # Sync meta-data such as schedule object, schedule tags, access mode etc
+ yield local_object.copyMetadata(remote_object)
+
+ # Purge the ones that remain
+ for local_object in local_objects.values():
+ yield local_object.purge()
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtest__init__py"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/__init__.py (0 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/__init__.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/__init__.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -0,0 +1,15 @@
</span><ins>+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (0 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -0,0 +1,300 @@
</span><ins>+##
+# Copyright (c) 2005-2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet.defer import inlineCallbacks
+from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
+from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
+from pycalendar.datetime import DateTime
+from twistedcaldav.ical import Component, normalize_iCalStr
+
+
+class TestConduitAPI(MultiStoreConduitTest):
+ """
+ Test that the conduit api works.
+ """
+
+ nowYear = {"now": DateTime.getToday().getYear()}
+
+ caldata1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid1
+DTSTART:{now:04d}0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**nowYear)
+
+ caldata1_changed = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid1
+DTSTART:{now:04d}0102T150000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:instance changed
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**nowYear)
+
+ caldata2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid2
+DTSTART:{now:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**nowYear)
+
+ caldata3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid3
+DTSTART:{now:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**nowYear)
+
+ @inlineCallbacks
+ def test_remote_home(self):
+ """
+ Test that a remote home can be accessed.
+ """
+
+ home01 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ self.assertTrue(home01 is not None)
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ home = yield syncer._remoteHome(self.theTransactionUnderTest(1))
+ self.assertTrue(home is not None)
+ self.assertEqual(home.id(), home01.id())
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_prepare_home(self):
+ """
+ Test that L{prepareCalendarHome} creates a home.
+ """
+
+ # No home present
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ self.assertTrue(home is None)
+ yield self.commitTransaction(1)
+
+ yield syncer.prepareCalendarHome()
+
+ # Home is present
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ self.assertTrue(home is not None)
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_prepare_home_external_txn(self):
+ """
+ Test that L{prepareCalendarHome} creates a home.
+ """
+
+ # No home present
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ self.assertTrue(home is None)
+ yield self.commitTransaction(1)
+
+ yield syncer.prepareCalendarHome(txn=self.theTransactionUnderTest(1))
+ yield self.commitTransaction(1)
+
+ # Home is present
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ self.assertTrue(home is not None)
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_get_calendar_sync_list(self):
+ """
+ Test that L{getCalendarSyncList} returns the correct results.
+ """
+
+ yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ yield self.commitTransaction(0)
+ home01 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01")
+ self.assertTrue(home01 is not None)
+ calendars01 = yield home01.loadChildren()
+ results01 = {}
+ for calendar in calendars01:
+ if calendar.owned():
+ sync_token = yield calendar.syncToken()
+ results01[calendar.name()] = sync_token
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ results = yield syncer.getCalendarSyncList()
+ self.assertEqual(results, results01)
+
+
+ @inlineCallbacks
+ def test_sync_calendar_initial_empty(self):
+ """
+ Test that L{syncCalendar} syncs an initially non-existent local calendar with
+ an empty remote calendar.
+ """
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ remote_sync_token = yield calendar0.syncToken()
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ yield syncer.prepareCalendarHome()
+
+ # No local calendar exists yet
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is None)
+ yield self.commitTransaction(1)
+
+ # Trigger sync of the one calendar
+ local_sync_state = {}
+ remote_sync_state = {"calendar": remote_sync_token}
+ yield syncer.syncCalendar(
+ "calendar",
+ local_sync_state,
+ remote_sync_state,
+ )
+ self.assertTrue("calendar" in local_sync_state)
+ self.assertEqual(local_sync_state["calendar"], remote_sync_state["calendar"])
+
+ # Local calendar exists
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is not None)
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_sync_calendar_initial_with_data(self):
+ """
+ Test that L{syncCalendar} syncs an initially non-existent local calendar with
+ a remote calendar containing data. Also check a change to one event is then
+ sync'd the second time.
+ """
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
+ yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ yield syncer.prepareCalendarHome()
+
+ # No local calendar exists yet
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is None)
+ yield self.commitTransaction(1)
+
+ # Trigger sync of the one calendar
+ local_sync_state = {}
+ remote_sync_state = yield syncer.getCalendarSyncList()
+ yield syncer.syncCalendar(
+ "calendar",
+ local_sync_state,
+ remote_sync_state,
+ )
+ self.assertTrue("calendar" in local_sync_state)
+ self.assertEqual(local_sync_state["calendar"], remote_sync_state["calendar"])
+
+ # Local calendar exists
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is not None)
+ children = yield calendar1.listObjectResources()
+ self.assertEqual(set(children), set(("1.ics", "2.ics", "3.ics",)))
+ yield self.commitTransaction(1)
+
+ # Change one resource
+ object0 = yield self.calendarObjectUnderTest(
+ txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics"
+ )
+ yield object0.setComponent(Component.fromString(self.caldata1_changed))
+ yield self.commitTransaction(0)
+
+ remote_sync_state = yield syncer.getCalendarSyncList()
+ yield syncer.syncCalendar(
+ "calendar",
+ local_sync_state,
+ remote_sync_state,
+ )
+
+ object1 = yield self.calendarObjectUnderTest(
+ txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), calendar_name="calendar", name="1.ics"
+ )
+ caldata = yield object1.component()
+ self.assertEqual(normalize_iCalStr(caldata), normalize_iCalStr(self.caldata1_changed))
+ yield self.commitTransaction(1)
+
+ # Remove one resource
+ object0 = yield self.calendarObjectUnderTest(
+ txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="2.ics"
+ )
+ yield object0.remove()
+ yield self.commitTransaction(0)
+
+ remote_sync_state = yield syncer.getCalendarSyncList()
+ yield syncer.syncCalendar(
+ "calendar",
+ local_sync_state,
+ remote_sync_state,
+ )
+
+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), name="calendar")
+ children = yield calendar1.listObjectResources()
+ self.assertEqual(set(children), set(("1.ics", "3.ics",)))
+ yield self.commitTransaction(1)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -350,7 +350,7 @@
</span><span class="cx"> StoreAPIConduitMixin._make_simple_action("homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
</span><span class="cx"> StoreAPIConduitMixin._make_simple_action("homechild_movehere", "moveObjectResourceHere")
</span><span class="cx"> StoreAPIConduitMixin._make_simple_action("homechild_moveaway", "moveObjectResourceAway")
</span><del>-StoreAPIConduitMixin._make_simple_action("homechild_synctoken", "syncToken")
</del><ins>+StoreAPIConduitMixin._make_simple_action("homechild_synctokenrevision", "syncTokenRevision")
</ins><span class="cx"> StoreAPIConduitMixin._make_simple_action("homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=StoreAPIConduitMixin._to_tuple)
</span><span class="cx"> StoreAPIConduitMixin._make_simple_action("homechild_search", "search")
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -65,7 +65,7 @@
</span><span class="cx"> _BIND_MODE_INDIRECT, _BIND_MODE_OWN, _BIND_STATUS_ACCEPTED, \
</span><span class="cx"> _BIND_STATUS_DECLINED, _BIND_STATUS_DELETED, _BIND_STATUS_INVALID, \
</span><span class="cx"> _BIND_STATUS_INVITED, _HOME_STATUS_EXTERNAL, _HOME_STATUS_NORMAL, \
</span><del>- _HOME_STATUS_PURGING, schema, splitSQLString
</del><ins>+ _HOME_STATUS_PURGING, schema, splitSQLString, _HOME_STATUS_MIGRATING
</ins><span class="cx"> from txdav.common.icommondatastore import ConcurrentModification, \
</span><span class="cx"> RecordNotAllowedError, ExternalShareFailed, ShareNotAllowed, \
</span><span class="cx"> IndexedSearchException, NotFoundError
</span><span class="lines">@@ -695,7 +695,7 @@
</span><span class="cx"> ).on(self)
</span><span class="cx">
</span><span class="cx">
</span><del>- def _determineMemo(self, storeType, uid, create=False, authzUID=None):
</del><ins>+ def _determineMemo(self, storeType, uid, create=False, authzUID=None, migratingUID=None):
</ins><span class="cx"> """
</span><span class="cx"> Determine the memo dictionary to use for homeWithUID.
</span><span class="cx"> """
</span><span class="lines">@@ -721,7 +721,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @memoizedKey("uid", _determineMemo)
</span><del>- def homeWithUID(self, storeType, uid, create=False, authzUID=None):
</del><ins>+ def homeWithUID(self, storeType, uid, create=False, authzUID=None, migratingUID=None):
</ins><span class="cx"> """
</span><span class="cx"> We need to distinguish between various different users "looking" at a home and its
</span><span class="cx"> child resources because we have per-user properties that depend on which user is "looking".
</span><span class="lines">@@ -733,15 +733,15 @@
</span><span class="cx"> if storeType not in (ECALENDARTYPE, EADDRESSBOOKTYPE):
</span><span class="cx"> raise RuntimeError("Unknown home type.")
</span><span class="cx">
</span><del>- return self._homeClass[storeType].homeWithUID(self, uid, create, authzUID)
</del><ins>+ return self._homeClass[storeType].homeWithUID(self, uid, create, authzUID, migratingUID)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def calendarHomeWithUID(self, uid, create=False, authzUID=None):
- return self.homeWithUID(ECALENDARTYPE, uid, create=create, authzUID=authzUID)
</del><ins>+ def calendarHomeWithUID(self, uid, create=False, authzUID=None, migratingUID=None):
+ return self.homeWithUID(ECALENDARTYPE, uid, create=create, authzUID=authzUID, migratingUID=migratingUID)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def addressbookHomeWithUID(self, uid, create=False, authzUID=None):
- return self.homeWithUID(EADDRESSBOOKTYPE, uid, create=create, authzUID=authzUID)
</del><ins>+ def addressbookHomeWithUID(self, uid, create=False, authzUID=None, migratingUID=None):
+ return self.homeWithUID(EADDRESSBOOKTYPE, uid, create=create, authzUID=authzUID, migratingUID=migratingUID)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -3231,7 +3231,7 @@
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="cx"> @inlineCallbacks
</span><del>- def homeWithUID(cls, txn, uid, create=False, authzUID=None):
</del><ins>+ def homeWithUID(cls, txn, uid, create=False, authzUID=None, migratingUID=None):
</ins><span class="cx"> """
</span><span class="cx"> @param uid: I'm going to assume uid is utf-8 encoded bytes
</span><span class="cx"> """
</span><span class="lines">@@ -3243,11 +3243,17 @@
</span><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="cx"> # Determine if the user is local or external
</span><del>- record = yield txn.directoryService().recordWithUID(uid.decode("utf-8"))
</del><ins>+ diruid = uid if migratingUID is None else migratingUID
+ record = yield txn.directoryService().recordWithUID(diruid.decode("utf-8"))
</ins><span class="cx"> if record is None:
</span><del>- raise DirectoryRecordNotFoundError("Cannot create home for UID since no directory record exists: {}".format(uid))
</del><ins>+ raise DirectoryRecordNotFoundError("Cannot create home for UID since no directory record exists: {}".format(diruid))
</ins><span class="cx">
</span><del>- state = _HOME_STATUS_NORMAL if record.thisServer() else _HOME_STATUS_EXTERNAL
</del><ins>+ if migratingUID is None:
+ state = _HOME_STATUS_NORMAL if record.thisServer() else _HOME_STATUS_EXTERNAL
+ else:
+ if record.thisServer():
+ raise RecordNotAllowedError("Cannot migrate a user data for a user already hosted on this server")
+ state = _HOME_STATUS_MIGRATING
</ins><span class="cx">
</span><span class="cx"> # Use savepoint so we can do a partial rollback if there is a race condition
</span><span class="cx"> # where this row has already been inserted
</span><span class="lines">@@ -3282,7 +3288,8 @@
</span><span class="cx"> # mapping in _cacher when creating as we don't want that to appear
</span><span class="cx"> # until AFTER the commit
</span><span class="cx"> home = yield cls.makeClass(txn, uid, no_cache=True, authzUID=authzUID)
</span><del>- yield home.createdHome()
</del><ins>+ if migratingUID is None:
+ yield home.createdHome()
</ins><span class="cx"> returnValue(home)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -3357,6 +3364,15 @@
</span><span class="cx"> return self._status == _HOME_STATUS_PURGING
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def migrating(self):
+ """
+ Is this an external home.
+
+ @return: a string.
+ """
+ return self._status == _HOME_STATUS_MIGRATING
+
+
</ins><span class="cx"> def purge(self):
</span><span class="cx"> """
</span><span class="cx"> Mark this home as being purged.
</span><span class="lines">@@ -3364,6 +3380,13 @@
</span><span class="cx"> return self.setStatus(_HOME_STATUS_PURGING)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def migrate(self):
+ """
+ Mark this home as being purged.
+ """
+ return self.setStatus(_HOME_STATUS_MIGRATING)
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def setStatus(self, newStatus):
</span><span class="cx"> """
</span><span class="lines">@@ -3598,13 +3621,18 @@
</span><span class="cx"> taken to invalid the cached value properly.
</span><span class="cx"> """
</span><span class="cx"> if self._syncTokenRevision is None:
</span><del>- self._syncTokenRevision = (yield self._syncTokenQuery.on(
- self._txn, resourceID=self._resourceID))[0][0]
- if self._syncTokenRevision is None:
- self._syncTokenRevision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
</del><ins>+ self._syncTokenRevision = yield self.syncTokenRevision()
</ins><span class="cx"> returnValue("%s_%s" % (self._resourceID, self._syncTokenRevision))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def syncTokenRevision(self):
+ revision = (yield self._syncTokenQuery.on(self._txn, resourceID=self._resourceID))[0][0]
+ if revision is None:
+ revision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
+ returnValue(revision)
+
+
</ins><span class="cx"> @classproperty
</span><span class="cx"> def _changesQuery(cls):
</span><span class="cx"> bind = cls._bindSchema
</span><span class="lines">@@ -4127,13 +4155,18 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def syncToken(self):
</span><span class="cx"> if self._syncTokenRevision is None:
</span><del>- self._syncTokenRevision = (yield self._childSyncTokenQuery.on(
- self._txn, resourceID=self._resourceID))[0][0]
- if self._syncTokenRevision is None:
- self._syncTokenRevision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
</del><ins>+ self._syncTokenRevision = yield self.syncTokenRevision()
</ins><span class="cx"> returnValue(("%s_%s" % (self._resourceID, self._syncTokenRevision,)))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def syncTokenRevision(self):
+ revision = (yield self._childSyncTokenQuery.on(self._txn, resourceID=self._resourceID))[0][0]
+ if revision is None:
+ revision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
+ returnValue(revision)
+
+
</ins><span class="cx"> def objectResourcesSinceToken(self, token):
</span><span class="cx"> raise NotImplementedError()
</span><span class="cx">
</span><span class="lines">@@ -7410,6 +7443,13 @@
</span><span class="cx"> self._cachedComponent = None
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def purge(self):
+ """
+ Do a "silent" removal of this object resource.
+ """
+ return self.remove()
+
+
</ins><span class="cx"> def removeNotifyCategory(self):
</span><span class="cx"> """
</span><span class="cx"> Indicates what category to use when determining the priority of push
</span><span class="lines">@@ -7775,15 +7815,18 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def syncToken(self):
</span><span class="cx"> if self._syncTokenRevision is None:
</span><del>- self._syncTokenRevision = (
- yield self._syncTokenQuery.on(
- self._txn, resourceID=self._resourceID)
- )[0][0]
- if self._syncTokenRevision is None:
- self._syncTokenRevision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
</del><ins>+ self._syncTokenRevision = yield self.syncTokenRevision()
</ins><span class="cx"> returnValue("%s_%s" % (self._resourceID, self._syncTokenRevision))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def syncTokenRevision(self):
+ revision = (yield self._syncTokenQuery.on(self._txn, resourceID=self._resourceID))[0][0]
+ if revision is None:
+ revision = int((yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
+ returnValue(revision)
+
+
</ins><span class="cx"> def properties(self):
</span><span class="cx"> return self._propertyStore
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -310,15 +310,14 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def syncToken(self):
</del><ins>+ def syncTokenRevision(self):
</ins><span class="cx"> if self._syncTokenRevision is None:
</span><span class="cx"> try:
</span><del>- token = yield self._txn.store().conduit.send_homechild_synctoken(self)
- self._syncTokenRevision = self.revisionFromToken(token)
</del><ins>+ revision = yield self._txn.store().conduit.send_homechild_synctokenrevision(self)
</ins><span class="cx"> except NonExistentExternalShare:
</span><span class="cx"> yield self.fixNonExistentExternalShare()
</span><span class="cx"> raise ExternalShareFailed("External share does not exist")
</span><del>- returnValue(("%s_%s" % (self._externalID, self._syncTokenRevision,)))
</del><ins>+ returnValue(revision)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -85,6 +85,7 @@
</span><span class="cx"> insert into HOME_STATUS values (0, 'normal' );
</span><span class="cx"> insert into HOME_STATUS values (1, 'external');
</span><span class="cx"> insert into HOME_STATUS values (2, 'purging');
</span><ins>+insert into HOME_STATUS values (3, 'migrating');
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> --------------
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_tablespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py (14338 => 14339)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py 2015-01-24 00:48:46 UTC (rev 14338)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py 2015-01-24 00:51:33 UTC (rev 14339)
</span><span class="lines">@@ -187,6 +187,7 @@
</span><span class="cx"> _HOME_STATUS_NORMAL = _homeStatus('normal')
</span><span class="cx"> _HOME_STATUS_EXTERNAL = _homeStatus('external')
</span><span class="cx"> _HOME_STATUS_PURGING = _homeStatus('purging')
</span><ins>+_HOME_STATUS_MIGRATING = _homeStatus('migrating')
</ins><span class="cx">
</span><span class="cx"> _bindStatus = _schemaConstants(
</span><span class="cx"> schema.CALENDAR_BIND_STATUS.DESCRIPTION,
</span></span></pre>
</div>
</div>
</body>
</html>