<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14783] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14783">14783</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-05-13 11:18:41 -0700 (Wed, 13 May 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Implement final pod migration clean-up steps.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkrequirementsstabletxt">CalendarServer/trunk/requirements-stable.txt</a></li>
<li><a href="#CalendarServertrunktxdavcaldavdatastoreschedulingworkpy">CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py</a></li>
<li><a href="#CalendarServertrunktxdavcaldavdatastoresqlpy">CalendarServer/trunk/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastorepoddingmigrationsync_metadatapy">CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastorepoddingmigrationtesttest_migrationpy">CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastorepoddingstore_apipy">CalendarServer/trunk/txdav/common/datastore/podding/store_api.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresqlpy">CalendarServer/trunk/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_notificationpy">CalendarServer/trunk/txdav/common/datastore/sql_notification.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemacurrentoracledialectsql">CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemacurrentsql">CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_54_to_55sql">CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_54_to_55sql">CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_sharingpy">CalendarServer/trunk/txdav/common/datastore/sql_sharing.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServertrunktxdavcommondatastorepoddingmigrationworkpy">CalendarServer/trunk/txdav/common/datastore/podding/migration/work.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkrequirementsstabletxt"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/requirements-stable.txt (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/requirements-stable.txt 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/requirements-stable.txt 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -36,7 +36,7 @@
</span><span class="cx"> #pyOpenSSL
</span><span class="cx"> pycrypto==2.6.1
</span><span class="cx">
</span><del>- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14775#egg=twextpy
</del><ins>+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@14782#egg=twextpy
</ins><span class="cx"> cffi==0.8.6
</span><span class="cx"> pycparser==2.10
</span><span class="cx"> #twisted
</span></span></pre></div>
<a id="CalendarServertrunktxdavcaldavdatastoreschedulingworkpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -1108,5 +1108,6 @@
</span><span class="cx"> log.debug("ImplicitProcessing - skipping auto-reply of missing ID: '{rid}'", rid=self.resourceID)
</span><span class="cx">
</span><span class="cx">
</span><del>-for workClass in (ScheduleOrganizerWork, ScheduleOrganizerSendWork, ScheduleReplyWork, ScheduleRefreshWork, ScheduleAutoReplyWork,):
</del><ins>+allScheduleWork = (ScheduleOrganizerWork, ScheduleOrganizerSendWork, ScheduleReplyWork, ScheduleRefreshWork, ScheduleAutoReplyWork,)
+for workClass in allScheduleWork:
</ins><span class="cx"> ScheduleWork._classForWorkType[workClass.__name__] = workClass
</span></span></pre></div>
<a id="CalendarServertrunktxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -14,9 +14,6 @@
</span><span class="cx"> # See the License for the specific language governing permissions and
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><del>-from txdav.caldav.datastore.scheduling.work import ScheduleOrganizerWork, \
- ScheduleWork, ScheduleAutoReplyWork, ScheduleOrganizerSendWork, \
- ScheduleRefreshWork, ScheduleReplyWork
</del><span class="cx">
</span><span class="cx">
</span><span class="cx"> """
</span><span class="lines">@@ -65,6 +62,7 @@
</span><span class="cx"> from txdav.caldav.datastore.scheduling.imip.token import iMIPTokenRecord
</span><span class="cx"> from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
</span><span class="cx"> from txdav.caldav.datastore.scheduling.utils import uidFromCalendarUserAddress
</span><ins>+from txdav.caldav.datastore.scheduling.work import allScheduleWork, ScheduleWork
</ins><span class="cx"> from txdav.caldav.datastore.sql_attachment import Attachment, DropBoxAttachment, \
</span><span class="cx"> AttachmentLink, ManagedAttachment
</span><span class="cx"> from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord, \
</span><span class="lines">@@ -538,6 +536,19 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def purgeAll(self):
+ """
+ Do a complete purge of all data associated with this calendar home. For now this will assume
+ a "silent" non-implicit behavior. In the future we will want to build in some of the options
+ the current set of "purge" CLI tools have to allow for cancels of future events etc.
+ """
+ # delete attachments corresponding to this home, also removing from disk
+ yield Attachment.removedHome(self._txn, self._resourceID)
+
+ yield super(CalendarHome, self).purgeAll()
+
+
+ @inlineCallbacks
</ins><span class="cx"> def copyMetadata(self, other, calendarIDMap):
</span><span class="cx"> """
</span><span class="cx"> Copy metadata from one L{CalendarObjectResource} to another. This is only
</span><span class="lines">@@ -1065,7 +1076,7 @@
</span><span class="cx"> L{ScheduleOrganizerSendWork}, L{ScheduleReplyWork}, L{ScheduleRefreshWork}, L{ScheduleAutoReplyWork}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- for workType in (ScheduleOrganizerWork, ScheduleOrganizerSendWork, ScheduleReplyWork, ScheduleRefreshWork, ScheduleAutoReplyWork,):
</del><ins>+ for workType in allScheduleWork:
</ins><span class="cx"> yield JobItem.updatesome(
</span><span class="cx"> self._txn,
</span><span class="cx"> where=JobItem.jobID.In(
</span><span class="lines">@@ -1083,7 +1094,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> results = collections.defaultdict(list)
</span><del>- for workType in (ScheduleOrganizerWork, ScheduleOrganizerSendWork, ScheduleReplyWork, ScheduleRefreshWork, ScheduleAutoReplyWork,):
</del><ins>+ for workType in allScheduleWork:
</ins><span class="cx"> workItems = yield workType.query(self._txn, workType.homeResourceID == self.id())
</span><span class="cx"> for item in workItems:
</span><span class="cx"> serialized = yield item.serializeWithAncillaryData()
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -14,23 +14,26 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><del>-from functools import wraps
</del><ins>+from twext.python.log import Logger
</ins><span class="cx">
</span><del>-from twext.python.log import Logger
</del><span class="cx"> from twisted.internet.defer import returnValue, inlineCallbacks
</span><span class="cx"> from twisted.python.failure import Failure
</span><ins>+
</ins><span class="cx"> from twistedcaldav.accounting import emitAccounting
</span><ins>+
+from txdav.caldav.datastore.sql import ManagedAttachment, CalendarBindRecord
</ins><span class="cx"> from txdav.caldav.icalendarstore import ComponentUpdateState
</span><span class="cx"> from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord, \
</span><span class="cx"> CalendarObjectMigrationRecord, AttachmentMigrationRecord, \
</span><span class="cx"> MigrationCleanupWork
</span><del>-from txdav.caldav.datastore.sql import ManagedAttachment, CalendarBindRecord
</del><ins>+from txdav.common.datastore.podding.migration.work import HomeCleanupWork
</ins><span class="cx"> from txdav.common.datastore.sql_external import NotificationCollectionExternal
</span><span class="cx"> from txdav.common.datastore.sql_notification import NotificationCollection
</span><span class="cx"> from txdav.common.datastore.sql_tables import _HOME_STATUS_MIGRATING, _HOME_STATUS_DISABLED, \
</span><span class="cx"> _HOME_STATUS_EXTERNAL, _HOME_STATUS_NORMAL
</span><span class="cx"> from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
</span><span class="cx">
</span><ins>+from functools import wraps
</ins><span class="cx"> from uuid import uuid4
</span><span class="cx"> import datetime
</span><span class="cx">
</span><span class="lines">@@ -287,8 +290,12 @@
</span><span class="cx"> homeResourceID=newhome.id(),
</span><span class="cx"> )
</span><span class="cx">
</span><del>- # TODO: purge the old ones
- pass
</del><ins>+ # Purge the old ones
+ yield HomeCleanupWork.reschedule(
+ txn,
+ HomeCleanupWork.notBeforeDelay,
+ ownerUID=newhome.uid(),
+ )
</ins><span class="cx">
</span><span class="cx"> self.accounting("Completed: enableLocalHome.\n")
</span><span class="cx">
</span><span class="lines">@@ -305,10 +312,19 @@
</span><span class="cx"> yield self.loadRecord()
</span><span class="cx"> self.accounting("Starting: removeRemoteHome...")
</span><span class="cx"> yield self.prepareCalendarHome()
</span><ins>+ yield self._migratedHome()
</ins><span class="cx">
</span><span class="cx"> self.accounting("Completed: removeRemoteHome.\n")
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inTransactionWrapper
+ def _migratedHome(self, txn):
+ """
+ Send cross-pod message to tell the old pod to remove the migrated data.
+ """
+ return txn.store().conduit.send_migrated_home(txn, self.diruid)
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def loadRecord(self):
</span><span class="cx"> """
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastorepoddingmigrationsync_metadatapy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -16,7 +16,7 @@
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.record import Record, fromTable
</span><span class="cx"> from twext.enterprise.dal.syntax import Parameter, Delete
</span><del>-from twext.enterprise.jobqueue import SingletonWorkItem
</del><ins>+from twext.enterprise.jobqueue import WorkItem
</ins><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema
</span><span class="cx">
</span><span class="lines">@@ -60,7 +60,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-class MigrationCleanupWork(SingletonWorkItem, fromTable(schema.MIGRATION_CLEANUP_WORK)):
</del><ins>+class MigrationCleanupWork(WorkItem, fromTable(schema.MIGRATION_CLEANUP_WORK)):
</ins><span class="cx">
</span><span class="cx"> group = "homeResourceID"
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastorepoddingmigrationtesttest_migrationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -27,12 +27,16 @@
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.scheduling.ischedule.delivery import IScheduleRequest
</span><span class="cx"> from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
</span><ins>+from txdav.caldav.datastore.scheduling.work import allScheduleWork
</ins><span class="cx"> from txdav.caldav.datastore.test.common import CaptureProtocol
</span><span class="cx"> from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
</span><span class="cx"> from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord, \
</span><span class="cx"> AttachmentMigrationRecord, CalendarObjectMigrationRecord, \
</span><span class="cx"> MigrationCleanupWork
</span><ins>+from txdav.common.datastore.podding.migration.work import HomeCleanupWork, MigratedHomeCleanupWork
</ins><span class="cx"> from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
</span><ins>+from txdav.common.datastore.sql_directory import DelegateRecord,\
+ DelegateGroupsRecord, ExternalDelegateGroupsRecord
</ins><span class="cx"> from txdav.common.datastore.sql_tables import _BIND_MODE_READ, \
</span><span class="cx"> _HOME_STATUS_DISABLED, _HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL, \
</span><span class="cx"> _HOME_STATUS_MIGRATING
</span><span class="lines">@@ -78,6 +82,8 @@
</span><span class="cx">
</span><span class="cx"> # Speed up work
</span><span class="cx"> self.patch(MigrationCleanupWork, "notBeforeDelay", 1)
</span><ins>+ self.patch(HomeCleanupWork, "notBeforeDelay", 1)
+ self.patch(MigratedHomeCleanupWork, "notBeforeDelay", 1)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def configure(self):
</span><span class="lines">@@ -375,6 +381,7 @@
</span><span class="cx">
</span><span class="cx"> # Data for user01
</span><span class="cx"> home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><ins>+ self.stash["user01_pod0_home_id"] = home.id()
</ins><span class="cx"> calendar = yield home.childWithName("calendar")
</span><span class="cx"> yield calendar.createCalendarObjectWithName("01_1.ics", Component.fromString(self.data01_1))
</span><span class="cx"> yield calendar.createCalendarObjectWithName("01_2.ics", Component.fromString(self.data01_2))
</span><span class="lines">@@ -661,14 +668,48 @@
</span><span class="cx"> data = yield self.attachmentToString(attachment)
</span><span class="cx"> self.assertEqual(data, "Here is some text #1.")
</span><span class="cx">
</span><del>- # No migration data left
</del><ins>+ # Check removal of data from new pod
+
+ # Make sure all jobs are done
+ yield JobItem.waitEmpty(self.theStoreUnderTest(1).newTransaction, reactor, 60)
+
+ # No migration state data left
</ins><span class="cx"> txn = self.theTransactionUnderTest(1)
</span><del>- yield JobItem.waitEmpty(self.theStoreUnderTest(1).newTransaction, reactor, 60)
</del><span class="cx"> for migrationType in (CalendarMigrationRecord, CalendarObjectMigrationRecord, AttachmentMigrationRecord,):
</span><span class="cx"> records = yield migrationType.all(txn)
</span><span class="cx"> self.assertEqual(len(records), 0, msg=migrationType.__name__)
</span><ins>+ yield self.commitTransaction(1)
</ins><span class="cx">
</span><ins>+ # No homes
+ txn = self.theTransactionUnderTest(1)
+ oldhome = yield txn.calendarHomeWithUID("user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(oldhome is None)
+ oldhome = yield txn.notificationsWithUID("user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(oldhome is None)
</ins><span class="cx">
</span><ins>+ # Check removal of data from old pod
+
+ # Make sure all jobs are done
+ yield JobItem.waitEmpty(self.theStoreUnderTest(0).newTransaction, reactor, 60)
+
+ # No homes
+ txn = self.theTransactionUnderTest(0)
+ oldhome = yield txn.calendarHomeWithUID("user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(oldhome is None)
+ oldhome = yield txn.notificationsWithUID("user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(oldhome is None)
+
+ # No delegates
+ for delegateType in (DelegateRecord, DelegateGroupsRecord, ExternalDelegateGroupsRecord):
+ records = yield delegateType.query(txn, delegateType.delegator == "user01")
+ self.assertEqual(len(records), 0, msg=delegateType.__name__)
+
+ # No work items
+ for workType in allScheduleWork:
+ records = yield workType.query(txn, workType.homeResourceID == self.stash["user01_pod0_home_id"])
+ self.assertEqual(len(records), 0, msg=workType.__name__)
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def test_migration(self):
</span><span class="cx"> """
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastorepoddingmigrationworkpy"></a>
<div class="addfile"><h4>Added: CalendarServer/trunk/txdav/common/datastore/podding/migration/work.py (0 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/podding/migration/work.py (rev 0)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/work.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -0,0 +1,101 @@
</span><ins>+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.record import fromTable
+from twext.enterprise.jobqueue import WorkItem
+
+from twisted.internet.defer import inlineCallbacks
+
+from txdav.caldav.datastore.scheduling.imip.token import iMIPTokenRecord
+from txdav.caldav.datastore.scheduling.work import allScheduleWork
+from txdav.common.datastore.sql_directory import DelegateRecord, \
+ DelegateGroupsRecord, ExternalDelegateGroupsRecord
+from txdav.common.datastore.sql_tables import schema, _HOME_STATUS_DISABLED
+
+
+class HomeCleanupWork(WorkItem, fromTable(schema.HOME_CLEANUP_WORK)):
+ """
+ Work item to clean up any previously "external" homes on the pod to which data was migrated to. Those
+ old homes will now be marked as disabled and need to be silently removed without any side effects
+ (i.e., no implicit scheduling, no sharing cancels, etc).
+ """
+
+ group = "ownerUID"
+
+ notBeforeDelay = 300 # 5 minutes
+
+ @inlineCallbacks
+ def doWork(self):
+ """
+ Delete all the corresponding homes.
+ """
+
+ oldhome = yield self.transaction.calendarHomeWithUID(self.ownerUID, status=_HOME_STATUS_DISABLED)
+ if oldhome is not None:
+ yield oldhome.purgeAll()
+
+ oldnotifications = yield self.transaction.notificationsWithUID(self.ownerUID, status=_HOME_STATUS_DISABLED)
+ if oldnotifications is not None:
+ yield oldnotifications.purge()
+
+
+
+class MigratedHomeCleanupWork(WorkItem, fromTable(schema.MIGRATED_HOME_CLEANUP_WORK)):
+ """
+ Work item to clean up the old home data left behind after migration, as well
+ as other unwanted items like iMIP tokens, delegates etc. The old homes will
+ now be marked as disabled and need to be silently removed without any side
+ effects (i.e., no implicit scheduling, no sharing cancels, etc).
+ """
+
+ group = "ownerUID"
+
+ notBeforeDelay = 300 # 5 minutes
+
+ @inlineCallbacks
+ def doWork(self):
+ """
+ Delete all the corresponding homes, then the ancillary data.
+ """
+
+ oldhome = yield self.transaction.calendarHomeWithUID(self.ownerUID, status=_HOME_STATUS_DISABLED)
+ if oldhome is not None:
+ # Work items - we need to clean these up before the home goes away because we have an "on delete cascade" on the WorkItem
+ # table, and if that ran it would leave orphaned Job rows set to a pause state and those would remain for ever in the table.
+ for workType in allScheduleWork:
+ items = yield workType.query(self.transaction, workType.homeResourceID == oldhome.id())
+ for item in items:
+ yield item.remove()
+
+ yield oldhome.purgeAll()
+
+ oldnotifications = yield self.transaction.notificationsWithUID(self.ownerUID, status=_HOME_STATUS_DISABLED)
+ if oldnotifications is not None:
+ yield oldnotifications.purge()
+
+ # These are things that reference the home id or the user UID but don't get removed via a cascade
+
+ # iMIP tokens
+ cuaddr = "urn:x-uid:{}".format(self.ownerUID)
+ yield iMIPTokenRecord.deletesome(
+ self.transaction,
+ iMIPTokenRecord.organizer == cuaddr,
+ )
+
+ # Delegators - individual and group
+ yield DelegateRecord.deletesome(self.transaction, DelegateRecord.delegator == self.ownerUID)
+ yield DelegateGroupsRecord.deletesome(self.transaction, DelegateGroupsRecord.delegator == self.ownerUID)
+ yield ExternalDelegateGroupsRecord.deletesome(self.transaction, ExternalDelegateGroupsRecord.delegator == self.ownerUID)
</ins></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastorepoddingstore_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/podding/store_api.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/podding/store_api.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/podding/store_api.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -135,6 +135,39 @@
</span><span class="cx"> })
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def send_migrated_home(self, txn, ownerUID):
+ """
+ Tell another pod that user data migration is done.
+
+ @param txn: transaction to use
+ @type txn: L{CommonStoreTransaction}
+ @param server: server to query
+ @type server: L{Server}
+ @param ownerUID: directory UID of the user whose data has been migrated
+ @type ownerUID: L{str}
+ """
+
+ recipient = yield self.store.directoryService().recordWithUID(ownerUID)
+ request = {
+ "action": "migrated-home",
+ "ownerUID": ownerUID,
+ }
+ yield self.sendRequestToServer(txn, recipient.server(), request)
+
+
+ @inlineCallbacks
+ def recv_migrated_home(self, txn, request):
+ """
+ Process a migrated homes cross-pod request. Request arguments as per L{send_migrated_home}.
+
+ @param request: request arguments
+ @type request: C{dict}
+ """
+
+ yield txn.migratedHome(request["ownerUID"])
+
+
</ins><span class="cx"> @staticmethod
</span><span class="cx"> def _to_serialize_pair_list(value):
</span><span class="cx"> """
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -58,6 +58,7 @@
</span><span class="cx"> from txdav.carddav.iaddressbookstore import IAddressBookTransaction
</span><span class="cx"> from txdav.common.datastore.common import HomeChildBase
</span><span class="cx"> from txdav.common.datastore.podding.conduit import PoddingConduit
</span><ins>+from txdav.common.datastore.podding.migration.work import MigratedHomeCleanupWork
</ins><span class="cx"> from txdav.common.datastore.sql_apn import APNSubscriptionsMixin
</span><span class="cx"> from txdav.common.datastore.sql_directory import DelegatesAPIMixin, \
</span><span class="cx"> GroupsAPIMixin, GroupCacherAPIMixin
</span><span class="lines">@@ -802,6 +803,23 @@
</span><span class="cx"> returnValue(result)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def migratedHome(self, ownerUID):
+ """
+ This pod is being told that user data migration to another pod has been completed, and the old data now
+ needs to be removed.
+
+ @param ownerUID: directory UID of the user whose data has been migrated
+ @type ownerUID: L{str}
+ """
+
+ # All we do is schedule a work item to run the actual clean-up
+ return MigratedHomeCleanupWork.reschedule(
+ self,
+ MigratedHomeCleanupWork.notBeforeDelay,
+ ownerUID=ownerUID,
+ )
+
+
</ins><span class="cx"> def preCommit(self, operation):
</span><span class="cx"> """
</span><span class="cx"> Run things before C{commit}. (Note: only provided by SQL
</span><span class="lines">@@ -2137,6 +2155,62 @@
</span><span class="cx"> self._children.pop(child.id(), None)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def purgeAll(self):
+ """
+ Do a complete purge of all data associated with this calendar home. For now this will assume
+ a "silent" non-implicit behavior. In the future we will want to build in some of the options
+ the current set of "purge" CLI tools have to allow for cancels of future events etc.
+ """
+
+ # Removing the home table entry does NOT remove the child class entry - it does remove
+ # the associated bind entry. So manually remove each child.
+ yield self.purgeAllChildren()
+
+ r = self._childClass._revisionsSchema
+ yield Delete(
+ From=r,
+ Where=r.HOME_RESOURCE_ID == self._resourceID,
+ ).on(self._txn)
+
+ h = self._homeSchema
+ yield Delete(
+ From=h,
+ Where=h.RESOURCE_ID == self._resourceID,
+ ).on(self._txn)
+
+ yield self.properties()._removeResource()
+
+ if self._txn._queryCacher:
+ yield self._txn._queryCacher.delete(self._txn._queryCacher.keyForHomeWithUID(
+ self._homeType,
+ self.uid(),
+ self._status,
+ ))
+ yield self._txn._queryCacher.delete(self._txn._queryCacher.keyForHomeWithID(
+ self._homeType,
+ self.id(),
+ self._status,
+ ))
+
+
+ @inlineCallbacks
+ def purgeAllChildren(self):
+ """
+ Purge each child (non-implicit).
+ """
+
+ children = yield self.loadChildren()
+ for child in children:
+ yield child.unshare()
+ if child.owned():
+ yield child.purge()
+ self._children.pop(child.name(), None)
+ self._children.pop(child.id(), None)
+
+ yield self.removeUnacceptedShares()
+
+
</ins><span class="cx"> def transaction(self):
</span><span class="cx"> return self._txn
</span><span class="cx">
</span><span class="lines">@@ -3599,6 +3673,11 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _reallyRemove(self):
</span><ins>+ """
+ Actually remove this collection from the database. All the child resources will be automatically
+ removed by virtue of an on delete cascade. Note that means no implicit scheduling cancels will
+ occur.
+ """
</ins><span class="cx">
</span><span class="cx"> # Stop sharing first
</span><span class="cx"> yield self.ownerDeleteShare()
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_notificationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_notification.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_notification.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql_notification.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -586,8 +586,10 @@
</span><span class="cx"> ),
</span><span class="cx"> ).on(self._txn, **kwds)
</span><span class="cx">
</span><ins>+ purge = remove
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class NotificationObjectRecord(SerializableRecord, fromTable(schema.NOTIFICATION)):
</span><span class="cx"> """
</span><span class="cx"> @DynamicAttrs
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemacurrentoracledialectsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -652,6 +652,18 @@
</span><span class="cx"> "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create table HOME_CLEANUP_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "OWNER_UID" nvarchar2(255)
+);
+
+create table MIGRATED_HOME_CLEANUP_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "OWNER_UID" nvarchar2(255)
+);
+
</ins><span class="cx"> create table CALENDARSERVER (
</span><span class="cx"> "NAME" nvarchar2(255) primary key,
</span><span class="cx"> "VALUE" nvarchar2(255)
</span><span class="lines">@@ -1011,4 +1023,12 @@
</span><span class="cx"> "HOME_RESOURCE_ID"
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create index HOME_CLEANUP_WORK_JOB_9631dfb0 on HOME_CLEANUP_WORK (
+ "JOB_ID"
+);
+
+create index MIGRATED_HOME_CLEANUP_4c714fd4 on MIGRATED_HOME_CLEANUP_WORK (
+ "JOB_ID"
+);
+
</ins><span class="cx"> -- Extra schema to add to current-oracle-dialect.sql
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -1234,6 +1234,32 @@
</span><span class="cx"> create index MIGRATION_CLEANUP_WORK_HOME_RESOURCE_ID on
</span><span class="cx"> MIGRATION_CLEANUP_WORK(HOME_RESOURCE_ID);
</span><span class="cx">
</span><ins>+-----------------------
+-- Home Cleanup Work --
+-----------------------
+
+create table HOME_CLEANUP_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ OWNER_UID varchar(255) not null
+);
+
+create index HOME_CLEANUP_WORK_JOB_ID on
+ HOME_CLEANUP_WORK(JOB_ID);
+
+--------------------------------
+-- Migrated Home Cleanup Work --
+--------------------------------
+
+create table MIGRATED_HOME_CLEANUP_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ OWNER_UID varchar(255) not null
+);
+
+create index MIGRATED_HOME_CLEANUP_WORK_JOB_ID on
+ MIGRATED_HOME_CLEANUP_WORK(JOB_ID);
+
</ins><span class="cx"> --------------------
</span><span class="cx"> -- Schema Version --
</span><span class="cx"> --------------------
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_54_to_55sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -22,7 +22,7 @@
</span><span class="cx"> alter table JOB
</span><span class="cx"> add ("PAUSE" integer default 0);
</span><span class="cx">
</span><del>--- New table
</del><ins>+-- New tables
</ins><span class="cx"> create table MIGRATION_CLEANUP_WORK (
</span><span class="cx"> "WORK_ID" integer primary key,
</span><span class="cx"> "JOB_ID" integer not null references JOB,
</span><span class="lines">@@ -36,5 +36,25 @@
</span><span class="cx"> "HOME_RESOURCE_ID"
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create table HOME_CLEANUP_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "OWNER_UID" nvarchar2(255)
+);
+
+create index HOME_CLEANUP_WORK_JOB_9631dfb0 on HOME_CLEANUP_WORK (
+ "JOB_ID"
+);
+
+create table MIGRATED_HOME_CLEANUP_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "OWNER_UID" nvarchar2(255)
+);
+
+create index MIGRATED_HOME_CLEANUP_4c714fd4 on MIGRATED_HOME_CLEANUP_WORK (
+ "JOB_ID"
+);
+
</ins><span class="cx"> -- update the version
</span><span class="cx"> update CALENDARSERVER set VALUE = '55' where NAME = 'VERSION';
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_54_to_55sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -22,7 +22,7 @@
</span><span class="cx"> alter table JOB
</span><span class="cx"> add column PAUSE integer default 0;
</span><span class="cx">
</span><del>--- New Table
</del><ins>+-- New tables
</ins><span class="cx"> create table MIGRATION_CLEANUP_WORK (
</span><span class="cx"> WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
</span><span class="cx"> JOB_ID integer references JOB not null,
</span><span class="lines">@@ -34,5 +34,23 @@
</span><span class="cx"> create index MIGRATION_CLEANUP_WORK_HOME_RESOURCE_ID on
</span><span class="cx"> MIGRATION_CLEANUP_WORK(HOME_RESOURCE_ID);
</span><span class="cx">
</span><ins>+ create table HOME_CLEANUP_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ OWNER_UID varchar(255) not null
+);
+
+create index HOME_CLEANUP_WORK_JOB_ID on
+ HOME_CLEANUP_WORK(JOB_ID);
+
+create table MIGRATED_HOME_CLEANUP_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ OWNER_UID varchar(255) not null
+);
+
+create index MIGRATED_HOME_CLEANUP_WORK_JOB_ID on
+ MIGRATED_HOME_CLEANUP_WORK(JOB_ID);
+
</ins><span class="cx"> -- update the version
</span><span class="cx"> update CALENDARSERVER set VALUE = '55' where NAME = 'VERSION';
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_sharing.py (14782 => 14783)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_sharing.py 2015-05-13 18:09:46 UTC (rev 14782)
+++ CalendarServer/trunk/txdav/common/datastore/sql_sharing.py 2015-05-13 18:18:41 UTC (rev 14783)
</span><span class="lines">@@ -29,7 +29,7 @@
</span><span class="cx"> from txdav.common.datastore.sql_tables import _BIND_MODE_OWN, _BIND_MODE_DIRECT, \
</span><span class="cx"> _BIND_MODE_INDIRECT, _BIND_STATUS_ACCEPTED, _BIND_STATUS_DECLINED, \
</span><span class="cx"> _BIND_STATUS_INVITED, _BIND_STATUS_INVALID, _BIND_STATUS_DELETED, \
</span><del>- _HOME_STATUS_EXTERNAL
</del><ins>+ _HOME_STATUS_EXTERNAL, _HOME_STATUS_DISABLED
</ins><span class="cx"> from txdav.common.icommondatastore import ExternalShareFailed, \
</span><span class="cx"> HomeChildNameAlreadyExistsError, AllRetriesFailed
</span><span class="cx"> from txdav.xml import element
</span><span class="lines">@@ -479,8 +479,9 @@
</span><span class="cx"> yield self._sendExternalUninvite(shareeView)
</span><span class="cx"> else:
</span><span class="cx"> # If current user state is accepted then we send an invite with the new state, otherwise
</span><del>- # we cancel any existing invites for the user
- if not shareeView.direct():
</del><ins>+ # we cancel any existing invites for the user. Also, if the ownerHome is disabled, we assume
+ # that no sharing invites are sent.
+ if not shareeView.direct() and shareeView.ownerHome().status() != _HOME_STATUS_DISABLED:
</ins><span class="cx"> if shareeView.shareStatus() != _BIND_STATUS_ACCEPTED:
</span><span class="cx"> yield self._removeInviteNotification(shareeView)
</span><span class="cx"> else:
</span></span></pre>
</div>
</div>
</body>
</html>