[CalendarServer-changes] [14507] CalendarServer/branches/users/cdaboo/pod2pod-migration
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 4 18:41:22 PST 2015
Revision: 14507
http://trac.calendarserver.org//changeset/14507
Author: cdaboo at apple.com
Date: 2015-03-04 18:41:22 -0800 (Wed, 04 Mar 2015)
Log Message:
-----------
Implement a test that covers the complete migration cycle.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/push/test/test_notifier.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -297,7 +297,7 @@
@inlineCallbacks
def test_notificationNotifier(self):
- notifications = yield self.transactionUnderTest().notificationsWithUID("user01")
+ notifications = yield self.transactionUnderTest().notificationsWithUID("user01", create=True)
yield notifications.notifyChanged(category=ChangeCategory.default)
self.assertEquals(
set(self.notifierFactory.history),
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/purge.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -992,7 +992,7 @@
if not self.dryrun:
yield storeCalHome.removeUnacceptedShares()
- notificationHome = yield txn.notificationsWithUID(storeCalHome.uid(), create=False)
+ notificationHome = yield txn.notificationsWithUID(storeCalHome.uid())
if notificationHome is not None:
yield notificationHome.remove()
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/resource.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -2139,7 +2139,7 @@
@inlineCallbacks
def createNotificationsCollection(self):
txn = self._associatedTransaction
- notifications = yield txn.notificationsWithUID(self._newStoreHome.uid())
+ notifications = yield txn.notificationsWithUID(self._newStoreHome.uid(), create=True)
from twistedcaldav.storebridge import StoreNotificationCollectionResource
similar = StoreNotificationCollectionResource(
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/scheduling/ischedule/delivery.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -448,15 +448,6 @@
@inlineCallbacks
def _processRequest(self, ssl, host, port, path):
- from twisted.internet import reactor
- f = Factory()
- f.protocol = HTTPClientProtocol
- if ssl:
- ep = GAIEndpoint(reactor, host, port, _configuredClientContextFactory())
- else:
- ep = GAIEndpoint(reactor, host, port)
- proto = (yield ep.connect(f))
-
if not self.server.podding() and config.Scheduling.iSchedule.DKIM.Enabled:
domain, selector, key_file, algorithm, useDNSKey, useHTTPKey, usePrivateExchangeKey, expire = DKIMUtils.getConfiguration(config)
request = DKIMRequest(
@@ -481,6 +472,21 @@
if accountingEnabledForCategory("iSchedule"):
self.loggedRequest = yield self.logRequest(request)
+ response = yield self._submitRequest(ssl, host, port, request)
+ returnValue(response)
+
+
+ @inlineCallbacks
+ def _submitRequest(self, ssl, host, port, request):
+ from twisted.internet import reactor
+ f = Factory()
+ f.protocol = HTTPClientProtocol
+ if ssl:
+ ep = GAIEndpoint(reactor, host, port, _configuredClientContextFactory())
+ else:
+ ep = GAIEndpoint(reactor, host, port)
+ proto = (yield ep.connect(f))
+
response = (yield proto.submitRequest(request))
returnValue(response)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/common.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -375,7 +375,7 @@
@inlineCallbacks
def notificationUnderTest(self):
txn = self.transactionUnderTest()
- notifications = yield txn.notificationsWithUID("home1")
+ notifications = yield txn.notificationsWithUID("home1", create=True)
yield notifications.writeNotificationObject(
"abc",
json.loads("{\"notification-type\":\"invite-notification\"}"),
@@ -402,7 +402,7 @@
objects changed or deleted since
"""
txn = self.transactionUnderTest()
- coll = yield txn.notificationsWithUID("home1")
+ coll = yield txn.notificationsWithUID("home1", create=True)
yield coll.writeNotificationObject(
"1",
json.loads("{\"notification-type\":\"invite-notification\"}"),
@@ -435,7 +435,7 @@
overwrite the notification object.
"""
notifications = yield self.transactionUnderTest().notificationsWithUID(
- "home1"
+ "home1", create=True
)
yield notifications.writeNotificationObject(
"abc",
@@ -462,7 +462,7 @@
"""
# Prime the home collection first
yield self.transactionUnderTest().notificationsWithUID(
- "home1"
+ "home1", create=True
)
yield self.commit()
@@ -512,7 +512,7 @@
overwrite the notification object.
"""
notifications = yield self.transactionUnderTest().notificationsWithUID(
- "home1"
+ "home1", create=True
)
yield notifications.writeNotificationObject(
"abc",
@@ -555,7 +555,7 @@
L{INotificationCollection} that the object was retrieved from.
"""
txn = self.transactionUnderTest()
- collection = yield txn.notificationsWithUID("home1")
+ collection = yield txn.notificationsWithUID("home1", create=True)
notification = yield self.notificationUnderTest()
self.assertIdentical(collection, notification.notificationCollection())
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -767,13 +767,13 @@
txn2 = calendarStore.newTransaction()
notification_uid1_1 = yield txn1.notificationsWithUID(
- "uid1",
+ "uid1", create=True
)
@inlineCallbacks
def _defer_notification_uid1_2():
notification_uid1_2 = yield txn2.notificationsWithUID(
- "uid1",
+ "uid1", create=True
)
yield txn2.commit()
returnValue(notification_uid1_2)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_sql_sharing.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -464,7 +464,7 @@
shared = yield self.calendarUnderTest(home="user02", name=sharedName)
self.assertTrue(shared is not None)
- notifyHome = yield self.transactionUnderTest().notificationsWithUID("user02")
+ notifyHome = yield self.transactionUnderTest().notificationsWithUID("user02", create=True)
notifications = yield notifyHome.listNotificationObjects()
self.assertEqual(len(notifications), 0)
@@ -654,7 +654,7 @@
@inlineCallbacks
def _check_notifications(self, uid, items):
- notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid)
+ notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid, create=True)
notifications = yield notifyHome.listNotificationObjects()
self.assertEqual(set(notifications), set(items))
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -364,13 +364,13 @@
txn2 = addressbookStore.newTransaction()
notification_uid1_1 = yield txn1.notificationsWithUID(
- "uid1",
+ "uid1", create=True,
)
@inlineCallbacks
def _defer_notification_uid1_2():
notification_uid1_2 = yield txn2.notificationsWithUID(
- "uid1",
+ "uid1", create=True,
)
yield txn2.commit()
returnValue(notification_uid1_2)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -198,7 +198,7 @@
@inlineCallbacks
def _check_notifications(self, home, items):
- notifyHome = yield self.transactionUnderTest().notificationsWithUID(home)
+ notifyHome = yield self.transactionUnderTest().notificationsWithUID(home, create=True)
notifications = yield notifyHome.listNotificationObjects()
self.assertEqual(set(notifications), set(items))
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -376,7 +376,7 @@
@memoizedKey("uid", "_notificationHomes", deferredResult=False)
- def notificationsWithUID(self, uid, home=None):
+ def notificationsWithUID(self, uid, home=None, create=False):
if home is None:
home = self.homeWithUID(self._notificationHomeType, uid, create=True)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -85,17 +85,22 @@
BATCH_SIZE = 50
- def __init__(self, store, diruid):
+ def __init__(self, store, diruid, final=False):
"""
@param store: the data store
@type store: L{CommonDataStore}
@param diruid: directory uid of the user whose home is to be sync'd
@type diruid: L{str}
+ @param final: indicates whether this is in the final sync stage with the remote home
+ already disabled
+ @type final: L{bool}
"""
self.store = store
self.diruid = diruid
- self.disabledRemote = False
+ self.disabledRemote = final
+ self.record = None
+ self.homeId = None
def label(self, detail):
@@ -133,7 +138,7 @@
# Step 6 - enable new home
yield self.enableLocalHome()
- # Step 7 - remote remote home
+ # Step 7 - remove remote home
yield self.removeRemoteHome()
# Step 8 - say phew! TODO: Actually alert everyone else
@@ -168,6 +173,9 @@
rows, recalculate quota etc.
"""
+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
+
# Link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
yield self.linkAttachments()
@@ -198,6 +206,9 @@
Mark the remote home as disabled.
"""
+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
+
# Calendar home
remote_home = yield self._remoteHome(txn)
yield remote_home.setStatus(_HOME_STATUS_DISABLED)
@@ -216,13 +227,16 @@
Mark the local home as enabled and remove any previously existing external home.
"""
+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
+
# Disable any local external homes
oldhome = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_EXTERNAL)
if oldhome is not None:
- yield oldhome.setStatus(_HOME_STATUS_DISABLED)
+ yield oldhome.setLocalStatus(_HOME_STATUS_DISABLED)
oldnotifications = yield txn.notificationsWithUID(self.diruid, status=_HOME_STATUS_EXTERNAL)
if oldnotifications:
- yield oldnotifications.setStatus(_HOME_STATUS_DISABLED)
+ yield oldnotifications.setLocalStatus(_HOME_STATUS_DISABLED)
# Enable the migrating ones
newhome = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING)
@@ -244,7 +258,8 @@
# TODO: implement API on CommonHome to purge the old data without
# any side-effects (scheduling, sharing etc).
- pass
+ yield self.loadRecord()
+ yield self.prepareCalendarHome()
@inlineCallbacks
@@ -253,11 +268,12 @@
Initiate a sync of the home.
"""
- self.record = yield self.store.directoryService().recordWithUID(self.diruid)
if self.record is None:
- raise DirectoryRecordNotFoundError("Cross-pod Migration Sync missing directory record for {}".format(self.diruid))
- if self.record.thisServer():
- raise ValueError("Cross-pod Migration Sync cannot sync with user already on this server: {}".format(self.diruid))
+ self.record = yield self.store.directoryService().recordWithUID(self.diruid)
+ if self.record is None:
+ raise DirectoryRecordNotFoundError("Cross-pod Migration Sync missing directory record for {}".format(self.diruid))
+ if self.record.thisServer():
+ raise ValueError("Cross-pod Migration Sync cannot sync with user already on this server: {}".format(self.diruid))
@inTransactionWrapper
@@ -267,10 +283,14 @@
Make sure the inactive home to migrate into is present on this pod.
"""
- home = yield self._localHome(txn)
- if home is None:
- home = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
- self.homeId = home.id()
+ if self.homeId is None:
+ home = yield self._localHome(txn)
+ if home is None:
+ if self.disabledRemote:
+ self.homeId = None
+ else:
+ home = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
+ self.homeId = home.id() if home is not None else None
@inTransactionWrapper
@@ -1004,6 +1024,8 @@
len_records = 0
for calendar in calendars.values():
records, bindUID = yield self.sharedByCollectionRecords(calendar.remoteResourceID, calendar.localResourceID)
+ if not records:
+ continue
records = records.items()
# Batch setting resources for the local home
@@ -1039,7 +1061,7 @@
calendarHomeResourceID=self.homeId,
calendarResourceID=local_id,
)
- if not local_records[0].bindUID:
+ if records and not local_records[0].bindUID:
yield local_records[0].update(bindUID=str(uuid4()))
returnValue((records, local_records[0].bindUID,))
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/accounts/groupAccounts.xml 2015-03-05 02:41:22 UTC (rev 14507)
@@ -106,6 +106,7 @@
<full-name>Group 01</full-name>
<email>group01 at example.com</email>
<member-uid>user01</member-uid>
+ <member-uid>puser01</member-uid>
</record>
<record type="group">
<short-name>group02</short-name>
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -744,6 +744,7 @@
"""
home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
calendar0 = yield home0.childWithName("calendar")
object0_1 = yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
object0_2 = yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
@@ -854,6 +855,7 @@
# Create remote home
yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
yield self.commitTransaction(0)
# Add some delegates
@@ -941,7 +943,7 @@
# Create remote home - and add some fake notifications
yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
- notifications = yield self.theTransactionUnderTest(0).notificationsWithUID("user01")
+ notifications = yield self.theTransactionUnderTest(0).notificationsWithUID("user01", create=True)
uid1 = str(uuid4())
obj1 = yield notifications.writeNotificationObject(uid1, "type1", "data1")
id1 = obj1.id()
@@ -1047,6 +1049,7 @@
# Create home
yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
yield self.commitTransaction(0)
# Shared by migrating user
@@ -1119,6 +1122,7 @@
# Create home
yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ yield self.notificationCollectionUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
yield self.commitTransaction(0)
# Shared by migrating user
Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -0,0 +1,691 @@
+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from pycalendar.datetime import DateTime
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python.filepath import FilePath
+from twistedcaldav.config import config
+from twistedcaldav.ical import Component
+from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
+from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
+from txdav.common.datastore.sql_tables import _BIND_MODE_READ, \
+ _HOME_STATUS_DISABLED, _HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL, \
+ _HOME_STATUS_MIGRATING
+from txdav.common.datastore.test.util import populateCalendarsFrom
+from txdav.who.delegates import Delegates
+from txweb2.http_headers import MimeType
+from txweb2.stream import MemoryStream
+from txdav.caldav.datastore.scheduling.ischedule.delivery import IScheduleRequest
+from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
+from txweb2.dav.test.util import SimpleRequest
+from txdav.caldav.datastore.test.common import CaptureProtocol
+
+
+class TestCompleteMigrationCycle(MultiStoreConduitTest):
+ """
+ Test that a full migration cycle using L{CrossPodHomeSync} works.
+ """
+
+ def __init__(self, methodName='runTest'):
+ super(TestCompleteMigrationCycle, self).__init__(methodName)
+ self.stash = {}
+
+
+ @inlineCallbacks
+ def setUp(self):
+ @inlineCallbacks
+ def _fakeSubmitRequest(iself, ssl, host, port, request):
+ pod = (port - 8008) / 100
+ inbox = IScheduleInboxResource(self.site.resource, self.theStoreUnderTest(pod), podding=True)
+ response = yield inbox.http_POST(SimpleRequest(
+ self.site,
+ "POST",
+ "http://{host}:{port}/podding".format(host=host, port=port),
+ request.headers,
+ request.stream.mem,
+ ))
+ returnValue(response)
+
+
+ self.patch(IScheduleRequest, "_submitRequest", _fakeSubmitRequest)
+ self.accounts = FilePath(__file__).sibling("accounts").child("groupAccounts.xml")
+ self.augments = FilePath(__file__).sibling("accounts").child("augments.xml")
+ yield super(TestCompleteMigrationCycle, self).setUp()
+ yield self.populate()
+
+
+ def configure(self):
+ super(TestCompleteMigrationCycle, self).configure()
+ config.GroupAttendees.Enabled = True
+ config.GroupAttendees.ReconciliationDelaySeconds = 0
+ config.GroupAttendees.AutoUpdateSecondsFromNow = 0
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateCalendarsFrom(self.requirements0, self.theStoreUnderTest(0))
+ yield populateCalendarsFrom(self.requirements1, self.theStoreUnderTest(1))
+
+ requirements0 = {
+ "user01" : None,
+ "user02" : None,
+ "user03" : None,
+ "user04" : None,
+ "user05" : None,
+ "user06" : None,
+ "user07" : None,
+ "user08" : None,
+ "user09" : None,
+ "user10" : None,
+ }
+
+ requirements1 = {
+ "puser01" : None,
+ "puser02" : None,
+ "puser03" : None,
+ "puser04" : None,
+ "puser05" : None,
+ "puser06" : None,
+ "puser07" : None,
+ "puser08" : None,
+ "puser09" : None,
+ "puser10" : None,
+ }
+
+
+ @inlineCallbacks
+ def _createShare(self, shareFrom, shareTo, accept=True):
+ # Invite
+ txnindex = 1 if shareFrom[0] == "p" else 0
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(txnindex), name=shareFrom, create=True)
+ calendar = yield home.childWithName("calendar")
+ shareeView = yield calendar.inviteUIDToShare(shareTo, _BIND_MODE_READ, "summary")
+ yield self.commitTransaction(txnindex)
+
+ # Accept
+ if accept:
+ inviteUID = shareeView.shareUID()
+ txnindex = 1 if shareTo[0] == "p" else 0
+ shareeHome = yield self.homeUnderTest(txn=self.theTransactionUnderTest(txnindex), name=shareTo)
+ shareeView = yield shareeHome.acceptShare(inviteUID)
+ sharedName = shareeView.name()
+ yield self.commitTransaction(txnindex)
+ else:
+ sharedName = None
+
+ returnValue(sharedName)
+
+
+ def attachmentToString(self, attachment):
+ """
+ Convenience to convert an L{IAttachment} to a string.
+
+ @param attachment: an L{IAttachment} provider to convert into a string.
+
+ @return: a L{Deferred} that fires with the contents of the attachment.
+
+ @rtype: L{Deferred} firing C{bytes}
+ """
+ capture = CaptureProtocol()
+ attachment.retrieve(capture)
+ return capture.deferred
+
+
+ now = {
+ "now": DateTime.getToday().getYear(),
+ "now1": DateTime.getToday().getYear() + 1,
+ }
+
+ data01_1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_1
+DTSTART:{now1:04d}0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:data01_1
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data01_1_changed = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_1
+DTSTART:{now1:04d}0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:data01_1_changed
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data01_2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_2
+DTSTART:{now1:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_2
+ORGANIZER:mailto:user01 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE:mailto:puser02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data01_3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data01_3
+DTSTART:{now1:04d}0102T180000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_3
+ORGANIZER:mailto:user01 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:group02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data02_1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data02_1
+DTSTART:{now1:04d}0103T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:data02_1
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data02_2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data02_2
+DTSTART:{now1:04d}0103T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data02_2
+ORGANIZER:mailto:user02 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:puser02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ data02_3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data02_3
+DTSTART:{now1:04d}0103T180000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data02_3
+ORGANIZER:mailto:user02 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE:mailto:group01 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ datap02_1 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_datap02_1
+DTSTART:{now1:04d}0103T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:datap02_1
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ datap02_2 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_datap02_2
+DTSTART:{now1:04d}0103T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:datap02_2
+ORGANIZER:mailto:puser02 at example.com
+ATTENDEE:mailto:puser02 at example.com
+ATTENDEE:mailto:user01 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+ datap02_3 = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_datap02_3
+DTSTART:{now1:04d}0103T180000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:datap02_3
+ORGANIZER:mailto:puser02 at example.com
+ATTENDEE:mailto:puser02 at example.com
+ATTENDEE:mailto:group01 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**now)
+
+
+ @inlineCallbacks
+ def preCheck(self):
+ """
+ Checks prior to starting any tests
+ """
+
+ for i in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(i)
+ record = yield txn.directoryService().recordWithUID(u"user01")
+ self.assertEqual(record.serviceNodeUID, "A")
+ self.assertEqual(record.thisServer(), i == 0)
+ record = yield txn.directoryService().recordWithUID(u"user02")
+ self.assertEqual(record.serviceNodeUID, "A")
+ self.assertEqual(record.thisServer(), i == 0)
+ record = yield txn.directoryService().recordWithUID(u"puser02")
+ self.assertEqual(record.serviceNodeUID, "B")
+ self.assertEqual(record.thisServer(), i == 1)
+ yield self.commitTransaction(i)
+
+
+ @inlineCallbacks
+ def initialState(self):
+ """
+ Setup the server with an initial set of data
+
+ user01 - migrating user
+ user02 - has a calendar shared with user01
+ user03 - shared to by user01
+
+ puser01 - user on other pod
+ puser02 - has a calendar shared with user01
+ puser03 - shared to by user01
+ """
+
+ # Data for user01
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("01_1.ics", Component.fromString(self.data01_1))
+ yield calendar.createCalendarObjectWithName("01_2.ics", Component.fromString(self.data01_2))
+ obj3 = yield calendar.createCalendarObjectWithName("01_3.ics", Component.fromString(self.data01_3))
+ attachment, _ignore_location = yield obj3.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1."))
+ self.stash["user01_attachment_id"] = attachment.id()
+ self.stash["user01_attachment_md5"] = attachment.md5()
+ self.stash["user01_attachment_mid"] = attachment.managedID()
+ yield self.commitTransaction(0)
+
+ # Data for user02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user02", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("02_1.ics", Component.fromString(self.data02_1))
+ yield calendar.createCalendarObjectWithName("02_2.ics", Component.fromString(self.data02_2))
+ yield calendar.createCalendarObjectWithName("02_3.ics", Component.fromString(self.data02_3))
+ yield self.commitTransaction(0)
+
+ # Data for puser02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("p02_1.ics", Component.fromString(self.datap02_1))
+ yield calendar.createCalendarObjectWithName("p02_2.ics", Component.fromString(self.datap02_2))
+ yield calendar.createCalendarObjectWithName("p02_3.ics", Component.fromString(self.datap02_3))
+ yield self.commitTransaction(1)
+
+ # Share calendars
+ self.stash["sharename_user01_to_user03"] = yield self._createShare("user01", "user03")
+ self.stash["sharename_user01_to_puser03"] = yield self._createShare("user01", "puser03")
+ self.stash["sharename_user02_to_user01"] = yield self._createShare("user02", "user01")
+ self.stash["sharename_puser02_to_user01"] = yield self._createShare("puser02", "user01")
+
+ # Add some delegates
+ txn = self.theTransactionUnderTest(0)
+ record01 = yield txn.directoryService().recordWithUID(u"user01")
+ record02 = yield txn.directoryService().recordWithUID(u"user02")
+ record03 = yield txn.directoryService().recordWithUID(u"user03")
+ precord01 = yield txn.directoryService().recordWithUID(u"puser01")
+
+ group02 = yield txn.directoryService().recordWithUID(u"group02")
+ group03 = yield txn.directoryService().recordWithUID(u"group03")
+
+ # Add user02 and user03 as individual delegates
+ yield Delegates.addDelegate(txn, record01, record02, True)
+ yield Delegates.addDelegate(txn, record01, record03, False)
+ yield Delegates.addDelegate(txn, record01, precord01, False)
+
+ # Add group delegates
+ yield Delegates.addDelegate(txn, record01, group02, True)
+ yield Delegates.addDelegate(txn, record01, group03, False)
+
+ # Add external delegates
+ yield txn.assignExternalDelegates(u"user01", None, None, u"external1", u"external2")
+
+ yield self.commitTransaction(0)
+
+ yield self.waitAllEmpty()
+
+
+ @inlineCallbacks
+ def secondState(self):
+ """
+ Setup the server with data changes appearing after the first sync
+ """
+ txn = self.theTransactionUnderTest(0)
+ obj = yield self.calendarObjectUnderTest(txn, name="01_1.ics", calendar_name="calendar", home="user01")
+ yield obj.setComponent(self.data01_1_changed)
+
+ obj = yield self.calendarObjectUnderTest(txn, name="02_2.ics", calendar_name="calendar", home="user02")
+ attachment, _ignore_location = yield obj.addAttachment(None, MimeType.fromString("text/plain"), "test_02.txt", MemoryStream("Here is some text #02."))
+ self.stash["user02_attachment_id"] = attachment.id()
+ self.stash["user02_attachment_md5"] = attachment.md5()
+ self.stash["user02_attachment_mid"] = attachment.managedID()
+
+ yield self.commitTransaction(0)
+
+ yield self.waitAllEmpty()
+
+
+ @inlineCallbacks
+ def finalState(self):
+ """
+ Setup the server with data changes appearing before the final sync
+ """
+ txn = self.theTransactionUnderTest(1)
+ obj = yield self.calendarObjectUnderTest(txn, name="p02_2.ics", calendar_name="calendar", home="puser02")
+ attachment, _ignore_location = yield obj.addAttachment(None, MimeType.fromString("text/plain"), "test_p02.txt", MemoryStream("Here is some text #p02."))
+ self.stash["puser02_attachment_id"] = attachment.id()
+ self.stash["puser02_attachment_mid"] = attachment.managedID()
+ self.stash["puser02_attachment_md5"] = attachment.md5()
+
+ yield self.commitTransaction(1)
+
+ yield self.waitAllEmpty()
+
+
+ @inlineCallbacks
+ def switchAccounts(self):
+ """
+ Switch the migrated user accounts to point to the new pod
+ """
+
+ for i in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(i)
+ record = yield txn.directoryService().recordWithUID(u"user01")
+ yield self.changeRecord(record, txn.directoryService().fieldName.serviceNodeUID, u"B", directory=txn.directoryService())
+ yield self.commitTransaction(i)
+
+ for i in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(i)
+ record = yield txn.directoryService().recordWithUID(u"user01")
+ self.assertEqual(record.serviceNodeUID, "B")
+ self.assertEqual(record.thisServer(), i == 1)
+ record = yield txn.directoryService().recordWithUID(u"user02")
+ self.assertEqual(record.serviceNodeUID, "A")
+ self.assertEqual(record.thisServer(), i == 0)
+ record = yield txn.directoryService().recordWithUID(u"puser02")
+ self.assertEqual(record.serviceNodeUID, "B")
+ self.assertEqual(record.thisServer(), i == 1)
+ yield self.commitTransaction(i)
+
+
+ @inlineCallbacks
+ def postCheck(self):
+ """
+ Checks after migration is done
+ """
+
+ # Check that the home has been moved
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01")
+ self.assertTrue(home.external())
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(home is None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_MIGRATING)
+ self.assertTrue(home is None)
+ yield self.commitTransaction(0)
+
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01")
+ self.assertTrue(home.normal())
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(home is None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(home is not None)
+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
+ self.assertTrue(home is None)
+ yield self.commitTransaction(1)
+
+ # Check that the notifications have been moved
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(notifications is None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(notifications is None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(notifications is not None)
+ yield self.commitTransaction(0)
+
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(notifications is not None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_EXTERNAL)
+ self.assertTrue(notifications is None)
+ notifications = yield self.notificationCollectionUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(notifications is not None)
+ yield self.commitTransaction(1)
+
+ # New pod data
+ homes = {}
+ homes["user01"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01")
+ homes["user02"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user02")
+ self.assertTrue(homes["user02"].external())
+ homes["user03"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user03")
+ self.assertTrue(homes["user03"].external())
+ homes["puser01"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="puser01")
+ self.assertTrue(homes["puser01"].normal())
+ homes["puser02"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="puser02")
+ self.assertTrue(homes["puser02"].normal())
+ homes["puser03"] = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="puser03")
+ self.assertTrue(homes["puser03"].normal())
+
+ # Check calendar data on new pod
+ calendars = yield homes["user01"].loadChildren()
+ calnames = dict([(calendar.name(), calendar) for calendar in calendars])
+ self.assertEqual(
+ set(calnames.keys()),
+ set(("calendar", "tasks", "inbox", self.stash["sharename_user02_to_user01"], self.stash["sharename_puser02_to_user01"],))
+ )
+
+ # Check shared-by user01 on new pod
+ shared = calnames["calendar"]
+ invitations = yield shared.sharingInvites()
+ by_sharee = dict([(invitation.shareeUID, invitation) for invitation in invitations])
+ self.assertEqual(len(invitations), 2)
+ self.assertEqual(set(by_sharee.keys()), set(("user03", "puser03",)))
+ self.assertEqual(by_sharee["user03"].shareeHomeID, homes["user03"].id())
+ self.assertEqual(by_sharee["puser03"].shareeHomeID, homes["puser03"].id())
+
+ # Check shared-to user01 on new pod
+ shared = calnames[self.stash["sharename_user02_to_user01"]]
+ self.assertEqual(shared.ownerHome().uid(), "user02")
+ self.assertEqual(shared.ownerHome().id(), homes["user02"].id())
+
+ shared = calnames[self.stash["sharename_puser02_to_user01"]]
+ self.assertEqual(shared.ownerHome().uid(), "puser02")
+ self.assertEqual(shared.ownerHome().id(), homes["puser02"].id())
+
+ shared = yield homes["puser02"].calendarWithName("calendar")
+ invitations = yield shared.sharingInvites()
+ self.assertEqual(len(invitations), 1)
+ self.assertEqual(invitations[0].shareeHomeID, homes["user01"].id())
+
+ yield self.commitTransaction(1)
+
+ # Old pod data
+ homes = {}
+ homes["user01"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user01")
+ homes["user02"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user02")
+ self.assertTrue(homes["user02"].normal())
+ homes["user03"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="user03")
+ self.assertTrue(homes["user03"].normal())
+ homes["puser01"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="puser01")
+ self.assertTrue(homes["puser01"] is None)
+ homes["puser02"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="puser02")
+ self.assertTrue(homes["puser02"].external())
+ homes["puser03"] = yield self.homeUnderTest(self.theTransactionUnderTest(0), name="puser03")
+ self.assertTrue(homes["puser03"].external())
+
+ # Check shared-by user01 on old pod
+ shared = yield homes["user03"].calendarWithName(self.stash["sharename_user01_to_user03"])
+ self.assertEqual(shared.ownerHome().uid(), "user01")
+ self.assertEqual(shared.ownerHome().id(), homes["user01"].id())
+
+ # Check shared-to user01 on old pod
+ shared = yield homes["user02"].calendarWithName("calendar")
+ invitations = yield shared.sharingInvites()
+ self.assertEqual(len(invitations), 1)
+ self.assertEqual(invitations[0].shareeHomeID, homes["user01"].id())
+
+ yield self.commitTransaction(0)
+
+ # Delegates on each pod
+ for pod in range(self.numberOfStores):
+ txn = self.theTransactionUnderTest(pod)
+ records = {}
+ for ctr in range(10):
+ uid = u"user{:02d}".format(ctr + 1)
+ records[uid] = yield txn.directoryService().recordWithUID(uid)
+ for ctr in range(10):
+ uid = u"puser{:02d}".format(ctr + 1)
+ records[uid] = yield txn.directoryService().recordWithUID(uid)
+ for ctr in range(10):
+ uid = u"group{:02d}".format(ctr + 1)
+ records[uid] = yield txn.directoryService().recordWithUID(uid)
+
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], True, False)
+ self.assertTrue(records["user02"] in delegates)
+ self.assertTrue(records["group02"] in delegates)
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], True, True)
+ self.assertTrue(records["user02"] in delegates)
+ self.assertTrue(records["user06"] in delegates)
+ self.assertTrue(records["user07"] in delegates)
+ self.assertTrue(records["user08"] in delegates)
+
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], False, False)
+ self.assertTrue(records["user03"] in delegates)
+ self.assertTrue(records["group03"] in delegates)
+ self.assertTrue(records["puser01"] in delegates)
+ delegates = yield Delegates.delegatesOf(txn, records["user01"], False, True)
+ self.assertTrue(records["user03"] in delegates)
+ self.assertTrue(records["user07"] in delegates)
+ self.assertTrue(records["user08"] in delegates)
+ self.assertTrue(records["user09"] in delegates)
+ self.assertTrue(records["puser01"] in delegates)
+
+ # Attachments
+ obj = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), name="01_3.ics", calendar_name="calendar", home="user01")
+ attachment = yield obj.attachmentWithManagedID(self.stash["user01_attachment_mid"])
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), self.stash["user01_attachment_md5"])
+ data = yield self.attachmentToString(attachment)
+ self.assertEqual(data, "Here is some text #1.")
+
+
+ @inlineCallbacks
+ def test_migration(self):
+ """
+ Full migration cycle.
+ """
+
+ yield self.preCheck()
+
+ # Step 1. Live full sync
+ yield self.initialState()
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.sync()
+
+ # Step 2. Live incremental sync
+ yield self.secondState()
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.sync()
+
+ # Step 3. Disable home after final changes
+ yield self.finalState()
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.disableRemoteHome()
+
+ # Step 4. Final incremental sync
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.sync()
+
+ # Step 5. Final reconcile sync
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.finalSync()
+
+ # Step 6. Enable new home
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.enableLocalHome()
+
+ # Step 7. Remove old home
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01", final=True)
+ yield syncer.removeRemoteHome()
+
+ yield self.switchAccounts()
+
+ yield self.postCheck()
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -14,6 +14,7 @@
# limitations under the License.
##
+from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.protocol import Protocol
@@ -34,6 +35,7 @@
from txweb2.stream import ProducerStream
from twext.enterprise.ienterprise import AlreadyFinishedError
+from twext.enterprise.jobqueue import JobItem
import json
@@ -225,6 +227,12 @@
self.activeTransactions[count] = None
+ @inlineCallbacks
+ def waitAllEmpty(self):
+ for i in range(self.numberOfStores):
+ yield JobItem.waitEmpty(self.theStoreUnderTest(i).newTransaction, reactor, 60.0)
+
+
def makeConduit(self, store):
conduit = PoddingConduit(store)
conduit.conduitRequestClass = FakeConduitRequest
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -579,7 +579,10 @@
"byID": collections.defaultdict(dict),
},
}
- self._notificationHomes = {}
+ self._notificationHomes = {
+ "byUID": collections.defaultdict(dict),
+ "byID": collections.defaultdict(dict),
+ }
self._notifierFactories = notifierFactories
self._notifiedAlready = set()
self._bumpedRevisionAlready = set()
@@ -753,7 +756,7 @@
result = yield self._homeClass[storeType].homeWithResourceID(self, rid)
if result:
self._determineMemo(storeType, "byID", None)[rid] = result
- self._determineMemo(storeType, "byUID", result._status)[result.uid()] = result
+ self._determineMemo(storeType, "byUID", result.status())[result.uid()] = result
returnValue(result)
@@ -765,22 +768,36 @@
return self.homeWithResourceID(EADDRESSBOOKTYPE, rid)
- @memoizedKey("uid", "_notificationHomes")
- def notificationsWithUID(self, uid, status=None, create=True):
+ @inlineCallbacks
+ def notificationsWithUID(self, uid, status=None, create=False):
"""
Implement notificationsWithUID.
"""
- return NotificationCollection.notificationsWithUID(self, uid, create=create)
+ result = self._notificationHomes["byUID"][status].get(uid)
+ if result is None:
+ result = yield NotificationCollection.notificationsWithUID(self, uid, status=status, create=create)
+ if result:
+ self._notificationHomes["byUID"][status][uid] = result
+ self._notificationHomes["byID"][None][result.id()] = result
+ returnValue(result)
- @memoizedKey("rid", "_notificationHomes")
+
+ @inlineCallbacks
def notificationsWithResourceID(self, rid):
"""
Implement notificationsWithResourceID.
"""
- return NotificationCollection.notificationsWithResourceID(self, rid)
+ result = self._notificationHomes["byID"][None].get(rid)
+ if result is None:
+ result = yield NotificationCollection.notificationsWithResourceID(self, rid)
+ if result:
+ self._notificationHomes["byID"][None][rid] = result
+ self._notificationHomes["byUID"][result.status()][result.uid()] = result
+ returnValue(result)
+
def preCommit(self, operation):
"""
Run things before C{commit}. (Note: only provided by SQL
@@ -1968,6 +1985,10 @@
return self._authzUID
+ def status(self):
+ return self._status
+
+
def normal(self):
"""
Is this an normal (internal) home.
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -89,6 +89,16 @@
return self._txn.store().conduit.send_home_set_status(self, newStatus)
+ def setLocalStatus(self, newStatus):
+ """
+ Set the status on the object in the local store not the remote one.
+
+ @param newStatus: the new status to set
+ @type newStatus: L{int}
+ """
+ return super(CommonHomeExternal, self).setStatus(newStatus)
+
+
def external(self):
"""
Is this an external home.
@@ -517,4 +527,14 @@
def setStatus(self, newStatus):
return self._txn.store().conduit.send_notification_set_status(self, newStatus)
+
+ def setLocalStatus(self, newStatus):
+ """
+ Set the status on the object in the local store not the remote one.
+
+ @param newStatus: the new status to set
+ @type newStatus: L{int}
+ """
+ return super(NotificationCollectionExternal, self).setStatus(newStatus)
+
NotificationCollection._externalClass = NotificationCollectionExternal
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_notification.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -147,8 +147,8 @@
@classmethod
- def notificationsWithUID(cls, txn, uid, status=None, create=True):
- return cls.notificationsWith(txn, None, uid, status, create=create)
+ def notificationsWithUID(cls, txn, uid, status=None, create=False):
+ return cls.notificationsWith(txn, None, uid, status=status, create=create)
@classmethod
@@ -158,7 +158,7 @@
@classmethod
@inlineCallbacks
- def notificationsWith(cls, txn, rid, uid, status=None, create=True):
+ def notificationsWith(cls, txn, rid, uid, status=None, create=False):
"""
@param uid: I'm going to assume uid is utf-8 encoded bytes
"""
@@ -318,6 +318,10 @@
return self._ownerUID
+ def status(self):
+ return self._status
+
+
@inlineCallbacks
def setStatus(self, newStatus):
"""
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -601,7 +601,7 @@
notificationdata["supported-components"] = self.getSupportedComponents()
# Add to sharee's collection
- notifications = yield self._txn.notificationsWithUID(shareeView.viewerHome().uid())
+ notifications = yield self._txn.notificationsWithUID(shareeView.viewerHome().uid(), create=True)
yield notifications.writeNotificationObject(shareeView.shareUID(), notificationtype, notificationdata)
@@ -632,7 +632,7 @@
}
# Add to owner notification collection
- notifications = yield self._txn.notificationsWithUID(self.ownerHome().uid())
+ notifications = yield self._txn.notificationsWithUID(self.ownerHome().uid(), create=True)
yield notifications.writeNotificationObject(notificationUID, notificationtype, notificationdata)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_util.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -691,7 +691,7 @@
L{NotificationHome} when it has been retrieved.
"""
if homeType == ENOTIFICATIONTYPE:
- return txn.notificationsWithUID(uid, create=False)
+ return txn.notificationsWithUID(uid)
else:
return txn.homeWithUID(homeType, uid)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -939,6 +939,13 @@
@inlineCallbacks
+ def notificationCollectionUnderTest(self, txn=None, name="home1", status=None, create=False):
+ if txn is None:
+ txn = self.transactionUnderTest()
+ returnValue((yield txn.notificationsWithUID(name, status=status, create=create)))
+
+
+ @inlineCallbacks
def userRecordWithShortName(self, shortname):
record = yield self.directory.recordWithShortName(self.directory.recordType.user, shortname)
returnValue(record)
@@ -962,11 +969,13 @@
@inlineCallbacks
- def changeRecord(self, record, fieldname, value):
+ def changeRecord(self, record, fieldname, value, directory=None):
+ if directory is None:
+ directory = self.directory
fields = record.fields.copy()
fields[fieldname] = value
- updatedRecord = DirectoryRecord(self.directory, fields)
- yield self.directory.updateRecords((updatedRecord,))
+ updatedRecord = DirectoryRecord(directory, fields)
+ yield directory.updateRecords((updatedRecord,))
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/upgrade/sql/upgrades/test/test_notification_upgrade_from_0_to_1.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -169,7 +169,7 @@
)
for uid, notificationtype, _ignore_jtype, notificationdata, _ignore_jdata in data:
- notifications = yield self.transactionUnderTest().notificationsWithUID("user01")
+ notifications = yield self.transactionUnderTest().notificationsWithUID("user01", create=True)
yield notifications.writeNotificationObject(uid, notificationtype, notificationdata)
# Force data version to previous
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py 2015-03-04 22:48:03 UTC (rev 14506)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/who/test/test_group_sharees.py 2015-03-05 02:41:22 UTC (rev 14507)
@@ -84,7 +84,7 @@
@inlineCallbacks
def _check_notifications(self, uid, items):
- notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid)
+ notifyHome = yield self.transactionUnderTest().notificationsWithUID(uid, create=True)
notifications = yield notifyHome.listNotificationObjects()
self.assertEqual(set(notifications), set(items))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150304/44e9d80b/attachment-0001.html>
More information about the calendarserver-changes
mailing list