[CalendarServer-changes] [14519] CalendarServer/branches/users/cdaboo/pod2pod-migration
source_changes at macosforge.org
source_changes at macosforge.org
Fri Mar 6 12:13:59 PST 2015
Revision: 14519
http://trac.calendarserver.org//changeset/14519
Author: cdaboo at apple.com
Date: 2015-03-06 12:13:58 -0800 (Fri, 06 Mar 2015)
Log Message:
-----------
Detailed migration logging. Command line tool to drive migration.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/pod_migration.py
Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/pod_migration.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/pod_migration.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/pod_migration.py 2015-03-06 20:13:58 UTC (rev 14519)
@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# -*- test-case-name: calendarserver.tools.test.test_calverify -*-
+##
+# Copyright (c) 2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from __future__ import print_function
+
+"""
+This tool manages an overall pod migration. Migration is done in a series of steps,
+with the system admin triggering each step individually by running this tool.
+"""
+
+import os
+import sys
+
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.text import wordWrap
+from twisted.python.usage import Options, UsageError
+
+from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
+from twistedcaldav.timezones import TimezoneCache
+
+from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
+
+from twext.python.log import Logger
+from twext.who.idirectory import RecordType
+
+from calendarserver.tools.cmdline import utilityMain, WorkerService
+
+
+log = Logger()
+
+VERSION = "1"
+
+
+
+def usage(e=None):
+ if e:
+ print(e)
+ print("")
+ try:
+ PodMigrationOptions().opt_help()
+ except SystemExit:
+ pass
+ if e:
+ sys.exit(64)
+ else:
+ sys.exit(0)
+
+
+description = ''.join(
+ wordWrap(
+ """
+ Usage: calendarserver_pod_migration [options] [input specifiers]
+ """,
+ int(os.environ.get('COLUMNS', '80'))
+ )
+)
+description += "\nVersion: %s" % (VERSION,)
+
+
+
+class ConfigError(Exception):
+ pass
+
+
+
+class PodMigrationOptions(Options):
+ """
+ Command-line options for 'calendarserver_pod_migration'
+ """
+
+ synopsis = description
+
+ optFlags = [
+ ['verbose', 'v', "Verbose logging."],
+ ['debug', 'D', "Debug logging."],
+ ['step1', '1', "Run step 1 of the migration (initial sync)"],
+ ['step2', '2', "Run step 2 of the migration (incremental sync)"],
+ ['step3', '3', "Run step 3 of the migration (prepare for final sync)"],
+ ['step4', '4', "Run step 4 of the migration (final incremental sync)"],
+ ['step5', '5', "Run step 5 of the migration (final reconcile sync)"],
+ ['step6', '6', "Run step 6 of the migration (enable new home)"],
+ ['step7', '7', "Run step 7 of the migration (remove old home)"],
+ ]
+
+ optParameters = [
+ ['config', 'f', DEFAULT_CONFIG_FILE, "Specify caldavd.plist configuration path."],
+ ['uid', 'u', "", "Directory record uid of user to migrate [REQUIRED]"],
+ ]
+
+ longdesc = "Only one step option is allowed."
+
+ def __init__(self):
+ super(PodMigrationOptions, self).__init__()
+ self.outputName = '-'
+
+
+ def opt_output(self, filename):
+ """
+ Specify output file path (default: '-', meaning stdout).
+ """
+ self.outputName = filename
+
+ opt_o = opt_output
+
+
+ def openOutput(self):
+ """
+ Open the appropriate output file based on the '--output' option.
+ """
+ if self.outputName == '-':
+ return sys.stdout
+ else:
+ return open(self.outputName, 'wb')
+
+
+ def postOptions(self):
+ runstep = None
+ for step in range(7):
+ if self["step{}".format(step + 1)]:
+ if runstep is None:
+ runstep = step
+ self["runstep"] = step + 1
+ else:
+ raise UsageError("Only one step option allowed")
+ else:
+ if runstep is None:
+ raise UsageError("One step option must be present")
+ if not self["uid"]:
+ raise UsageError("A uid is required")
+
+
+
+class PodMigrationService(WorkerService, object):
+ """
+ Service which runs, does its stuff, then stops the reactor.
+ """
+
+ def __init__(self, store, options, output, reactor, config):
+ super(PodMigrationService, self).__init__(store)
+ self.options = options
+ self.output = output
+ self.reactor = reactor
+ self.config = config
+ TimezoneCache.create()
+
+
+ @inlineCallbacks
+ def doWork(self):
+ """
+ Do the work, stopping the reactor when done.
+ """
+ self.output.write("\n---- Pod Migration version: %s ----\n" % (VERSION,))
+
+ # Map short name to uid
+ record = yield self.store.directoryService().recordWithUID(self.options["uid"])
+ if record is None:
+ record = yield self.store.directoryService().recordWithShortName(RecordType.user, self.options["uid"])
+ if record is not None:
+ self.options["uid"] = record.uid
+
+ try:
+ yield getattr(self, "step{}".format(self.options["runstep"]))()
+ self.output.close()
+ except ConfigError:
+ pass
+ except:
+ log.failure("doWork()")
+
+
+ @inlineCallbacks
+ def step1(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 1\n")
+ yield syncer.sync()
+
+
+ @inlineCallbacks
+ def step2(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 2\n")
+ yield syncer.sync()
+
+
+ @inlineCallbacks
+ def step3(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 3\n")
+ yield syncer.disableRemoteHome()
+
+
+ @inlineCallbacks
+ def step4(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ final=True,
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 4\n")
+ yield syncer.sync()
+
+
+ @inlineCallbacks
+ def step5(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ final=True,
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 5\n")
+ yield syncer.finalSync()
+
+
+ @inlineCallbacks
+ def step6(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 6\n")
+ yield syncer.enableLocalHome()
+
+
+ @inlineCallbacks
+ def step7(self):
+ syncer = CrossPodHomeSync(
+ self.store,
+ self.options["uid"],
+ final=True,
+ uselog=self.output if self.options["verbose"] else None
+ )
+ syncer.accounting("Pod Migration Step 7\n")
+ yield syncer.removeRemoteHome()
+
+
+
+def main(argv=sys.argv, stderr=sys.stderr, reactor=None):
+ """
+ Do the export.
+ """
+ if reactor is None:
+ from twisted.internet import reactor
+ options = PodMigrationOptions()
+ try:
+ options.parseOptions(argv[1:])
+ except UsageError as e:
+ stderr.write("Invalid options specified\n")
+ options.opt_help()
+
+ try:
+ output = options.openOutput()
+ except IOError, e:
+ stderr.write("Unable to open output file for writing: %s\n" % (e))
+ sys.exit(1)
+
+
+ def makeService(store):
+ from twistedcaldav.config import config
+ config.TransactionTimeoutSeconds = 0
+ return PodMigrationService(store, options, output, reactor, config)
+
+ utilityMain(options['config'], makeService, reactor, verbose=options["debug"])
+
+if __name__ == '__main__':
+ main()
Property changes on: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/pod_migration.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/stdconfig.py 2015-03-06 19:34:30 UTC (rev 14518)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/stdconfig.py 2015-03-06 20:13:58 UTC (rev 14519)
@@ -405,6 +405,7 @@
"Implicit Errors": False,
"AutoScheduling": False,
"iSchedule": False,
+ "migration": False,
},
"AccountingPrincipals": [],
"AccountingLogRoot" : "accounting",
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-03-06 19:34:30 UTC (rev 14518)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-03-06 20:13:58 UTC (rev 14519)
@@ -19,6 +19,7 @@
from twext.python.log import Logger
from twisted.internet.defer import returnValue, inlineCallbacks
from twisted.python.failure import Failure
+from twistedcaldav.accounting import emitAccounting
from txdav.caldav.icalendarstore import ComponentUpdateState
from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord, \
CalendarObjectMigrationRecord, AttachmentMigrationRecord
@@ -30,9 +31,12 @@
from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
from uuid import uuid4
+import datetime
log = Logger()
+ACCOUNTING_TYPE = "migration"
+ACCOUNTING_LOG = "migration.log"
def inTransactionWrapper(operation):
"""
@@ -85,7 +89,7 @@
BATCH_SIZE = 50
- def __init__(self, store, diruid, final=False):
+ def __init__(self, store, diruid, final=False, uselog=None):
"""
@param store: the data store
@type store: L{CommonDataStore}
@@ -94,11 +98,14 @@
@param final: indicates whether this is in the final sync stage with the remote home
already disabled
@type final: L{bool}
+ @param uselog: additional logging written to this object
+ @type: L{File}
"""
self.store = store
self.diruid = diruid
self.disabledRemote = final
+ self.uselog = uselog
self.record = None
self.homeId = None
@@ -107,6 +114,12 @@
return "Cross-pod Migration Sync for {}: {}".format(self.diruid, detail)
+ def accounting(self, logstr):
+ emitAccounting(ACCOUNTING_TYPE, self.record, "{} {}\n".format(datetime.datetime.now().isoformat(), logstr), filename=ACCOUNTING_LOG)
+ if self.uselog is not None:
+ self.uselog.write("CrossPodHomeSync: {}\n".format(logstr))
+
+
@inlineCallbacks
def migrateHere(self):
"""
@@ -154,6 +167,7 @@
"""
yield self.loadRecord()
+ self.accounting("Starting: sync...")
yield self.prepareCalendarHome()
# Calendar list and calendar data
@@ -165,7 +179,9 @@
# Sync attachments
yield self.syncAttachments()
+ self.accounting("Completed: sync.\n")
+
@inlineCallbacks
def finalSync(self):
"""
@@ -174,6 +190,7 @@
"""
yield self.loadRecord()
+ self.accounting("Starting: finalSync...")
yield self.prepareCalendarHome()
# Link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
@@ -198,7 +215,9 @@
# TODO: work items
pass
+ self.accounting("Completed: finalSync.\n")
+
@inTransactionWrapper
@inlineCallbacks
def disableRemoteHome(self, txn):
@@ -207,6 +226,7 @@
"""
yield self.loadRecord()
+ self.accounting("Starting: disableRemoteHome...")
yield self.prepareCalendarHome()
# Calendar home
@@ -219,7 +239,9 @@
self.disabledRemote = True
+ self.accounting("Completed: disableRemoteHome.\n")
+
@inTransactionWrapper
@inlineCallbacks
def enableLocalHome(self, txn):
@@ -228,6 +250,7 @@
"""
yield self.loadRecord()
+ self.accounting("Starting: enableLocalHome...")
yield self.prepareCalendarHome()
# Disable any local external homes
@@ -249,7 +272,9 @@
# TODO: purge the old ones
pass
+ self.accounting("Completed: enableLocalHome.\n")
+
@inlineCallbacks
def removeRemoteHome(self):
"""
@@ -259,9 +284,12 @@
# TODO: implement API on CommonHome to purge the old data without
# any side-effects (scheduling, sharing etc).
yield self.loadRecord()
+ self.accounting("Starting: removeRemoteHome...")
yield self.prepareCalendarHome()
+ self.accounting("Completed: removeRemoteHome.\n")
+
@inlineCallbacks
def loadRecord(self):
"""
@@ -290,6 +318,7 @@
self.homeId = None
else:
home = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
+ self.accounting(" Created new home collection to migrate into.")
self.homeId = home.id() if home is not None else None
@@ -300,6 +329,7 @@
Make sure the home meta-data (alarms, default calendars) is properly sync'd
"""
+ self.accounting("Starting: syncCalendarHomeMetaData...")
remote_home = yield self._remoteHome(txn)
yield remote_home.readMetaData()
@@ -309,7 +339,9 @@
local_home = yield self._localHome(txn)
yield local_home.copyMetadata(remote_home, calendarIDMap)
+ self.accounting("Completed: syncCalendarHomeMetaData.")
+
@inlineCallbacks
def _remoteHome(self, txn):
"""
@@ -350,11 +382,15 @@
Synchronize each owned calendar.
"""
+ self.accounting("Starting: syncCalendarList...")
+
# Remote sync details
remote_sync_state = yield self.getCalendarSyncList()
+ self.accounting(" Found {} remote calendars to sync.".format(len(remote_sync_state)))
# Get local sync details from local DB
local_sync_state = yield self.getSyncState()
+ self.accounting(" Found {} local calendars to sync.".format(len(local_sync_state)))
# Remove local calendars no longer on the remote side
yield self.purgeLocal(local_sync_state, remote_sync_state)
@@ -363,7 +399,9 @@
for remoteID in remote_sync_state.keys():
yield self.syncCalendar(remoteID, local_sync_state, remote_sync_state)
+ self.accounting("Completed: syncCalendarList.")
+
@inTransactionWrapper
@inlineCallbacks
def getCalendarSyncList(self, txn):
@@ -433,11 +471,12 @@
@type remote_sync_state: L{dict}
"""
home = yield self._localHome(txn)
- for remoteID in set(local_sync_state.keys()) - set(remote_sync_state.keys()):
- calendar = yield home.childWithID(local_sync_state[remoteID].localResourceID)
+ for localID in set(local_sync_state.keys()) - set(remote_sync_state.keys()):
+ calendar = yield home.childWithID(local_sync_state[localID].localResourceID)
if calendar is not None:
yield calendar.purge()
- del local_sync_state[remoteID]
+ del local_sync_state[localID]
+ self.accounting(" Purged calendar local-id={} that no longer exists on the remote pod.".format(localID))
@inlineCallbacks
@@ -454,6 +493,8 @@
@type remote_sync_state: L{dict}
"""
+ self.accounting("Starting: syncCalendar.")
+
# See if we need to create the local one first
if remoteID not in local_sync_state:
localID = yield self.newCalendar()
@@ -463,6 +504,10 @@
localResourceID=localID,
lastSyncToken=None,
)
+ self.accounting(" Created new calendar local-id={}, remote-id={}.".format(localID, remoteID))
+ else:
+ localID = local_sync_state.get(remoteID).localResourceID
+ self.accounting(" Updating calendar local-id={}, remote-id={}.".format(localID, remoteID))
local_record = local_sync_state.get(remoteID)
remote_token = remote_sync_state[remoteID].lastSyncToken
@@ -471,11 +516,13 @@
yield self.syncCalendarMetaData(local_record)
# Sync object resources
- changed, deleted = yield self.findObjectsToSync(local_record)
- yield self.purgeDeletedObjectsInBatches(local_record, deleted)
+ changed, removed = yield self.findObjectsToSync(local_record)
+ self.accounting(" Calendar objects changed={}, removed={}.".format(len(changed), len(removed)))
+ yield self.purgeDeletedObjectsInBatches(local_record, removed)
yield self.updateChangedObjectsInBatches(local_record, changed)
yield self.updateSyncState(local_record, remote_token)
+ self.accounting("Completed: syncCalendar.")
@inTransactionWrapper
@@ -500,6 +547,7 @@
@param migrationRecord: current migration record
@type localID: L{CalendarMigrationRecord}
"""
+
# Remote changes
remote_home = yield self._remoteHome(txn)
remote_calendar = yield remote_home.childWithID(migrationRecord.remoteResourceID)
@@ -510,6 +558,7 @@
local_home = yield self._localHome(txn)
local_calendar = yield local_home.childWithID(migrationRecord.localResourceID)
yield local_calendar.copyMetadata(remote_calendar)
+ self.accounting(" Copied calendar meta-data for calendar local-id={0.localResourceID}, remote-id={0.remoteResourceID}.".format(migrationRecord))
@inTransactionWrapper
@@ -592,6 +641,7 @@
for local_object in local_objects:
yield local_object.purge()
+ self.accounting(" Purged calendar object local-id={}.".format(local_object.id()))
@inlineCallbacks
@@ -660,6 +710,7 @@
local_object = yield local_objects[obj_name]
yield local_object._setComponentInternal(remote_data, internal_state=ComponentUpdateState.RAW)
del local_objects[obj_name]
+ log_op = "Updated"
else:
local_object = yield local_calendar._createCalendarObjectWithNameInternal(obj_name, remote_data, internal_state=ComponentUpdateState.RAW)
@@ -671,13 +722,16 @@
remoteResourceID=remote_object.id(),
localResourceID=local_object.id()
)
+ log_op = "Created"
# Sync meta-data such as schedule object, schedule tags, access mode etc
yield local_object.copyMetadata(remote_object)
+ self.accounting(" {} calendar object local-id={}, remote-id={}.".format(log_op, local_object.id(), remote_object.id()))
# Purge the ones that remain
for local_object in local_objects.values():
yield local_object.purge()
+ self.accounting(" Purged calendar object local-id={}.".format(local_object.id()))
@inlineCallbacks
@@ -686,12 +740,17 @@
Sync attachments (both metadata and actual attachment data) for the home being migrated.
"""
+ self.accounting("Starting: syncAttachments...")
+
# Two steps - sync the table first in one txn, then sync each attachment's data
changed_ids, removed_ids = yield self.syncAttachmentTable()
+ self.accounting(" Attachments changed={}, removed={}".format(len(changed_ids), len(removed_ids)))
for local_id in changed_ids:
yield self.syncAttachmentData(local_id)
+ self.accounting("Completed: syncAttachments.")
+
returnValue((changed_ids, removed_ids,))
@@ -772,6 +831,7 @@
if records:
# Read the data from the conduit
yield remote_home.readAttachmentData(records[0].remoteResourceID, attachment)
+ self.accounting(" Read attachment local-id={0.localResourceID}, remote-id={0.remoteResourceID}".format(records[0]))
@inlineCallbacks
@@ -780,8 +840,11 @@
Link attachments to the calendar objects they belong to.
"""
+ self.accounting("Starting: linkAttachments...")
+
# Get the map of links for the remote home
links = yield self.getAttachmentLinks()
+ self.accounting(" Linking {} attachments".format(len(links)))
# Get remote->local ID mappings
attachmentIDMap, objectIDMap = yield self.getAttachmentMappings()
@@ -792,6 +855,8 @@
yield self.makeAttachmentLinks(links[:50], attachmentIDMap, objectIDMap)
links = links[50:]
+ self.accounting("Completed: linkAttachments.")
+
returnValue(len_links)
@@ -854,11 +919,15 @@
a fake directory UID locally.
"""
+ self.accounting("Starting: delegateReconcile...")
+
yield self.individualDelegateReconcile()
yield self.groupDelegateReconcile()
yield self.externalDelegateReconcile()
+ self.accounting("Completed: delegateReconcile.")
+
@inTransactionWrapper
@inlineCallbacks
def individualDelegateReconcile(self, txn):
@@ -870,7 +939,9 @@
for record in remote_records:
yield record.insert(txn)
+ self.accounting(" Found {} individual delegates".format(len(remote_records)))
+
@inTransactionWrapper
@inlineCallbacks
def groupDelegateReconcile(self, txn):
@@ -885,7 +956,9 @@
delegator.groupID = local_group.groupID
yield delegator.insert(txn)
+ self.accounting(" Found {} group delegates".format(len(remote_records)))
+
@inTransactionWrapper
@inlineCallbacks
def externalDelegateReconcile(self, txn):
@@ -897,15 +970,20 @@
for record in remote_records:
yield record.insert(txn)
+ self.accounting(" Found {} external delegates".format(len(remote_records)))
+
@inlineCallbacks
def groupAttendeeReconcile(self):
"""
Sync the remote group attendee links to the local store.
"""
+ self.accounting("Starting: groupAttendeeReconcile...")
+
# Get remote data and local mapping information
remote_group_attendees, objectIDMap = yield self.groupAttendeeData()
+ self.accounting(" Found {} group attendees".format(len(remote_group_attendees)))
# Map each result to a local resource (in batches)
number_of_links = len(remote_group_attendees)
@@ -913,6 +991,8 @@
yield self.groupAttendeeProcess(remote_group_attendees[:50], objectIDMap)
remote_group_attendees = remote_group_attendees[50:]
+ self.accounting("Completed: groupAttendeeReconcile.")
+
returnValue(number_of_links)
@@ -957,7 +1037,9 @@
Sync all the existing L{NotificationObject} resources from the remote store.
"""
+ self.accounting("Starting: notificationsReconcile...")
records = yield self.notificationRecords()
+ self.accounting(" Found {} notifications".format(len(records)))
# Batch setting resources for the local home
len_records = len(records)
@@ -965,6 +1047,8 @@
yield self.makeNotifications(records[:50])
records = records[50:]
+ self.accounting("Completed: notificationsReconcile.")
+
returnValue(len_records)
@@ -998,7 +1082,8 @@
for record in records:
# Do this via the "write" API so that sync revisions are updated properly, rather than just
# inserting the records directly.
- yield notifications.writeNotificationObject(record.notificationUID, record.notificationType, record.notificationData)
+ notification = yield notifications.writeNotificationObject(record.notificationUID, record.notificationType, record.notificationData)
+ self.accounting(" Added notification local-id={}.".format(notification.id()))
@inlineCallbacks
@@ -1019,6 +1104,7 @@
A -> C | B -> C (new) | (removed)
"""
+ self.accounting("Starting: sharedByCollectionsReconcile...")
calendars = yield self.getSyncState()
len_records = 0
@@ -1028,6 +1114,10 @@
continue
records = records.items()
+ self.accounting(" Found shared by calendar local-id={0.localResourceID}, remote-id={0.remoteResourceID} with {1} sharees".format(
+ calendar, len(records),
+ ))
+
# Batch setting resources for the local home
len_records += len(records)
while records:
@@ -1040,6 +1130,8 @@
# Update the remote pod to switch over the shares
yield self.updatedRemoteSharedByCollections(calendar.remoteResourceID, bindUID)
+ self.accounting("Completed: sharedByCollectionsReconcile.")
+
returnValue(len_records)
@@ -1093,12 +1185,14 @@
calendarResourceID=calendar_id,
bindRevision=0,
)
+ self.accounting(" Updating existing sharee {}".format(shareeHome.uid()))
else:
# Map the record resource ids and insert a new record
record.calendarHomeResourceID = shareeHome.id()
record.calendarResourceID = calendar_id
record.bindRevision = 0
yield record.insert(txn)
+ self.accounting(" Adding new sharee {}".format(shareeHome.uid()))
@inTransactionWrapper
@@ -1116,6 +1210,7 @@
share.groupID = local_group.groupID
share.calendarID = local_id
yield share.insert(txn)
+ self.accounting(" Adding group sharee {}".format(local_group.groupUID))
@inTransactionWrapper
@@ -1128,6 +1223,7 @@
remote_home = yield self._remoteHome(txn)
remote_calendar = yield remote_home.childWithID(remote_id)
records = yield remote_calendar.migrateBindRecords(bindUID)
+ self.accounting(" Updating remote records")
returnValue(records)
@@ -1147,14 +1243,20 @@
B -> A | B -> B (modify existing) | (removed)
C -> A | C -> B (new) | (removed)
"""
+
+ self.accounting("Starting: sharedToCollectionsReconcile...")
+
records = yield self.sharedToCollectionRecords()
records = records.items()
len_records = len(records)
+ self.accounting(" Found {} shared to collections".format(len_records))
while records:
yield self.makeSharedToCollections(records[:50])
records = records[50:]
+ self.accounting("Completed: sharedToCollectionsReconcile.")
+
returnValue(len_records)
@@ -1198,6 +1300,7 @@
yield oldrecord[0].update(
calendarHomeResourceID=self.homeId,
)
+ self.accounting(" Updated existing local sharer record {}".format(sharerHome.uid()))
else:
raise AssertionError("An existing share must be present")
else:
@@ -1217,6 +1320,7 @@
if oldrecord:
# Map the record resource ids and insert a new record
calendar_id = oldrecord.calendarResourceID
+ log_op = "Updated"
else:
sharerView = yield sharerHome.createCollectionForExternalShare(
ownerRecord.calendarResourceName,
@@ -1224,11 +1328,13 @@
metadataRecord.supportedComponents,
)
calendar_id = sharerView.id()
+ log_op = "Created"
shareeRecord.calendarHomeResourceID = self.homeId
shareeRecord.calendarResourceID = calendar_id
shareeRecord.bindRevision = 0
yield shareeRecord.insert(txn)
+ self.accounting(" {} remote sharer record {}".format(log_op, sharerHome.uid()))
yield self.updatedRemoteSharedToCollection(remote_id, txn=txn)
@@ -1243,4 +1349,5 @@
remote_home = yield self._remoteHome(txn)
remote_calendar = yield remote_home.childWithID(remote_id)
records = yield remote_calendar.migrateBindRecords(None)
+ self.accounting(" Updating remote records")
returnValue(records)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-03-06 19:34:30 UTC (rev 14518)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-03-06 20:13:58 UTC (rev 14519)
@@ -32,7 +32,8 @@
ExternalDelegateGroupsRecord, DelegateGroupsRecord, GroupsRecord
from txdav.common.datastore.sql_notification import NotificationCollection
from txdav.common.datastore.sql_tables import schema, _HOME_STATUS_EXTERNAL, \
- _BIND_MODE_READ, _HOME_STATUS_MIGRATING
+ _BIND_MODE_READ, _HOME_STATUS_MIGRATING, _HOME_STATUS_NORMAL, \
+ _HOME_STATUS_DISABLED
from txdav.common.datastore.test.util import populateCalendarsFrom
from txdav.who.delegates import Delegates
from txweb2.http_headers import MimeType
@@ -981,7 +982,35 @@
yield self.commitTransaction(1)
+ @inlineCallbacks
+ def test_disable_remote_home(self):
+ """
+ Test that L{disableRemoteHome} changes the remote status and prevents a normal state
+ home from being created.
+ """
+ # Create remote home - and add some fake notifications
+ yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ yield self.theTransactionUnderTest(0).notificationsWithUID("user01", create=True)
+ yield self.commitTransaction(0)
+
+ # Sync from remote side
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ yield syncer.prepareCalendarHome()
+ yield syncer.disableRemoteHome()
+
+ # It is disabled
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01")
+ self.assertTrue(home is None)
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_NORMAL)
+ self.assertTrue(home is None)
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_DISABLED)
+ self.assertTrue(home is not None)
+ yield self.commitTransaction(0)
+
+
+
class TestSharingSync(MultiStoreConduitTest):
"""
Test that L{CrossPodHomeSync} sharing sync works.
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py 2015-03-06 19:34:30 UTC (rev 14518)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_migration.py 2015-03-06 20:13:58 UTC (rev 14519)
@@ -72,6 +72,8 @@
config.GroupAttendees.Enabled = True
config.GroupAttendees.ReconciliationDelaySeconds = 0
config.GroupAttendees.AutoUpdateSecondsFromNow = 0
+ config.AccountingCategories.migration = True
+ config.AccountingPrincipals = ["*"]
@inlineCallbacks
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150306/dfd5db13/attachment-0001.html>
More information about the calendarserver-changes
mailing list