<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14481] CalendarServer/branches/users/cdaboo/pod2pod-migration</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14481">14481</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-02-27 11:59:39 -0800 (Fri, 27 Feb 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Checkpoint: final sharing state sync. Also includes major re-factor of the way home objects are created to allow for multiple with the same onwerUID but different status.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationcalendarservertoolsexportpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/export.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtwistedcaldavtesttest_wrappingpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/test/test_wrapping.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavbasedatastoreutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/base/datastore/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sql_sharingpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorefilepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtesttest_store_apipy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentoracledialectsql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_51_to_52sql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_51_to_52sql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_sharingpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_tablespy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoretestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoreworktesttest_revision_cleanuppy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/work/test/test_revision_cleanup.py</a></li>
</ul>
<h3>Removed Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoreschedulepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/schedule.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretesttest_schedulepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_schedule.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationcalendarservertoolsexportpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/export.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/export.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/calendarserver/tools/export.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -203,7 +203,7 @@
</span><span class="cx"> for this calendar home.
</span><span class="cx"> """
</span><span class="cx"> uid = yield self.getHomeUID(exportService)
</span><del>- home = yield txn.calendarHomeWithUID(uid, True)
</del><ins>+ home = yield txn.calendarHomeWithUID(uid, create=True)
</ins><span class="cx"> result = []
</span><span class="cx"> if self.collections:
</span><span class="cx"> for collection in self.collections:
</span><span class="lines">@@ -303,6 +303,7 @@
</span><span class="cx"> fileobj.write(comp.getTextWithTimezones(True))
</span><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def exportToDirectory(calendars, dirname):
</span><span class="cx"> """
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtwistedcaldavtesttest_wrappingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/test/test_wrapping.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/test/test_wrapping.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/twistedcaldav/test/test_wrapping.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -120,7 +120,7 @@
</span><span class="cx"> record = yield self.directory.recordWithShortName(RecordType.user, u"wsanchez")
</span><span class="cx"> uid = record.uid
</span><span class="cx"> txn = self.transactionUnderTest()
</span><del>- home = yield txn.calendarHomeWithUID(uid, True)
</del><ins>+ home = yield txn.calendarHomeWithUID(uid, create=True)
</ins><span class="cx"> cal = yield home.calendarWithName("calendar")
</span><span class="cx"> yield cal.createCalendarObjectWithName(objectName, VComponent.fromString(objectText))
</span><span class="cx"> yield self.commit()
</span><span class="lines">@@ -139,7 +139,7 @@
</span><span class="cx"> record = yield self.directory.recordWithShortName(RecordType.user, u"wsanchez")
</span><span class="cx"> uid = record.uid
</span><span class="cx"> txn = self.transactionUnderTest()
</span><del>- home = yield txn.addressbookHomeWithUID(uid, True)
</del><ins>+ home = yield txn.addressbookHomeWithUID(uid, create=True)
</ins><span class="cx"> adbk = yield home.addressbookWithName("addressbook")
</span><span class="cx"> yield adbk.createAddressBookObjectWithName(objectName, VCComponent.fromString(objectText))
</span><span class="cx"> yield self.commit()
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavbasedatastoreutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/base/datastore/util.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/base/datastore/util.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/base/datastore/util.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -86,6 +86,18 @@
</span><span class="cx"> return self.delete(key)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ # Home objects by UID
+
+ def keyForHomeWithUID(self, homeType, ownerUID, status):
+ return "homeWithUID:%s:%s:%s" % (homeType, status, ownerUID)
+
+
+ # Home objects by id
+
+ def keyForHomeWithID(self, homeType, homeResourceID, status):
+ return "homeWithID:%s:%s:%s" % (homeType, status, homeResourceID)
+
+
</ins><span class="cx"> # Home child objects by name
</span><span class="cx">
</span><span class="cx"> def keyForObjectWithName(self, homeResourceID, name):
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoreschedulepy"></a>
<div class="delfile"><h4>Deleted: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/schedule.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/schedule.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/schedule.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -1,214 +0,0 @@
</span><del>-# -*- test-case-name: txdav.caldav.datastore.test.test_scheduling -*-
-##
-# Copyright (c) 2010-2015 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-from zope.interface.declarations import implements
-from txdav.caldav.icalendarstore import ICalendarHome, ICalendar, ICalendarObject, \
- ICalendarTransaction, ICalendarStore
-
-from twisted.python.util import FancyEqMixin
-from twisted.python.components import proxyForInterface
-from twisted.internet.defer import inlineCallbacks, returnValue
-
-
-
-class ImplicitTransaction(
- proxyForInterface(ICalendarTransaction,
- originalAttribute="_transaction")):
- """
- Wrapper around an L{ICalendarStoreTransaction}.
- """
-
- def __init__(self, transaction):
- """
- Initialize an L{ImplicitTransaction}.
-
- @type transaction: L{ICalendarStoreTransaction}
- """
- self._transaction = transaction
-
-
- @inlineCallbacks
- def calendarHomeWithUID(self, uid, create=False):
- # FIXME: 'create' flag
- newHome = yield super(ImplicitTransaction, self).calendarHomeWithUID(uid, create)
-# return ImplicitCalendarHome(newHome, self)
- if newHome is None:
- returnValue(None)
- else:
- # FIXME: relay transaction
- returnValue(ImplicitCalendarHome(newHome, None))
-
-
-
-class ImplicitCalendarHome(proxyForInterface(ICalendarHome, "_calendarHome")):
-
- implements(ICalendarHome)
-
- def __init__(self, calendarHome, transaction):
- """
- Initialize L{ImplicitCalendarHome} with an underlying
- calendar home and L{ImplicitTransaction}.
- """
- self._calendarHome = calendarHome
- self._transaction = transaction
-
-
-# def properties(self):
-# # FIXME: wrap?
-# return self._calendarHome.properties()
-
- @inlineCallbacks
- def calendars(self):
- superCalendars = (yield super(ImplicitCalendarHome, self).calendars())
- wrapped = []
- for calendar in superCalendars:
- wrapped.append(ImplicitCalendar(self, calendar))
- returnValue(wrapped)
-
-
- @inlineCallbacks
- def loadCalendars(self):
- superCalendars = (yield super(ImplicitCalendarHome, self).loadCalendars())
- wrapped = []
- for calendar in superCalendars:
- wrapped.append(ImplicitCalendar(self, calendar))
- returnValue(wrapped)
-
-
- def createCalendarWithName(self, name):
- self._calendarHome.createCalendarWithName(name)
-
-
- def removeCalendarWithName(self, name):
- self._calendarHome.removeCalendarWithName(name)
-
-
- @inlineCallbacks
- def calendarWithName(self, name):
- calendar = yield self._calendarHome.calendarWithName(name)
- if calendar is not None:
- returnValue(ImplicitCalendar(self, calendar))
- else:
- returnValue(None)
-
-
- def hasCalendarResourceUIDSomewhereElse(self, uid, ok_object, type):
- return self._calendarHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object, type)
-
-
- def getCalendarResourcesForUID(self, uid):
- return self._calendarHome.getCalendarResourcesForUID(uid)
-
-
-
-class ImplicitCalendarObject(object):
- implements(ICalendarObject)
-
- def setComponent(self, component):
- pass
-
-
- def component(self):
- pass
-
-
- def uid(self):
- pass
-
-
- def componentType(self):
- pass
-
-
- def organizer(self):
- pass
-
-
- def properties(self):
- pass
-
-
-
-class ImplicitCalendar(FancyEqMixin,
- proxyForInterface(ICalendar, "_subCalendar")):
-
- compareAttributes = (
- "_subCalendar",
- "_parentHome",
- )
-
- def __init__(self, parentHome, subCalendar):
- self._parentHome = parentHome
- self._subCalendar = subCalendar
- self._supportedComponents = None
-
-# def ownerCalendarHome(self):
-# return self._parentHome
-# def calendarObjects(self):
-# # FIXME: wrap
-# return self._subCalendar.calendarObjects()
-# def calendarObjectWithUID(self, uid): ""
-# def createCalendarObjectWithName(self, name, component):
-# # FIXME: implement most of StoreCalendarObjectResource here!
-# self._subCalendar.createCalendarObjectWithName(name, component)
-# def syncToken(self): ""
-# def calendarObjectsInTimeRange(self, start, end, timeZone): ""
-# def calendarObjectsSinceToken(self, token): ""
-# def properties(self):
-# # FIXME: probably need to wrap this as well
-# return self._subCalendar.properties()
-#
-# def calendarObjectWithName(self, name):
-# #FIXME: wrap
-# return self._subCalendar.calendarObjectWithName(name)
-
-
- def _createCalendarObjectWithNameInternal(self, name, component, internal_state, options=None):
- return self.createCalendarObjectWithName(name, component, options)
-
-
- def setSupportedComponents(self, supported_components):
- """
- Update the database column with the supported components. Technically this should only happen once
- on collection creation, but for migration we may need to change after the fact - hence a separate api.
- """
- self._supportedComponents = supported_components
-
-
- def getSupportedComponents(self):
- return self._supportedComponents
-
-
-
-class ImplicitStore(proxyForInterface(ICalendarStore, "_calendarStore")):
- """
- This is a wrapper around an L{ICalendarStore} that implements implicit
- scheduling.
- """
-
- def __init__(self, calendarStore):
- """
- Create an L{ImplicitStore} wrapped around another
- L{ICalendarStore} provider.
- """
- self._calendarStore = calendarStore
-
-
- def newTransaction(self, label="unlabeled"):
- """
- Wrap an underlying L{ITransaction}.
- """
- return ImplicitTransaction(self._calendarStore.newTransaction(label))
</del></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -52,7 +52,6 @@
</span><span class="cx"> pyCalendarTodatetime, parseSQLDateToPyCalendar
</span><span class="cx"> from twistedcaldav.ical import Component, InvalidICalendarDataError, Property
</span><span class="cx"> from twistedcaldav.instance import InvalidOverriddenInstanceError
</span><del>-from twistedcaldav.memcacher import Memcacher
</del><span class="cx"> from twistedcaldav.timezones import TimezoneException
</span><span class="cx">
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="lines">@@ -445,8 +444,6 @@
</span><span class="cx"> _notifierPrefix = "CalDAV"
</span><span class="cx"> _dataVersionKey = "CALENDAR-DATAVERSION"
</span><span class="cx">
</span><del>- _cacher = Memcacher("SQL.calhome", pickle=True, key_normalization=False)
-
</del><span class="cx"> _componentCalendarName = {
</span><span class="cx"> "VEVENT": "calendar",
</span><span class="cx"> "VTODO": "tasks",
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -14,19 +14,19 @@
</span><span class="cx"> # See the License for the specific language governing permissions and
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><del>-from txdav.common.datastore.sql_directory import GroupsRecord
-from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord
-from txdav.caldav.datastore.sql_attachment import Attachment, AttachmentLink
</del><span class="cx"> """
</span><span class="cx"> SQL backend for CalDAV storage when resources are external.
</span><span class="cx"> """
</span><span class="cx">
</span><del>-from twisted.internet.defer import succeed, inlineCallbacks, returnValue
</del><ins>+from twisted.internet.defer import inlineCallbacks, returnValue
</ins><span class="cx">
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.sql import CalendarHome, Calendar, CalendarObject
</span><ins>+from txdav.caldav.datastore.sql_attachment import Attachment, AttachmentLink
+from txdav.caldav.datastore.sql_directory import GroupAttendeeRecord
</ins><span class="cx"> from txdav.caldav.icalendarstore import ComponentUpdateState, ComponentRemoveState
</span><ins>+from txdav.common.datastore.sql_directory import GroupsRecord
</ins><span class="cx"> from txdav.common.datastore.sql_external import CommonHomeExternal, CommonHomeChildExternal, \
</span><span class="cx"> CommonObjectResourceExternal
</span><span class="cx">
</span><span class="lines">@@ -37,10 +37,10 @@
</span><span class="cx"> Wrapper for a CalendarHome that is external and only supports a limited set of operations.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def __init__(self, transaction, ownerUID, resourceID):
</del><ins>+ def __init__(self, transaction, homeData):
</ins><span class="cx">
</span><del>- CalendarHome.__init__(self, transaction, ownerUID)
- CommonHomeExternal.__init__(self, transaction, ownerUID, resourceID)
</del><ins>+ CalendarHome.__init__(self, transaction, homeData)
+ CommonHomeExternal.__init__(self, transaction, homeData)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def hasCalendarResourceUIDSomewhereElse(self, uid, ok_object, mode):
</span><span class="lines">@@ -126,13 +126,6 @@
</span><span class="cx"> returnValue([(GroupAttendeeRecord.deserialize(item[0]), GroupsRecord.deserialize(item[1]),) for item in raw_results])
</span><span class="cx">
</span><span class="cx">
</span><del>- def createdHome(self):
- """
- No children - make this a no-op.
- """
- return succeed(None)
-
-
</del><span class="cx"> def splitCalendars(self):
</span><span class="cx"> """
</span><span class="cx"> No children.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoretesttest_schedulepy"></a>
<div class="delfile"><h4>Deleted: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_schedule.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_schedule.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/test/test_schedule.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -1,70 +0,0 @@
</span><del>-##
-# Copyright (c) 2010-2015 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-Tests for L{txdav.caldav.datastore.scheduling}.
-
-The aforementioned module is intended to eventually support implicit
-scheduling; however, it does not currently. The interim purpose of this module
-and accompanying tests is to effectively test the interface specifications to
-make sure that the common tests don't require anything I{not} specified in the
-interface, so that dynamic proxies specified with a tool like
-C{proxyForInterface} can be used to implement features such as implicit
-scheduling or data caching as middleware in the data-store layer.
-"""
-
-from twisted.trial.unittest import TestCase, SkipTest
-from txdav.caldav.datastore.test.test_file import FileStorageTests
-from txdav.caldav.datastore.schedule import ImplicitStore
-
-simpleEvent = """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER:mailto:user1@example.com
-ATTENDEE:mailto:user1@example.com
-ATTENDEE:mailto:user2@example.com
-END:VEVENT
-END:VCALENDAR
-"""
-
-class ImplicitStoreTests(FileStorageTests, TestCase):
- """
- Tests for L{ImplicitSchedulingStore}.
- """
-
- implicitStore = None
-
- def storeUnderTest(self):
- if self.implicitStore is None:
- sut = super(ImplicitStoreTests, self).storeUnderTest()
- self.implicitStore = ImplicitStore(sut)
- return self.implicitStore
-
-
- def skipit(self):
- raise SkipTest("No private attribute tests.")
-
- test_calendarObjectsWithDotFile = skipit
- test_countComponentTypes = skipit
- test_init = skipit
- test_calendarObjectsWithDirectory = skipit
- test_hasCalendarResourceUIDSomewhereElse = skipit
-
-del FileStorageTests
</del></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -40,7 +40,6 @@
</span><span class="cx"> from twisted.python import hashlib
</span><span class="cx">
</span><span class="cx"> from twistedcaldav.config import config
</span><del>-from twistedcaldav.memcacher import Memcacher
</del><span class="cx"> from twistedcaldav.vcard import Component as VCard, InvalidVCardDataError, Property, \
</span><span class="cx"> vCardProductID
</span><span class="cx">
</span><span class="lines">@@ -86,13 +85,12 @@
</span><span class="cx">
</span><span class="cx"> _notifierPrefix = "CardDAV"
</span><span class="cx"> _dataVersionKey = "ADDRESSBOOK-DATAVERSION"
</span><del>- _cacher = Memcacher("SQL.adbkhome", pickle=True, key_normalization=False)
</del><span class="cx">
</span><span class="cx">
</span><del>- def __init__(self, transaction, ownerUID, authzUID=None):
</del><ins>+ def __init__(self, transaction, homeData, authzUID=None):
</ins><span class="cx">
</span><del>- super(AddressBookHome, self).__init__(transaction, ownerUID, authzUID=authzUID)
</del><span class="cx"> self._addressbookPropertyStoreID = None
</span><ins>+ super(AddressBookHome, self).__init__(transaction, homeData, authzUID=authzUID)
</ins><span class="cx"> self._addressbook = None
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -118,6 +116,7 @@
</span><span class="cx"> return (
</span><span class="cx"> cls._homeSchema.RESOURCE_ID,
</span><span class="cx"> cls._homeSchema.OWNER_UID,
</span><ins>+ cls._homeSchema.STATUS,
</ins><span class="cx"> cls._homeSchema.ADDRESSBOOK_PROPERTY_STORE_ID,
</span><span class="cx"> )
</span><span class="cx">
</span><span class="lines">@@ -133,19 +132,20 @@
</span><span class="cx"> return (
</span><span class="cx"> "_resourceID",
</span><span class="cx"> "_ownerUID",
</span><ins>+ "_status",
</ins><span class="cx"> "_addressbookPropertyStoreID",
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def initFromStore(self, no_cache=False):
</del><ins>+ def initFromStore(self):
</ins><span class="cx"> """
</span><span class="cx"> Initialize this object from the store. We read in and cache all the
</span><span class="cx"> extra meta-data from the DB to avoid having to do DB queries for those
</span><span class="cx"> individually later.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- result = yield super(AddressBookHome, self).initFromStore(no_cache)
</del><ins>+ result = yield super(AddressBookHome, self).initFromStore()
</ins><span class="cx"> if result is not None:
</span><span class="cx"> # Created owned address book
</span><span class="cx"> addressbook = AddressBook(
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql_external.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql_external.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/sql_external.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -18,8 +18,6 @@
</span><span class="cx"> SQL backend for CardDAV storage when resources are external.
</span><span class="cx"> """
</span><span class="cx">
</span><del>-from twisted.internet.defer import succeed
-
</del><span class="cx"> from twext.python.log import Logger
</span><span class="cx">
</span><span class="cx"> from txdav.carddav.datastore.sql import AddressBookHome, AddressBook, \
</span><span class="lines">@@ -31,10 +29,10 @@
</span><span class="cx">
</span><span class="cx"> class AddressBookHomeExternal(CommonHomeExternal, AddressBookHome):
</span><span class="cx">
</span><del>- def __init__(self, transaction, ownerUID, resourceID):
</del><ins>+ def __init__(self, transaction, homeData):
</ins><span class="cx">
</span><del>- AddressBookHome.__init__(self, transaction, ownerUID)
- CommonHomeExternal.__init__(self, transaction, ownerUID, resourceID)
</del><ins>+ AddressBookHome.__init__(self, transaction, homeData)
+ CommonHomeExternal.__init__(self, transaction, homeData)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def hasAddressBookResourceUIDSomewhereElse(self, uid, ok_object, mode):
</span><span class="lines">@@ -51,13 +49,6 @@
</span><span class="cx"> raise AssertionError("CommonHomeExternal: not supported")
</span><span class="cx">
</span><span class="cx">
</span><del>- def createdHome(self):
- """
- No children - make this a no-op.
- """
- return succeed(None)
-
-
</del><span class="cx"> def addressbook(self):
</span><span class="cx"> """
</span><span class="cx"> No children.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -70,7 +70,7 @@
</span><span class="cx"> populateTxn = self.storeUnderTest().newTransaction()
</span><span class="cx"> for homeUID in self.requirements:
</span><span class="cx"> addressbooks = self.requirements[homeUID]
</span><del>- home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.addressbookHomeWithUID(homeUID, create=True)
</ins><span class="cx"> if addressbooks is not None:
</span><span class="cx"> addressbook = home.addressbook()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcarddavdatastoretesttest_sql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/carddav/datastore/test/test_sql_sharing.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -45,7 +45,7 @@
</span><span class="cx"> for homeUID in self.requirements:
</span><span class="cx"> addressbooks = self.requirements[homeUID]
</span><span class="cx"> if addressbooks is not None:
</span><del>- home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.addressbookHomeWithUID(homeUID, create=True)
</ins><span class="cx"> addressbook = home.addressbook()
</span><span class="cx">
</span><span class="cx"> addressbookObjNames = addressbooks[addressbook.name()]
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorefilepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/file.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -334,15 +334,15 @@
</span><span class="cx"> CommonStoreTransaction._homeClass[EADDRESSBOOKTYPE] = AddressBookHome
</span><span class="cx">
</span><span class="cx">
</span><del>- def calendarHomeWithUID(self, uid, create=False):
- return self.homeWithUID(ECALENDARTYPE, uid, create=create)
</del><ins>+ def calendarHomeWithUID(self, uid, status=None, create=False):
+ return self.homeWithUID(ECALENDARTYPE, uid, status=status, create=create)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def addressbookHomeWithUID(self, uid, create=False):
- return self.homeWithUID(EADDRESSBOOKTYPE, uid, create=create)
</del><ins>+ def addressbookHomeWithUID(self, uid, status=None, create=False):
+ return self.homeWithUID(EADDRESSBOOKTYPE, uid, status=status, create=create)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def _determineMemo(self, storeType, uid, create=False):
</del><ins>+ def _determineMemo(self, storeType, uid, status=None, create=False):
</ins><span class="cx"> """
</span><span class="cx"> Determine the memo dictionary to use for homeWithUID.
</span><span class="cx"> """
</span><span class="lines">@@ -365,7 +365,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @memoizedKey("uid", _determineMemo, deferredResult=False)
</span><del>- def homeWithUID(self, storeType, uid, create=False):
</del><ins>+ def homeWithUID(self, storeType, uid, status=None, create=False):
</ins><span class="cx"> if uid.startswith("."):
</span><span class="cx"> return None
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -25,10 +25,11 @@
</span><span class="cx"> from txdav.caldav.datastore.sql import ManagedAttachment, CalendarBindRecord
</span><span class="cx"> from txdav.common.datastore.sql_external import NotificationCollectionExternal
</span><span class="cx"> from txdav.common.datastore.sql_notification import NotificationCollection
</span><del>-from txdav.common.datastore.sql_tables import _HOME_STATUS_EXTERNAL
</del><ins>+from txdav.common.datastore.sql_tables import _HOME_STATUS_EXTERNAL, \
+ _HOME_STATUS_MIGRATING
</ins><span class="cx"> from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
</span><span class="cx">
</span><del>-import uuid
</del><ins>+from uuid import uuid4
</ins><span class="cx">
</span><span class="cx"> log = Logger()
</span><span class="cx">
</span><span class="lines">@@ -100,10 +101,6 @@
</span><span class="cx"> return "Cross-pod Migration Sync for {}: {}".format(self.diruid, detail)
</span><span class="cx">
</span><span class="cx">
</span><del>- def migratingUid(self):
- return "Migrating-{}".format(self.diruid)
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def migrateHere(self):
</span><span class="cx"> """
</span><span class="lines">@@ -151,7 +148,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> yield self.loadRecord()
</span><del>- self.homeId = yield self.prepareCalendarHome()
</del><ins>+ yield self.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx"> # Calendar list and calendar data
</span><span class="cx"> yield self.syncCalendarList()
</span><span class="lines">@@ -249,10 +246,10 @@
</span><span class="cx"> Make sure the inactive home to migrate into is present on this pod.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ home = yield self._localHome(txn)
</ins><span class="cx"> if home is None:
</span><del>- home = yield txn.calendarHomeWithUID(self.migratingUid(), create=True, migratingUID=self.diruid)
- returnValue(home.id())
</del><ins>+ home = yield txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING, create=True)
+ self.homeId = home.id()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inTransactionWrapper
</span><span class="lines">@@ -268,7 +265,7 @@
</span><span class="cx"> calendars = yield CalendarMigrationRecord.querysimple(txn, calendarHomeResourceID=self.homeId)
</span><span class="cx"> calendarIDMap = dict((item.remoteResourceID, item.localResourceID) for item in calendars)
</span><span class="cx">
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> yield local_home.copyMetadata(remote_home, calendarIDMap)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -276,19 +273,22 @@
</span><span class="cx"> def _remoteHome(self, txn):
</span><span class="cx"> """
</span><span class="cx"> Create a synthetic external home object that maps to the actual remote home.
</span><del>-
- @param ownerUID: directory uid of the user's home
- @type ownerUID: L{str}
</del><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.sql_external import CalendarHomeExternal
</span><span class="cx"> resourceID = yield txn.store().conduit.send_home_resource_id(txn, self.record)
</span><del>- home = CalendarHomeExternal(txn, self.record.uid, resourceID) if resourceID is not None else None
- if home:
- home._childClass = home._childClass._externalClass
</del><ins>+ home = CalendarHomeExternal.makeSyntheticExternalHome(txn, self.record.uid, resourceID) if resourceID is not None else None
</ins><span class="cx"> returnValue(home)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def _localHome(self, txn):
+ """
+ Get the home on this pod that will have data migrated to it.
+ """
+
+ return txn.calendarHomeWithUID(self.diruid, status=_HOME_STATUS_MIGRATING)
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def syncCalendarList(self):
</span><span class="cx"> """
</span><span class="lines">@@ -377,7 +377,7 @@
</span><span class="cx"> @param remote_sync_state: remote sync state
</span><span class="cx"> @type remote_sync_state: L{dict}
</span><span class="cx"> """
</span><del>- home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ home = yield self._localHome(txn)
</ins><span class="cx"> for remoteID in set(local_sync_state.keys()) - set(remote_sync_state.keys()):
</span><span class="cx"> calendar = yield home.childWithID(local_sync_state[remoteID].localResourceID)
</span><span class="cx"> if calendar is not None:
</span><span class="lines">@@ -431,8 +431,8 @@
</span><span class="cx"> of the calendar right now - it will be sync'd later.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- home = yield txn.calendarHomeWithUID(self.migratingUid())
- calendar = yield home.createChildWithName(str(uuid.uuid4()))
</del><ins>+ home = yield self._localHome(txn)
+ calendar = yield home.createChildWithName(str(uuid4()))
</ins><span class="cx"> returnValue(calendar.id())
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -452,7 +452,7 @@
</span><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="cx"> # Check whether the deleted set items
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> local_calendar = yield local_home.childWithID(migrationRecord.localResourceID)
</span><span class="cx"> yield local_calendar.copyMetadata(remote_calendar)
</span><span class="cx">
</span><span class="lines">@@ -478,7 +478,7 @@
</span><span class="cx"> changed, deleted, _ignore_invalid = yield remote_calendar.resourceNamesSinceToken(migrationRecord.lastSyncToken)
</span><span class="cx">
</span><span class="cx"> # Check whether the deleted set items
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> local_calendar = yield local_home.childWithID(migrationRecord.localResourceID)
</span><span class="cx">
</span><span class="cx"> # Check the md5's on each changed remote with the local one to filter out ones
</span><span class="lines">@@ -531,7 +531,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Check whether the deleted set items
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> local_calendar = yield local_home.childWithID(localID)
</span><span class="cx"> local_objects = yield local_calendar.objectResourcesWithNames(purge_names)
</span><span class="cx">
</span><span class="lines">@@ -587,7 +587,7 @@
</span><span class="cx"> remote_objects = dict([(obj.name(), obj) for obj in remote_objects])
</span><span class="cx">
</span><span class="cx"> # Get local objects
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> local_calendar = yield local_home.childWithID(localID)
</span><span class="cx"> local_objects = yield local_calendar.objectResourcesWithNames(remaining)
</span><span class="cx"> local_objects = dict([(obj.name(), obj) for obj in local_objects])
</span><span class="lines">@@ -652,7 +652,7 @@
</span><span class="cx"> rattachments = yield remote_home.getAllAttachments()
</span><span class="cx"> rmap = dict([(attachment.id(), attachment) for attachment in rattachments])
</span><span class="cx">
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> lattachments = yield local_home.getAllAttachments()
</span><span class="cx"> lmap = dict([(attachment.id(), attachment) for attachment in lattachments])
</span><span class="cx">
</span><span class="lines">@@ -706,7 +706,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> remote_home = yield self._remoteHome(txn)
</span><del>- local_home = yield txn.calendarHomeWithUID(self.migratingUid())
</del><ins>+ local_home = yield self._localHome(txn)
</ins><span class="cx"> attachment = yield local_home.getAttachmentByID(local_id)
</span><span class="cx"> if attachment is None:
</span><span class="cx"> returnValue(None)
</span><span class="lines">@@ -951,13 +951,24 @@
</span><span class="cx"> """
</span><span class="cx"> Sync all the collections shared by the migrating user from the remote store. We will do this one calendar at a time since
</span><span class="cx"> there could be a large number of sharees per calendar.
</span><ins>+
+ Here is the logic we need: first assume we have three pods: A, B, C, and we are migrating a user from A->B. We start
+ with a set of shares (X -> Y - where X is the sharer and Y the sharee) on pod A. We migrate the sharer to pod B. We
+ then need to have a set of bind records on pod B, and adjust the set on pod A. Note that no changes are required on pod C.
+
+ Original | Changes | Changes
+ Shares | on B | on A
+ --------------|------------------------------|---------------------
+ A -> A | B -> A (new) | B -> A (modify existing)
+ A -> B | B -> B (modify existing) | (removed)
+ A -> C | B -> C (new) | (removed)
</ins><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> calendars = yield self.getSyncState()
</span><span class="cx">
</span><span class="cx"> len_records = 0
</span><span class="cx"> for calendar in calendars.values():
</span><del>- records = yield self.sharedByCollectionRecords(calendar.remoteResourceID)
</del><ins>+ records, bindUID = yield self.sharedByCollectionRecords(calendar.remoteResourceID, calendar.localResourceID)
</ins><span class="cx"> records = records.items()
</span><span class="cx">
</span><span class="cx"> # Batch setting resources for the local home
</span><span class="lines">@@ -966,22 +977,36 @@
</span><span class="cx"> yield self.makeSharedByCollections(records[:50], calendar.localResourceID)
</span><span class="cx"> records = records[50:]
</span><span class="cx">
</span><ins>+ # Update the remote pod to switch over the shares
+ yield self.updatedRemoteSharedByCollections(calendar.remoteResourceID, bindUID)
+
</ins><span class="cx"> returnValue(len_records)
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inTransactionWrapper
</span><span class="cx"> @inlineCallbacks
</span><del>- def sharedByCollectionRecords(self, txn, remote_id):
</del><ins>+ def sharedByCollectionRecords(self, txn, remote_id, local_id):
</ins><span class="cx"> """
</span><del>- Get all the existing L{CalendarBindRecord}'s from the remote store.
</del><ins>+ Get all the existing L{CalendarBindRecord}'s from the remote store. Also make sure a
+ bindUID exists for the local calendar.
</ins><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> remote_home = yield self._remoteHome(txn)
</span><span class="cx"> remote_calendar = yield remote_home.childWithID(remote_id)
</span><span class="cx"> records = yield remote_calendar.sharingBindRecords()
</span><del>- returnValue(records)
</del><span class="cx">
</span><ins>+ # Check bindUID
+ local_records = yield CalendarBindRecord.querysimple(
+ txn,
+ calendarHomeResourceID=self.homeId,
+ calendarResourceID=local_id,
+ )
+ if not local_records[0].bindUID:
+ yield local_records[0].update(bindUID=str(uuid4()))
</ins><span class="cx">
</span><ins>+ returnValue((records, local_records[0].bindUID,))
+
+
</ins><span class="cx"> @inTransactionWrapper
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def makeSharedByCollections(self, txn, records, calendar_id):
</span><span class="lines">@@ -1016,10 +1041,34 @@
</span><span class="cx"> yield record.insert(txn)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inTransactionWrapper
</ins><span class="cx"> @inlineCallbacks
</span><ins>+ def updatedRemoteSharedByCollections(self, txn, remote_id, bindUID):
+ """
+ Get all the existing L{CalendarBindRecord}'s from the remote store.
+ """
+
+ remote_home = yield self._remoteHome(txn)
+ remote_calendar = yield remote_home.childWithID(remote_id)
+ records = yield remote_calendar.migrateBindRecords(bindUID)
+ returnValue(records)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def sharedToCollectionsReconcile(self):
</span><span class="cx"> """
</span><span class="cx"> Sync all the collections shared to the migrating user from the remote store.
</span><ins>+
+ Here is the logic we need: first assume we have three pods: A, B, C, and we are migrating a user from A->B. We start
+ with a set of shares (X -> Y - where X is the sharer and Y the sharee) with sharee on pod A. We migrate the sharee to pod B. We
+ then need to have a set of bind records on pod B, and adjust the set on pod A. Note that no changes are required on pod C.
+
+ Original | Changes | Changes
+ Shares | on B | on A
+ --------------|------------------------------|---------------------
+ A -> A | A -> B (new) | A -> B (modify existing)
+ B -> A | B -> B (modify existing) | (removed)
+ C -> A | C -> B (new) | (removed)
</ins><span class="cx"> """
</span><span class="cx"> records = yield self.sharedToCollectionRecords()
</span><span class="cx"> records = records.items()
</span><span class="lines">@@ -1080,6 +1129,8 @@
</span><span class="cx"> # resource ID from that to use in the new CALENDAR_BIND record we create. If a pre-existing share
</span><span class="cx"> # is not present, then we have to create the CALENDAR table entry and associated pieces
</span><span class="cx">
</span><ins>+ remote_id = shareeRecord.calendarResourceID
+
</ins><span class="cx"> # Look for pre-existing share with the same external ID
</span><span class="cx"> oldrecord = yield CalendarBindRecord.querysimple(
</span><span class="cx"> txn,
</span><span class="lines">@@ -1101,3 +1152,18 @@
</span><span class="cx"> shareeRecord.calendarResourceID = calendar_id
</span><span class="cx"> shareeRecord.bindRevision = 0
</span><span class="cx"> yield shareeRecord.insert(txn)
</span><ins>+
+ yield self.updatedRemoteSharedToCollection(remote_id, txn=txn)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def updatedRemoteSharedToCollection(self, txn, remote_id):
+ """
+ Get all the existing L{CalendarBindRecord}'s from the remote store.
+ """
+
+ remote_home = yield self._remoteHome(txn)
+ remote_calendar = yield remote_home.childWithID(remote_id)
+ records = yield remote_calendar.migrateBindRecords(None)
+ returnValue(records)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -31,7 +31,7 @@
</span><span class="cx"> ExternalDelegateGroupsRecord, DelegateGroupsRecord
</span><span class="cx"> from txdav.common.datastore.sql_notification import NotificationCollection
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema, _HOME_STATUS_EXTERNAL, \
</span><del>- _BIND_MODE_READ
</del><ins>+ _BIND_MODE_READ, _HOME_STATUS_MIGRATING
</ins><span class="cx"> from txdav.common.datastore.test.util import populateCalendarsFrom
</span><span class="cx"> from txdav.who.delegates import Delegates
</span><span class="cx"> from txweb2.http_headers import MimeType
</span><span class="lines">@@ -154,14 +154,14 @@
</span><span class="cx">
</span><span class="cx"> # No home present
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><del>- home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> self.assertTrue(home is None)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="cx"> yield syncer.prepareCalendarHome()
</span><span class="cx">
</span><span class="cx"> # Home is present
</span><del>- home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> self.assertTrue(home is not None)
</span><span class="cx"> children = yield home.listChildren()
</span><span class="cx"> self.assertEqual(len(children), 0)
</span><span class="lines">@@ -176,7 +176,7 @@
</span><span class="cx">
</span><span class="cx"> # No home present
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><del>- home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> self.assertTrue(home is None)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="lines">@@ -184,7 +184,7 @@
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="cx"> # Home is present
</span><del>- home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> self.assertTrue(home is not None)
</span><span class="cx"> children = yield home.listChildren()
</span><span class="cx"> self.assertEqual(len(children), 0)
</span><span class="lines">@@ -257,7 +257,7 @@
</span><span class="cx"> yield syncer.sync()
</span><span class="cx">
</span><span class="cx"> # Home is present with correct metadata
</span><del>- home1 = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> self.assertTrue(home1 is not None)
</span><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> events1 = yield home1.childWithName("events")
</span><span class="lines">@@ -284,7 +284,7 @@
</span><span class="cx"> yield syncer.sync()
</span><span class="cx">
</span><span class="cx"> # Home is present with correct metadata
</span><del>- home1 = yield self.homeUnderTest(self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> self.assertTrue(home1 is not None)
</span><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> events1 = yield home1.childWithName("events")
</span><span class="lines">@@ -345,10 +345,10 @@
</span><span class="cx">
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><del>- syncer.homeId = yield syncer.prepareCalendarHome()
</del><ins>+ yield syncer.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx"> # No local calendar exists yet
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> children = yield home1.listChildren()
</span><span class="cx"> self.assertEqual(len(children), 0)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="lines">@@ -370,7 +370,7 @@
</span><span class="cx"> self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
</span><span class="cx">
</span><span class="cx"> # Local calendar exists
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="lines">@@ -395,10 +395,10 @@
</span><span class="cx">
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><del>- syncer.homeId = yield syncer.prepareCalendarHome()
</del><ins>+ yield syncer.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx"> # No local calendar exists yet
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is None)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="lines">@@ -427,7 +427,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> # Local calendar exists
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> children = yield calendar1.objectResources()
</span><span class="lines">@@ -451,7 +451,7 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> object1 = yield self.calendarObjectUnderTest(
</span><del>- txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), calendar_name="calendar", name="1.ics"
</del><ins>+ txn=self.theTransactionUnderTest(1), home="user01", status=_HOME_STATUS_MIGRATING, calendar_name="calendar", name="1.ics"
</ins><span class="cx"> )
</span><span class="cx"> caldata = yield object1.component()
</span><span class="cx"> self.assertEqual(normalize_iCalStr(caldata), normalize_iCalStr(self.caldata1_changed))
</span><span class="lines">@@ -472,7 +472,7 @@
</span><span class="cx"> remote_sync_state,
</span><span class="cx"> )
</span><span class="cx">
</span><del>- calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), name="calendar")
</del><ins>+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home="user01", status=_HOME_STATUS_MIGRATING, name="calendar")
</ins><span class="cx"> children = yield calendar1.objectResources()
</span><span class="cx"> self.assertEqual(set([child.name() for child in children]), set(("1.ics", "3.ics",)))
</span><span class="cx"> mapping1 = dict([(o.name(), o.id()) for o in children])
</span><span class="lines">@@ -492,7 +492,7 @@
</span><span class="cx"> remote_sync_state,
</span><span class="cx"> )
</span><span class="cx">
</span><del>- calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), name="calendar")
</del><ins>+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home="user01", status=_HOME_STATUS_MIGRATING, name="calendar")
</ins><span class="cx"> children = yield calendar1.objectResources()
</span><span class="cx"> self.assertEqual(set([child.name() for child in children]), set(("1.ics", "3.ics", "4.ics")))
</span><span class="cx"> mapping1 = dict([(o.name(), o.id()) for o in children])
</span><span class="lines">@@ -515,17 +515,17 @@
</span><span class="cx">
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><del>- syncer.homeId = yield syncer.prepareCalendarHome()
</del><ins>+ yield syncer.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx"> # No local calendar exists yet
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> children1 = yield home1.loadChildren()
</span><span class="cx"> self.assertEqual(len(children1), 0)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="cx"> # Trigger sync
</span><span class="cx"> yield syncer.syncCalendarList()
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> children1 = yield home1.loadChildren()
</span><span class="cx"> details1 = dict([(child.id(), child.name()) for child in children1])
</span><span class="cx"> self.assertEqual(set(details1.values()), set(details0.values()))
</span><span class="lines">@@ -539,7 +539,7 @@
</span><span class="cx">
</span><span class="cx"> # Trigger sync
</span><span class="cx"> yield syncer.syncCalendarList()
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> children1 = yield home1.loadChildren()
</span><span class="cx"> details1 = dict([(child.id(), child.name()) for child in children1])
</span><span class="cx"> self.assertTrue("new-calendar" in details1.values())
</span><span class="lines">@@ -555,7 +555,7 @@
</span><span class="cx">
</span><span class="cx"> # Trigger sync
</span><span class="cx"> yield syncer.syncCalendarList()
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> children1 = yield home1.loadChildren()
</span><span class="cx"> details1 = dict([(child.id(), child.name()) for child in children1])
</span><span class="cx"> self.assertTrue("new-calendar" not in details1.values())
</span><span class="lines">@@ -582,7 +582,7 @@
</span><span class="cx">
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><del>- syncer.homeId = yield syncer.prepareCalendarHome()
</del><ins>+ yield syncer.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx"> # Trigger sync of the one calendar
</span><span class="cx"> local_sync_state = {}
</span><span class="lines">@@ -623,7 +623,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> # Local calendar exists
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> children = yield calendar1.objectResources()
</span><span class="lines">@@ -649,7 +649,7 @@
</span><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> attachments = yield home1.getAllAttachments()
</span><span class="cx"> mapping1 = dict([(o.md5(), o.id()) for o in attachments])
</span><span class="cx"> yield _checkAttachmentObjectMigrationState(home1, mapping1)
</span><span class="lines">@@ -668,7 +668,7 @@
</span><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> attachments = yield home1.getAllAttachments()
</span><span class="cx"> mapping1 = dict([(o.md5(), o.id()) for o in attachments])
</span><span class="cx"> yield _checkAttachmentObjectMigrationState(home1, mapping1)
</span><span class="lines">@@ -689,7 +689,7 @@
</span><span class="cx"> self.assertEqual(removed, set((id0_1,)))
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> attachments = yield home1.getAllAttachments()
</span><span class="cx"> mapping1 = dict([(o.md5(), o.id()) for o in attachments])
</span><span class="cx"> yield _checkAttachmentObjectMigrationState(home1, mapping1)
</span><span class="lines">@@ -711,7 +711,7 @@
</span><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> attachments = yield home1.getAllAttachments()
</span><span class="cx"> mapping1 = dict([(o.md5(), o.id()) for o in attachments])
</span><span class="cx"> yield _checkAttachmentObjectMigrationState(home1, mapping1)
</span><span class="lines">@@ -730,7 +730,7 @@
</span><span class="cx"> self.assertEqual(removed, set())
</span><span class="cx">
</span><span class="cx"> # Validate changes
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> attachments = yield home1.getAllAttachments()
</span><span class="cx"> mapping1 = dict([(o.md5(), o.id()) for o in attachments])
</span><span class="cx"> yield _checkAttachmentObjectMigrationState(home1, mapping1)
</span><span class="lines">@@ -776,7 +776,7 @@
</span><span class="cx">
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><del>- syncer.homeId = yield syncer.prepareCalendarHome()
</del><ins>+ yield syncer.prepareCalendarHome()
</ins><span class="cx">
</span><span class="cx"> # Trigger sync of the one calendar
</span><span class="cx"> local_sync_state = {}
</span><span class="lines">@@ -810,7 +810,7 @@
</span><span class="cx"> self.assertEqual(len_links, 3)
</span><span class="cx">
</span><span class="cx"> # Local calendar exists
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> children = yield calendar1.objectResources()
</span><span class="lines">@@ -950,7 +950,7 @@
</span><span class="cx"> # Sync from remote side
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="cx"> yield syncer.loadRecord()
</span><del>- syncer.homeId = yield syncer.prepareCalendarHome()
</del><ins>+ yield syncer.prepareCalendarHome()
</ins><span class="cx"> changes = yield syncer.notificationsReconcile()
</span><span class="cx"> self.assertEqual(changes, 2)
</span><span class="cx">
</span><span class="lines">@@ -1030,28 +1030,50 @@
</span><span class="cx"> self.assertEqual(changes, 2)
</span><span class="cx">
</span><span class="cx"> # Local calendar exists with shares
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> invites1 = yield calendar1.sharingInvites()
</span><span class="cx"> self.assertEqual(len(invites1), 2)
</span><span class="cx"> self.assertEqual(set([invite.uid for invite in invites1]), set((shared_name_02, shared_name_03,)))
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><ins>+ # Remote sharee can access it
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user02")
+ calendar0 = yield home0.childWithName(shared_name_02)
+ self.assertTrue(calendar0 is not None)
+
</ins><span class="cx"> # Local sharee can access it
</span><span class="cx"> home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser03")
</span><span class="cx"> calendar1 = yield home1.childWithName(shared_name_03)
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx">
</span><span class="cx"> # Local shared calendars exist
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName(shared_name_04)
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> calendar1 = yield home1.childWithName(shared_name_05)
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><ins>+ # Sharers see migrated user as sharee
+ externalHome0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", status=_HOME_STATUS_EXTERNAL)
+ calendar0 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user04", name="calendar")
+ invites = yield calendar0.allInvitations()
+ self.assertEqual(len(invites), 1)
+ self.assertEqual(invites[0].shareeUID, "user01")
+ self.assertEqual(invites[0].shareeHomeID, externalHome0.id())
+ yield self.commitTransaction(0)
</ins><span class="cx">
</span><ins>+ shareeHome1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home="puser05", name="calendar")
+ invites = yield calendar1.allInvitations()
+ self.assertEqual(len(invites), 1)
+ self.assertEqual(invites[0].shareeUID, "user01")
+ self.assertEqual(invites[0].shareeHomeID, shareeHome1.id())
+ yield self.commitTransaction(1)
</ins><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class TestGroupAttendeeSync(MultiStoreConduitTest):
</span><span class="cx"> """
</span><span class="cx"> GroupAttendeeReconciliation tests
</span><span class="lines">@@ -1160,7 +1182,7 @@
</span><span class="cx"> self.assertEqual(len_links, 2)
</span><span class="cx">
</span><span class="cx"> # Local calendar exists
</span><del>- home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</del><ins>+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="user01", status=_HOME_STATUS_MIGRATING)
</ins><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><span class="cx"> children = yield calendar1.objectResources()
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -168,6 +168,7 @@
</span><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=UtilityConduitMixin._to_tuple)
</span><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_search", "search")
</span><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_sharing_records", "sharingBindRecords", transform_recv_result=StoreAPIConduitMixin._to_serialize_dict_value)
</span><ins>+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_migrate_sharing_records", "migrateBindRecords")
</ins><span class="cx">
</span><span class="cx"> # Calls on L{CommonObjectResource} objects
</span><span class="cx"> UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_serialize_list)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtesttest_store_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_store_api.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -105,7 +105,7 @@
</span><span class="cx"> from txdav.caldav.datastore.sql_external import CalendarHomeExternal
</span><span class="cx"> recipient = yield txn.store().directoryService().recordWithUID(uid)
</span><span class="cx"> resourceID = yield txn.store().conduit.send_home_resource_id(txn, recipient)
</span><del>- home = CalendarHomeExternal(txn, recipient.uid, resourceID) if resourceID is not None else None
</del><ins>+ home = CalendarHomeExternal.makeSyntheticExternalHome(txn, recipient.uid, resourceID) if resourceID is not None else None
</ins><span class="cx"> if home:
</span><span class="cx"> home._childClass = home._childClass._externalClass
</span><span class="cx"> returnValue(home)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -14,9 +14,6 @@
</span><span class="cx"> # See the License for the specific language governing permissions and
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><del>-from txdav.common.datastore.sql_notification import NotificationCollection
-from txdav.common.datastore.sql_util import _EmptyCacher, _SharedSyncLogic
-from txdav.common.datastore.sql_sharing import SharingHomeMixIn, SharingMixIn
</del><span class="cx">
</span><span class="cx"> """
</span><span class="cx"> SQL data store.
</span><span class="lines">@@ -64,9 +61,12 @@
</span><span class="cx"> from txdav.common.datastore.sql_directory import DelegatesAPIMixin, \
</span><span class="cx"> GroupsAPIMixin, GroupCacherAPIMixin
</span><span class="cx"> from txdav.common.datastore.sql_imip import imipAPIMixin
</span><ins>+from txdav.common.datastore.sql_notification import NotificationCollection
</ins><span class="cx"> from txdav.common.datastore.sql_tables import _BIND_MODE_OWN, _BIND_STATUS_ACCEPTED, \
</span><span class="cx"> _HOME_STATUS_EXTERNAL, _HOME_STATUS_NORMAL, \
</span><span class="cx"> _HOME_STATUS_PURGING, schema, splitSQLString, _HOME_STATUS_MIGRATING
</span><ins>+from txdav.common.datastore.sql_util import _SharedSyncLogic
+from txdav.common.datastore.sql_sharing import SharingHomeMixIn, SharingMixIn
</ins><span class="cx"> from txdav.common.icommondatastore import ConcurrentModification, \
</span><span class="cx"> RecordNotAllowedError, ShareNotAllowed, \
</span><span class="cx"> IndexedSearchException, EADDRESSBOOKTYPE, ECALENDARTYPE
</span><span class="lines">@@ -81,6 +81,7 @@
</span><span class="cx">
</span><span class="cx"> from zope.interface import implements, directlyProvides
</span><span class="cx">
</span><ins>+import collections
</ins><span class="cx"> import inspect
</span><span class="cx"> import itertools
</span><span class="cx"> import sys
</span><span class="lines">@@ -567,8 +568,16 @@
</span><span class="cx">
</span><span class="cx"> self._store = store
</span><span class="cx"> self._queuer = self._store.queuer
</span><del>- self._calendarHomes = {}
- self._addressbookHomes = {}
</del><ins>+ self._cachedHomes = {
+ ECALENDARTYPE: {
+ "byUID": collections.defaultdict(dict),
+ "byID": collections.defaultdict(dict),
+ },
+ EADDRESSBOOKTYPE: {
+ "byUID": collections.defaultdict(dict),
+ "byID": collections.defaultdict(dict),
+ },
+ }
</ins><span class="cx"> self._notificationHomes = {}
</span><span class="cx"> self._notifierFactories = notifierFactories
</span><span class="cx"> self._notifiedAlready = set()
</span><span class="lines">@@ -677,14 +686,11 @@
</span><span class="cx"> ).on(self)
</span><span class="cx">
</span><span class="cx">
</span><del>- def _determineMemo(self, storeType, uid, create=False, authzUID=None, migratingUID=None):
</del><ins>+ def _determineMemo(self, storeType, lookupMode, status):
</ins><span class="cx"> """
</span><span class="cx"> Determine the memo dictionary to use for homeWithUID.
</span><span class="cx"> """
</span><del>- if storeType == ECALENDARTYPE:
- return self._calendarHomes
- else:
- return self._addressbookHomes
</del><ins>+ return self._cachedHomes[storeType][lookupMode][status]
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -699,11 +705,11 @@
</span><span class="cx"> yield self.homeWithUID(storeType, uid, create=False)
</span><span class="cx">
</span><span class="cx"> # Return the memoized list directly
</span><del>- returnValue([kv[1] for kv in sorted(self._determineMemo(storeType, None).items(), key=lambda x: x[0])])
</del><ins>+ returnValue([kv[1] for kv in sorted(self._determineMemo(storeType, "byUID", _HOME_STATUS_NORMAL).items(), key=lambda x: x[0])])
</ins><span class="cx">
</span><span class="cx">
</span><del>- @memoizedKey("uid", _determineMemo)
- def homeWithUID(self, storeType, uid, create=False, authzUID=None, migratingUID=None):
</del><ins>+ @inlineCallbacks
+ def homeWithUID(self, storeType, uid, status=None, create=False, authzUID=None):
</ins><span class="cx"> """
</span><span class="cx"> We need to distinguish between various different users "looking" at a home and its
</span><span class="cx"> child resources because we have per-user properties that depend on which user is "looking".
</span><span class="lines">@@ -715,15 +721,21 @@
</span><span class="cx"> if storeType not in (ECALENDARTYPE, EADDRESSBOOKTYPE):
</span><span class="cx"> raise RuntimeError("Unknown home type.")
</span><span class="cx">
</span><del>- return self._homeClass[storeType].homeWithUID(self, uid, create, authzUID, migratingUID)
</del><ins>+ result = self._determineMemo(storeType, "byUID", status).get(uid)
+ if result is None:
+ result = yield self._homeClass[storeType].homeWithUID(self, uid, status, create, authzUID)
+ if result:
+ self._determineMemo(storeType, "byUID", status)[uid] = result
+ self._determineMemo(storeType, "byID", None)[result.id()] = result
+ returnValue(result)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def calendarHomeWithUID(self, uid, create=False, authzUID=None, migratingUID=None):
- return self.homeWithUID(ECALENDARTYPE, uid, create=create, authzUID=authzUID, migratingUID=migratingUID)
</del><ins>+ def calendarHomeWithUID(self, uid, status=None, create=False, authzUID=None):
+ return self.homeWithUID(ECALENDARTYPE, uid, status=status, create=create, authzUID=authzUID)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def addressbookHomeWithUID(self, uid, create=False, authzUID=None, migratingUID=None):
- return self.homeWithUID(EADDRESSBOOKTYPE, uid, create=create, authzUID=authzUID, migratingUID=migratingUID)
</del><ins>+ def addressbookHomeWithUID(self, uid, status=None, create=False, authzUID=None):
+ return self.homeWithUID(EADDRESSBOOKTYPE, uid, status=status, create=create, authzUID=authzUID)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -731,12 +743,15 @@
</span><span class="cx"> """
</span><span class="cx"> Load a calendar or addressbook home by its integer resource ID.
</span><span class="cx"> """
</span><del>- uid = (yield self._homeClass[storeType].homeUIDWithResourceID(self, rid))
- if uid:
- # Always get the owner's view of the home = i.e., authzUID=uid
- result = (yield self.homeWithUID(storeType, uid, authzUID=uid))
- else:
- result = None
</del><ins>+ if storeType not in (ECALENDARTYPE, EADDRESSBOOKTYPE):
+ raise RuntimeError("Unknown home type.")
+
+ result = self._determineMemo(storeType, "byID", None).get(rid)
+ if result is None:
+ result = yield self._homeClass[storeType].homeWithResourceID(self, rid)
+ if result:
+ self._determineMemo(storeType, "byID", None)[rid] = result
+ self._determineMemo(storeType, "byUID", result._status)[result.uid()] = result
</ins><span class="cx"> returnValue(result)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -1530,38 +1545,38 @@
</span><span class="cx"> _dataVersionKey = None
</span><span class="cx"> _dataVersionValue = None
</span><span class="cx">
</span><del>- _cacher = None # Initialize in derived classes
-
</del><span class="cx"> @classmethod
</span><del>- @inlineCallbacks
- def makeClass(cls, transaction, ownerUID, no_cache=False, authzUID=None):
</del><ins>+ def makeClass(cls, transaction, homeData, authzUID=None):
</ins><span class="cx"> """
</span><span class="cx"> Build the actual home class taking into account the possibility that we might need to
</span><span class="cx"> switch in the external version of the class.
</span><span class="cx">
</span><span class="cx"> @param transaction: transaction
</span><span class="cx"> @type transaction: L{CommonStoreTransaction}
</span><del>- @param ownerUID: owner UID of home to load
- @type ownerUID: C{str}
- @param no_cache: should cached query be used
- @type no_cache: C{bool}
</del><ins>+ @param homeData: home table column data
+ @type homeData: C{list}
</ins><span class="cx"> """
</span><del>- home = cls(transaction, ownerUID, authzUID=authzUID)
- actualHome = yield home.initFromStore(no_cache)
- returnValue(actualHome)
</del><span class="cx">
</span><ins>+ status = homeData[cls.homeColumns().index(cls._homeSchema.STATUS)]
+ if status == _HOME_STATUS_EXTERNAL:
+ home = cls._externalClass(transaction, homeData)
+ else:
+ home = cls(transaction, homeData, authzUID=authzUID)
+ return home.initFromStore()
</ins><span class="cx">
</span><del>- def __init__(self, transaction, ownerUID, authzUID=None):
</del><ins>+
+ def __init__(self, transaction, homeData, authzUID=None):
</ins><span class="cx"> self._txn = transaction
</span><del>- self._ownerUID = ownerUID
</del><ins>+
+ for attr, value in zip(self.homeAttributes(), homeData):
+ setattr(self, attr, value)
+
</ins><span class="cx"> self._authzUID = authzUID
</span><span class="cx"> if self._authzUID is None:
</span><span class="cx"> if self._txn._authz_uid is not None:
</span><span class="cx"> self._authzUID = self._txn._authz_uid
</span><span class="cx"> else:
</span><span class="cx"> self._authzUID = self._ownerUID
</span><del>- self._resourceID = None
- self._status = _HOME_STATUS_NORMAL
</del><span class="cx"> self._dataVersion = None
</span><span class="cx"> self._childrenLoaded = False
</span><span class="cx"> self._children = {}
</span><span class="lines">@@ -1570,15 +1585,13 @@
</span><span class="cx"> self._created = None
</span><span class="cx"> self._modified = None
</span><span class="cx"> self._syncTokenRevision = None
</span><del>- if transaction._disableCache:
- self._cacher = _EmptyCacher()
</del><span class="cx">
</span><span class="cx"> # This is used to track whether the originating request is from the store associated
</span><span class="cx"> # by the transaction, or from a remote store. We need to be able to distinguish store
</span><span class="cx"> # objects that are locally hosted (_HOME_STATUS_NORMAL) or remotely hosted
</span><span class="cx"> # (_HOME_STATUS_EXTERNAL). For the later we need to know whether the object is being
</span><span class="cx"> # accessed from the local store (in which case requests for child objects etc will be
</span><del>- # directed at a remote store) or whether it is being accessed as the tresult of a remote
</del><ins>+ # directed at a remote store) or whether it is being accessed as the result of a remote
</ins><span class="cx"> # request (in which case requests for child objects etc will be directed at the local store).
</span><span class="cx"> self._internalRequest = True
</span><span class="cx">
</span><span class="lines">@@ -1603,14 +1616,16 @@
</span><span class="cx"> return Select(
</span><span class="cx"> cls.homeColumns(),
</span><span class="cx"> From=home,
</span><del>- Where=home.OWNER_UID == Parameter("ownerUID")
</del><ins>+ Where=(home.OWNER_UID == Parameter("ownerUID")).And(
+ home.STATUS == Parameter("status")
+ )
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classproperty
</span><span class="cx"> def _ownerFromResourceID(cls):
</span><span class="cx"> home = cls._homeSchema
</span><del>- return Select([home.OWNER_UID],
</del><ins>+ return Select([home.OWNER_UID, home.STATUS],
</ins><span class="cx"> From=home,
</span><span class="cx"> Where=home.RESOURCE_ID == Parameter("resourceID"))
</span><span class="cx">
</span><span class="lines">@@ -1686,41 +1701,22 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def initFromStore(self, no_cache=False):
</del><ins>+ def initFromStore(self):
</ins><span class="cx"> """
</span><span class="cx"> Initialize this object from the store. We read in and cache all the
</span><span class="cx"> extra meta-data from the DB to avoid having to do DB queries for those
</span><span class="cx"> individually later.
</span><span class="cx"> """
</span><del>- result = yield self._cacher.get(self._ownerUID)
- if result is None:
- result = yield self._homeColumnsFromOwnerQuery.on(self._txn, ownerUID=self._ownerUID)
- if result:
- result = result[0]
- if not no_cache:
- yield self._cacher.set(self._ownerUID, result)
</del><span class="cx">
</span><del>- if result:
- for attr, value in zip(self.homeAttributes(), result):
- setattr(self, attr, value)
</del><ins>+ yield self.initMetaDataFromStore()
+ yield self._loadPropertyStore()
</ins><span class="cx">
</span><del>- # STOP! If the status is external we need to convert this object to a CommonHomeExternal class which will
- # have the right behavior for non-hosted external users.
- if self._status == _HOME_STATUS_EXTERNAL:
- actualHome = self._externalClass(self._txn, self._ownerUID, self._resourceID)
- else:
- actualHome = self
- yield actualHome.initMetaDataFromStore()
- yield actualHome._loadPropertyStore()
</del><ins>+ for factory_type, factory in self._txn._notifierFactories.items():
+ self.addNotifier(factory_type, factory.newNotifier(self))
</ins><span class="cx">
</span><del>- for factory_type, factory in self._txn._notifierFactories.items():
- actualHome.addNotifier(factory_type, factory.newNotifier(actualHome))
</del><ins>+ returnValue(self)
</ins><span class="cx">
</span><del>- returnValue(actualHome)
- else:
- returnValue(None)
</del><span class="cx">
</span><del>-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def initMetaDataFromStore(self):
</span><span class="cx"> """
</span><span class="lines">@@ -1780,31 +1776,108 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><ins>+ def homeWithUID(cls, txn, uid, status=None, create=False, authzUID=None):
+ return cls.homeWith(txn, None, uid, status, create=create, authzUID=authzUID)
+
+
+ @classmethod
+ def homeWithResourceID(cls, txn, rid):
+ return cls.homeWith(txn, rid, None)
+
+
+ @classmethod
</ins><span class="cx"> @inlineCallbacks
</span><del>- def homeWithUID(cls, txn, uid, create=False, authzUID=None, migratingUID=None):
</del><ins>+ def homeWith(cls, txn, rid, uid, status=None, create=False, authzUID=None):
</ins><span class="cx"> """
</span><del>- @param uid: I'm going to assume uid is utf-8 encoded bytes
</del><ins>+ Lookup or create a home based in either its resource id or uid. If a status is given,
+ return only the one matching that status. If status is L{None} we lookup any regular
+ status type (normal, external or purging). When creating with status L{None} we create
+ one with a status matching the current directory record thisServer() value. The only
+ other status that can be directly created is migrating.
</ins><span class="cx"> """
</span><del>- homeObject = yield cls.makeClass(txn, uid, authzUID=authzUID)
- if homeObject is not None:
</del><ins>+
+ # Setup the SQL query and query cacher keys
+ queryCacher = txn._queryCacher
+ cacheKeys = []
+ if rid is not None:
+ query = cls._homeSchema.RESOURCE_ID == rid
+ if queryCacher:
+ cacheKeys.append(queryCacher.keyForHomeWithID(cls._homeType, rid, status))
+ elif uid is not None:
+ query = cls._homeSchema.OWNER_UID == uid
+ if status is not None:
+ query = query.And(cls._homeSchema.STATUS == status)
+ if queryCacher:
+ cacheKeys.append(queryCacher.keyForHomeWithUID(cls._homeType, uid, status))
+ else:
+ statusSet = (_HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL, _HOME_STATUS_PURGING)
+ query = query.And(cls._homeSchema.STATUS.In(statusSet))
+ if queryCacher:
+ for item in statusSet:
+ cacheKeys.append(queryCacher.keyForHomeWithUID(cls._homeType, uid, item))
+ else:
+ raise AssertionError("One of rid or uid must be set")
+
+ # Try to fetch a result from the query cache first
+ for cacheKey in cacheKeys:
+ result = (yield queryCacher.get(cacheKey))
+ if result is not None:
+ break
+ else:
+ result = None
+
+ # If nothing in thr cache, do the SQL query and cache the result
+ if result is None:
+ results = yield Select(
+ cls.homeColumns(),
+ From=cls._homeSchema,
+ Where=query,
+ ).on(txn)
+
+ # Pick the internal one
+ if len(results) > 1:
+ if results[0][cls.homeColumns().index(cls._homeSchema.STATUS)] == _HOME_STATUS_NORMAL:
+ result = results[0]
+ else:
+ result = results[1]
+ elif results:
+ result = results[0]
+ else:
+ result = None
+
+ if result and queryCacher:
+ if rid is not None:
+ cacheKey = cacheKeys[0]
+ elif uid is not None:
+ cacheKey = queryCacher.keyForHomeWithUID(cls._homeType, uid, result[cls.homeColumns().index(cls._homeSchema.STATUS)])
+ yield queryCacher.set(cacheKey, result)
+
+ if result:
+ # Return object that already exists in the store
+ homeObject = yield cls.makeClass(txn, result, authzUID=authzUID)
</ins><span class="cx"> returnValue(homeObject)
</span><span class="cx"> else:
</span><del>- if not create:
</del><ins>+ # Can only create when uid is specified
+ if not create or uid is None:
</ins><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="cx"> # Determine if the user is local or external
</span><del>- diruid = uid if migratingUID is None else migratingUID
- record = yield txn.directoryService().recordWithUID(diruid.decode("utf-8"))
</del><ins>+ record = yield txn.directoryService().recordWithUID(uid.decode("utf-8"))
</ins><span class="cx"> if record is None:
</span><del>- raise DirectoryRecordNotFoundError("Cannot create home for UID since no directory record exists: {}".format(diruid))
</del><ins>+ raise DirectoryRecordNotFoundError("Cannot create home for UID since no directory record exists: {}".format(uid))
</ins><span class="cx">
</span><del>- if migratingUID is None:
- state = _HOME_STATUS_NORMAL if record.thisServer() else _HOME_STATUS_EXTERNAL
- else:
</del><ins>+ if status is None:
+ createStatus = _HOME_STATUS_NORMAL if record.thisServer() else _HOME_STATUS_EXTERNAL
+ elif status == _HOME_STATUS_MIGRATING:
</ins><span class="cx"> if record.thisServer():
</span><span class="cx"> raise RecordNotAllowedError("Cannot migrate a user data for a user already hosted on this server")
</span><del>- state = _HOME_STATUS_MIGRATING
</del><ins>+ createStatus = status
+ elif status in (_HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL,):
+ createStatus = status
+ else:
+ raise RecordNotAllowedError("Cannot create home with status {}: {}".format(status, uid))
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> # Use savepoint so we can do a partial rollback if there is a race condition
</span><span class="cx"> # where this row has already been inserted
</span><span class="cx"> savepoint = SavepointAction("homeWithUID")
</span><span class="lines">@@ -1816,7 +1889,7 @@
</span><span class="cx"> resourceid = (yield Insert(
</span><span class="cx"> {
</span><span class="cx"> cls._homeSchema.OWNER_UID: uid,
</span><del>- cls._homeSchema.STATUS: state,
</del><ins>+ cls._homeSchema.STATUS: createStatus,
</ins><span class="cx"> cls._homeSchema.DATAVERSION: cls._dataVersionValue,
</span><span class="cx"> },
</span><span class="cx"> Return=cls._homeSchema.RESOURCE_ID
</span><span class="lines">@@ -1826,8 +1899,13 @@
</span><span class="cx"> yield savepoint.rollback(txn)
</span><span class="cx">
</span><span class="cx"> # Retry the query - row may exist now, if not re-raise
</span><del>- homeObject = yield cls.makeClass(txn, uid, authzUID=authzUID)
- if homeObject:
</del><ins>+ results = yield Select(
+ cls.homeColumns(),
+ From=cls._homeSchema,
+ Where=query,
+ ).on(txn)
+ if results:
+ homeObject = yield cls.makeClass(txn, results[0], authzUID=authzUID)
</ins><span class="cx"> returnValue(homeObject)
</span><span class="cx"> else:
</span><span class="cx"> raise
</span><span class="lines">@@ -1835,28 +1913,27 @@
</span><span class="cx"> yield savepoint.release(txn)
</span><span class="cx">
</span><span class="cx"> # Note that we must not cache the owner_uid->resource_id
</span><del>- # mapping in _cacher when creating as we don't want that to appear
</del><ins>+ # mapping in the query cacher when creating as we don't want that to appear
</ins><span class="cx"> # until AFTER the commit
</span><del>- home = yield cls.makeClass(txn, uid, no_cache=True, authzUID=authzUID)
- if migratingUID is None:
- yield home.createdHome()
- returnValue(home)
</del><ins>+ results = yield Select(
+ cls.homeColumns(),
+ From=cls._homeSchema,
+ Where=cls._homeSchema.RESOURCE_ID == resourceid,
+ ).on(txn)
+ homeObject = yield cls.makeClass(txn, results[0], authzUID=authzUID)
+ if homeObject.normal():
+ yield homeObject.createdHome()
+ returnValue(homeObject)
</ins><span class="cx">
</span><span class="cx">
</span><del>- @classmethod
- @inlineCallbacks
- def homeUIDWithResourceID(cls, txn, rid):
- rows = (yield cls._ownerFromResourceID.on(txn, resourceID=rid))
- if rows:
- returnValue(rows[0][0])
- else:
- returnValue(None)
-
-
</del><span class="cx"> def __repr__(self):
</span><span class="cx"> return "<%s: %s, %s>" % (self.__class__.__name__, self._resourceID, self._ownerUID)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def cacheKey(self):
+ return "{}-{}".format(self._status, self._ownerUID)
+
+
</ins><span class="cx"> def id(self):
</span><span class="cx"> """
</span><span class="cx"> Retrieve the store identifier for this home.
</span><span class="lines">@@ -1957,8 +2034,18 @@
</span><span class="cx"> {self._homeSchema.STATUS: newStatus},
</span><span class="cx"> Where=(self._homeSchema.RESOURCE_ID == self._resourceID),
</span><span class="cx"> ).on(self._txn)
</span><ins>+ if self._txn._queryCacher:
+ yield self._txn._queryCacher.delete(self._txn._queryCacher.keyForHomeWithUID(
+ self._homeType,
+ self.uid(),
+ self._status,
+ ))
+ yield self._txn._queryCacher.delete(self._txn._queryCacher.keyForHomeWithID(
+ self._homeType,
+ self.id(),
+ self._status,
+ ))
</ins><span class="cx"> self._status = newStatus
</span><del>- yield self._cacher.delete(self._ownerUID)
</del><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1982,7 +2069,17 @@
</span><span class="cx">
</span><span class="cx"> yield self.properties()._removeResource()
</span><span class="cx">
</span><del>- yield self._cacher.delete(str(self._ownerUID))
</del><ins>+ if self._txn._queryCacher:
+ yield self._txn._queryCacher.delete(self._txn._queryCacher.keyForHomeWithUID(
+ self._homeType,
+ self.uid(),
+ self._status,
+ ))
+ yield self._txn._queryCacher.delete(self._txn._queryCacher.keyForHomeWithID(
+ self._homeType,
+ self.id(),
+ self._status,
+ ))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_external.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -42,17 +42,38 @@
</span><span class="cx"> are all stubbed out since no data for the user is actually hosted in this store.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def __init__(self, transaction, ownerUID, resourceID):
- super(CommonHomeExternal, self).__init__(transaction, ownerUID)
- self._resourceID = resourceID
- self._status = _HOME_STATUS_EXTERNAL
</del><ins>+ @classmethod
+ def makeSyntheticExternalHome(cls, transaction, diruid, resourceID):
+ """
+ During migration we need to refer to the remote home as an external home but without have a local representation
+ of it in the store. There will be a new local store home for the migrating user that will operate on local store
+ objects. The synthetic home operates only on remote objects.
</ins><span class="cx">
</span><ins>+ @param diruid: directory UID of user
+ @type diruid: L{str}
+ @param resourceID: resource ID in the remote store
+ @type resourceID: L{int}
+ """
+ attrMap = {
+ "_resourceID": resourceID,
+ "_ownerUID": diruid,
+ "_status": _HOME_STATUS_EXTERNAL,
+ }
+ homeData = [attrMap.get(attr) for attr in cls.homeAttributes()]
+ result = cls(transaction, homeData)
+ result._childClass = result._childClass._externalClass
+ return result
</ins><span class="cx">
</span><del>- def initFromStore(self, no_cache=False):
</del><ins>+
+ def __init__(self, transaction, homeData):
+ super(CommonHomeExternal, self).__init__(transaction, homeData)
+
+
+ def initFromStore(self):
</ins><span class="cx"> """
</span><del>- Never called - this should be done by CommonHome.initFromStore only.
</del><ins>+ NoOp for an external share as there is no metadata or properties.
</ins><span class="cx"> """
</span><del>- raise AssertionError("CommonHomeExternal: not supported")
</del><ins>+ return succeed(self)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -363,7 +384,11 @@
</span><span class="cx"> returnValue(dict([(k, self._bindRecordClass.deserialize(v),) for k, v in results.items()]))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def migrateBindRecords(self, bindUID):
+ return self._txn.store().conduit.send_homechild_migrate_sharing_records(self, bindUID)
</ins><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class CommonObjectResourceExternal(CommonObjectResource):
</span><span class="cx"> """
</span><span class="cx"> A CommonObjectResource for a resource not hosted on this system, but on another pod. This will forward
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentoracledialectsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -29,9 +29,10 @@
</span><span class="cx">
</span><span class="cx"> create table CALENDAR_HOME (
</span><span class="cx"> "RESOURCE_ID" integer primary key,
</span><del>- "OWNER_UID" nvarchar2(255) unique,
</del><ins>+ "OWNER_UID" nvarchar2(255),
</ins><span class="cx"> "STATUS" integer default 0 not null,
</span><del>- "DATAVERSION" integer default 0 not null
</del><ins>+ "DATAVERSION" integer default 0 not null,
+ unique ("OWNER_UID", "STATUS")
</ins><span class="cx"> );
</span><span class="cx">
</span><span class="cx"> create table HOME_STATUS (
</span><span class="lines">@@ -43,6 +44,7 @@
</span><span class="cx"> insert into HOME_STATUS (DESCRIPTION, ID) values ('external', 1);
</span><span class="cx"> insert into HOME_STATUS (DESCRIPTION, ID) values ('purging', 2);
</span><span class="cx"> insert into HOME_STATUS (DESCRIPTION, ID) values ('migrating', 3);
</span><ins>+insert into HOME_STATUS (DESCRIPTION, ID) values ('disabled', 4);
</ins><span class="cx"> create table CALENDAR (
</span><span class="cx"> "RESOURCE_ID" integer primary key
</span><span class="cx"> );
</span><span class="lines">@@ -79,9 +81,10 @@
</span><span class="cx">
</span><span class="cx"> create table NOTIFICATION_HOME (
</span><span class="cx"> "RESOURCE_ID" integer primary key,
</span><del>- "OWNER_UID" nvarchar2(255) unique,
</del><ins>+ "OWNER_UID" nvarchar2(255),
</ins><span class="cx"> "STATUS" integer default 0 not null,
</span><del>- "DATAVERSION" integer default 0 not null
</del><ins>+ "DATAVERSION" integer default 0 not null,
+ unique ("OWNER_UID", "STATUS")
</ins><span class="cx"> );
</span><span class="cx">
</span><span class="cx"> create table NOTIFICATION (
</span><span class="lines">@@ -262,9 +265,10 @@
</span><span class="cx"> create table ADDRESSBOOK_HOME (
</span><span class="cx"> "RESOURCE_ID" integer primary key,
</span><span class="cx"> "ADDRESSBOOK_PROPERTY_STORE_ID" integer not null,
</span><del>- "OWNER_UID" nvarchar2(255) unique,
</del><ins>+ "OWNER_UID" nvarchar2(255),
</ins><span class="cx"> "STATUS" integer default 0 not null,
</span><del>- "DATAVERSION" integer default 0 not null
</del><ins>+ "DATAVERSION" integer default 0 not null,
+ unique ("OWNER_UID", "STATUS")
</ins><span class="cx"> );
</span><span class="cx">
</span><span class="cx"> create table ADDRESSBOOK_HOME_METADATA (
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -70,9 +70,11 @@
</span><span class="cx">
</span><span class="cx"> create table CALENDAR_HOME (
</span><span class="cx"> RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
</span><del>- OWNER_UID varchar(255) not null unique, -- implicit index
</del><ins>+ OWNER_UID varchar(255) not null, -- implicit index
</ins><span class="cx"> STATUS integer default 0 not null, -- enum HOME_STATUS
</span><del>- DATAVERSION integer default 0 not null
</del><ins>+ DATAVERSION integer default 0 not null,
+
+ unique (OWNER_UID, STATUS) -- implicit index
</ins><span class="cx"> );
</span><span class="cx">
</span><span class="cx"> -- Enumeration of statuses
</span><span class="lines">@@ -86,6 +88,7 @@
</span><span class="cx"> insert into HOME_STATUS values (1, 'external');
</span><span class="cx"> insert into HOME_STATUS values (2, 'purging');
</span><span class="cx"> insert into HOME_STATUS values (3, 'migrating');
</span><ins>+insert into HOME_STATUS values (4, 'disabled');
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> --------------
</span><span class="lines">@@ -159,9 +162,11 @@
</span><span class="cx">
</span><span class="cx"> create table NOTIFICATION_HOME (
</span><span class="cx"> RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
</span><del>- OWNER_UID varchar(255) not null unique, -- implicit index
</del><ins>+ OWNER_UID varchar(255) not null, -- implicit index
</ins><span class="cx"> STATUS integer default 0 not null, -- enum HOME_STATUS
</span><del>- DATAVERSION integer default 0 not null
</del><ins>+ DATAVERSION integer default 0 not null,
+
+ unique (OWNER_UID, STATUS) -- implicit index
</ins><span class="cx"> );
</span><span class="cx">
</span><span class="cx"> create table NOTIFICATION (
</span><span class="lines">@@ -475,9 +480,11 @@
</span><span class="cx"> create table ADDRESSBOOK_HOME (
</span><span class="cx"> RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
</span><span class="cx"> ADDRESSBOOK_PROPERTY_STORE_ID integer default nextval('RESOURCE_ID_SEQ') not null, -- implicit index
</span><del>- OWNER_UID varchar(255) not null unique, -- implicit index
</del><ins>+ OWNER_UID varchar(255) not null,
</ins><span class="cx"> STATUS integer default 0 not null, -- enum HOME_STATUS
</span><del>- DATAVERSION integer default 0 not null
</del><ins>+ DATAVERSION integer default 0 not null,
+
+ unique (OWNER_UID, STATUS) -- implicit index
</ins><span class="cx"> );
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_51_to_52sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -20,7 +20,24 @@
</span><span class="cx">
</span><span class="cx"> -- New status value
</span><span class="cx"> insert into HOME_STATUS (DESCRIPTION, ID) values ('migrating', 3);
</span><ins>+insert into HOME_STATUS (DESCRIPTION, ID) values ('disabled', 4);
</ins><span class="cx">
</span><ins>+-- Home constraints
+alter table CALENDAR_HOME
+ drop unique (OWNER_UID);
+alter table CALENDAR_HOME
+ add unique (OWNER_UID, STATUS);
+
+alter table ADDRESSBOOK_HOME
+ drop unique (OWNER_UID);
+alter table ADDRESSBOOK_HOME
+ add unique (OWNER_UID, STATUS);
+
+alter table NOTIFICATION_HOME
+ drop unique (OWNER_UID);
+alter table NOTIFICATION_HOME
+ add unique (OWNER_UID, STATUS);
+
</ins><span class="cx"> -- Change columns
</span><span class="cx"> alter table CALENDAR_BIND
</span><span class="cx"> drop column EXTERNAL_ID
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_51_to_52sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -20,7 +20,21 @@
</span><span class="cx">
</span><span class="cx"> -- New status value
</span><span class="cx"> insert into HOME_STATUS values (3, 'migrating');
</span><ins>+insert into HOME_STATUS values (4, 'disabled');
</ins><span class="cx">
</span><ins>+-- Home constraints
+alter table CALENDAR_HOME
+ drop constraint CALENDAR_HOME_OWNER_UID_KEY,
+ add unique (OWNER_UID, STATUS);
+
+alter table ADDRESSBOOK_HOME
+ drop constraint ADDRESSBOOK_HOME_OWNER_UID_KEY,
+ add unique (OWNER_UID, STATUS);
+
+alter table NOTIFICATION_HOME
+ drop constraint NOTIFICATION_HOME_OWNER_UID_KEY,
+ add unique (OWNER_UID, STATUS);
+
</ins><span class="cx"> -- Change columns
</span><span class="cx"> alter table CALENDAR_BIND
</span><span class="cx"> drop column EXTERNAL_ID,
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_sharingpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_sharing.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -28,7 +28,8 @@
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="cx"> from txdav.common.datastore.sql_tables import _BIND_MODE_OWN, _BIND_MODE_DIRECT, \
</span><span class="cx"> _BIND_MODE_INDIRECT, _BIND_STATUS_ACCEPTED, _BIND_STATUS_DECLINED, \
</span><del>- _BIND_STATUS_INVITED, _BIND_STATUS_INVALID, _BIND_STATUS_DELETED
</del><ins>+ _BIND_STATUS_INVITED, _BIND_STATUS_INVALID, _BIND_STATUS_DELETED, \
+ _HOME_STATUS_EXTERNAL
</ins><span class="cx"> from txdav.common.icommondatastore import ExternalShareFailed, \
</span><span class="cx"> HomeChildNameAlreadyExistsError, AllRetriesFailed
</span><span class="cx"> from txdav.xml import element
</span><span class="lines">@@ -87,7 +88,7 @@
</span><span class="cx">
</span><span class="cx"> # Get the owner home - create external one if not present
</span><span class="cx"> ownerHome = yield self._txn.homeWithUID(
</span><del>- self._homeType, ownerUID, create=True
</del><ins>+ self._homeType, ownerUID, status=_HOME_STATUS_EXTERNAL, create=True
</ins><span class="cx"> )
</span><span class="cx"> if ownerHome is None or not ownerHome.external():
</span><span class="cx"> raise ExternalShareFailed("Invalid owner UID: {}".format(ownerUID))
</span><span class="lines">@@ -117,7 +118,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Get the owner home
</span><del>- ownerHome = yield self._txn.homeWithUID(self._homeType, ownerUID)
</del><ins>+ ownerHome = yield self._txn.homeWithUID(self._homeType, ownerUID, status=_HOME_STATUS_EXTERNAL)
</ins><span class="cx"> if ownerHome is None or not ownerHome.external():
</span><span class="cx"> raise ExternalShareFailed("Invalid owner UID: {}".format(ownerUID))
</span><span class="cx">
</span><span class="lines">@@ -147,7 +148,7 @@
</span><span class="cx"> # Make sure the shareeUID and shareUID match
</span><span class="cx">
</span><span class="cx"> # Get the owner home - create external one if not present
</span><del>- shareeHome = yield self._txn.homeWithUID(self._homeType, shareeUID)
</del><ins>+ shareeHome = yield self._txn.homeWithUID(self._homeType, shareeUID, status=_HOME_STATUS_EXTERNAL)
</ins><span class="cx"> if shareeHome is None or not shareeHome.external():
</span><span class="cx"> raise ExternalShareFailed(
</span><span class="cx"> "Invalid sharee UID: {}".format(shareeUID)
</span><span class="lines">@@ -761,7 +762,7 @@
</span><span class="cx"> @return: the name of the shared calendar in the new calendar home.
</span><span class="cx"> @rtype: L{str}
</span><span class="cx"> """
</span><del>- shareeHome = yield self._txn.calendarHomeWithUID(shareeUID, create=True)
</del><ins>+ shareeHome = yield self._txn.homeWithUID(self._home._homeType, shareeUID, create=True)
</ins><span class="cx"> returnValue(
</span><span class="cx"> (yield self.shareWith(shareeHome, mode, status, summary, shareName))
</span><span class="cx"> )
</span><span class="lines">@@ -1041,7 +1042,112 @@
</span><span class="cx"> returnValue(dict([(homeMap[getattr(record, self._bindHomeIDAttributeName)], record,) for record in records if record.bindMode != _BIND_MODE_OWN]))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def migrateBindRecords(self, bindUID):
+ """
+ The user that owns this collection is being migrated to another pod. We need to switch over
+ the sharing details to point to the new external user.
+ """
+ if self.owned():
+ return self.migrateSharedByRecords(bindUID)
+ else:
+ return self.migrateSharedToRecords()
+
+
</ins><span class="cx"> @inlineCallbacks
</span><ins>+ def migrateSharedByRecords(self, bindUID):
+ """
+ The user that owns this collection is being migrated to another pod. We need to switch over
+ the sharing details to point to the new external user. For sharees hosted on this pod, we
+ update their bind record to point to a new external home/calendar for the sharer. For sharees
+ hosted on other pods, we simply remove their bind entries.
+ """
+
+ # Get the external home and make sure there is a "fake" calendar associated with it
+ home = yield self.externalHome()
+ calendar = yield home.childWithBindUID(bindUID)
+ if calendar is None:
+ calendar = yield home.createCollectionForExternalShare(
+ self.name(),
+ bindUID,
+ self.getSupportedComponents() if hasattr(self, "getSupportedComponents") else None,
+ )
+
+ remaining = False
+ records = yield self._bindRecordClass.querysimple(self._txn, **{self._bindResourceIDAttributeName: self.id()})
+ for record in records:
+ if record.bindMode == _BIND_MODE_OWN:
+ continue
+ shareeHome = yield self._txn.homeWithResourceID(home._homeType, getattr(record, self._bindHomeIDAttributeName))
+ if shareeHome.normal():
+ remaining = True
+ yield record.update(**{
+ self._bindResourceIDAttributeName: calendar.id(),
+ })
+ else:
+ # It is OK to just delete (as opposed to doing a full "unshare") without adjusting other things
+ # like sync revisions since those would not have been used for an external share anyway. Also,
+ # revisions are tied to the calendar id and the original calendar will be removed after migration
+ # is complete.
+ yield record.delete()
+
+ # If there are no external shares remaining, we can remove the external calendar
+ if not remaining:
+ yield calendar.remove()
+
+
+ @inlineCallbacks
+ def migrateSharedToRecords(self):
+ """
+ The user that owns this collection is being migrated to another pod. We need to switch over
+ the sharing details to point to the new external user.
+ """
+
+ # Update the bind record for this calendar to point to the external home
+ records = yield self._bindRecordClass.querysimple(
+ self._txn,
+ **{
+ self._bindHomeIDAttributeName: self.viewerHome().id(),
+ self._bindResourceIDAttributeName: self.id(),
+ }
+ )
+
+ if len(records) == 1:
+
+ # What we do depends on whether the sharer is local to this pod or not
+ if self.ownerHome().normal():
+ # Get the external home for the sharee
+ home = yield self.externalHome()
+
+ yield records[0].update(**{
+ self._bindHomeIDAttributeName: home.id(),
+ })
+ else:
+ # It is OK to just delete (as opposed to doing a full "unshare") without adjusting other things
+ # like sync revisions since those would not have been used for an external share anyway. Also,
+ # revisions are tied to the sharee calendar home id and that will be removed after migration
+ # is complete.
+ yield records[0].delete()
+
+ # Clean up external calendar if no sharees left
+ calendar = yield self.ownerView()
+ invites = yield calendar.sharingInvites()
+ if len(invites) == 0:
+ yield calendar.remove()
+ else:
+ raise AssertionError("We must have a bind record for this calendar.")
+
+
+ def externalHome(self):
+ """
+ Create and return an L{CommonHome} for the user being migrated. Note that when called, the user
+ directory record may still indicate that they are hosted on this pod, so we have to forcibly create
+ a home for the external user.
+ """
+ currentHome = self.viewerHome()
+ return self._txn.homeWithUID(currentHome._homeType, currentHome.uid(), status=_HOME_STATUS_EXTERNAL, create=True)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def _initBindRevision(self):
</span><span class="cx"> yield self.syncToken() # init self._syncTokenRevision if None
</span><span class="cx"> self._bindRevision = self._syncTokenRevision
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_tablespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_tables.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -188,6 +188,7 @@
</span><span class="cx"> _HOME_STATUS_EXTERNAL = _homeStatus('external')
</span><span class="cx"> _HOME_STATUS_PURGING = _homeStatus('purging')
</span><span class="cx"> _HOME_STATUS_MIGRATING = _homeStatus('migrating')
</span><ins>+_HOME_STATUS_DISABLED = _homeStatus('disabled')
</ins><span class="cx">
</span><span class="cx"> _bindStatus = _schemaConstants(
</span><span class="cx"> schema.CALENDAR_BIND_STATUS.DESCRIPTION,
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoretestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/test/util.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -303,10 +303,7 @@
</span><span class="cx"> yield cleanupTxn.commit()
</span><span class="cx">
</span><span class="cx"> # Deal with memcached items that must be cleared
</span><del>- from txdav.caldav.datastore.sql import CalendarHome
- CalendarHome._cacher.flushAll()
- from txdav.carddav.datastore.sql import AddressBookHome
- AddressBookHome._cacher.flushAll()
</del><ins>+ storeToClean.queryCacher.flushAll()
</ins><span class="cx"> from txdav.base.propertystore.sql import PropertyStore
</span><span class="cx"> PropertyStore._cacher.flushAll()
</span><span class="cx">
</span><span class="lines">@@ -449,7 +446,7 @@
</span><span class="cx"> populateTxn._migrating = True
</span><span class="cx"> for homeUID in requirements:
</span><span class="cx"> calendars = requirements[homeUID]
</span><del>- home = yield populateTxn.calendarHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.calendarHomeWithUID(homeUID, create=True)
</ins><span class="cx"> if calendars is not None:
</span><span class="cx"> # We don't want the default calendar or inbox to appear unless it's
</span><span class="cx"> # explicitly listed.
</span><span class="lines">@@ -534,7 +531,7 @@
</span><span class="cx"> for homeUID in md5s:
</span><span class="cx"> calendars = md5s[homeUID]
</span><span class="cx"> if calendars is not None:
</span><del>- home = yield populateTxn.calendarHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.calendarHomeWithUID(homeUID, create=True)
</ins><span class="cx"> for calendarName in calendars:
</span><span class="cx"> calendarObjNames = calendars[calendarName]
</span><span class="cx"> if calendarObjNames is not None:
</span><span class="lines">@@ -565,7 +562,7 @@
</span><span class="cx"> for homeUID in requirements:
</span><span class="cx"> addressbooks = requirements[homeUID]
</span><span class="cx"> if addressbooks is not None:
</span><del>- home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.addressbookHomeWithUID(homeUID, create=True)
</ins><span class="cx"> # We don't want the default addressbook
</span><span class="cx"> try:
</span><span class="cx"> yield home.removeAddressBookWithName("addressbook")
</span><span class="lines">@@ -602,7 +599,7 @@
</span><span class="cx"> for homeUID in md5s:
</span><span class="cx"> addressbooks = md5s[homeUID]
</span><span class="cx"> if addressbooks is not None:
</span><del>- home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.addressbookHomeWithUID(homeUID, create=True)
</ins><span class="cx"> for addressbookName in addressbooks:
</span><span class="cx"> addressbookObjNames = addressbooks[addressbookName]
</span><span class="cx"> if addressbookObjNames is not None:
</span><span class="lines">@@ -883,32 +880,32 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def homeUnderTest(self, txn=None, name="home1", create=False):
</del><ins>+ def homeUnderTest(self, txn=None, name="home1", status=None, create=False):
</ins><span class="cx"> """
</span><span class="cx"> Get the calendar home detailed by C{requirements['home1']}.
</span><span class="cx"> """
</span><span class="cx"> if txn is None:
</span><span class="cx"> txn = self.transactionUnderTest()
</span><del>- returnValue((yield txn.calendarHomeWithUID(name, create=create)))
</del><ins>+ returnValue((yield txn.calendarHomeWithUID(name, status=status, create=create)))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def calendarUnderTest(self, txn=None, name="calendar_1", home="home1"):
</del><ins>+ def calendarUnderTest(self, txn=None, name="calendar_1", home="home1", status=None):
</ins><span class="cx"> """
</span><span class="cx"> Get the calendar detailed by C{requirements['home1']['calendar_1']}.
</span><span class="cx"> """
</span><span class="cx"> returnValue((
</span><del>- yield (yield self.homeUnderTest(txn, home)).calendarWithName(name)
</del><ins>+ yield (yield self.homeUnderTest(txn, name=home, status=status)).calendarWithName(name)
</ins><span class="cx"> ))
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def calendarObjectUnderTest(self, txn=None, name="1.ics", calendar_name="calendar_1", home="home1"):
</del><ins>+ def calendarObjectUnderTest(self, txn=None, name="1.ics", calendar_name="calendar_1", home="home1", status=None):
</ins><span class="cx"> """
</span><span class="cx"> Get the calendar detailed by
</span><span class="cx"> C{requirements[home][calendar_name][name]}.
</span><span class="cx"> """
</span><del>- returnValue((yield (yield self.calendarUnderTest(txn, name=calendar_name, home=home))
</del><ins>+ returnValue((yield (yield self.calendarUnderTest(txn, name=calendar_name, home=home, status=status))
</ins><span class="cx"> .calendarObjectWithName(name)))
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoreworktesttest_revision_cleanuppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/work/test/test_revision_cleanup.py (14480 => 14481)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/work/test/test_revision_cleanup.py 2015-02-27 19:42:41 UTC (rev 14480)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/work/test/test_revision_cleanup.py 2015-02-27 19:59:39 UTC (rev 14481)
</span><span class="lines">@@ -55,7 +55,7 @@
</span><span class="cx"> for homeUID in addressookRequirements:
</span><span class="cx"> addressbooks = addressookRequirements[homeUID]
</span><span class="cx"> if addressbooks is not None:
</span><del>- home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
</del><ins>+ home = yield populateTxn.addressbookHomeWithUID(homeUID, create=True)
</ins><span class="cx"> addressbook = home.addressbook()
</span><span class="cx">
</span><span class="cx"> addressbookObjNames = addressbooks[addressbook.name()]
</span></span></pre>
</div>
</div>
</body>
</html>