<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[12732] CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade</title>
</head>
<body>

<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt;  }
#msg dl a { font-weight: bold}
#msg dl a:link    { color:#fc3; }
#msg dl a:active  { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff  {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/12732">12732</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2014-02-20 10:08:01 -0800 (Thu, 20 Feb 2014)</dd>
</dl>

<h3>Log Message</h3>
<pre>Make sure file store migration applies SQL data upgrades.</pre>

<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtxdavcommondatastoreupgrademigratepy">CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_1_to_2py">CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py</a></li>
<li><a href="#CalendarServerbranchesreleaseCalendarServer52devtxdavcommondatastoreupgradetesttest_migratepy">CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py</a></li>
</ul>

</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesreleaseCalendarServer52devtxdavcommondatastoreupgrademigratepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py (12731 => 12732)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py        2014-02-20 18:07:19 UTC (rev 12731)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py        2014-02-20 18:08:01 UTC (rev 12732)
</span><span class="lines">@@ -23,31 +23,35 @@
</span><span class="cx"> import errno
</span><span class="cx"> import xattr
</span><span class="cx"> 
</span><ins>+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twisted.internet.defer import maybeDeferred
+from twisted.protocols.amp import AMP, Command, String, Boolean
</ins><span class="cx"> from twisted.python.failure import Failure
</span><del>-from twext.python.log import Logger
-
</del><ins>+from twisted.python.reflect import namedAny, qual
</ins><span class="cx"> from twisted.python.runtime import platform
</span><del>-from twisted.python.reflect import namedAny, qual
</del><span class="cx"> 
</span><del>-from twisted.internet.defer import inlineCallbacks, returnValue, succeed
-from twisted.internet.defer import maybeDeferred
-
</del><ins>+from twext.enterprise.dal.syntax import Update
+from twext.internet.spawnsvc import SpawnerService
</ins><span class="cx"> from twext.python.filepath import CachingFilePath
</span><del>-from twext.internet.spawnsvc import SpawnerService
</del><ins>+from twext.python.log import Logger
</ins><span class="cx"> 
</span><del>-from twisted.protocols.amp import AMP, Command, String, Boolean
-
</del><ins>+from txdav.base.datastore.util import normalizeUUIDOrNot
+from txdav.base.propertystore.appledouble_xattr import PropertyStore as AppleDoubleStore
+from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
+from txdav.caldav.datastore.util import fixOneCalendarObject
</ins><span class="cx"> from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
</span><span class="cx"> from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
</span><span class="cx"> from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
</span><del>-from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
-from txdav.base.propertystore.appledouble_xattr import (PropertyStore
-                                                        as AppleDoubleStore)
-from txdav.caldav.datastore.util import fixOneCalendarObject
-from txdav.base.datastore.util import normalizeUUIDOrNot
</del><ins>+from txdav.common.datastore.sql import EADDRESSBOOKTYPE, ECALENDARTYPE
</ins><span class="cx"> 
</span><ins>+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_1_to_2 import doUpgrade as doCalendarUpgrade_1_to_2
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_2_to_3 import doUpgrade as doCalendarUpgrade_2_to_3
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_3_to_4 import doUpgrade as doCalendarUpgrade_3_to_4
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_4_to_5 import doUpgrade as doCalendarUpgrade_4_to_5
+from txdav.common.datastore.upgrade.sql.upgrades.addressbook_upgrade_from_1_to_2 import doUpgrade as doAddressbookUpgrade_1_to_2
</ins><span class="cx"> 
</span><span class="cx"> 
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _getFixedComponent(cobj):
</span><span class="cx">     &quot;&quot;&quot;
</span><span class="lines">@@ -374,6 +378,8 @@
</span><span class="cx">             for fp in sqlAttachmentsPath.walk():
</span><span class="cx">                 os.chown(fp.path, uid, gid)
</span><span class="cx"> 
</span><ins>+        yield self.doDataUpgradeSteps()
+
</ins><span class="cx">         self.sqlStore.setMigrating(False)
</span><span class="cx"> 
</span><span class="cx">         self.log.warn(
</span><span class="lines">@@ -392,3 +398,35 @@
</span><span class="cx">         if self.fileStore is None:
</span><span class="cx">             return succeed(None)
</span><span class="cx">         return self.doMigration()
</span><ins>+
+
+    @inlineCallbacks
+    def doDataUpgradeSteps(self):
+        &quot;&quot;&quot;
+        Do SQL store data upgrades to make sure properties etc that were moved from the property store
+        into columns also get migrated to the current schema.
+        &quot;&quot;&quot;
+
+        # First force each home to v1 data format so the upgrades will be triggered
+        self.log.warn(&quot;Migration extra steps.&quot;)
+        txn = self.sqlStore.newTransaction()
+        for storeType in (ECALENDARTYPE, EADDRESSBOOKTYPE):
+            schema = txn._homeClass[storeType]._homeSchema
+            yield Update(
+                {schema.DATAVERSION: 1},
+                Where=None,
+            ).on(txn)
+        yield txn.commit()
+
+        # Now apply each required data upgrade
+        self.sqlStore.setUpgrading(True)
+        for upgrade, description  in (
+            (doCalendarUpgrade_1_to_2, &quot;Calendar data upgrade from v1 to v2&quot;),
+            (doCalendarUpgrade_2_to_3, &quot;Calendar data upgrade from v2 to v3&quot;),
+            (doCalendarUpgrade_3_to_4, &quot;Calendar data upgrade from v3 to v4&quot;),
+            (doCalendarUpgrade_4_to_5, &quot;Calendar data upgrade from v4 to v5&quot;),
+            (doAddressbookUpgrade_1_to_2, &quot;Addressbook data upgrade from v1 to v2&quot;),
+        ):
+            self.log.warn(&quot;Migration extra step: {}&quot;.format(description))
+            yield upgrade(self.sqlStore)
+        self.sqlStore.setUpgrading(False)
</ins></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devtxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_1_to_2py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py (12731 => 12732)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py        2014-02-20 18:07:19 UTC (rev 12731)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py        2014-02-20 18:08:01 UTC (rev 12732)
</span><span class="lines">@@ -17,7 +17,7 @@
</span><span class="cx"> 
</span><span class="cx"> from twext.enterprise.dal.syntax import Update
</span><span class="cx"> 
</span><del>-from twisted.internet.defer import inlineCallbacks
</del><ins>+from twisted.internet.defer import inlineCallbacks, returnValue
</ins><span class="cx"> 
</span><span class="cx"> from twistedcaldav import caldavxml
</span><span class="cx"> 
</span><span class="lines">@@ -58,22 +58,25 @@
</span><span class="cx"> 
</span><span class="cx">     sqlTxn = sqlStore.newTransaction()
</span><span class="cx">     try:
</span><del>-        calendar_rid = None
-        rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
-        total = len(rows)
-        count = 0
-        for calendar_rid, value in rows:
-            prop = WebDAVDocument.fromString(value).root_element
-            supported_components = &quot;,&quot;.join(sorted([comp.attributes[&quot;name&quot;].upper() for comp in prop.children]))
-            meta = schema.CALENDAR_METADATA
-            yield Update(
-                {
-                    meta.SUPPORTED_COMPONENTS : supported_components
-                },
-                Where=(meta.RESOURCE_ID == calendar_rid)
-            ).on(sqlTxn)
-            count += 1
-            logUpgradeStatus(&quot;Move supported-component-set&quot;, count, total)
</del><ins>+        # Do not move the properties if migrating, as migration will do a split and set supported-components,
+        # however we still need to remove the old properties.
+        if not sqlStore._migrating:
+            calendar_rid = None
+            rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
+            total = len(rows)
+            count = 0
+            for calendar_rid, value in rows:
+                prop = WebDAVDocument.fromString(value).root_element
+                supported_components = &quot;,&quot;.join(sorted([comp.attributes[&quot;name&quot;].upper() for comp in prop.children]))
+                meta = schema.CALENDAR_METADATA
+                yield Update(
+                    {
+                        meta.SUPPORTED_COMPONENTS : supported_components
+                    },
+                    Where=(meta.RESOURCE_ID == calendar_rid)
+                ).on(sqlTxn)
+                count += 1
+                logUpgradeStatus(&quot;Move supported-component-set&quot;, count, total)
</ins><span class="cx"> 
</span><span class="cx">         yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)
</span><span class="cx">         yield sqlTxn.commit()
</span><span class="lines">@@ -95,6 +98,11 @@
</span><span class="cx">     Split all calendars by component type.
</span><span class="cx">     &quot;&quot;&quot;
</span><span class="cx"> 
</span><ins>+    # This is already done when doing file-&gt;sql migration
+    if sqlStore._migrating:
+        returnValue(None)
+
+
</ins><span class="cx">     @inlineCallbacks
</span><span class="cx">     def doIt(txn, homeResourceID):
</span><span class="cx">         &quot;&quot;&quot;
</span></span></pre></div>
<a id="CalendarServerbranchesreleaseCalendarServer52devtxdavcommondatastoreupgradetesttest_migratepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py (12731 => 12732)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py        2014-02-20 18:07:19 UTC (rev 12731)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py        2014-02-20 18:08:01 UTC (rev 12732)
</span><span class="lines">@@ -30,8 +30,11 @@
</span><span class="cx"> from twisted.python.reflect import qual, namedAny
</span><span class="cx"> from twisted.trial.unittest import TestCase
</span><span class="cx"> 
</span><ins>+from twistedcaldav import customxml, caldavxml
</ins><span class="cx"> from twistedcaldav.config import config
</span><ins>+from twistedcaldav.ical import Component
</ins><span class="cx"> 
</span><ins>+from txdav.base.propertystore.base import PropertyName
</ins><span class="cx"> from txdav.caldav.datastore.test.common import CommonTests
</span><span class="cx"> from txdav.carddav.datastore.test.common import CommonTests as ABCommonTests
</span><span class="cx"> from txdav.common.datastore.file import CommonDataStore
</span><span class="lines">@@ -44,7 +47,9 @@
</span><span class="cx">     withSpecialValue
</span><span class="cx"> from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseStep, \
</span><span class="cx">     StoreSpawnerService, swapAMP
</span><ins>+from txdav.xml import element
</ins><span class="cx"> 
</span><ins>+
</ins><span class="cx"> import copy
</span><span class="cx"> 
</span><span class="cx"> 
</span><span class="lines">@@ -130,6 +135,28 @@
</span><span class="cx">     Tests for L{UpgradeToDatabaseStep}.
</span><span class="cx">     &quot;&quot;&quot;
</span><span class="cx"> 
</span><ins>+    av1 = Component.fromString(&quot;&quot;&quot;BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//calendarserver.org//Zonal//EN
+BEGIN:VAVAILABILITY
+ORGANIZER:mailto:user01@example.com
+UID:1@example.com
+DTSTAMP:20061005T133225Z
+DTEND:20140101T000000Z
+BEGIN:AVAILABLE
+UID:1-1@example.com
+DTSTAMP:20061005T133225Z
+SUMMARY:Monday to Friday from 9:00 to 17:00
+DTSTART:20130101T090000Z
+DTEND:20130101T170000Z
+RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
+END:AVAILABLE
+END:VAVAILABILITY
+END:VCALENDAR
+&quot;&quot;&quot;)
+
+
</ins><span class="cx">     @inlineCallbacks
</span><span class="cx">     def setUp(self):
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="lines">@@ -166,7 +193,32 @@
</span><span class="cx">             &quot;__uids__&quot;).child(&quot;ho&quot;).child(&quot;me&quot;).child(&quot;home1&quot;).child(
</span><span class="cx">             &quot;.some-extra-data&quot;).setContent(&quot;some extra data&quot;)
</span><span class="cx"> 
</span><ins>+        # Add some properties we want to check get migrated over
+        txn = self.fileStore.newTransaction()
+        home = yield txn.calendarHomeWithUID(&quot;home_defaults&quot;)
</ins><span class="cx"> 
</span><ins>+        cal = yield home.calendarWithName(&quot;calendar_1&quot;)
+        props = cal.properties()
+        props[PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)] = caldavxml.SupportedCalendarComponentSet(
+            caldavxml.CalendarComponent(name=&quot;VEVENT&quot;),
+            caldavxml.CalendarComponent(name=&quot;VTODO&quot;),
+        )
+        props[PropertyName.fromElement(element.ResourceType)] = element.ResourceType(
+            element.Collection(),
+            caldavxml.Calendar(),
+        )
+        props[PropertyName.fromElement(customxml.GETCTag)] = customxml.GETCTag.fromString(&quot;foobar&quot;)
+
+        inbox = yield home.calendarWithName(&quot;inbox&quot;)
+        props = inbox.properties()
+        props[PropertyName.fromElement(customxml.CalendarAvailability)] = customxml.CalendarAvailability.fromString(str(self.av1))
+        props[PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)] = caldavxml.ScheduleDefaultCalendarURL(
+            element.HRef.fromString(&quot;/calendars/__uids__/home_defaults/calendar_1&quot;),
+        )
+
+        yield txn.commit()
+
+
</ins><span class="cx">     def mergeRequirements(self, a, b):
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="cx">         Merge two requirements dictionaries together, modifying C{a} and
</span><span class="lines">@@ -370,6 +422,40 @@
</span><span class="cx">             self.assertEquals(object.md5(), md5)
</span><span class="cx"> 
</span><span class="cx"> 
</span><ins>+    @inlineCallbacks
+    def test_upgradeProperties(self):
+        &quot;&quot;&quot;
+        L{UpgradeToDatabaseService.startService} will do the upgrade, then
+        start its dependent service by adding it to its service hierarchy.
+        &quot;&quot;&quot;
+        yield self.upgrader.stepWithResult(None)
+        txn = self.sqlStore.newTransaction()
+        self.addCleanup(txn.commit)
+
+        # Want metadata preserved
+        home = (yield txn.calendarHomeWithUID(&quot;home_defaults&quot;))
+        cal = (yield home.calendarWithName(&quot;calendar_1&quot;))
+        inbox = (yield home.calendarWithName(&quot;inbox&quot;))
+
+        # Supported components
+        self.assertEqual(cal.getSupportedComponents(), &quot;VEVENT&quot;)
+        self.assertTrue(cal.properties().get(PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)) is None)
+
+        # Resource type removed
+        self.assertTrue(cal.properties().get(PropertyName.fromElement(element.ResourceType)) is None)
+
+        # Ctag removed
+        self.assertTrue(cal.properties().get(PropertyName.fromElement(customxml.GETCTag)) is None)
+
+        # Availability
+        self.assertEquals(str(home.getAvailability()), str(self.av1))
+        self.assertTrue(inbox.properties().get(PropertyName.fromElement(customxml.CalendarAvailability)) is None)
+
+        # Default calendar
+        self.assertTrue(home.isDefaultCalendar(cal))
+        self.assertTrue(inbox.properties().get(PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)) is None)
+
+
</ins><span class="cx">     def test_fileStoreFromPath(self):
</span><span class="cx">         &quot;&quot;&quot;
</span><span class="cx">         Verify that fileStoreFromPath() will return a CommonDataStore if
</span></span></pre>
</div>
</div>

</body>
</html>