[CalendarServer-changes] [12732] CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common /datastore/upgrade
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:19:03 PDT 2014
Revision: 12732
http://trac.calendarserver.org//changeset/12732
Author: cdaboo at apple.com
Date: 2014-02-20 10:08:01 -0800 (Thu, 20 Feb 2014)
Log Message:
-----------
Make sure file store migration applies SQL data upgrades.
Modified Paths:
--------------
CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py
CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py
CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py
Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py 2014-02-20 18:07:19 UTC (rev 12731)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/migrate.py 2014-02-20 18:08:01 UTC (rev 12732)
@@ -23,31 +23,35 @@
import errno
import xattr
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+from twisted.internet.defer import maybeDeferred
+from twisted.protocols.amp import AMP, Command, String, Boolean
from twisted.python.failure import Failure
-from twext.python.log import Logger
-
+from twisted.python.reflect import namedAny, qual
from twisted.python.runtime import platform
-from twisted.python.reflect import namedAny, qual
-from twisted.internet.defer import inlineCallbacks, returnValue, succeed
-from twisted.internet.defer import maybeDeferred
-
+from twext.enterprise.dal.syntax import Update
+from twext.internet.spawnsvc import SpawnerService
from twext.python.filepath import CachingFilePath
-from twext.internet.spawnsvc import SpawnerService
+from twext.python.log import Logger
-from twisted.protocols.amp import AMP, Command, String, Boolean
-
+from txdav.base.datastore.util import normalizeUUIDOrNot
+from txdav.base.propertystore.appledouble_xattr import PropertyStore as AppleDoubleStore
+from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
+from txdav.caldav.datastore.util import fixOneCalendarObject
from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
-from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
-from txdav.base.propertystore.appledouble_xattr import (PropertyStore
- as AppleDoubleStore)
-from txdav.caldav.datastore.util import fixOneCalendarObject
-from txdav.base.datastore.util import normalizeUUIDOrNot
+from txdav.common.datastore.sql import EADDRESSBOOKTYPE, ECALENDARTYPE
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_1_to_2 import doUpgrade as doCalendarUpgrade_1_to_2
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_2_to_3 import doUpgrade as doCalendarUpgrade_2_to_3
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_3_to_4 import doUpgrade as doCalendarUpgrade_3_to_4
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_4_to_5 import doUpgrade as doCalendarUpgrade_4_to_5
+from txdav.common.datastore.upgrade.sql.upgrades.addressbook_upgrade_from_1_to_2 import doUpgrade as doAddressbookUpgrade_1_to_2
+
@inlineCallbacks
def _getFixedComponent(cobj):
"""
@@ -374,6 +378,8 @@
for fp in sqlAttachmentsPath.walk():
os.chown(fp.path, uid, gid)
+ yield self.doDataUpgradeSteps()
+
self.sqlStore.setMigrating(False)
self.log.warn(
@@ -392,3 +398,35 @@
if self.fileStore is None:
return succeed(None)
return self.doMigration()
+
+
+ @inlineCallbacks
+ def doDataUpgradeSteps(self):
+ """
+ Do SQL store data upgrades to make sure properties etc that were moved from the property store
+ into columns also get migrated to the current schema.
+ """
+
+ # First force each home to v1 data format so the upgrades will be triggered
+ self.log.warn("Migration extra steps.")
+ txn = self.sqlStore.newTransaction()
+ for storeType in (ECALENDARTYPE, EADDRESSBOOKTYPE):
+ schema = txn._homeClass[storeType]._homeSchema
+ yield Update(
+ {schema.DATAVERSION: 1},
+ Where=None,
+ ).on(txn)
+ yield txn.commit()
+
+ # Now apply each required data upgrade
+ self.sqlStore.setUpgrading(True)
+ for upgrade, description in (
+ (doCalendarUpgrade_1_to_2, "Calendar data upgrade from v1 to v2"),
+ (doCalendarUpgrade_2_to_3, "Calendar data upgrade from v2 to v3"),
+ (doCalendarUpgrade_3_to_4, "Calendar data upgrade from v3 to v4"),
+ (doCalendarUpgrade_4_to_5, "Calendar data upgrade from v4 to v5"),
+ (doAddressbookUpgrade_1_to_2, "Addressbook data upgrade from v1 to v2"),
+ ):
+ self.log.warn("Migration extra step: {}".format(description))
+ yield upgrade(self.sqlStore)
+ self.sqlStore.setUpgrading(False)
Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py 2014-02-20 18:07:19 UTC (rev 12731)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py 2014-02-20 18:08:01 UTC (rev 12732)
@@ -17,7 +17,7 @@
from twext.enterprise.dal.syntax import Update
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import inlineCallbacks, returnValue
from twistedcaldav import caldavxml
@@ -58,22 +58,25 @@
sqlTxn = sqlStore.newTransaction()
try:
- calendar_rid = None
- rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
- total = len(rows)
- count = 0
- for calendar_rid, value in rows:
- prop = WebDAVDocument.fromString(value).root_element
- supported_components = ",".join(sorted([comp.attributes["name"].upper() for comp in prop.children]))
- meta = schema.CALENDAR_METADATA
- yield Update(
- {
- meta.SUPPORTED_COMPONENTS : supported_components
- },
- Where=(meta.RESOURCE_ID == calendar_rid)
- ).on(sqlTxn)
- count += 1
- logUpgradeStatus("Move supported-component-set", count, total)
+ # Do not move the properties if migrating, as migration will do a split and set supported-components,
+ # however we still need to remove the old properties.
+ if not sqlStore._migrating:
+ calendar_rid = None
+ rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
+ total = len(rows)
+ count = 0
+ for calendar_rid, value in rows:
+ prop = WebDAVDocument.fromString(value).root_element
+ supported_components = ",".join(sorted([comp.attributes["name"].upper() for comp in prop.children]))
+ meta = schema.CALENDAR_METADATA
+ yield Update(
+ {
+ meta.SUPPORTED_COMPONENTS : supported_components
+ },
+ Where=(meta.RESOURCE_ID == calendar_rid)
+ ).on(sqlTxn)
+ count += 1
+ logUpgradeStatus("Move supported-component-set", count, total)
yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)
yield sqlTxn.commit()
@@ -95,6 +98,11 @@
Split all calendars by component type.
"""
+ # This is already done when doing file->sql migration
+ if sqlStore._migrating:
+ returnValue(None)
+
+
@inlineCallbacks
def doIt(txn, homeResourceID):
"""
Modified: CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py 2014-02-20 18:07:19 UTC (rev 12731)
+++ CalendarServer/branches/release/CalendarServer-5.2-dev/txdav/common/datastore/upgrade/test/test_migrate.py 2014-02-20 18:08:01 UTC (rev 12732)
@@ -30,8 +30,11 @@
from twisted.python.reflect import qual, namedAny
from twisted.trial.unittest import TestCase
+from twistedcaldav import customxml, caldavxml
from twistedcaldav.config import config
+from twistedcaldav.ical import Component
+from txdav.base.propertystore.base import PropertyName
from txdav.caldav.datastore.test.common import CommonTests
from txdav.carddav.datastore.test.common import CommonTests as ABCommonTests
from txdav.common.datastore.file import CommonDataStore
@@ -44,7 +47,9 @@
withSpecialValue
from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseStep, \
StoreSpawnerService, swapAMP
+from txdav.xml import element
+
import copy
@@ -130,6 +135,28 @@
Tests for L{UpgradeToDatabaseStep}.
"""
+ av1 = Component.fromString("""BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//calendarserver.org//Zonal//EN
+BEGIN:VAVAILABILITY
+ORGANIZER:mailto:user01 at example.com
+UID:1 at example.com
+DTSTAMP:20061005T133225Z
+DTEND:20140101T000000Z
+BEGIN:AVAILABLE
+UID:1-1 at example.com
+DTSTAMP:20061005T133225Z
+SUMMARY:Monday to Friday from 9:00 to 17:00
+DTSTART:20130101T090000Z
+DTEND:20130101T170000Z
+RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
+END:AVAILABLE
+END:VAVAILABILITY
+END:VCALENDAR
+""")
+
+
@inlineCallbacks
def setUp(self):
"""
@@ -166,7 +193,32 @@
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
+ # Add some properties we want to check get migrated over
+ txn = self.fileStore.newTransaction()
+ home = yield txn.calendarHomeWithUID("home_defaults")
+ cal = yield home.calendarWithName("calendar_1")
+ props = cal.properties()
+ props[PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)] = caldavxml.SupportedCalendarComponentSet(
+ caldavxml.CalendarComponent(name="VEVENT"),
+ caldavxml.CalendarComponent(name="VTODO"),
+ )
+ props[PropertyName.fromElement(element.ResourceType)] = element.ResourceType(
+ element.Collection(),
+ caldavxml.Calendar(),
+ )
+ props[PropertyName.fromElement(customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
+
+ inbox = yield home.calendarWithName("inbox")
+ props = inbox.properties()
+ props[PropertyName.fromElement(customxml.CalendarAvailability)] = customxml.CalendarAvailability.fromString(str(self.av1))
+ props[PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)] = caldavxml.ScheduleDefaultCalendarURL(
+ element.HRef.fromString("/calendars/__uids__/home_defaults/calendar_1"),
+ )
+
+ yield txn.commit()
+
+
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
@@ -370,6 +422,40 @@
self.assertEquals(object.md5(), md5)
+ @inlineCallbacks
+ def test_upgradeProperties(self):
+ """
+ L{UpgradeToDatabaseService.startService} will do the upgrade, then
+ start its dependent service by adding it to its service hierarchy.
+ """
+ yield self.upgrader.stepWithResult(None)
+ txn = self.sqlStore.newTransaction()
+ self.addCleanup(txn.commit)
+
+ # Want metadata preserved
+ home = (yield txn.calendarHomeWithUID("home_defaults"))
+ cal = (yield home.calendarWithName("calendar_1"))
+ inbox = (yield home.calendarWithName("inbox"))
+
+ # Supported components
+ self.assertEqual(cal.getSupportedComponents(), "VEVENT")
+ self.assertTrue(cal.properties().get(PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)) is None)
+
+ # Resource type removed
+ self.assertTrue(cal.properties().get(PropertyName.fromElement(element.ResourceType)) is None)
+
+ # Ctag removed
+ self.assertTrue(cal.properties().get(PropertyName.fromElement(customxml.GETCTag)) is None)
+
+ # Availability
+ self.assertEquals(str(home.getAvailability()), str(self.av1))
+ self.assertTrue(inbox.properties().get(PropertyName.fromElement(customxml.CalendarAvailability)) is None)
+
+ # Default calendar
+ self.assertTrue(home.isDefaultCalendar(cal))
+ self.assertTrue(inbox.properties().get(PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)) is None)
+
+
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/311e22af/attachment.html>
More information about the calendarserver-changes
mailing list