[CalendarServer-changes] [11860] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:15:35 PDT 2014
Revision: 11860
http://trac.calendarserver.org//changeset/11860
Author: cdaboo at apple.com
Date: 2013-10-31 14:40:44 -0700 (Thu, 31 Oct 2013)
Log Message:
-----------
Allow for incremental upgrade of specific calendar homes.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tools/upgrade.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/util.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -1429,7 +1429,9 @@
# Conditionally stop after upgrade at this point
pps.addStep(
- QuitAfterUpgradeStep(config.StopAfterUpgradeTriggerFile)
+ QuitAfterUpgradeStep(
+ config.StopAfterUpgradeTriggerFile or config.UpgradeHomePrefix
+ )
)
pps.addStep(
Modified: CalendarServer/trunk/calendarserver/tools/upgrade.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/upgrade.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/calendarserver/tools/upgrade.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -82,6 +82,7 @@
optParameters = [
['config', 'f', DEFAULT_CONFIG_FILE, "Specify caldavd.plist configuration path."],
+ ['prefix', 'x', "", "Only upgrade homes with the specified GUID prefix - partial upgrade only."],
]
def __init__(self):
@@ -197,9 +198,11 @@
data.MergeUpgrades = True
config.addPostUpdateHooks([setMerge])
+
def makeService(store):
return UpgraderService(store, options, output, reactor, config)
+
def onlyUpgradeEvents(eventDict):
text = formatEvent(eventDict)
output.write(logDateString() + " " + text + "\n")
@@ -209,14 +212,19 @@
log.publisher.levels.setLogLevelForNamespace(None, LogLevel.debug)
addObserver(onlyUpgradeEvents)
+
def customServiceMaker():
customService = CalDAVServiceMaker()
customService.doPostImport = options["postprocess"]
return customService
+
def _patchConfig(config):
config.FailIfUpgradeNeeded = options["status"]
+ if options["prefix"]:
+ config.UpgradeHomePrefix = options["prefix"]
+
def _onShutdown():
if not UpgraderService.started:
print("Failed to start service.")
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -307,9 +307,15 @@
"FailIfUpgradeNeeded" : True, # Set to True to prevent the server or utility tools
# tools from running if the database needs a schema
# upgrade.
- "StopAfterUpgradeTriggerFile" : "stop_after_upgrade", # if this file exists
- # in ConfigRoot, stop the service after finishing upgrade phase
+ "StopAfterUpgradeTriggerFile" : "stop_after_upgrade", # if this file exists in ConfigRoot, stop
+ # the service after finishing upgrade phase
+ "UpgradeHomePrefix" : "", # When upgrading, only upgrade homes where the owner UID starts with
+ # with the specified prefix. The upgrade will only be partial and only
+ # apply to upgrade pieces that affect entire homes. The upgrade will
+ # need to be run again without this prefix set to complete the overall
+ # upgrade.
+
#
# Types of service provided
#
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -1444,6 +1444,7 @@
self._txn = transaction
self._ownerUID = ownerUID
self._resourceID = None
+ self._dataVersion = None
self._childrenLoaded = False
self._children = {}
self._notifiers = None
@@ -1689,6 +1690,23 @@
yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+ @classproperty
+ def _dataVersionQuery(cls): #@NoSelf
+ ch = cls._homeSchema
+ return Select(
+ [ch.DATAVERSION], From=ch,
+ Where=ch.RESOURCE_ID == Parameter("resourceID")
+ )
+
+
+ @inlineCallbacks
+ def dataVersion(self):
+ if self._dataVersion is None:
+ self._dataVersion = (yield self._dataVersionQuery.on(
+ self._txn, resourceID=self._resourceID))[0][0]
+ returnValue(self._dataVersion)
+
+
def name(self):
"""
Implement L{IDataStoreObject.name} to return the uid.
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -15,21 +15,17 @@
# limitations under the License.
##
-from twext.enterprise.dal.syntax import Select, Delete, Parameter
-
from twisted.internet.defer import inlineCallbacks
from twistedcaldav import caldavxml, customxml
from txdav.base.propertystore.base import PropertyName
from txdav.caldav.icalendarstore import InvalidDefaultCalendar
-from txdav.common.datastore.sql_tables import schema, _BIND_MODE_OWN
-from txdav.common.datastore.upgrade.sql.upgrades.util import rowsForProperty, updateCalendarDataVersion, \
- updateAllCalendarHomeDataVersions, removeProperty, countProperty, cleanPropertyStore, \
- logUpgradeStatus, logUpgradeError
-from txdav.xml.parser import WebDAVDocument
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.upgrade.sql.upgrades.util import updateCalendarDataVersion, \
+ removeProperty, cleanPropertyStore, logUpgradeStatus, doToEachHomeNotAtVersion
from txdav.xml import element
-from twisted.python.failure import Failure
+from twistedcaldav.config import config
"""
Data upgrade from database version 3 to 4
@@ -43,212 +39,111 @@
"""
Do the required upgrade steps.
"""
- yield moveDefaultCalendarProperties(sqlStore)
- yield moveCalendarTranspProperties(sqlStore)
- yield moveDefaultAlarmProperties(sqlStore)
- yield removeResourceType(sqlStore)
+ yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix)
- # Always bump the DB value
- yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
- yield updateAllCalendarHomeDataVersions(sqlStore, UPGRADE_TO_VERSION)
+ # Don't do remaining upgrade if we are only process a subset of the homes
+ if not config.UpgradeHomePrefix:
+ yield removeResourceType(sqlStore)
+ # Always bump the DB value
+ yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
+
@inlineCallbacks
-def moveDefaultCalendarProperties(sqlStore):
+def updateCalendarHomes(sqlStore, prefix=None):
"""
- Need to move all the CalDAV:default-calendar and CS:default-tasks properties in the
- RESOURCE_PROPERTY table to the new CALENDAR_HOME_METADATA table columns, extracting
- the new value from the XML property.
+ For each calendar home, update the associated properties on the home or its owned calendars.
"""
- meta = schema.CALENDAR_HOME_METADATA
- yield _processDefaultCalendarProperty(sqlStore, caldavxml.ScheduleDefaultCalendarURL, meta.DEFAULT_EVENTS)
- yield _processDefaultCalendarProperty(sqlStore, customxml.ScheduleDefaultTasksURL, meta.DEFAULT_TASKS)
+ yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix)
@inlineCallbacks
-def _processDefaultCalendarProperty(sqlStore, propname, colname):
+def updateCalendarHome(txn, homeResourceID):
"""
- Move the specified property value to the matching CALENDAR_HOME_METADATA table column.
-
- Since the number of calendar homes may well be large, we need to do this in batches.
+ For this calendar home, update the associated properties on the home or its owned calendars.
"""
- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
+ home = yield txn.calendarHomeWithResourceID(homeResourceID)
+ yield moveDefaultCalendarProperties(home)
+ yield moveCalendarTranspProperties(home)
+ yield moveDefaultAlarmProperties(home)
+ yield cleanPropertyStore()
- logUpgradeStatus("Starting Process {}".format(propname.qname()))
- sqlTxn = sqlStore.newTransaction()
- total = (yield countProperty(sqlTxn, propname))
- yield sqlTxn.commit()
- count = 0
- try:
- inbox_rid = None
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
+ at inlineCallbacks
+def moveDefaultCalendarProperties(home):
+ """
+ Need to move any the CalDAV:default-calendar and CS:default-tasks properties in the
+ RESOURCE_PROPERTY table to the new CALENDAR_HOME_METADATA table columns, extracting
+ the new value from the XML property.
+ """
- delete_ids = []
- for inbox_rid, value in rows:
- delete_ids.append(inbox_rid)
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == inbox_rid,
- ).on(sqlTxn)
- if len(ids) > 0:
+ yield _processDefaultCalendarProperty(home, caldavxml.ScheduleDefaultCalendarURL)
+ yield _processDefaultCalendarProperty(home, customxml.ScheduleDefaultTasksURL)
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(ids[0][0]))
- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- defaultCalendar = str(prop.children[0])
- parts = defaultCalendar.split("/")
- if len(parts) == 5:
- calendarName = parts[-1]
- calendarHomeUID = parts[-2]
- expectedHome = (yield sqlTxn.calendarHomeWithUID(calendarHomeUID))
- if expectedHome is not None and expectedHome.id() == calendarHome.id():
+ at inlineCallbacks
+def _processDefaultCalendarProperty(home, propname):
+ """
+ Move the specified property value to the matching CALENDAR_HOME_METADATA table column.
+ """
- calendar = (yield calendarHome.calendarWithName(calendarName))
- if calendar is not None:
- try:
- yield calendarHome.setDefaultCalendar(
- calendar, tasks=(propname == customxml.ScheduleDefaultTasksURL)
- )
- except InvalidDefaultCalendar:
- # Ignore these - the server will recover
- pass
+ inbox = (yield home.calendarWithName("inbox"))
+ prop = inbox.properties().get(PropertyName.fromElement(propname))
+ if prop is not None:
+ defaultCalendar = str(prop.children[0])
+ parts = defaultCalendar.split("/")
+ if len(parts) == 5:
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(propname).toString()),
- ).on(sqlTxn, ids=delete_ids)
+ calendarName = parts[-1]
+ calendarHomeUID = parts[-2]
+ if calendarHomeUID == home.uid():
- yield sqlTxn.commit()
+ calendar = (yield home.calendarWithName(calendarName))
+ if calendar is not None:
+ try:
+ yield home.setDefaultCalendar(
+ calendar, tasks=(propname == customxml.ScheduleDefaultTasksURL)
+ )
+ except InvalidDefaultCalendar:
+ # Ignore these - the server will recover
+ pass
- count += len(rows)
- logUpgradeStatus(
- "Process {}".format(propname.qname()),
- count,
- total
- )
+ del inbox.properties()[PropertyName.fromElement(propname)]
- yield cleanPropertyStore()
- logUpgradeStatus("End Process {}".format(propname.qname()))
- except RuntimeError as e:
- f = Failure()
- logUpgradeError(
- "Process {}".format(propname.qname()),
- "Inbox: {}, error: {}".format(inbox_rid, e),
- )
- yield sqlTxn.abort()
- f.raiseException()
-
-
@inlineCallbacks
-def moveCalendarTranspProperties(sqlStore):
+def moveCalendarTranspProperties(home):
"""
Need to move all the CalDAV:schedule-calendar-transp properties in the
RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
the new value from the XML property.
"""
- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
+ # Iterate over each calendar (both owned and shared)
+ calendars = (yield home.loadChildren())
+ for calendar in calendars:
+ if calendar.isInbox():
+ continue
+ prop = calendar.properties().get(PropertyName.fromElement(caldavxml.ScheduleCalendarTransp))
+ if prop is not None:
+ yield calendar.setUsedForFreeBusy(prop == caldavxml.ScheduleCalendarTransp(caldavxml.Opaque()))
+ del calendar.properties()[PropertyName.fromElement(caldavxml.ScheduleCalendarTransp)]
+ inbox = (yield home.calendarWithName("inbox"))
+ prop = inbox.properties().get(PropertyName.fromElement(caldavxml.CalendarFreeBusySet))
+ if prop is not None:
+ del inbox.properties()[PropertyName.fromElement(caldavxml.CalendarFreeBusySet)]
- propname = caldavxml.ScheduleCalendarTransp
- logUpgradeStatus("Starting Process {}".format(propname.qname()))
- sqlTxn = sqlStore.newTransaction()
- total = (yield countProperty(sqlTxn, propname))
- yield sqlTxn.commit()
- count = 0
- try:
- calendar_rid = None
- calendars_for_id = {}
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, with_uid=True, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
-
- delete_ids = []
- for calendar_rid, value, viewer in rows:
- delete_ids.append(calendar_rid)
- if calendar_rid not in calendars_for_id:
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, cb.BIND_MODE, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == calendar_rid,
- ).on(sqlTxn)
- calendars_for_id[calendar_rid] = ids
-
- if viewer:
- calendarHome = (yield sqlTxn.calendarHomeWithUID(viewer))
- else:
- calendarHome = None
- for row in calendars_for_id[calendar_rid]:
- home_id, bind_mode = row
- if bind_mode == _BIND_MODE_OWN:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(home_id))
- break
-
- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- calendar = (yield calendarHome.childWithID(calendar_rid))
- if calendar is not None:
- yield calendar.setUsedForFreeBusy(prop == caldavxml.ScheduleCalendarTransp(caldavxml.Opaque()))
-
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(caldavxml.ScheduleCalendarTransp).toString()),
- ).on(sqlTxn, ids=delete_ids)
-
- yield sqlTxn.commit()
-
- count += len(rows)
- logUpgradeStatus(
- "Process {}".format(propname.qname()),
- count,
- total,
- )
-
- sqlTxn = sqlStore.newTransaction()
- yield removeProperty(sqlTxn, PropertyName.fromElement(caldavxml.CalendarFreeBusySet))
- yield sqlTxn.commit()
- yield cleanPropertyStore()
- logUpgradeStatus("End Process {}".format(propname.qname()))
-
- except RuntimeError as e:
- f = Failure()
- logUpgradeError(
- "Process {}".format(propname.qname()),
- "Inbox: {}, error: {}".format(calendar_rid, e),
- )
- yield sqlTxn.abort()
- f.raiseException()
-
-
-
@inlineCallbacks
-def moveDefaultAlarmProperties(sqlStore):
+def moveDefaultAlarmProperties(home):
"""
Need to move all the CalDAV:default-calendar and CS:default-tasks properties in the
RESOURCE_PROPERTY table to the new CALENDAR_HOME_METADATA table columns, extracting
@@ -256,25 +151,25 @@
"""
yield _processDefaultAlarmProperty(
- sqlStore,
+ home,
caldavxml.DefaultAlarmVEventDateTime,
True,
True,
)
yield _processDefaultAlarmProperty(
- sqlStore,
+ home,
caldavxml.DefaultAlarmVEventDate,
True,
False,
)
yield _processDefaultAlarmProperty(
- sqlStore,
+ home,
caldavxml.DefaultAlarmVToDoDateTime,
False,
True,
)
yield _processDefaultAlarmProperty(
- sqlStore,
+ home,
caldavxml.DefaultAlarmVToDoDate,
False,
False,
@@ -283,108 +178,33 @@
@inlineCallbacks
-def _processDefaultAlarmProperty(sqlStore, propname, vevent, timed):
+def _processDefaultAlarmProperty(home, propname, vevent, timed):
"""
Move the specified property value to the matching CALENDAR_HOME_METADATA or CALENDAR_BIND table column.
Since the number of properties may well be large, we need to do this in batches.
"""
- hm = schema.CALENDAR_HOME_METADATA
- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
+ # Check the home first
+ prop = home.properties().get(PropertyName.fromElement(propname))
+ if prop is not None:
+ alarm = str(prop.children[0]) if prop.children else None
+ yield home.setDefaultAlarm(alarm, vevent, timed)
+ del home.properties()[PropertyName.fromElement(propname)]
- logUpgradeStatus("Starting Process {} {}".format(propname.qname(), vevent))
+ # Now each child
+ calendars = (yield home.loadChildren())
+ for calendar in calendars:
+ if calendar.isInbox():
+ continue
+ prop = calendar.properties().get(PropertyName.fromElement(propname))
+ if prop is not None:
+ alarm = str(prop.children[0]) if prop.children else None
+ yield calendar.setDefaultAlarm(alarm, vevent, timed)
+ del calendar.properties()[PropertyName.fromElement(propname)]
- sqlTxn = sqlStore.newTransaction()
- total = (yield countProperty(sqlTxn, propname))
- yield sqlTxn.commit()
- count = 0
- try:
- rid = None
- calendars_for_id = {}
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, with_uid=True, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
- delete_ids = []
- for rid, value, viewer in rows:
- delete_ids.append(rid)
-
- prop = WebDAVDocument.fromString(value).root_element
- alarm = str(prop.children[0]) if prop.children else None
-
- # First check if the rid is a home - this is the most common case
- ids = yield Select(
- [hm.RESOURCE_ID, ],
- From=hm,
- Where=hm.RESOURCE_ID == rid,
- ).on(sqlTxn)
-
- if len(ids) > 0:
- # Home object
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(ids[0][0]))
- if calendarHome is not None:
- yield calendarHome.setDefaultAlarm(alarm, vevent, timed)
- else:
- # rid is a calendar - we need to find the per-user calendar for the resource viewer
- if rid not in calendars_for_id:
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, cb.BIND_MODE, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == rid,
- ).on(sqlTxn)
- calendars_for_id[rid] = ids
-
- if viewer:
- calendarHome = (yield sqlTxn.calendarHomeWithUID(viewer))
- else:
- calendarHome = None
- for row in calendars_for_id[rid]:
- home_id, bind_mode = row
- if bind_mode == _BIND_MODE_OWN:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(home_id))
- break
-
- if calendarHome is not None:
- calendar = yield calendarHome.childWithID(rid)
- if calendar is not None:
- yield calendar.setDefaultAlarm(alarm, vevent, timed)
-
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(propname).toString()),
- ).on(sqlTxn, ids=delete_ids)
-
- yield sqlTxn.commit()
-
- count += len(rows)
- logUpgradeStatus(
- "Process {} {}".format(propname.qname(), vevent),
- count,
- total,
- )
-
- yield cleanPropertyStore()
- logUpgradeStatus("End Process {} {}".format(propname.qname(), vevent))
-
- except RuntimeError as e:
- f = Failure()
- logUpgradeError(
- "Process {} {}".format(propname.qname(), vevent),
- "Rid: {}, error: {}".format(rid, e),
- )
- yield sqlTxn.abort()
- f.raiseException()
-
-
-
@inlineCallbacks
def removeResourceType(sqlStore):
logUpgradeStatus("Starting Calendar Remove Resource Type")
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -15,22 +15,18 @@
# limitations under the License.
##
-from twext.enterprise.dal.syntax import Select, Delete, Parameter
+from twext.web2.dav.resource import TwistedQuotaUsedProperty, TwistedGETContentMD5
from twisted.internet.defer import inlineCallbacks
-from twisted.python.failure import Failure
from twistedcaldav import caldavxml, customxml
+from twistedcaldav.config import config
from txdav.base.propertystore.base import PropertyName
-from txdav.common.datastore.sql_tables import schema, _BIND_MODE_OWN
-from txdav.common.datastore.upgrade.sql.upgrades.util import rowsForProperty, updateCalendarDataVersion, \
- updateAllCalendarHomeDataVersions, removeProperty, cleanPropertyStore, \
- logUpgradeStatus, countProperty, logUpgradeError
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.upgrade.sql.upgrades.util import updateCalendarDataVersion, \
+ removeProperty, cleanPropertyStore, logUpgradeStatus, doToEachHomeNotAtVersion
from txdav.xml import element
-from txdav.xml.parser import WebDAVDocument
-from twext.web2.dav.resource import TwistedQuotaUsedProperty, \
- TwistedGETContentMD5
"""
Data upgrade from database version 4 to 5
@@ -44,178 +40,75 @@
"""
Do the required upgrade steps.
"""
- yield moveCalendarTimezoneProperties(sqlStore)
- yield moveCalendarAvailabilityProperties(sqlStore)
- yield removeOtherProperties(sqlStore)
+ yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix)
- # Always bump the DB value
- yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
- yield updateAllCalendarHomeDataVersions(sqlStore, UPGRADE_TO_VERSION)
+ # Don't do remaining upgrade if we are only process a subset of the homes
+ if not config.UpgradeHomePrefix:
+ yield removeOtherProperties(sqlStore)
+ # Always bump the DB value
+ yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
+
@inlineCallbacks
-def moveCalendarTimezoneProperties(sqlStore):
+def updateCalendarHomes(sqlStore, prefix=None):
"""
- Need to move all the CalDAV:calendar-timezone properties in the
- RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
- the new value from the XML property.
+ For each calendar home, update the associated properties on the home or its owned calendars.
"""
- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
+ yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix)
- propname = caldavxml.CalendarTimeZone
- logUpgradeStatus("Starting Process {}".format(propname.qname()))
- sqlTxn = sqlStore.newTransaction()
- total = (yield countProperty(sqlTxn, propname))
- yield sqlTxn.commit()
- count = 0
- try:
- calendar_rid = None
- calendars_for_id = {}
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, with_uid=True, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
+ at inlineCallbacks
+def updateCalendarHome(txn, homeResourceID):
+ """
+ For this calendar home, update the associated properties on the home or its owned calendars.
+ """
- delete_ids = []
- for calendar_rid, value, viewer in rows:
- delete_ids.append(calendar_rid)
- if calendar_rid not in calendars_for_id:
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, cb.BIND_MODE, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == calendar_rid,
- ).on(sqlTxn)
- calendars_for_id[calendar_rid] = ids
+ home = yield txn.calendarHomeWithResourceID(homeResourceID)
+ yield moveCalendarTimezoneProperties(home)
+ yield moveCalendarAvailabilityProperties(home)
+ yield cleanPropertyStore()
- if viewer:
- calendarHome = (yield sqlTxn.calendarHomeWithUID(viewer))
- else:
- calendarHome = None
- for row in calendars_for_id[calendar_rid]:
- home_id, bind_mode = row
- if bind_mode == _BIND_MODE_OWN:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(home_id))
- break
- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- calendar = (yield calendarHome.childWithID(calendar_rid))
- if calendar is not None:
- yield calendar.setTimezone(prop.calendar())
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(caldavxml.CalendarTimeZone).toString()),
- ).on(sqlTxn, ids=delete_ids)
+ at inlineCallbacks
+def moveCalendarTimezoneProperties(home):
+ """
+ Need to move all the CalDAV:calendar-timezone properties in the
+ RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
+ the new value from the XML property.
+ """
- yield sqlTxn.commit()
- count += len(rows)
- logUpgradeStatus(
- "Process {}".format(propname.qname()),
- count,
- total,
- )
+ # Iterate over each calendar (both owned and shared)
+ calendars = (yield home.loadChildren())
+ for calendar in calendars:
+ if calendar.isInbox():
+ continue
+ prop = calendar.properties().get(PropertyName.fromElement(caldavxml.CalendarTimeZone))
+ if prop is not None:
+ yield calendar.setTimezone(prop.calendar())
+ del calendar.properties()[PropertyName.fromElement(caldavxml.CalendarTimeZone)]
- yield cleanPropertyStore()
- logUpgradeStatus("End Process {}".format(propname.qname()))
- except RuntimeError as e:
- f = Failure()
- logUpgradeError(
- "Process {}".format(propname.qname()),
- "Rid: {}, error: {}".format(calendar_rid, e),
- )
- yield sqlTxn.abort()
- f.raiseException()
-
-
@inlineCallbacks
-def moveCalendarAvailabilityProperties(sqlStore):
+def moveCalendarAvailabilityProperties(home):
"""
Need to move all the CS:calendar-availability properties in the
RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
the new value from the XML property.
"""
+ inbox = (yield home.calendarWithName("inbox"))
+ prop = inbox.properties().get(PropertyName.fromElement(customxml.CalendarAvailability))
+ if prop is not None:
+ yield home.setAvailability(prop.calendar())
+ del inbox.properties()[customxml.CalendarAvailability]
- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
- propname = customxml.CalendarAvailability
- logUpgradeStatus("Starting Process {}".format(propname.qname()))
- sqlTxn = sqlStore.newTransaction()
- total = (yield countProperty(sqlTxn, propname))
- yield sqlTxn.commit()
- count = 0
-
- try:
- calendar_rid = None
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
-
- # Map each calendar to a home id using a single query for efficiency
- calendar_ids = [row[0] for row in rows]
-
- home_map = yield Select(
- [cb.CALENDAR_RESOURCE_ID, cb.CALENDAR_HOME_RESOURCE_ID, ],
- From=cb,
- Where=(cb.CALENDAR_RESOURCE_ID.In(Parameter("ids", len(calendar_ids)))).And(cb.BIND_MODE == _BIND_MODE_OWN),
- ).on(sqlTxn, ids=calendar_ids)
- calendar_to_home = dict(home_map)
-
- # Move property to each home
- for calendar_rid, value in rows:
- if calendar_rid in calendar_to_home:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(calendar_to_home[calendar_rid]))
-
- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- yield calendarHome.setAvailability(prop.calendar())
-
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(calendar_ids)))).And
- (rp.NAME == PropertyName.fromElement(customxml.CalendarAvailability).toString()),
- ).on(sqlTxn, ids=calendar_ids)
-
- yield sqlTxn.commit()
-
- count += len(rows)
- logUpgradeStatus(
- "Process {}".format(propname.qname()),
- count,
- total,
- )
-
- yield cleanPropertyStore()
- logUpgradeStatus("End Process {}".format(propname.qname()))
-
- except RuntimeError as e:
- f = Failure()
- logUpgradeError(
- "Process {}".format(propname.qname()),
- "Rid: {}, error: {}".format(calendar_rid, e),
- )
- yield sqlTxn.abort()
- f.raiseException()
-
-
-
@inlineCallbacks
def removeOtherProperties(sqlStore):
"""
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -13,23 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
+
+from twext.enterprise.dal.syntax import Update, Insert
+
+from twistedcaldav import caldavxml
from twistedcaldav.caldavxml import ScheduleDefaultCalendarURL, \
- CalendarFreeBusySet, Opaque, ScheduleCalendarTransp
+ CalendarFreeBusySet, Opaque, ScheduleCalendarTransp, Transparent
+
from txdav.base.propertystore.base import PropertyName
from txdav.caldav.datastore.test.util import CommonStoreTests
+from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_3_to_4 import updateCalendarHomes, \
+ doUpgrade
+from txdav.xml import element
from txdav.xml.element import HRef
-from twext.enterprise.dal.syntax import Update, Insert
-from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_3_to_4 import moveDefaultCalendarProperties, \
- moveCalendarTranspProperties, removeResourceType, moveDefaultAlarmProperties
-from txdav.xml import element
-from twistedcaldav import caldavxml
-from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
+from twistedcaldav.config import config
"""
Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
"""
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import inlineCallbacks, returnValue
class Upgrade_from_3_to_4(CommonStoreTests):
"""
@@ -37,7 +41,7 @@
"""
@inlineCallbacks
- def test_defaultCalendarUpgrade(self):
+ def _defaultCalendarUpgrade_setup(self):
# Set dead property on inbox
for user in ("user01", "user02",):
@@ -52,30 +56,56 @@
Where=chm.RESOURCE_ID == home._resourceID,
).on(self.transactionUnderTest())
- # Force data version to previous
- ch = home._homeSchema
- yield Update(
- {ch.DATAVERSION: 3},
- Where=ch.RESOURCE_ID == home._resourceID,
- ).on(self.transactionUnderTest())
+ # Force data version to previous
+ ch = home._homeSchema
+ yield Update(
+ {ch.DATAVERSION: 3},
+ Where=ch.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
yield self.commit()
- # Trigger upgrade
- yield moveDefaultCalendarProperties(self._sqlCalendarStore)
+ @inlineCallbacks
+ def _defaultCalendarUpgrade_check(self, changed_users, unchanged_users):
+
# Test results
- for user in ("user01", "user02",):
+ for user in changed_users:
home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
self.assertTrue(home.isDefaultCalendar(calendar))
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) not in inbox.properties())
+ for user in unchanged_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertFalse(home.isDefaultCalendar(calendar))
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) in inbox.properties())
+
@inlineCallbacks
- def test_invalidDefaultCalendarUpgrade(self):
+ def test_defaultCalendarUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01", "user02",), ())
+
+ @inlineCallbacks
+ def test_partialDefaultCalendarUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._defaultCalendarUpgrade_check(("user01",), ("user02",))
+
+
+ @inlineCallbacks
+ def _invalidDefaultCalendarUpgrade_setup(self):
+
# Set dead property on inbox
for user in ("user01", "user02",):
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
@@ -93,30 +123,56 @@
tasks = (yield home.createCalendarWithName("tasks_1"))
yield tasks.setSupportedComponents("VTODO")
- # Force data version to previous
- ch = home._homeSchema
- yield Update(
- {ch.DATAVERSION: 3},
- Where=ch.RESOURCE_ID == home._resourceID,
- ).on(self.transactionUnderTest())
+ # Force data version to previous
+ ch = home._homeSchema
+ yield Update(
+ {ch.DATAVERSION: 3},
+ Where=ch.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
yield self.commit()
- # Trigger upgrade
- yield moveDefaultCalendarProperties(self._sqlCalendarStore)
+ @inlineCallbacks
+ def _invalidDefaultCalendarUpgrade_check(self, changed_users, unchanged_users):
+
# Test results
- for user in ("user01", "user02",):
+ for user in changed_users:
home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
calendar = (yield self.calendarUnderTest(name="tasks_1", home=user))
self.assertFalse(home.isDefaultCalendar(calendar))
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) not in inbox.properties())
+ for user in unchanged_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ calendar = (yield self.calendarUnderTest(name="tasks_1", home=user))
+ self.assertFalse(home.isDefaultCalendar(calendar))
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) in inbox.properties())
+
@inlineCallbacks
- def test_calendarTranspUpgrade(self):
+ def test_invalidDefaultCalendarUpgrade(self):
+ yield self._invalidDefaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._invalidDefaultCalendarUpgrade_check(("user01", "user02",), ())
+
+ @inlineCallbacks
+ def test_partialInvalidDefaultCalendarUpgrade(self):
+ yield self._invalidDefaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._invalidDefaultCalendarUpgrade_check(("user01",), ("user02",))
+
+
+ @inlineCallbacks
+ def _calendarTranspUpgrade_setup(self):
+
# Set dead property on inbox
for user in ("user01", "user02",):
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
@@ -125,7 +181,7 @@
# Force current to transparent
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
yield calendar.setUsedForFreeBusy(False)
- calendar.properties()[PropertyName.fromElement(ScheduleCalendarTransp)] = ScheduleCalendarTransp(Opaque())
+ calendar.properties()[PropertyName.fromElement(ScheduleCalendarTransp)] = ScheduleCalendarTransp(Opaque() if user == "user01" else Transparent())
# Force data version to previous
home = (yield self.homeUnderTest(name=user))
@@ -159,21 +215,55 @@
).on(txn)
yield self.commit()
- # Trigger upgrade
- yield moveCalendarTranspProperties(self._sqlCalendarStore)
+ @inlineCallbacks
+ def _calendarTranspUpgrade_check(self, changed_users, unchanged_users):
+
# Test results
- for user in ("user01", "user02",):
+ for user in changed_users:
home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
- self.assertTrue(calendar.isUsedForFreeBusy())
+ if user == "user01":
+ self.assertTrue(calendar.isUsedForFreeBusy())
+ else:
+ self.assertFalse(calendar.isUsedForFreeBusy())
+ self.assertTrue(PropertyName.fromElement(caldavxml.ScheduleCalendarTransp) not in calendar.properties())
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
self.assertTrue(PropertyName.fromElement(CalendarFreeBusySet) not in inbox.properties())
+ for user in unchanged_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ if user == "user01":
+ self.assertFalse(calendar.isUsedForFreeBusy())
+ else:
+ self.assertFalse(calendar.isUsedForFreeBusy())
+ self.assertTrue(PropertyName.fromElement(caldavxml.ScheduleCalendarTransp) in calendar.properties())
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(CalendarFreeBusySet) in inbox.properties())
+
@inlineCallbacks
- def test_defaultAlarmUpgrade(self):
+ def test_calendarTranspUpgrade(self):
+ yield self._calendarTranspUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarTranspUpgrade_check(("user01", "user02",), ())
+
+ @inlineCallbacks
+ def test_partialCalendarTranspUpgrade(self):
+ yield self._calendarTranspUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarTranspUpgrade_check(("user01",), ("user02",))
+
+
+ @inlineCallbacks
+ def _defaultAlarmUpgrade_setup(self):
+
alarmhome1 = """BEGIN:VALARM
ACTION:AUDIO
TRIGGER;RELATED=START:-PT1M
@@ -277,13 +367,28 @@
shared = yield self.calendarUnderTest(name=shared_name, home="user02")
for _ignore_vevent, _ignore_timed, alarm, prop in detailsshared:
shared.properties()[PropertyName.fromElement(prop)] = prop(alarm)
+
+ for user in ("user01", "user02",):
+ # Force data version to previous
+ home = (yield self.homeUnderTest(name=user))
+ ch = home._homeSchema
+ yield Update(
+ {ch.DATAVERSION: 3},
+ Where=ch.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
+
yield self.commit()
- # Trigger upgrade
- yield moveDefaultAlarmProperties(self._sqlCalendarStore)
+ returnValue((detailshome, detailscalendar, detailsshared, shared_name,))
+
+ @inlineCallbacks
+ def _defaultAlarmUpgrade_check(self, changed_users, unchanged_users, detailshome, detailscalendar, detailsshared, shared_name):
+
# Check each type of collection
home = yield self.homeUnderTest(name="user01")
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
for vevent, timed, alarm, prop in detailshome:
alarm_result = (yield home.getDefaultAlarm(vevent, timed))
self.assertEquals(alarm_result, alarm)
@@ -293,18 +398,67 @@
for vevent, timed, alarm, prop in detailscalendar:
alarm_result = (yield calendar.getDefaultAlarm(vevent, timed))
self.assertEquals(alarm_result, alarm)
- self.assertTrue(PropertyName.fromElement(prop) not in home.properties())
+ self.assertTrue(PropertyName.fromElement(prop) not in calendar.properties())
- shared = yield self.calendarUnderTest(name=shared_name, home="user02")
- for vevent, timed, alarm, prop in detailsshared:
- alarm_result = (yield shared.getDefaultAlarm(vevent, timed))
- self.assertEquals(alarm_result, alarm)
- self.assertTrue(PropertyName.fromElement(prop) not in home.properties())
+ if "user02" in changed_users:
+ home = (yield self.homeUnderTest(name="user02"))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ shared = yield self.calendarUnderTest(name=shared_name, home="user02")
+ for vevent, timed, alarm, prop in detailsshared:
+ alarm_result = (yield shared.getDefaultAlarm(vevent, timed))
+ self.assertEquals(alarm_result, alarm)
+ self.assertTrue(PropertyName.fromElement(prop) not in shared.properties())
+ else:
+ home = (yield self.homeUnderTest(name="user02"))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ shared = yield self.calendarUnderTest(name=shared_name, home="user02")
+ for vevent, timed, alarm, prop in detailsshared:
+ alarm_result = (yield shared.getDefaultAlarm(vevent, timed))
+ self.assertEquals(alarm_result, None)
+ self.assertTrue(PropertyName.fromElement(prop) in shared.properties())
@inlineCallbacks
- def test_resourceTypeUpgrade(self):
+ def test_defaultAlarmUpgrade(self):
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._defaultAlarmUpgrade_check(("user01", "user02",), (), detailshome, detailscalendar, detailsshared, shared_name)
+
+ @inlineCallbacks
+ def test_partialDefaultAlarmUpgrade(self):
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._defaultAlarmUpgrade_check(("user01",), ("user02",), detailshome, detailscalendar, detailsshared, shared_name)
+
+
+ @inlineCallbacks
+ def test_combinedUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01", "user02",), ())
+ yield self._calendarTranspUpgrade_check(("user01", "user02",), ())
+ yield self._defaultAlarmUpgrade_check(("user01", "user02",), (), detailshome, detailscalendar, detailsshared, shared_name)
+
+
+ @inlineCallbacks
+ def test_partialCombinedUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._defaultCalendarUpgrade_check(("user01",), ("user02",))
+ yield self._calendarTranspUpgrade_check(("user01",), ("user02",))
+ yield self._defaultAlarmUpgrade_check(("user01",), ("user02",), detailshome, detailscalendar, detailsshared, shared_name)
+
+
+ @inlineCallbacks
+ def _resourceTypeUpgrade_setup(self):
+
# Set dead property on calendar
for user in ("user01", "user02",):
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
@@ -314,12 +468,60 @@
for user in ("user01", "user02",):
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
self.assertTrue(PropertyName.fromElement(element.ResourceType) in calendar.properties())
+
+ yield self.transactionUnderTest().updateCalendarserverValue("CALENDAR-DATAVERSION", "3")
+
yield self.commit()
- # Trigger upgrade
- yield removeResourceType(self._sqlCalendarStore)
+ @inlineCallbacks
+ def _resourceTypeUpgrade_check(self, full=True):
+
# Test results
- for user in ("user01", "user02",):
- calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
- self.assertTrue(PropertyName.fromElement(element.ResourceType) not in calendar.properties())
+ if full:
+ for user in ("user01", "user02",):
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceType) not in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 4)
+ else:
+ for user in ("user01", "user02",):
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceType) in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 3)
+
+
+ @inlineCallbacks
+ def test_resourceTypeUpgrade(self):
+ yield self._resourceTypeUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._resourceTypeUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_fullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "")
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield self._resourceTypeUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01", "user02",), ())
+ yield self._calendarTranspUpgrade_check(("user01", "user02",), ())
+ yield self._defaultAlarmUpgrade_check(("user01", "user02",), (), detailshome, detailscalendar, detailsshared, shared_name)
+ yield self._resourceTypeUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_partialFullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "user01")
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ yield self._resourceTypeUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01",), ("user02",))
+ yield self._calendarTranspUpgrade_check(("user01",), ("user02",))
+ yield self._defaultAlarmUpgrade_check(("user01",), ("user02",), detailshome, detailscalendar, detailsshared, shared_name)
+ yield self._resourceTypeUpgrade_check(False)
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -13,21 +13,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
-from twistedcaldav import caldavxml, customxml
-from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_4_to_5 import moveCalendarTimezoneProperties, \
- removeOtherProperties, moveCalendarAvailabilityProperties
-from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
-from txdav.xml import element
"""
Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
"""
from twext.enterprise.dal.syntax import Update, Insert
-from twisted.internet.defer import inlineCallbacks
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from twistedcaldav import caldavxml, customxml
+from twistedcaldav.config import config
from twistedcaldav.ical import Component
+
from txdav.base.propertystore.base import PropertyName
from txdav.caldav.datastore.test.util import CommonStoreTests
+from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_4_to_5 import updateCalendarHomes, doUpgrade
+from txdav.xml import element
class Upgrade_from_4_to_5(CommonStoreTests):
"""
@@ -35,7 +38,7 @@
"""
@inlineCallbacks
- def test_calendarTimezoneUpgrade(self):
+ def _calendarTimezoneUpgrade_setup(self):
tz1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
@@ -137,19 +140,47 @@
).on(txn)
yield self.commit()
- # Trigger upgrade
- yield moveCalendarTimezoneProperties(self._sqlCalendarStore)
+ returnValue(user_details)
+
+ @inlineCallbacks
+ def _calendarTimezoneUpgrade_check(self, changed_users, unchanged_users, user_details):
+
# Test results
for user, calname, tz in user_details:
- calendar = (yield self.calendarUnderTest(name=calname, home=user))
- self.assertEqual(calendar.getTimezone(), tz)
- self.assertTrue(PropertyName.fromElement(caldavxml.CalendarTimeZone) not in calendar.properties())
+ if user in changed_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 5)
+ calendar = (yield self.calendarUnderTest(name=calname, home=user))
+ self.assertEqual(calendar.getTimezone(), tz)
+ self.assertTrue(PropertyName.fromElement(caldavxml.CalendarTimeZone) not in calendar.properties())
+ else:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ calendar = (yield self.calendarUnderTest(name=calname, home=user))
+ self.assertEqual(calendar.getTimezone(), None)
+ self.assertTrue(PropertyName.fromElement(caldavxml.CalendarTimeZone) in calendar.properties())
@inlineCallbacks
- def test_calendarAvailabilityUpgrade(self):
+ def test_calendarTimezoneUpgrade(self):
+ user_details = yield self._calendarTimezoneUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01", "user02", "user03",), (), user_details)
+
+ @inlineCallbacks
+ def test_partialCalendarTimezoneUpgrade(self):
+ user_details = yield self._calendarTimezoneUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarTimezoneUpgrade_check(("user01",), ("user02", "user03",), user_details)
+
+
+ @inlineCallbacks
+ def _calendarAvailabilityUpgrade_setup(self):
+
av1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
@@ -220,20 +251,65 @@
self.assertEqual(PropertyName.fromElement(customxml.CalendarAvailability) in calendar.properties(), av is not None)
yield self.commit()
- # Trigger upgrade
- yield moveCalendarAvailabilityProperties(self._sqlCalendarStore)
+ returnValue(user_details)
+
+ @inlineCallbacks
+ def _calendarAvailabilityUpgrade_check(self, changed_users, unchanged_users, user_details):
+
# Test results
for user, av in user_details:
- home = (yield self.homeUnderTest(name=user))
- calendar = (yield self.calendarUnderTest(name="inbox", home=user))
- self.assertEqual(home.getAvailability(), av)
- self.assertTrue(PropertyName.fromElement(customxml.CalendarAvailability) not in calendar.properties())
+ if user in changed_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 5)
+ calendar = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertEqual(home.getAvailability(), av)
+ self.assertTrue(PropertyName.fromElement(customxml.CalendarAvailability) not in calendar.properties())
+ else:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ calendar = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertEqual(home.getAvailability(), None)
+ self.assertTrue(PropertyName.fromElement(customxml.CalendarAvailability) in calendar.properties())
@inlineCallbacks
- def test_removeOtherPropertiesUpgrade(self):
+ def test_calendarAvailabilityUpgrade(self):
+ user_details = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarAvailabilityUpgrade_check(("user01", "user02", "user03",), (), user_details)
+
+ @inlineCallbacks
+ def test_partialCalendarAvailabilityUpgrade(self):
+ user_details = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarAvailabilityUpgrade_check(("user01",), ("user02", "user03",), user_details)
+
+
+ @inlineCallbacks
+ def test_combinedUpgrade(self):
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01", "user02", "user03",), (), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01", "user02", "user03",), (), user_details2)
+
+
+ @inlineCallbacks
+ def test_partialCombinedUpgrade(self):
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarTimezoneUpgrade_check(("user01",), ("user02", "user03",), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01",), ("user02", "user03",), user_details2)
+
+
+ @inlineCallbacks
+ def _removeOtherPropertiesUpgrade_setup(self):
+
# Set dead property on calendar
for user in ("user01", "user02",):
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
@@ -243,12 +319,55 @@
for user in ("user01", "user02",):
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
self.assertTrue(PropertyName.fromElement(element.ResourceID) in calendar.properties())
+
+ yield self.transactionUnderTest().updateCalendarserverValue("CALENDAR-DATAVERSION", "4")
+
yield self.commit()
- # Trigger upgrade
- yield removeOtherProperties(self._sqlCalendarStore)
+ @inlineCallbacks
+ def _removeOtherPropertiesUpgrade_check(self, full=True):
+
# Test results
for user in ("user01", "user02",):
- calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
- self.assertTrue(PropertyName.fromElement(element.ResourceID) not in calendar.properties())
+ if full:
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceID) not in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 5)
+ else:
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceID) in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 4)
+
+
+ @inlineCallbacks
+ def test_removeOtherPropertiesUpgrade(self):
+ yield self._removeOtherPropertiesUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._removeOtherPropertiesUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_fullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "")
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield self._removeOtherPropertiesUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01", "user02", "user03",), (), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01", "user02", "user03",), (), user_details2)
+ yield self._removeOtherPropertiesUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_partialFullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "user01")
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield self._removeOtherPropertiesUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01",), ("user02", "user03",), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01",), ("user02", "user03",), user_details2)
+ yield self._removeOtherPropertiesUpgrade_check(False)
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/util.py 2013-10-31 18:37:58 UTC (rev 11859)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrades/util.py 2013-10-31 21:40:44 UTC (rev 11860)
@@ -129,17 +129,21 @@
@inlineCallbacks
-def doToEachHomeNotAtVersion(store, homeSchema, version, doIt, logStr):
+def doToEachHomeNotAtVersion(store, homeSchema, version, doIt, logStr, filterOwnerUID=None):
"""
Do something to each home whose version column indicates it is older
- than the specified version. Do this in batches as there may be a lot of work to do.
+ than the specified version. Do this in batches as there may be a lot of work to do. Also,
+ allow the GUID to be filtered to support a parallel mode of operation.
"""
txn = store.newTransaction("updateDataVersion")
+ where = homeSchema.DATAVERSION < version
+ if filterOwnerUID:
+ where = where.And(homeSchema.OWNER_UID.StartsWith(filterOwnerUID))
total = (yield Select(
[Count(homeSchema.RESOURCE_ID), ],
From=homeSchema,
- Where=homeSchema.DATAVERSION < version,
+ Where=where,
).on(txn))[0][0]
yield txn.commit()
count = 0
@@ -154,7 +158,7 @@
rows = yield Select(
[homeSchema.RESOURCE_ID, homeSchema.OWNER_UID, ],
From=homeSchema,
- Where=homeSchema.DATAVERSION < version,
+ Where=where,
OrderBy=homeSchema.OWNER_UID,
Limit=1,
).on(txn)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/854aa9de/attachment.html>
More information about the calendarserver-changes
mailing list