Hi,
I am trying to upgrade from calendar server version 4.2 to version 7.0 by starting version 7.0 of the server connected to a database originating from version 4.2. The upgrade process seems to start correctly, but after a while the process is aborted. Some properties in the resource_property table contains UTF-8 characters and that seems to abort the upgrade process and make the server shut down. The following lines are an example from the error log when the problem happens:
2015-12-14 13:14:03+0000 [-] [calendarserver.tap.util#error] Step failure
Traceback (most recent call last):
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 442, in errback
self._startRunCallbacks(fail)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 509, in _startRunCallbacks
self._runCallbacks()
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 596, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1192, in gotResult
_inlineCallbacks(r, g, deferred)
--- <exception caught here> ---
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
de
yield self.upgradeVersion(actual_version, required_version, dialect)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
n
yield self.applyUpgrade(fp)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
yield doUpgrade(self.sqlStore)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
yield doIt(txn, homeResourceID)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
yield moveCalendarTimezoneProperties(home)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1136, in _inlineCallbacks
result = g.send(result)
prop = calendar.properties().get(PropertyName.fromElement(caldavxml.CalendarTimeZone))
File "/usr/share/python/calendarserver/lib/python2.7/UserDict.py", line 168, in get
return self[key]
return self._getitem_uid(key, self._perUser)
return WebDAVDocument.fromString(value).root_element
source = StringIO(source)
exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\u2212' in position 348: ordinal not in range(128)
The properties that I have found to be related to this problem are the following:
CALDAV:calendar-timezone
CS:calendar-availability
They are both processed in txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py.
The values of these properties seems to be valid xml data with the encoding attribute set to 'UTF-8', i.e.:
<?xml version='1.0' encoding='UTF-8'?>
<calendar-timezone xmlns='urn:ietf:params:xml:ns:caldav'><![CDATA[BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//Mac OS X 10.11//EN
CALSCALE:GREGORIAN
...
...
Best regards,
Kristoffer Møllerhøj.