[CalendarServer-users] Upgrade fails when specific properties containing UTF-8 characters are processed

[Kristoffer]

Hi,

I am trying to upgrade from calendar server version 4.2 to version 7.0 by starting version 7.0 of the server connected to a database originating from version 4.2. The upgrade process seems to start correctly, but after a while the process is aborted. Some properties in the resource_property table contains UTF-8 characters and that seems to abort the upgrade process and make the server shut down. The following lines are an example from the error log when the problem happens:

2015-12-14 13:14:03+0000 [-] [calendarserver.tap.util#error] Step failure
Traceback (most recent call last):
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 442, in errback
self._startRunCallbacks(fail)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 509, in _startRunCallbacks
self._runCallbacks()
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 596, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1192, in gotResult
_inlineCallbacks(r, g, deferred)
--- <exception caught here> ---
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrade.py>", line 147, in databaseUpgra
de
yield self.upgradeVersion(actual_version, required_version, dialect)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrade.py>", line 212, in upgradeVersio
n
yield self.applyUpgrade(fp)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrade.py>", line 361, in applyUpgrade
yield doUpgrade(self.sqlStore)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 43, in doUpgrade
yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 60, in updateCalendarHomes
yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/util.py>", line 179, in doToEachHomeNotAtVersion
yield doIt(txn, homeResourceID)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 71, in updateCalendarHome
yield moveCalendarTimezoneProperties(home)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1136, in _inlineCallbacks
result = g.send(result)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 90, in moveCalendarTimezoneProperties
prop = calendar.properties().get(PropertyName.fromElement(caldavxml.CalendarTimeZone))
File "/usr/share/python/calendarserver/lib/python2.7/UserDict.py", line 168, in get
return self[key]
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/base/propertystore/base.py>", line 227, in __getitem__
return self._getitem_uid(key, self._perUser)
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/base/propertystore/sql.py>", line 275, in _getitem_uid
return WebDAVDocument.fromString(value).root_element
File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/xml/parser_base.py>", line 43, in fromString
source = StringIO(source)
exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\u2212' in position 348: ordinal not in range(128)


The properties that I have found to be related to this problem are the following:

CALDAV:calendar-timezone
CS:calendar-availability


They are both processed in txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py.
The values of these properties seems to be valid xml data with the encoding attribute set to 'UTF-8', i.e.:

<?xml version='1.0' encoding='UTF-8'?>
<calendar-timezone xmlns='urn:ietf:params:xml:ns:caldav'><![CDATA[BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//Mac OS X 10.11//EN
CALSCALE:GREGORIAN
...

...
Best regards,
Kristoffer Møllerhøj.

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-users/attachments/20151222/dca669a8/attachment.html>