Revision: 3652 http://trac.macosforge.org/projects/calendarserver/changeset/3652 Author: sagen@apple.com Date: 2009-02-09 14:31:27 -0800 (Mon, 09 Feb 2009) Log Message: ----------- Handle UTF-16 encoded .strings files which start with a Byte Order Mark (BOM) Modified Paths: -------------- CalendarServer/trunk/twistedcaldav/localization.py Modified: CalendarServer/trunk/twistedcaldav/localization.py =================================================================== --- CalendarServer/trunk/twistedcaldav/localization.py 2009-02-09 17:09:00 UTC (rev 3651) +++ CalendarServer/trunk/twistedcaldav/localization.py 2009-02-09 22:31:27 UTC (rev 3652) @@ -21,6 +21,7 @@ import os import struct import array +import codecs from locale import normalize from twistedcaldav.config import config from twistedcaldav.log import Logger @@ -420,6 +421,12 @@ return (key, value) +boms = { + codecs.BOM_UTF8 : 'UTF8', + codecs.BOM_UTF16_BE : 'UTF-16BE', + codecs.BOM_UTF16_LE : 'UTF-16LE', +} + def convertStringsFile(src, dest): strings = { } @@ -433,9 +440,19 @@ return with open(src) as input: - lines = input.readlines() + contents = input.read() + for bom, encoding in boms.iteritems(): + if contents.startswith(bom): + contents = contents[len(bom):] + break + else: + encoding = "UTF8" + contents = contents.decode(encoding) + lines = contents.split("\n") + for num, line in enumerate(lines): + print num, line line = line.strip() if not line.startswith('"'): continue @@ -443,7 +460,7 @@ try: key, value = parseLine(line) except ParseError, err: - log.error("Error on line %d of %s: %s" % (num+1, src, str(err))) + log.info("Error on line %d of %s: %s" % (num+1, src, str(err))) raise strings[key] = value @@ -461,11 +478,14 @@ for original in originals: translation = strings[original] - descriptors.append((len(keys), len(original), len(values), - len(translation))) - keys += original + '\0' # <NUL> terminated - values += translation + '\0' + origStr = original.encode("UTF-8") + transStr = translation.encode("UTF-8") + descriptors.append((len(keys), len(origStr), len(values), + len(transStr))) + keys += origStr + '\0' # <NUL> terminated + values += transStr + '\0' + # The header is 28 bytes, each descriptor is 8 bytes, with two descriptors # per string (one pointing at original, one pointing at translation) keysOffset = 28 + len(originals) * 2 * 8