[CalendarServer-changes] [3652] CalendarServer/trunk/twistedcaldav/localization.py

[source_changes at macosforge.org]

Revision: 3652
          http://trac.macosforge.org/projects/calendarserver/changeset/3652
Author:   sagen at apple.com
Date:     2009-02-09 14:31:27 -0800 (Mon, 09 Feb 2009)
Log Message:
-----------
Handle UTF-16 encoded .strings files which start with a Byte Order Mark (BOM)

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/localization.py

Modified: CalendarServer/trunk/twistedcaldav/localization.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/localization.py	2009-02-09 17:09:00 UTC (rev 3651)
+++ CalendarServer/trunk/twistedcaldav/localization.py	2009-02-09 22:31:27 UTC (rev 3652)
@@ -21,6 +21,7 @@
 import os
 import struct
 import array
+import codecs
 from locale import normalize
 from twistedcaldav.config import config
 from twistedcaldav.log import Logger
@@ -420,6 +421,12 @@
     return (key, value)
 
 
+boms = {
+    codecs.BOM_UTF8 : 'UTF8',
+    codecs.BOM_UTF16_BE : 'UTF-16BE',
+    codecs.BOM_UTF16_LE : 'UTF-16LE',
+}
+
 def convertStringsFile(src, dest):
     strings = { }
 
@@ -433,9 +440,19 @@
             return
 
     with open(src) as input:
-        lines = input.readlines()
+        contents = input.read()
+        for bom, encoding in boms.iteritems():
+            if contents.startswith(bom):
+                contents = contents[len(bom):]
+                break
+        else:
+            encoding = "UTF8"
 
+    contents = contents.decode(encoding)
+    lines = contents.split("\n")
+
     for num, line in enumerate(lines):
+        print num, line
         line = line.strip()
         if not line.startswith('"'):
             continue
@@ -443,7 +460,7 @@
         try:
             key, value = parseLine(line)
         except ParseError, err:
-            log.error("Error on line %d of %s: %s" % (num+1, src, str(err)))
+            log.info("Error on line %d of %s: %s" % (num+1, src, str(err)))
             raise
 
         strings[key] = value
@@ -461,11 +478,14 @@
     for original in originals:
         translation = strings[original]
 
-        descriptors.append((len(keys), len(original), len(values),
-            len(translation)))
-        keys += original + '\0' # <NUL> terminated
-        values += translation + '\0'
+        origStr = original.encode("UTF-8")
+        transStr = translation.encode("UTF-8")
 
+        descriptors.append((len(keys), len(origStr), len(values),
+            len(transStr)))
+        keys += origStr + '\0' # <NUL> terminated
+        values += transStr + '\0'
+
     # The header is 28 bytes, each descriptor is 8 bytes, with two descriptors
     # per string (one pointing at original, one pointing at translation)
     keysOffset = 28 + len(originals) * 2 * 8
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090209/48cfcb0a/attachment.html>