[CalendarServer-changes] [8057] CalendarServer/trunk/twistedcaldav

[source_changes at macosforge.org]

Revision: 8057
          http://trac.macosforge.org/projects/calendarserver/changeset/8057
Author:   sagen at apple.com
Date:     2011-09-08 16:34:22 -0700 (Thu, 08 Sep 2011)
Log Message:
-----------
Remove illegal characters (below ASCII 32 aside from NL and CR) from calendar data during upgrade.

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
    CalendarServer/trunk/twistedcaldav/upgrade.py

Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py	2011-09-08 01:49:24 UTC (rev 8056)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py	2011-09-08 23:34:22 UTC (rev 8057)
@@ -23,7 +23,8 @@
 from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
 from twistedcaldav.mail import MailGatewayTokensDatabase
 from twistedcaldav.upgrade import (
-    xattrname, UpgradeError, upgradeData, updateFreeBusySet
+    xattrname, UpgradeError, upgradeData, updateFreeBusySet,
+    removeIllegalCharacters
 )
 from twistedcaldav.test.util import TestCase
 from calendarserver.tools.util import getDirectory
@@ -1393,8 +1394,21 @@
     test_migrateResourceInfo.todo = "FIXME: perhaps ProxySqliteDB isn't being set up correctly?"
 
 
+    def test_removeIllegalCharacters(self):
+        """
+        Control characters aside from NL and CR are removed.
+        """
+        data = "Contains\x03 control\x06 characters\x12 some\x0a allowed\x0d"
+        after, changed = removeIllegalCharacters(data)
+        self.assertEquals(after, "Contains control characters some\x0a allowed\x0d")
+        self.assertTrue(changed)
 
+        data = "Contains only\x0a legal\x0d"
+        after, changed = removeIllegalCharacters(data)
+        self.assertEquals(after, "Contains only\x0a legal\x0d")
+        self.assertFalse(changed)
 
+
 event01_before = """BEGIN:VCALENDAR
 VERSION:2.0
 PRODID:-//Apple Inc.//iCal 3.0//EN
@@ -1485,6 +1499,7 @@
 
 event01_after_md5 = hashlib.md5(event01_after).hexdigest()
 
+
 def isValidCTag(value):
     """
     Since ctag is generated from datetime.now(), let's make sure that at

Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py	2011-09-08 01:49:24 UTC (rev 8056)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py	2011-09-08 23:34:22 UTC (rev 8057)
@@ -178,6 +178,17 @@
                     continue
 
                 try:
+                    data, fixed = removeIllegalCharacters(data)
+                    if fixed:
+                        log.warn("Removing illegal characters in %s" % (resPath,))
+                        needsRewrite = True
+                except Exception, e:
+                    log.error("Error while removing illegal characters in %s: %s" %
+                        (resPath, e))
+                    errorOccurred = True
+                    continue
+
+                try:
                     data, fixed = normalizeCUAddrs(data, directory)
                     if fixed:
                         log.debug("Normalized CUAddrs in %s" % (resPath,))
@@ -808,7 +819,23 @@
         os.remove(srcPath)
 
 
+DELETECHARS = ''.join(chr(i) for i in xrange(32) if i not in (10, 13))
+def removeIllegalCharacters(data):
+    """
+    Remove all characters below ASCII 32 except NL and CR
 
+    Return tuple with the processed data, and a boolean indicating wether
+    the data changed.
+    """
+    beforeLen = len(data)
+    data =  data.translate(None, DELETECHARS)
+    afterLen = len(data)
+    if afterLen != beforeLen:
+        return data, True
+    else:
+        return data, False
+
+
 class UpgradeFileSystemFormatService(Service, object):
     """
     Upgrade filesystem from previous versions.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110908/180b1c28/attachment.html>