[CalendarServer-changes] [14915] PyCalendar/trunk/src

source_changes at macosforge.org source_changes at macosforge.org
Thu Jun 25 14:49:45 PDT 2015


Revision: 14915
          http://trac.calendarserver.org//changeset/14915
Author:   cdaboo at apple.com
Date:     2015-06-25 14:49:45 -0700 (Thu, 25 Jun 2015)
Log Message:
-----------
Fix JSON unicode encoding issues. Fix PyPy open files issue.

Modified Paths:
--------------
    PyCalendar/trunk/src/pycalendar/icalendar/calendar.py
    PyCalendar/trunk/src/pycalendar/icalendar/requeststatusvalue.py
    PyCalendar/trunk/src/pycalendar/icalendar/tests/test_json.py
    PyCalendar/trunk/src/pycalendar/plaintextvalue.py
    PyCalendar/trunk/src/pycalendar/property.py
    PyCalendar/trunk/src/pycalendar/timezonedb.py
    PyCalendar/trunk/src/pycalendar/validator.py
    PyCalendar/trunk/src/pycalendar/vcard/adr.py
    PyCalendar/trunk/src/pycalendar/vcard/n.py
    PyCalendar/trunk/src/pycalendar/vcard/orgvalue.py
    PyCalendar/trunk/src/zonal/tzconvert.py
    PyCalendar/trunk/src/zonal/tzdump.py
    PyCalendar/trunk/src/zonal/tzverify.py

Modified: PyCalendar/trunk/src/pycalendar/icalendar/calendar.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/icalendar/calendar.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/icalendar/calendar.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -456,10 +456,10 @@
         return root
 
 
-    def getTextJSON(self, includeTimezones=None):
+    def getTextJSON(self, includeTimezones=None, sort_keys=False):
         jobject = []
         self.writeJSON(jobject, includeTimezones)
-        return json.dumps(jobject[0], indent=2, separators=(',', ':'))
+        return json.dumps(jobject[0], indent=2, separators=(',', ':'), sort_keys=sort_keys)
 
 
     def writeJSON(self, jobject, includeTimezones=None):

Modified: PyCalendar/trunk/src/pycalendar/icalendar/requeststatusvalue.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/icalendar/requeststatusvalue.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/icalendar/requeststatusvalue.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -97,7 +97,7 @@
 
 
     def parseJSONValue(self, jobject):
-        self.mValue = jobject
+        self.mValue = map(lambda x: x.encode("utf-8"), jobject)
 
 
     def writeJSONValue(self, jobject):

Modified: PyCalendar/trunk/src/pycalendar/icalendar/tests/test_json.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/icalendar/tests/test_json.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/icalendar/tests/test_json.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -18,6 +18,7 @@
 from pycalendar.icalendar.property import Property
 import difflib
 import unittest
+import json
 
 class TestJSON(unittest.TestCase):
 
@@ -387,14 +388,380 @@
         ),
     )
 
+    i18ndata = (
+        (
+            """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VEVENT
+DTSTAMP:20080205T191224Z
+DTSTART;VALUE=DATE:20081006
+SUMMARY:Planning meeting å
+UID:4088E990AD89CB3DBB484909
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n"),
+
+            """[
+  "vcalendar",
+  [
+    [
+      "version",
+      {},
+      "text",
+      "2.0"
+    ],
+    [
+      "calscale",
+      {},
+      "text",
+      "GREGORIAN"
+    ],
+    [
+      "prodid",
+      {},
+      "text",
+      "-//Example Inc.//Example Calendar//EN"
+    ]
+  ],
+  [
+    [
+      "vevent",
+      [
+        [
+          "uid",
+          {},
+          "text",
+          "4088E990AD89CB3DBB484909"
+        ],
+        [
+          "dtstart",
+          {},
+          "date",
+          "2008-10-06"
+        ],
+        [
+          "dtstamp",
+          {},
+          "date-time",
+          "2008-02-05T19:12:24Z"
+        ],
+        [
+          "summary",
+          {},
+          "text",
+          "Planning meeting å"
+        ]
+      ],
+      []
+    ]
+  ]
+]""",
+        ),
+        (
+            """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Example Corp.//Example Client//EN
+BEGIN:VTIMEZONE
+LAST-MODIFIED:20040110T032845Z
+TZID:US/Eastern
+BEGIN:DAYLIGHT
+DTSTART:20000404T020000
+RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
+TZNAME:EDT
+TZOFFSETFROM:-0500
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20001026T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+TZNAME:EST
+TZOFFSETFROM:-0400
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+DTSTAMP:20060206T001121Z
+DTSTART;TZID=US/Eastern:20060102T120000
+DURATION:PT1H
+GEO:-123.45;67.89
+RRULE:FREQ=DAILY;COUNT=5
+RDATE;TZID=US/Eastern;VALUE=PERIOD:20060102T150000/PT2H
+SUMMARY:Event #2 å
+DESCRIPTION:We are having a meeting all this week at 12 pm fo
+ r one hour\\, with an additional meeting on the first day 2 h
+ ours long.\\nPlease bring your own lunch for the 12 pm meetin
+ gs.
+UID:00959BC664CA650E933C892C at example.com
+END:VEVENT
+BEGIN:VEVENT
+DTSTAMP:20060206T001121Z
+DTSTART;TZID=US/Eastern:20060104T140000
+DURATION:PT1H
+RECURRENCE-ID;TZID=US/Eastern:20060104T120000
+SUMMARY:Event #2 bis å
+UID:00959BC664CA650E933C892C at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n"),
+
+            """[
+  "vcalendar",
+  [
+    [
+      "version",
+      {},
+      "text",
+      "2.0"
+    ],
+    [
+      "prodid",
+      {},
+      "text",
+      "-//Example Corp.//Example Client//EN"
+    ]
+  ],
+  [
+    [
+      "vtimezone",
+      [
+        [
+          "tzid",
+          {},
+          "text",
+          "US/Eastern"
+        ],
+        [
+          "last-modified",
+          {},
+          "date-time",
+          "2004-01-10T03:28:45Z"
+        ]
+      ],
+      [
+        [
+          "daylight",
+          [
+            [
+              "dtstart",
+              {},
+              "date-time",
+              "2000-04-04T02:00:00"
+            ],
+            [
+              "rrule",
+              {},
+              "recur",
+              {
+                "bymonth":[
+                  4
+                ],
+                "freq":"YEARLY",
+                "byday":[
+                  "1SU"
+                ]
+              }
+            ],
+            [
+              "tzname",
+              {},
+              "text",
+              "EDT"
+            ],
+            [
+              "tzoffsetfrom",
+              {},
+              "utc-offset",
+              "-05:00"
+            ],
+            [
+              "tzoffsetto",
+              {},
+              "utc-offset",
+              "-04:00"
+            ]
+          ],
+          []
+        ],
+        [
+          "standard",
+          [
+            [
+              "dtstart",
+              {},
+              "date-time",
+              "2000-10-26T02:00:00"
+            ],
+            [
+              "rrule",
+              {},
+              "recur",
+              {
+                "bymonth":[
+                  10
+                ],
+                "freq":"YEARLY",
+                "byday":[
+                  "-1SU"
+                ]
+              }
+            ],
+            [
+              "tzname",
+              {},
+              "text",
+              "EST"
+            ],
+            [
+              "tzoffsetfrom",
+              {},
+              "utc-offset",
+              "-04:00"
+            ],
+            [
+              "tzoffsetto",
+              {},
+              "utc-offset",
+              "-05:00"
+            ]
+          ],
+          []
+        ]
+      ]
+    ],
+    [
+      "vevent",
+      [
+        [
+          "uid",
+          {},
+          "text",
+          "00959BC664CA650E933C892C at example.com"
+        ],
+        [
+          "dtstart",
+          {
+            "tzid":"US/Eastern"
+          },
+          "date-time",
+          "2006-01-02T12:00:00"
+        ],
+        [
+          "duration",
+          {},
+          "duration",
+          "PT1H"
+        ],
+        [
+          "description",
+          {},
+          "text",
+          "We are having a meeting all this week at 12 pm for one hour, with an additional meeting on the first day 2 hours long.\\nPlease bring your own lunch for the 12 pm meetings."
+        ],
+        [
+          "dtstamp",
+          {},
+          "date-time",
+          "2006-02-06T00:11:21Z"
+        ],
+        [
+          "geo",
+          {},
+          "float",
+          [
+            -123.45,
+            67.89
+          ]
+        ],
+        [
+          "rdate",
+          {
+            "tzid":"US/Eastern"
+          },
+          "period",
+          [
+            "2006-01-02T15:00:00",
+            "PT2H"
+          ]
+        ],
+        [
+          "rrule",
+          {},
+          "recur",
+          {
+            "count":5,
+            "freq":"DAILY"
+          }
+        ],
+        [
+          "summary",
+          {},
+          "text",
+          "Event #2 å"
+        ]
+      ],
+      []
+    ],
+    [
+      "vevent",
+      [
+        [
+          "uid",
+          {},
+          "text",
+          "00959BC664CA650E933C892C at example.com"
+        ],
+        [
+          "recurrence-id",
+          {
+            "tzid":"US/Eastern"
+          },
+          "date-time",
+          "2006-01-04T12:00:00"
+        ],
+        [
+          "dtstart",
+          {
+            "tzid":"US/Eastern"
+          },
+          "date-time",
+          "2006-01-04T14:00:00"
+        ],
+        [
+          "duration",
+          {},
+          "duration",
+          "PT1H"
+        ],
+        [
+          "dtstamp",
+          {},
+          "date-time",
+          "2006-02-06T00:11:21Z"
+        ],
+        [
+          "summary",
+          {},
+          "text",
+          "Event #2 bis å"
+        ]
+      ],
+      []
+    ]
+  ]
+]""",
+        ),
+    )
+
     def testGenerateJSON(self):
 
         def _doRoundtrip(caldata, resultdata):
-            test1 = resultdata
+            test1 = json.dumps(json.loads(resultdata), indent=2, separators=(',', ':'), sort_keys=True)
 
             cal = Calendar.parseText(caldata)
 
-            test2 = cal.getTextJSON()
+            test2 = cal.getTextJSON(sort_keys=True)
             self.assertEqual(
                 test1,
                 test2,
@@ -424,6 +791,25 @@
             _doRoundtrip(item1, item2)
 
 
+    def testParseJSONi18n(self):
+
+        def _doRoundtrip(caldata, jcaldata):
+            cal1 = Calendar.parseText(caldata)
+            test1 = cal1.getText()
+
+            cal2 = Calendar.parseJSONData(jcaldata)
+            test2 = cal2.getText()
+
+            self.assertEqual(
+                test1,
+                test2,
+                "\n".join(difflib.unified_diff(str(test1).splitlines(), test2.splitlines()))
+            )
+
+        for item1, item2 in self.i18ndata:
+            _doRoundtrip(item1, item2)
+
+
     def testjCalExample1(self):
 
         jcaldata = """["vcalendar",

Modified: PyCalendar/trunk/src/pycalendar/plaintextvalue.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/plaintextvalue.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/plaintextvalue.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -48,7 +48,7 @@
 
 
     def parseJSONValue(self, jobject):
-        self.mValue = str(jobject)
+        self.mValue = jobject.encode("utf-8")
 
 
     def writeJSONValue(self, jobject):

Modified: PyCalendar/trunk/src/pycalendar/property.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/property.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/property.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -472,21 +472,21 @@
             prop = cls()
 
             # Get the name
-            prop.mName = jobject[0].upper()
+            prop.mName = jobject[0].encode("utf-8").upper()
 
             # Get the parameters
             if jobject[1]:
                 for name, value in jobject[1].items():
                     # Now add parameter value
                     name = name.upper()
-                    attrvalue = Parameter(name=name, value=value)
+                    attrvalue = Parameter(name=name.encode("utf-8"), value=value.encode("utf-8"))
                     prop.mParameters.setdefault(name, []).append(attrvalue)
 
             # Get default value type from property name and insert a VALUE parameter if current value type is not default
             value_type = cls.sValueTypeMap.get(jobject[2].upper(), Value.VALUETYPE_UNKNOWN)
             default_type = cls.sDefaultValueTypeMap.get(prop.mName.upper(), Value.VALUETYPE_UNKNOWN)
             if default_type != value_type:
-                attrvalue = Parameter(name=cls.sValue, value=jobject[2].upper())
+                attrvalue = Parameter(name=cls.sValue, value=jobject[2].encode("utf-8").upper())
                 prop.mParameters.setdefault(cls.sValue, []).append(attrvalue)
 
             # Get value type from property name

Modified: PyCalendar/trunk/src/pycalendar/timezonedb.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/timezonedb.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/timezonedb.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -127,7 +127,8 @@
         tzpath = os.path.normpath(tzpath)
         if tzpath.startswith(self.dbpath) and os.path.isfile(tzpath):
             try:
-                self.calendar.parseComponent(open(tzpath))
+                with open(tzpath) as f:
+                    self.calendar.parseComponent(f)
             except (IOError, InvalidData):
                 raise NoTimezoneInDatabase(self.dbpath, tzid)
         else:

Modified: PyCalendar/trunk/src/pycalendar/validator.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/validator.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/validator.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -27,7 +27,8 @@
     Check whether the contents of the specified file is valid iCalendar or vCard data.
     """
 
-    data = open(fname).read()
+    with open(fname) as f:
+        data = f.read()
 
     ParserContext.allRaise()
 

Modified: PyCalendar/trunk/src/pycalendar/vcard/adr.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/vcard/adr.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/vcard/adr.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -120,7 +120,7 @@
 
 
     def parseJSON(self, jobject):
-        self.mValue = tuple(jobject)
+        self.mValue = tuple(map(lambda x: x.encode("utf-8"), jobject))
 
 
     def writeJSON(self, jobject):

Modified: PyCalendar/trunk/src/pycalendar/vcard/n.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/vcard/n.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/vcard/n.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -117,7 +117,7 @@
 
 
     def parseJSON(self, jobject):
-        self.mValue = tuple(jobject)
+        self.mValue = tuple(map(lambda x: x.encode("utf-8"), jobject))
 
 
     def writeJSON(self, jobject):

Modified: PyCalendar/trunk/src/pycalendar/vcard/orgvalue.py
===================================================================
--- PyCalendar/trunk/src/pycalendar/vcard/orgvalue.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/pycalendar/vcard/orgvalue.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -45,7 +45,7 @@
 
 
     def parseJSONValue(self, jobject):
-        self.mValue = tuple(jobject)
+        self.mValue = tuple(map(lambda x: x.encode("utf-8"), jobject))
 
 
     def writeJSONValue(self, jobject):

Modified: PyCalendar/trunk/src/zonal/tzconvert.py
===================================================================
--- PyCalendar/trunk/src/zonal/tzconvert.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/zonal/tzconvert.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -53,28 +53,28 @@
 
     def parse(self, file):
         try:
-            f = open(file, "r")
-            ctr = 0
-            for line in f:
-                ctr += 1
-                line = line[:-1]
-                while True:
-                    if line.startswith("#") or len(line) == 0:
-                        break
-                    elif line.startswith("Rule"):
-                        self.parseRule(line)
-                        break
-                    elif line.startswith("Zone"):
-                        line = self.parseZone(line, f)
-                        if line is None:
+            with open(file, "r") as f:
+                ctr = 0
+                for line in f:
+                    ctr += 1
+                    line = line[:-1]
+                    while True:
+                        if line.startswith("#") or len(line) == 0:
                             break
-                    elif line.startswith("Link"):
-                        self.parseLink(line)
-                        break
-                    elif len(line.strip()) != 0:
-                        assert False, "Could not parse line %d from tzconvert file: '%s'" % (ctr, line,)
-                    else:
-                        break
+                        elif line.startswith("Rule"):
+                            self.parseRule(line)
+                            break
+                        elif line.startswith("Zone"):
+                            line = self.parseZone(line, f)
+                            if line is None:
+                                break
+                        elif line.startswith("Link"):
+                            self.parseLink(line)
+                            break
+                        elif len(line.strip()) != 0:
+                            assert False, "Could not parse line %d from tzconvert file: '%s'" % (ctr, line,)
+                        else:
+                            break
         except:
             print("Failed to parse file %s" % (file,))
             raise
@@ -119,8 +119,8 @@
     def parseWindowsAliases(self, aliases):
 
         try:
-            xmlfile = open(aliases)
-            xmlroot = XML.ElementTree(file=xmlfile).getroot()
+            with open(aliases) as xmlfile:
+                xmlroot = XML.ElementTree(file=xmlfile).getroot()
         except (IOError, XMLParseError):
             raise ValueError("Unable to open or read windows alias file: {}".format(aliases))
 
@@ -217,7 +217,8 @@
 
             # Generate link mapping file
             linkPath = os.path.join(outputdir, "links.txt")
-            open(linkPath, "w").write("\n".join(link_list))
+            with open(linkPath, "w") as f:
+                f.write("\n".join(link_list))
 
 
 

Modified: PyCalendar/trunk/src/zonal/tzdump.py
===================================================================
--- PyCalendar/trunk/src/zonal/tzdump.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/zonal/tzdump.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -27,12 +27,12 @@
     cal = Calendar()
     if verbose:
         print "Parsing calendar data: %s" % (file,)
-    fin = open(file, "r")
-    try:
-        cal.parse(fin)
-    except InvalidData, e:
-        print "Failed to parse bad data: %s" % (e.mData,)
-        raise
+    with open(file, "r") as fin:
+        try:
+            cal.parse(fin)
+        except InvalidData, e:
+            print "Failed to parse bad data: %s" % (e.mData,)
+            raise
     return cal
 
 

Modified: PyCalendar/trunk/src/zonal/tzverify.py
===================================================================
--- PyCalendar/trunk/src/zonal/tzverify.py	2015-06-25 18:47:00 UTC (rev 14914)
+++ PyCalendar/trunk/src/zonal/tzverify.py	2015-06-25 21:49:45 UTC (rev 14915)
@@ -63,12 +63,12 @@
     for file in files:
         if verbose:
             print "Parsing calendar data: %s" % (file,)
-        fin = open(file, "r")
-        try:
-            cal.parse(fin)
-        except InvalidData, e:
-            print "Failed to parse bad data: %s" % (e.mData,)
-            raise
+        with open(file, "r") as fin:
+            try:
+                cal.parse(fin)
+            except InvalidData, e:
+                print "Failed to parse bad data: %s" % (e.mData,)
+                raise
     return CalendarZonesWrapper(calendar=cal)
 
 
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150625/d4428f44/attachment-0001.html>


More information about the calendarserver-changes mailing list