[CalendarServer-changes] [8503] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Tue Jan 10 11:55:28 PST 2012


Revision: 8503
          http://trac.macosforge.org/projects/calendarserver/changeset/8503
Author:   cdaboo at apple.com
Date:     2012-01-10 11:55:28 -0800 (Tue, 10 Jan 2012)
Log Message:
-----------
Re-work the fix for broken recurrence-ids to only fix during migration, and reject during PUT.

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/ical.py
    CalendarServer/trunk/twistedcaldav/test/test_icalendar.py
    CalendarServer/trunk/txdav/caldav/datastore/file.py
    CalendarServer/trunk/txdav/caldav/datastore/sql.py
    CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
    CalendarServer/trunk/txdav/caldav/datastore/util.py

Modified: CalendarServer/trunk/twistedcaldav/ical.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/ical.py	2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/twistedcaldav/ical.py	2012-01-10 19:55:28 UTC (rev 8503)
@@ -1286,7 +1286,7 @@
 
         return changed
 
-    def validCalendarData(self, doFix=True, doRaise=True):
+    def validCalendarData(self, doFix=True, doRaise=True, validateRecurrences=False):
         """
         @return: tuple of fixed, unfixed issues
         @raise InvalidICalendarDataError: if the given calendar data is not valid and
@@ -1299,6 +1299,27 @@
             log.debug("Unknown resource type: %s" % (self,))
             raise InvalidICalendarDataError("Unknown resource type")
 
+        # Do underlying iCalendar library validation with data fix
+        fixed, unfixed = self._pycalendar.validate(doFix=doFix)
+
+        # Detect invalid occurrences and fix by adding RDATEs for them
+        if validateRecurrences:
+            rfixed, runfixed = self.validRecurrenceIDs(doFix=doFix)
+            fixed.extend(rfixed)
+            unfixed.extend(runfixed)
+
+        if unfixed:
+            log.debug("Calendar data had unfixable problems:\n  %s" % ("\n  ".join(unfixed),))
+            if doRaise:
+                raise InvalidICalendarDataError("Calendar data had unfixable problems:\n  %s" % ("\n  ".join(unfixed),))
+        if fixed:
+            log.debug("Calendar data had fixable problems:\n  %s" % ("\n  ".join(fixed),))
+        
+        return fixed, unfixed
+
+
+    def validRecurrenceIDs(self, doFix=True):
+
         fixed = []
         unfixed = []
 
@@ -1309,39 +1330,32 @@
             all_rids = set(self.getComponentInstances())
             if None in all_rids:
                 all_rids.remove(None)
+
             # Get the set of all valid recurrence IDs
             valid_rids = self.validInstances(all_rids, ignoreInvalidInstances=True)
+
             # Get the set of all RDATEs and add those to the valid set
             rdates = []
             for property in master.properties("RDATE"):
                 rdates.extend([_rdate.getValue() for _rdate in property.value()])
             valid_rids.update(set(rdates))
+
             # Determine the invalid recurrence IDs by set subtraction
             invalid_rids = all_rids - valid_rids
+
             # Add RDATEs for the invalid ones.
             for invalid_rid in invalid_rids:
+                brokenComponent = self.overriddenComponent(invalid_rid)
+                brokenRID = brokenComponent.propertyValue("RECURRENCE-ID")
                 if doFix:
-                    master.addProperty(Property("RDATE", [invalid_rid,]))
+                    master.addProperty(Property("RDATE", [brokenRID,]))
                     fixed.append("Added RDATE for invalid occurrence: %s" %
-                        (invalid_rid,))
+                        (brokenRID,))
                 else:
-                    unfixed.append("Invalid occurrence: %s" % (invalid_rid,))
+                    unfixed.append("Invalid occurrence: %s" % (brokenRID,))
 
-        # Do underlying iCalendar library validation with data fix
-        pyfixed, pyunfixed = self._pycalendar.validate(doFix=doFix)
-        fixed.extend(pyfixed)
-        unfixed.extend(pyunfixed)
-        if unfixed:
-            log.debug("Calendar data had unfixable problems:\n  %s" % ("\n  ".join(unfixed),))
-            if doRaise:
-                raise InvalidICalendarDataError("Calendar data had unfixable problems:\n  %s" % ("\n  ".join(unfixed),))
-        if fixed:
-            log.debug("Calendar data had fixable problems:\n  %s" % ("\n  ".join(fixed),))
-        
         return fixed, unfixed
 
-        
-        
     def validCalendarForCalDAV(self, methodAllowed):
         """
         @param methodAllowed:     True if METHOD property is allowed, False otherwise.
@@ -1462,7 +1476,7 @@
         if len(s.translate(None, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F")) != len(s):
             raise InvalidICalendarDataError("iCalendar contains illegal control character")
 
-    def validOrganizerForScheduling(self):
+    def validOrganizerForScheduling(self, doFix=True):
         """
         Check that the ORGANIZER property is valid for scheduling 
         """
@@ -1487,10 +1501,13 @@
         
         # If there are some components without an ORGANIZER we will fix the data
         if foundOrganizer and missingRids:
-            log.debug("Fixing missing ORGANIZER properties")
-            organizerProperty = self.overriddenComponent(foundRid).getProperty("ORGANIZER")
-            for rid in missingRids:
-                self.overriddenComponent(rid).addProperty(organizerProperty)
+            if doFix:
+                log.debug("Fixing missing ORGANIZER properties")
+                organizerProperty = self.overriddenComponent(foundRid).getProperty("ORGANIZER")
+                for rid in missingRids:
+                    self.overriddenComponent(rid).addProperty(organizerProperty)
+            else:
+                raise InvalidICalendarDataError("iCalendar missing ORGANIZER properties in some instances")
 
         return foundOrganizer
 

Modified: CalendarServer/trunk/twistedcaldav/test/test_icalendar.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_icalendar.py	2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/twistedcaldav/test/test_icalendar.py	2012-01-10 19:55:28 UTC (rev 8503)
@@ -437,18 +437,18 @@
         # Ensure it starts off invalid
         calendar = Component.fromString(data)
         try:
-            calendar.validCalendarData(doFix=False)
+            calendar.validCalendarData(doFix=False, validateRecurrences=True)
         except InvalidICalendarDataError:
             pass
         else:
             self.fail("Shouldn't validate for CalDAV")
 
         # Fix it
-        calendar.validCalendarData(doFix=True)
-        self.assertTrue("RDATE:20111215T223000Z\r\n" in str(calendar))
+        calendar.validCalendarData(doFix=True, validateRecurrences=True)
+        self.assertTrue("RDATE;TZID=America/Los_Angeles:20111215T143000\r\n" in str(calendar))
 
         # Now it should pass without fixing
-        calendar.validCalendarData(doFix=False)
+        calendar.validCalendarData(doFix=False, validateRecurrences=True)
 
 
     def test_component_timeranges(self):

Modified: CalendarServer/trunk/txdav/caldav/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/file.py	2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/file.py	2012-01-10 19:55:28 UTC (rev 8503)
@@ -403,7 +403,7 @@
     @writeOperation
     def setComponent(self, component, inserting=False):
 
-        validateCalendarComponent(self, self._calendar, component, inserting)
+        validateCalendarComponent(self, self._calendar, component, inserting, self._transaction._migrating)
 
         self._calendar.retrieveOldIndex().addResource(
             self.name(), component

Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py	2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py	2012-01-10 19:55:28 UTC (rev 8503)
@@ -708,7 +708,7 @@
     @inlineCallbacks
     def setComponent(self, component, inserting=False):
 
-        validateCalendarComponent(self, self._calendar, component, inserting)
+        validateCalendarComponent(self, self._calendar, component, inserting, self._txn._migrating)
 
         yield self.updateDatabase(component, inserting=inserting)
         if inserting:

Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py	2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py	2012-01-10 19:55:28 UTC (rev 8503)
@@ -174,6 +174,82 @@
 
 
     @inlineCallbacks
+    def test_migrateRecurrenceFixCalendarFromFile(self):
+        """
+        C{_migrateCalendar()} can migrate a file-backed calendar to a database-
+        backed calendar. We need to test what happens when there is "bad" calendar data
+        present in the file-backed calendar with a broken recurrence-id that we can fix.
+        """
+        
+        self.storeUnderTest().setMigrating(True)
+        fromCalendar = yield (yield self.fileTransaction().calendarHomeWithUID(
+            "home_bad")).calendarWithName("calendar_fix_recurrence")
+        toHome = yield self.transactionUnderTest().calendarHomeWithUID(
+            "new-home", create=True)
+        toCalendar = yield toHome.calendarWithName("calendar")
+        ok, bad = (yield _migrateCalendar(fromCalendar, toCalendar,
+                               lambda x: x.component()))
+        self.assertEqual(ok, 1)
+        self.assertEqual(bad, 0)
+
+        self.transactionUnderTest().commit()
+        self.storeUnderTest().setMigrating(False)
+
+        toHome = yield self.transactionUnderTest().calendarHomeWithUID(
+            "new-home", create=True)
+        toCalendar = yield toHome.calendarWithName("calendar")
+        toResource = yield toCalendar.calendarObjectWithName("1.ics")
+        caldata = yield toResource.component()
+        self.assertEqual(str(caldata), """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VTIMEZONE
+TZID:US/Eastern
+LAST-MODIFIED:20040110T032845Z
+BEGIN:DAYLIGHT
+DTSTART:20000404T020000
+RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
+TZNAME:EDT
+TZOFFSETFROM:-0500
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20001026T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+TZNAME:EST
+TZOFFSETFROM:-0400
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+UID:uid2
+DTSTART;TZID=US/Eastern:20060102T140000
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RDATE;TZID=US/Eastern:20060104T160000
+RRULE:FREQ=DAILY;COUNT=5
+SUMMARY:event 6-%ctr
+END:VEVENT
+BEGIN:VEVENT
+UID:uid2
+RECURRENCE-ID;TZID=US/Eastern:20060104T160000
+DTSTART;TZID=US/Eastern:20060104T160000
+DURATION:PT1H
+CREATED:20060102T190000Z
+DESCRIPTION:Some notes
+DTSTAMP:20051222T210507Z
+SUMMARY:event 6-%ctr changed again
+BEGIN:VALARM
+ACTION:AUDIO
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n"))
+        
+    @inlineCallbacks
     def test_migrateDuplicateAttachmentsCalendarFromFile(self):
         """
         C{_migrateCalendar()} can migrate a file-backed calendar to a database-

Modified: CalendarServer/trunk/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/util.py	2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/util.py	2012-01-10 19:55:28 UTC (rev 8503)
@@ -40,7 +40,7 @@
 
 log = Logger()
 
-def validateCalendarComponent(calendarObject, calendar, component, inserting):
+def validateCalendarComponent(calendarObject, calendar, component, inserting, migrating):
     """
     Validate a calendar component for a particular calendar.
 
@@ -71,7 +71,7 @@
     try:
         # FIXME: This is a bad way to do this test (== 'inbox'), there should be a
         # Calendar-level API for it.
-        component.validCalendarData()
+        component.validCalendarData(validateRecurrences=migrating)
         component.validCalendarForCalDAV(methodAllowed=calendar.name() == 'inbox')
     except InvalidICalendarDataError, e:
         raise InvalidObjectResourceError(e)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120110/537b3ec2/attachment-0001.html>


More information about the calendarserver-changes mailing list