[CalendarServer-changes] [8503] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Jan 10 11:55:28 PST 2012
Revision: 8503
http://trac.macosforge.org/projects/calendarserver/changeset/8503
Author: cdaboo at apple.com
Date: 2012-01-10 11:55:28 -0800 (Tue, 10 Jan 2012)
Log Message:
-----------
Re-work the fix for broken recurrence-ids to only fix during migration, and reject during PUT.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/ical.py
CalendarServer/trunk/twistedcaldav/test/test_icalendar.py
CalendarServer/trunk/txdav/caldav/datastore/file.py
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/caldav/datastore/util.py
Modified: CalendarServer/trunk/twistedcaldav/ical.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/ical.py 2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/twistedcaldav/ical.py 2012-01-10 19:55:28 UTC (rev 8503)
@@ -1286,7 +1286,7 @@
return changed
- def validCalendarData(self, doFix=True, doRaise=True):
+ def validCalendarData(self, doFix=True, doRaise=True, validateRecurrences=False):
"""
@return: tuple of fixed, unfixed issues
@raise InvalidICalendarDataError: if the given calendar data is not valid and
@@ -1299,6 +1299,27 @@
log.debug("Unknown resource type: %s" % (self,))
raise InvalidICalendarDataError("Unknown resource type")
+ # Do underlying iCalendar library validation with data fix
+ fixed, unfixed = self._pycalendar.validate(doFix=doFix)
+
+ # Detect invalid occurrences and fix by adding RDATEs for them
+ if validateRecurrences:
+ rfixed, runfixed = self.validRecurrenceIDs(doFix=doFix)
+ fixed.extend(rfixed)
+ unfixed.extend(runfixed)
+
+ if unfixed:
+ log.debug("Calendar data had unfixable problems:\n %s" % ("\n ".join(unfixed),))
+ if doRaise:
+ raise InvalidICalendarDataError("Calendar data had unfixable problems:\n %s" % ("\n ".join(unfixed),))
+ if fixed:
+ log.debug("Calendar data had fixable problems:\n %s" % ("\n ".join(fixed),))
+
+ return fixed, unfixed
+
+
+ def validRecurrenceIDs(self, doFix=True):
+
fixed = []
unfixed = []
@@ -1309,39 +1330,32 @@
all_rids = set(self.getComponentInstances())
if None in all_rids:
all_rids.remove(None)
+
# Get the set of all valid recurrence IDs
valid_rids = self.validInstances(all_rids, ignoreInvalidInstances=True)
+
# Get the set of all RDATEs and add those to the valid set
rdates = []
for property in master.properties("RDATE"):
rdates.extend([_rdate.getValue() for _rdate in property.value()])
valid_rids.update(set(rdates))
+
# Determine the invalid recurrence IDs by set subtraction
invalid_rids = all_rids - valid_rids
+
# Add RDATEs for the invalid ones.
for invalid_rid in invalid_rids:
+ brokenComponent = self.overriddenComponent(invalid_rid)
+ brokenRID = brokenComponent.propertyValue("RECURRENCE-ID")
if doFix:
- master.addProperty(Property("RDATE", [invalid_rid,]))
+ master.addProperty(Property("RDATE", [brokenRID,]))
fixed.append("Added RDATE for invalid occurrence: %s" %
- (invalid_rid,))
+ (brokenRID,))
else:
- unfixed.append("Invalid occurrence: %s" % (invalid_rid,))
+ unfixed.append("Invalid occurrence: %s" % (brokenRID,))
- # Do underlying iCalendar library validation with data fix
- pyfixed, pyunfixed = self._pycalendar.validate(doFix=doFix)
- fixed.extend(pyfixed)
- unfixed.extend(pyunfixed)
- if unfixed:
- log.debug("Calendar data had unfixable problems:\n %s" % ("\n ".join(unfixed),))
- if doRaise:
- raise InvalidICalendarDataError("Calendar data had unfixable problems:\n %s" % ("\n ".join(unfixed),))
- if fixed:
- log.debug("Calendar data had fixable problems:\n %s" % ("\n ".join(fixed),))
-
return fixed, unfixed
-
-
def validCalendarForCalDAV(self, methodAllowed):
"""
@param methodAllowed: True if METHOD property is allowed, False otherwise.
@@ -1462,7 +1476,7 @@
if len(s.translate(None, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F")) != len(s):
raise InvalidICalendarDataError("iCalendar contains illegal control character")
- def validOrganizerForScheduling(self):
+ def validOrganizerForScheduling(self, doFix=True):
"""
Check that the ORGANIZER property is valid for scheduling
"""
@@ -1487,10 +1501,13 @@
# If there are some components without an ORGANIZER we will fix the data
if foundOrganizer and missingRids:
- log.debug("Fixing missing ORGANIZER properties")
- organizerProperty = self.overriddenComponent(foundRid).getProperty("ORGANIZER")
- for rid in missingRids:
- self.overriddenComponent(rid).addProperty(organizerProperty)
+ if doFix:
+ log.debug("Fixing missing ORGANIZER properties")
+ organizerProperty = self.overriddenComponent(foundRid).getProperty("ORGANIZER")
+ for rid in missingRids:
+ self.overriddenComponent(rid).addProperty(organizerProperty)
+ else:
+ raise InvalidICalendarDataError("iCalendar missing ORGANIZER properties in some instances")
return foundOrganizer
Modified: CalendarServer/trunk/twistedcaldav/test/test_icalendar.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_icalendar.py 2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/twistedcaldav/test/test_icalendar.py 2012-01-10 19:55:28 UTC (rev 8503)
@@ -437,18 +437,18 @@
# Ensure it starts off invalid
calendar = Component.fromString(data)
try:
- calendar.validCalendarData(doFix=False)
+ calendar.validCalendarData(doFix=False, validateRecurrences=True)
except InvalidICalendarDataError:
pass
else:
self.fail("Shouldn't validate for CalDAV")
# Fix it
- calendar.validCalendarData(doFix=True)
- self.assertTrue("RDATE:20111215T223000Z\r\n" in str(calendar))
+ calendar.validCalendarData(doFix=True, validateRecurrences=True)
+ self.assertTrue("RDATE;TZID=America/Los_Angeles:20111215T143000\r\n" in str(calendar))
# Now it should pass without fixing
- calendar.validCalendarData(doFix=False)
+ calendar.validCalendarData(doFix=False, validateRecurrences=True)
def test_component_timeranges(self):
Modified: CalendarServer/trunk/txdav/caldav/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/file.py 2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/file.py 2012-01-10 19:55:28 UTC (rev 8503)
@@ -403,7 +403,7 @@
@writeOperation
def setComponent(self, component, inserting=False):
- validateCalendarComponent(self, self._calendar, component, inserting)
+ validateCalendarComponent(self, self._calendar, component, inserting, self._transaction._migrating)
self._calendar.retrieveOldIndex().addResource(
self.name(), component
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-01-10 19:55:28 UTC (rev 8503)
@@ -708,7 +708,7 @@
@inlineCallbacks
def setComponent(self, component, inserting=False):
- validateCalendarComponent(self, self._calendar, component, inserting)
+ validateCalendarComponent(self, self._calendar, component, inserting, self._txn._migrating)
yield self.updateDatabase(component, inserting=inserting)
if inserting:
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-01-10 19:55:28 UTC (rev 8503)
@@ -174,6 +174,82 @@
@inlineCallbacks
+ def test_migrateRecurrenceFixCalendarFromFile(self):
+ """
+ C{_migrateCalendar()} can migrate a file-backed calendar to a database-
+ backed calendar. We need to test what happens when there is "bad" calendar data
+ present in the file-backed calendar with a broken recurrence-id that we can fix.
+ """
+
+ self.storeUnderTest().setMigrating(True)
+ fromCalendar = yield (yield self.fileTransaction().calendarHomeWithUID(
+ "home_bad")).calendarWithName("calendar_fix_recurrence")
+ toHome = yield self.transactionUnderTest().calendarHomeWithUID(
+ "new-home", create=True)
+ toCalendar = yield toHome.calendarWithName("calendar")
+ ok, bad = (yield _migrateCalendar(fromCalendar, toCalendar,
+ lambda x: x.component()))
+ self.assertEqual(ok, 1)
+ self.assertEqual(bad, 0)
+
+ self.transactionUnderTest().commit()
+ self.storeUnderTest().setMigrating(False)
+
+ toHome = yield self.transactionUnderTest().calendarHomeWithUID(
+ "new-home", create=True)
+ toCalendar = yield toHome.calendarWithName("calendar")
+ toResource = yield toCalendar.calendarObjectWithName("1.ics")
+ caldata = yield toResource.component()
+ self.assertEqual(str(caldata), """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VTIMEZONE
+TZID:US/Eastern
+LAST-MODIFIED:20040110T032845Z
+BEGIN:DAYLIGHT
+DTSTART:20000404T020000
+RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
+TZNAME:EDT
+TZOFFSETFROM:-0500
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20001026T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+TZNAME:EST
+TZOFFSETFROM:-0400
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+UID:uid2
+DTSTART;TZID=US/Eastern:20060102T140000
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RDATE;TZID=US/Eastern:20060104T160000
+RRULE:FREQ=DAILY;COUNT=5
+SUMMARY:event 6-%ctr
+END:VEVENT
+BEGIN:VEVENT
+UID:uid2
+RECURRENCE-ID;TZID=US/Eastern:20060104T160000
+DTSTART;TZID=US/Eastern:20060104T160000
+DURATION:PT1H
+CREATED:20060102T190000Z
+DESCRIPTION:Some notes
+DTSTAMP:20051222T210507Z
+SUMMARY:event 6-%ctr changed again
+BEGIN:VALARM
+ACTION:AUDIO
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n"))
+
+ @inlineCallbacks
def test_migrateDuplicateAttachmentsCalendarFromFile(self):
"""
C{_migrateCalendar()} can migrate a file-backed calendar to a database-
Modified: CalendarServer/trunk/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/util.py 2012-01-10 19:34:58 UTC (rev 8502)
+++ CalendarServer/trunk/txdav/caldav/datastore/util.py 2012-01-10 19:55:28 UTC (rev 8503)
@@ -40,7 +40,7 @@
log = Logger()
-def validateCalendarComponent(calendarObject, calendar, component, inserting):
+def validateCalendarComponent(calendarObject, calendar, component, inserting, migrating):
"""
Validate a calendar component for a particular calendar.
@@ -71,7 +71,7 @@
try:
# FIXME: This is a bad way to do this test (== 'inbox'), there should be a
# Calendar-level API for it.
- component.validCalendarData()
+ component.validCalendarData(validateRecurrences=migrating)
component.validCalendarForCalDAV(methodAllowed=calendar.name() == 'inbox')
except InvalidICalendarDataError, e:
raise InvalidObjectResourceError(e)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120110/537b3ec2/attachment-0001.html>
More information about the calendarserver-changes
mailing list