[CalendarServer-changes] [9410] CalendarServer/trunk/calendarserver/tools
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jul 6 12:55:02 PDT 2012
Revision: 9410
http://trac.macosforge.org/projects/calendarserver/changeset/9410
Author: cdaboo at apple.com
Date: 2012-07-06 12:55:01 -0700 (Fri, 06 Jul 2012)
Log Message:
-----------
Add option to "nuke" specific calendar object resources. Detect/fix issue where R-ID and EXDATE are the same.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/calverify.py
CalendarServer/trunk/calendarserver/tools/test/test_calverify.py
Modified: CalendarServer/trunk/calendarserver/tools/calverify.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/calverify.py 2012-07-05 19:13:42 UTC (rev 9409)
+++ CalendarServer/trunk/calendarserver/tools/calverify.py 2012-07-06 19:55:01 UTC (rev 9410)
@@ -50,8 +50,7 @@
from twext.enterprise.dal.syntax import Select, Parameter, Count
from twisted.application.service import Service
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
-from twisted.python import log
-from twisted.python.text import wordWrap
+from twisted.python import log, usage
from twisted.python.usage import Options
from twistedcaldav import caldavxml
from twistedcaldav.dateops import pyCalendarTodatetime
@@ -65,15 +64,111 @@
from txdav.common.icommondatastore import InternalDataStoreError
import base64
import collections
-import os
import sys
import time
import traceback
import uuid
-VERSION = "5"
+# Monkey patch
+def new_validRecurrenceIDs(self, doFix=True):
-def usage(e=None):
+ fixed = []
+ unfixed = []
+
+ # Detect invalid occurrences and fix by adding RDATEs for them
+ master = self.masterComponent()
+ if master is not None:
+ # Get the set of all recurrence IDs
+ all_rids = set(self.getComponentInstances())
+ if None in all_rids:
+ all_rids.remove(None)
+
+ # If the master has no recurrence properties treat any other components as invalid
+ if master.isRecurring():
+
+ # Remove all EXDATEs with a matching RECURRENCE-ID. Do this before we start
+ # processing of valid instances just in case the matching R-ID is also not valid and
+ # thus will need RDATE added.
+ exdates = {}
+ for property in list(master.properties("EXDATE")):
+ for exdate in property.value():
+ exdates[exdate.getValue()] = property
+ for rid in all_rids:
+ if rid in exdates:
+ if doFix:
+ property = exdates[rid]
+ for value in property.value():
+ if value.getValue() == rid:
+ property.value().remove(value)
+ break
+ master.removeProperty(property)
+ if len(property.value()) > 0:
+ master.addProperty(property)
+ del exdates[rid]
+ fixed.append("Removed EXDATE for valid override: %s" % (rid,))
+ else:
+ unfixed.append("EXDATE for valid override: %s" % (rid,))
+
+ # Get the set of all valid recurrence IDs
+ valid_rids = self.validInstances(all_rids, ignoreInvalidInstances=True)
+
+ # Get the set of all RDATEs and add those to the valid set
+ rdates = []
+ for property in master.properties("RDATE"):
+ rdates.extend([_rdate.getValue() for _rdate in property.value()])
+ valid_rids.update(set(rdates))
+
+
+ # Remove EXDATEs predating master
+ dtstart = master.propertyValue("DTSTART")
+ if dtstart is not None:
+ for property in list(master.properties("EXDATE")):
+ newValues = []
+ changed = False
+ for exdate in property.value():
+ exdateValue = exdate.getValue()
+ if exdateValue < dtstart:
+ if doFix:
+ fixed.append("Removed earlier EXDATE: %s" % (exdateValue,))
+ else:
+ unfixed.append("EXDATE earlier than master: %s" % (exdateValue,))
+ changed = True
+ else:
+ newValues.append(exdateValue)
+
+ if changed and doFix:
+ # Remove the property...
+ master.removeProperty(property)
+ if newValues:
+ # ...and add it back only if it still has values
+ property.setValue(newValues)
+ master.addProperty(property)
+
+
+ else:
+ valid_rids = set()
+
+ # Determine the invalid recurrence IDs by set subtraction
+ invalid_rids = all_rids - valid_rids
+
+ # Add RDATEs for the invalid ones, or remove any EXDATE.
+ for invalid_rid in invalid_rids:
+ brokenComponent = self.overriddenComponent(invalid_rid)
+ brokenRID = brokenComponent.propertyValue("RECURRENCE-ID")
+ if doFix:
+ master.addProperty(Property("RDATE", [brokenRID,]))
+ fixed.append("Added RDATE for invalid occurrence: %s" %
+ (brokenRID,))
+ else:
+ unfixed.append("Invalid occurrence: %s" % (brokenRID,))
+
+ return fixed, unfixed
+
+Component.validRecurrenceIDs = new_validRecurrenceIDs
+
+VERSION = "6"
+
+def printusage(e=None):
if e:
print e
print ""
@@ -87,17 +182,51 @@
sys.exit(0)
-description = ''.join(
- wordWrap(
- """
- Usage: calendarserver_verify_data [options] [input specifiers]
- """,
- int(os.environ.get('COLUMNS', '80'))
- )
-)
-description += "\nVersion: %s" % (VERSION,)
+description = """
+Usage: calendarserver_verify_data [options]
+Version: %s
+This tool scans the calendar store to look for and correct any
+problems.
+OPTIONS:
+
+Modes of operation:
+
+-h : print help and exit.
+--ical : verify iCalendar data.
+--mismatch : verify scheduling state.
+--missing : display orphaned calendar homes - can be used.
+ with either --ical or --mismatch.
+
+--nuke PATH|RID : remove specific calendar resources - can
+ only be used by itself. PATH is the full
+ /calendars/__uids__/XXX/YYY/ZZZ.ics object
+ resource path, RID is the SQL DB resource-id.
+
+Options for all modes:
+
+--fix : changes are only made when this is present.
+--config : caldavd.plist file for the server.
+-v : verbose logging
+
+Options for --ical:
+
+--badcua : only look for with bad CALENDARSERVER-OLD-CUA.
+--nobase64 : do not apply base64 encoding to CALENDARSERVER-OLD-CUA.
+--uuid : only scan specified calendar homes. Can be a partial GUID
+ to scan all GUIDs with that as a prefix.
+--uid : scan only calendar data with the specific iCalendar UID.
+
+Options for --mismatch:
+
+--uid : look for mismatches with the specified iCalendar UID only.
+--details : log extended details on each mismatch.
+--tzid : timezone to adjust details to.
+
+""" % (VERSION,)
+
+
def safePercent(x, y, multiplier=100.0):
return ((multiplier * x) / y) if y else 0
@@ -123,9 +252,9 @@
optParameters = [
['config', 'f', DEFAULT_CONFIG_FILE, "Specify caldavd.plist configuration path."],
- ['data', 'd', "./calverify-data", "Path where ancillary data is stored."],
['uuid', 'u', "", "Only check this user."],
['uid', 'U', "", "Only this event UID."],
+ ['nuke', 'e', "", "Remove event given its path"]
]
@@ -133,6 +262,8 @@
super(CalVerifyOptions, self).__init__()
self.outputName = '-'
+ def getUsage(self, width=None):
+ return ""
def opt_output(self, filename):
"""
@@ -153,7 +284,6 @@
return open(self.outputName, 'wb')
-
class CalVerifyService(Service, object):
"""
Service which runs, exports the appropriate records, then stops the reactor.
@@ -216,14 +346,17 @@
self.output.write("\n---- CalVerify version: %s ----\n" % (VERSION,))
try:
- if self.options["missing"]:
- yield self.doOrphans()
-
- if self.options["mismatch"] or self.options["ical"] or self.options["badcua"]:
- yield self.doScan(self.options["ical"] or self.options["badcua"], self.options["mismatch"], self.options["fix"])
+ if self.options["nuke"]:
+ yield self.doNuke()
+ else:
+ if self.options["missing"]:
+ yield self.doOrphans()
+
+ if self.options["mismatch"] or self.options["ical"] or self.options["badcua"]:
+ yield self.doScan(self.options["ical"] or self.options["badcua"], self.options["mismatch"], self.options["fix"])
+
+ self.printSummary()
- self.printSummary()
-
self.output.close()
except:
log.err()
@@ -232,6 +365,51 @@
@inlineCallbacks
+ def doNuke(self):
+ """
+ Remove a resource using either its path or resource id. When doing this do not
+ read the iCalendar data which may be corrupt.
+ """
+
+ self.output.write("\n---- Removing calendar resource ----\n")
+ self.txn = self.store.newTransaction()
+
+ nuke = self.options["nuke"]
+ if nuke.startswith("/calendars/__uids__/"):
+ pathbits = nuke.split("/")
+ if len(pathbits) != 6:
+ printusage("Not a valid calendar object resource path: %s" % (nuke,))
+ homeName = pathbits[3]
+ calendarName = pathbits[4]
+ resourceName = pathbits[5]
+
+ rid = yield self.getResourceID(homeName, calendarName, resourceName)
+ if rid is None:
+ yield self.txn.commit()
+ self.txn = None
+ self.output.write("\n")
+ self.output.write("Path does not exist. Nothing nuked.\n")
+ returnValue(None)
+ rid = int(rid)
+ else:
+ try:
+ rid = int(nuke)
+ except ValueError:
+ printusage("nuke argument must be a calendar object path or an SQL resource-id")
+
+ if self.options["fix"]:
+ result = yield self.fixByRemovingEvent(rid)
+ if result:
+ self.output.write("\n")
+ self.output.write("Removed resource: %s.\n" % (rid,))
+ else:
+ self.output.write("\n")
+ self.output.write("Resource: %s.\n" % (rid,))
+ yield self.txn.commit()
+ self.txn = None
+
+
+ @inlineCallbacks
def doOrphans(self):
"""
Report on home collections for which there are no directory records, or record is for user on
@@ -575,6 +753,30 @@
returnValue(rows[0])
+ @inlineCallbacks
+ def getResourceID(self, home, calendar, resource):
+ co = schema.CALENDAR_OBJECT
+ cb = schema.CALENDAR_BIND
+ ch = schema.CALENDAR_HOME
+
+ kwds = {
+ "home":home,
+ "calendar":calendar,
+ "resource":resource,
+ }
+ rows = (yield Select(
+ [co.RESOURCE_ID],
+ From=ch.join(
+ cb, type="inner", on=(ch.RESOURCE_ID == cb.CALENDAR_HOME_RESOURCE_ID)).join(
+ co, type="inner", on=(cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID)),
+ Where=(ch.OWNER_UID == Parameter("home")).And(
+ cb.CALENDAR_RESOURCE_NAME == Parameter("calendar")).And(
+ co.RESOURCE_NAME == Parameter("resource")
+ ),
+ ).on(self.txn, **kwds))
+ returnValue(rows[0][0] if rows else None)
+
+
def buildResourceInfo(self, rows, onlyOrganizer=False, onlyAttendee=False):
skipped = 0
inboxes = 0
@@ -672,18 +874,18 @@
yield self.txn.commit()
self.txn = None
if self.options["verbose"]:
- self.output.write((
- "\r" +
- ("%s" % badlen).rjust(rjust) +
- ("%s" % count).rjust(rjust) +
- ("%s" % total).rjust(rjust) +
- ("%d%%" % safePercent(count, total)).rjust(rjust)
- ).ljust(80) + "\n")
+ self.output.write((
+ "\r" +
+ ("%s" % badlen).rjust(rjust) +
+ ("%s" % count).rjust(rjust) +
+ ("%s" % total).rjust(rjust) +
+ ("%d%%" % safePercent(count, total)).rjust(rjust)
+ ).ljust(80) + "\n")
# Print table of results
table = tables.Table()
table.addHeader(("Owner", "Event UID", "RID", "Problem",))
- for item in results_bad:
+ for item in sorted(results_bad, key=lambda x:(x[0],x[1])):
owner, uid, resid, message = item
owner_record = self.directoryService().recordWithGUID(owner)
table.addRow((
@@ -704,7 +906,7 @@
diff_time = time.time() - t
self.output.write("Time: %.2f s Average: %.1f ms/resource\n" % (
diff_time,
- (1000.0 * diff_time) / total,
+ safePercent(diff_time, total, 1000.0),
))
errorPrefix = "Calendar data had unfixable problems:\n "
@@ -1654,16 +1856,22 @@
if reactor is None:
from twisted.internet import reactor
options = CalVerifyOptions()
- options.parseOptions(argv[1:])
try:
+ options.parseOptions(argv[1:])
+ except usage.UsageError, e:
+ printusage(e)
+
+ try:
output = options.openOutput()
except IOError, e:
stderr.write("Unable to open output file for writing: %s\n" % (e))
sys.exit(1)
+
def makeService(store):
from twistedcaldav.config import config
config.TransactionTimeoutSeconds = 0
return CalVerifyService(store, options, output, reactor, config)
+
utilityMain(options['config'], makeService, reactor)
if __name__ == '__main__':
Modified: CalendarServer/trunk/calendarserver/tools/test/test_calverify.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_calverify.py 2012-07-05 19:13:42 UTC (rev 9409)
+++ CalendarServer/trunk/calendarserver/tools/test/test_calverify.py 2012-07-06 19:55:01 UTC (rev 9410)
@@ -359,8 +359,39 @@
END:VCALENDAR
""".replace("\n", "\r\n")
+# Bad recurrence EXDATE
+BAD11_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VEVENT
+CREATED:20100303T181216Z
+UID:BAD11
+DTEND:20100307T151500Z
+TRANSP:OPAQUE
+SUMMARY:Ancient event
+DTSTART:20100307T111500Z
+DTSTAMP:20100303T181220Z
+EXDATE:20100314T111500Z
+RRULE:FREQ=WEEKLY
+SEQUENCE:2
+END:VEVENT
+BEGIN:VEVENT
+CREATED:20100303T181216Z
+UID:BAD11
+RECURRENCE-ID:20100314T111500Z
+DTEND:20100314T151500Z
+TRANSP:OPAQUE
+SUMMARY:Ancient event
+DTSTART:20100314T111500Z
+DTSTAMP:20100303T181220Z
+SEQUENCE:2
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
class CalVerifyDataTests(CommonCommonTests, unittest.TestCase):
"""
Tests calverify for iCalendar data problems.
@@ -388,6 +419,7 @@
"ok8.ics" : (OK8_ICS, metadata,),
"bad9.ics" : (BAD9_ICS, metadata,),
"bad10.ics" : (BAD10_ICS, metadata,),
+ "bad11.ics" : (BAD11_ICS, metadata,),
}
},
}
@@ -488,7 +520,7 @@
calverify.emailDomain = "example.com"
yield calverify.doScan(True, False, False)
- self.assertEqual(calverify.results["Number of events to process"], 11)
+ self.assertEqual(calverify.results["Number of events to process"], 12)
self.verifyResultsByUID(calverify.results["Bad iCalendar data"], set((
("home1", "BAD1",),
("home1", "BAD2",),
@@ -499,6 +531,7 @@
("home1", "BAD7",),
("home1", "BAD9",),
("home1", "BAD10",),
+ ("home1", "BAD11",),
)))
sync_token_new = (yield (yield self.calendarUnderTest()).syncToken())
@@ -532,7 +565,7 @@
calverify.emailDomain = "example.com"
yield calverify.doScan(True, False, True)
- self.assertEqual(calverify.results["Number of events to process"], 11)
+ self.assertEqual(calverify.results["Number of events to process"], 12)
self.verifyResultsByUID(calverify.results["Bad iCalendar data"], set((
("home1", "BAD1",),
("home1", "BAD2",),
@@ -543,6 +576,7 @@
("home1", "BAD7",),
("home1", "BAD9",),
("home1", "BAD10",),
+ ("home1", "BAD11",),
)))
# Do scan
@@ -550,7 +584,7 @@
calverify.emailDomain = "example.com"
yield calverify.doScan(True, False, False)
- self.assertEqual(calverify.results["Number of events to process"], 11)
+ self.assertEqual(calverify.results["Number of events to process"], 12)
self.verifyResultsByUID(calverify.results["Bad iCalendar data"], set((
("home1", "BAD1",),
)))
@@ -595,7 +629,7 @@
calverify.emailDomain = "example.com"
yield calverify.doScan(True, False, False)
- self.assertEqual(calverify.results["Number of events to process"], 11)
+ self.assertEqual(calverify.results["Number of events to process"], 12)
self.verifyResultsByUID(calverify.results["Bad iCalendar data"], set((
("home1", "BAD4",),
("home1", "BAD5",),
@@ -636,7 +670,7 @@
calverify.emailDomain = "example.com"
yield calverify.doScan(True, False, True)
- self.assertEqual(calverify.results["Number of events to process"], 11)
+ self.assertEqual(calverify.results["Number of events to process"], 12)
self.verifyResultsByUID(calverify.results["Bad iCalendar data"], set((
("home1", "BAD4",),
("home1", "BAD5",),
@@ -651,7 +685,7 @@
calverify.emailDomain = "example.com"
yield calverify.doScan(True, False, False)
- self.assertEqual(calverify.results["Number of events to process"], 11)
+ self.assertEqual(calverify.results["Number of events to process"], 12)
self.verifyResultsByUID(calverify.results["Bad iCalendar data"], set((
)))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120706/afd8c533/attachment-0001.html>
More information about the calendarserver-changes
mailing list