Revision: 9257 http://trac.macosforge.org/projects/calendarserver/changeset/9257 Author: sagen@apple.com Date: 2012-05-18 15:39:24 -0700 (Fri, 18 May 2012) Log Message: ----------- Updates purge tool to handle remove attachments Modified Paths: -------------- CalendarServer/trunk/calendarserver/tools/purge.py CalendarServer/trunk/calendarserver/tools/test/test_purge.py CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py CalendarServer/trunk/txdav/caldav/datastore/sql.py CalendarServer/trunk/txdav/carddav/datastore/sql.py Modified: CalendarServer/trunk/calendarserver/tools/purge.py =================================================================== --- CalendarServer/trunk/calendarserver/tools/purge.py 2012-05-18 17:44:13 UTC (rev 9256) +++ CalendarServer/trunk/calendarserver/tools/purge.py 2012-05-18 22:39:24 UTC (rev 9257) @@ -198,7 +198,7 @@ def doWork(self): rootResource = self.rootResource() directory = rootResource.getDirectory() - total = (yield purgeUIDs(directory, rootResource, self.uids, + total = (yield purgeUIDs(self._store, directory, rootResource, self.uids, verbose=self.verbose, dryrun=self.dryrun, completely=self.completely, doimplicit=self.doimplicit)) if self.verbose: @@ -503,14 +503,14 @@ @inlineCallbacks -def purgeUIDs(directory, root, uids, verbose=False, dryrun=False, +def purgeUIDs(store, directory, root, uids, verbose=False, dryrun=False, completely=False, doimplicit=True): total = 0 allAssignments = { } for uid in uids: - count, allAssignments[uid] = (yield purgeUID(uid, directory, root, + count, allAssignments[uid] = (yield purgeUID(store, uid, directory, root, verbose=verbose, dryrun=dryrun, completely=completely, doimplicit=doimplicit)) total += count @@ -627,7 +627,7 @@ @inlineCallbacks -def purgeUID(uid, directory, root, verbose=False, dryrun=False, proxies=True, +def purgeUID(store, uid, directory, root, verbose=False, dryrun=False, proxies=True, when=None, completely=False, doimplicit=True): if when is None: @@ -658,7 +658,10 @@ davxml.HRef.fromString("/principals/__uids__/%s/" % (uid,)) ) - calendarHome = yield principal.calendarHome(request) + # See if calendar home is provisioned + txn = store.newTransaction() + calHomeProvisioned = ((yield txn.calendarHomeWithUID(uid)) is not None) + (yield txn.commit()) # Anything in the past is left alone whenString = when.getText() @@ -679,102 +682,117 @@ perUserFilter = PerUserDataFilter(uid) try: - for collName in (yield calendarHome.listChildren()): - collection = (yield calendarHome.getChild(collName)) - if collection.isCalendarCollection() or collName == "inbox": + if calHomeProvisioned: + calendarHome = yield principal.calendarHome(request) + for collName in (yield calendarHome.listChildren()): + collection = (yield calendarHome.getChild(collName)) + if collection.isCalendarCollection() or collName == "inbox": - childNames = [] + childNames = [] - if completely: - # all events - for childName in (yield collection.listChildren()): - childNames.append(childName) - else: - # events matching filter - for childName, childUid, childType in (yield collection.index().indexedSearch(filter)): - childNames.append(childName) - - for childName in childNames: - - childResource = (yield collection.getChild(childName)) if completely: - action = CANCELEVENT_SHOULD_DELETE + # all events + for childName in (yield collection.listChildren()): + childNames.append(childName) else: - event = (yield childResource.iCalendar()) - event = perUserFilter.filter(event) - action = cancelEvent(event, when, cua) + # events matching filter + for childName, childUid, childType in (yield collection.index().indexedSearch(filter)): + childNames.append(childName) - uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName) - request.path = uri - if action == CANCELEVENT_MODIFIED: - count += 1 - request._rememberResource(childResource, uri) - storer = StoreCalendarObjectResource( - request=request, - destination=childResource, - destination_uri=uri, - destinationcal=True, - destinationparent=collection, - calendar=str(event), - ) - if verbose: - if dryrun: - print "Would modify: %s" % (uri,) - else: - print "Modifying: %s" % (uri,) - if not dryrun: - result = (yield storer.run()) + for childName in childNames: - elif action == CANCELEVENT_SHOULD_DELETE: - incrementCount = dryrun - request._rememberResource(childResource, uri) - if verbose: - if dryrun: - print "Would delete: %s" % (uri,) - else: - print "Deleting: %s" % (uri,) - if not dryrun: - retry = False - try: - result = (yield childResource.storeRemove(request, doimplicit, uri)) - if result != NO_CONTENT: - print "Error deleting %s/%s/%s: %s" % (uid, - collName, childName, result) - retry = True + childResource = (yield collection.getChild(childName)) + if completely: + action = CANCELEVENT_SHOULD_DELETE + else: + event = (yield childResource.iCalendar()) + event = perUserFilter.filter(event) + action = cancelEvent(event, when, cua) + + uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName) + request.path = uri + if action == CANCELEVENT_MODIFIED: + count += 1 + request._rememberResource(childResource, uri) + storer = StoreCalendarObjectResource( + request=request, + destination=childResource, + destination_uri=uri, + destinationcal=True, + destinationparent=collection, + calendar=str(event), + ) + if verbose: + if dryrun: + print "Would modify: %s" % (uri,) else: - incrementCount = True + print "Modifying: %s" % (uri,) + if not dryrun: + result = (yield storer.run()) - except Exception, e: - print "Exception deleting %s/%s/%s: %s" % (uid, - collName, childName, str(e)) - traceback.print_stack() - retry = True - - if retry and doimplicit: - # Try again with implicit scheduling off - print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName) + elif action == CANCELEVENT_SHOULD_DELETE: + incrementCount = dryrun + request._rememberResource(childResource, uri) + if verbose: + if dryrun: + print "Would delete: %s" % (uri,) + else: + print "Deleting: %s" % (uri,) + if not dryrun: + retry = False try: - result = (yield childResource.storeRemove(request, False, uri)) + result = (yield childResource.storeRemove(request, doimplicit, uri)) if result != NO_CONTENT: print "Error deleting %s/%s/%s: %s" % (uid, collName, childName, result) + retry = True else: incrementCount = True + except Exception, e: - print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e)) + print "Exception deleting %s/%s/%s: %s" % (uid, + collName, childName, str(e)) traceback.print_stack() + retry = True - if incrementCount: - count += 1 + if retry and doimplicit: + # Try again with implicit scheduling off + print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName) + try: + result = (yield childResource.storeRemove(request, False, uri)) + if result != NO_CONTENT: + print "Error deleting %s/%s/%s: %s" % (uid, + collName, childName, result) + else: + incrementCount = True + except Exception, e: + print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e)) + traceback.print_stack() + if incrementCount: + count += 1 - txn = request._newStoreTransaction + txn = getattr(request, "_newStoreTransaction", None) + # Commit + if txn is not None: + (yield txn.commit()) + + except Exception, e: + # Abort + txn = getattr(request, "_newStoreTransaction", None) + if txn is not None: + (yield txn.abort()) + raise e + + try: + txn = store.newTransaction() + # Remove empty calendar collections (and calendar home if no more # calendars) - calHome = (yield txn.calendarHomeWithUID(uid)) - if calHome is not None: - calendars = list((yield calHome.calendars())) + storeCalHome = (yield txn.calendarHomeWithUID(uid)) + if storeCalHome is not None: + calendars = list((yield storeCalHome.calendars())) remainingCalendars = len(calendars) for calColl in calendars: if len(list((yield calColl.calendarObjects()))) == 0: @@ -786,7 +804,7 @@ else: print "Deleting calendar: %s" % (calendarName,) if not dryrun: - (yield calHome.removeChildWithName(calendarName)) + (yield storeCalHome.removeChildWithName(calendarName)) if not remainingCalendars: if verbose: @@ -795,13 +813,13 @@ else: print "Deleting calendar home" if not dryrun: - (yield calHome.remove()) + (yield storeCalHome.remove()) # Remove VCards - abHome = (yield txn.addressbookHomeWithUID(uid)) - if abHome is not None: - for abColl in list( (yield abHome.addressbooks()) ): + storeAbHome = (yield txn.addressbookHomeWithUID(uid)) + if storeAbHome is not None: + for abColl in list( (yield storeAbHome.addressbooks()) ): for card in list( (yield abColl.addressbookObjects()) ): cardName = card.name() if verbose: @@ -821,7 +839,7 @@ print "Deleting addressbook: %s" % (abName,) if not dryrun: # Also remove the addressbook collection itself - (yield abHome.removeChildWithName(abColl.name())) + (yield storeAbHome.removeChildWithName(abColl.name())) if verbose: if dryrun: @@ -829,14 +847,13 @@ else: print "Deleting addressbook home" if not dryrun: - (yield abHome.remove()) + (yield storeAbHome.remove()) # Commit (yield txn.commit()) except Exception, e: # Abort - txn = request._newStoreTransaction (yield txn.abort()) raise e Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge.py =================================================================== --- CalendarServer/trunk/calendarserver/tools/test/test_purge.py 2012-05-18 17:44:13 UTC (rev 9256) +++ CalendarServer/trunk/calendarserver/tools/test/test_purge.py 2012-05-18 22:39:24 UTC (rev 9257) @@ -15,18 +15,26 @@ ## -from calendarserver.tools.purge import cancelEvent +from calendarserver.tap.util import getRootResource +from calendarserver.tools.purge import cancelEvent, purgeUID from calendarserver.tools.purge import CANCELEVENT_MODIFIED, CANCELEVENT_SHOULD_DELETE +from twistedcaldav.config import config from twistedcaldav.ical import Component from twistedcaldav.test.util import TestCase from pycalendar.datetime import PyCalendarDateTime from pycalendar.timezone import PyCalendarTimezone +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks +from txdav.common.datastore.test.util import buildStore, populateCalendarsFrom, CommonCommonTests +from twext.web2.http_headers import MimeType +import os + future = PyCalendarDateTime.getNowUTC() future.offsetDay(1) future = future.getText() @@ -727,3 +735,144 @@ END:VEVENT END:VCALENDAR """.replace("\n", "\r\n") + + + + + +ATTACHMENT_ICS = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//iCal 4.0.1//EN +CALSCALE:GREGORIAN +BEGIN:VTIMEZONE +TZID:US/Pacific +BEGIN:DAYLIGHT +TZOFFSETFROM:-0800 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU +DTSTART:20070311T020000 +TZNAME:PDT +TZOFFSETTO:-0700 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:-0700 +RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU +DTSTART:20071104T020000 +TZNAME:PST +TZOFFSETTO:-0800 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20100303T195159Z +UID:F2F14D94-B944-43D9-8F6F-97F95B2764CA +DTEND;TZID=US/Pacific:20100304T141500 +TRANSP:OPAQUE +SUMMARY:Attachment +DTSTART;TZID=US/Pacific:20100304T120000 +DTSTAMP:20100303T195203Z +SEQUENCE:2 +X-APPLE-DROPBOX:/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/dropbox/F2F14D94-B944-43D9-8F6F-97F95B2764CA.dropbox +END:VEVENT +END:VCALENDAR +""".replace("\n", "\r\n") + + + + +class PurgePrincipalTests(CommonCommonTests, unittest.TestCase): + """ + Tests for purging the data belonging to a given principal + """ + uid = "6423F94A-6B76-4A3A-815B-D52CFD77935D" + + metadata = { + "accessMode": "PUBLIC", + "isScheduleObject": True, + "scheduleTag": "abc", + "scheduleEtags": (), + "hasPrivateComment": False, + } + + requirements = { + uid : { + "calendar1" : { + "attachment.ics" : (ATTACHMENT_ICS, metadata,), + } + }, + } + + @inlineCallbacks + def setUp(self): + yield super(PurgePrincipalTests, self).setUp() + self._sqlCalendarStore = yield buildStore(self, self.notifierFactory) + yield self.populate() + + self.patch(config.DirectoryService.params, "xmlFile", + os.path.join( + os.path.dirname(__file__), "purge", "accounts.xml" + ) + ) + self.patch(config.ResourceService.params, "xmlFile", + os.path.join( + os.path.dirname(__file__), "purge", "resources.xml" + ) + ) + self.rootResource = getRootResource(config, self._sqlCalendarStore) + self.directory = self.rootResource.getDirectory() + + # Add attachment to attachment.ics + txn = self._sqlCalendarStore.newTransaction() + home = (yield txn.calendarHomeWithUID(self.uid)) + calendar = (yield home.calendarWithName("calendar1")) + event = (yield calendar.calendarObjectWithName("attachment.ics")) + attachment = (yield event.createAttachmentWithName("attachment.txt")) + t = attachment.store(MimeType("text", "x-fixture")) + t.write("attachment") + t.write(" text") + (yield t.loseConnection()) + (yield txn.commit()) + + + @inlineCallbacks + def populate(self): + yield populateCalendarsFrom(self.requirements, self.storeUnderTest()) + self.notifierFactory.reset() + + + def storeUnderTest(self): + """ + Create and return a L{CalendarStore} for testing. + """ + return self._sqlCalendarStore + + + @inlineCallbacks + def test_purgeUID(self): + """ + Verify purgeUID removes homes, and doesn't provision homes that don't exist + """ + + # Now you see it + txn = self._sqlCalendarStore.newTransaction() + home = (yield txn.calendarHomeWithUID(self.uid)) + self.assertNotEquals(home, None) + (yield txn.commit()) + + count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory, + self.rootResource, verbose=False, proxies=False, completely=True)) + self.assertEquals(count, 1) # 1 event + + # Now you don't + txn = self._sqlCalendarStore.newTransaction() + home = (yield txn.calendarHomeWithUID(self.uid)) + self.assertEquals(home, None) + (yield txn.commit()) + + count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory, + self.rootResource, verbose=False, proxies=False, completely=True)) + self.assertEquals(count, 0) + + # And you still don't (making sure it's not provisioned) + txn = self._sqlCalendarStore.newTransaction() + home = (yield txn.calendarHomeWithUID(self.uid)) + self.assertEquals(home, None) + (yield txn.commit()) Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py =================================================================== --- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2012-05-18 17:44:13 UTC (rev 9256) +++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2012-05-18 22:39:24 UTC (rev 9257) @@ -531,7 +531,7 @@ (yield txn.commit()) # Purge home1 - total, ignored = (yield purgeUID("home1", self.directory, + total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory, self.rootResource, verbose=False, proxies=False, when=PyCalendarDateTime(2010, 4, 1, 12, 0, 0, 0, PyCalendarTimezone(utc=True)))) @@ -568,7 +568,7 @@ (yield txn.commit()) # Purge home1 completely - total, ignored = (yield purgeUID("home1", self.directory, + total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory, self.rootResource, verbose=False, proxies=False, completely=True)) # 4 items deleted: 3 events and 1 vcard Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py =================================================================== --- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-05-18 17:44:13 UTC (rev 9256) +++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-05-18 22:39:24 UTC (rev 9257) @@ -129,12 +129,27 @@ def remove(self): ch = schema.CALENDAR_HOME cb = schema.CALENDAR_BIND - chm = schema.CALENDAR_HOME_METADATA cor = schema.CALENDAR_OBJECT_REVISIONS + at = schema.ATTACHMENT + # delete attachments corresponding to this home, also removing from disk + rows = (yield Select( + [at.DROPBOX_ID, at.PATH, ], + From=at, + Where=( + at.CALENDAR_HOME_RESOURCE_ID == self._resourceID + ), + ).on(self._txn)) + for dropboxID, path in rows: + attachment = Attachment._attachmentPathRoot(self._txn, dropboxID).child(path) + if attachment.exists(): + self._txn.postCommit(attachment.remove) + yield Delete( - From=chm, - Where=chm.RESOURCE_ID == self._resourceID + From=at, + Where=( + at.CALENDAR_HOME_RESOURCE_ID == self._resourceID + ), ).on(self._txn) yield Delete( Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py =================================================================== --- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2012-05-18 17:44:13 UTC (rev 9256) +++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2012-05-18 22:39:24 UTC (rev 9257) @@ -106,15 +106,9 @@ def remove(self): ah = schema.ADDRESSBOOK_HOME ab = schema.ADDRESSBOOK_BIND - ahm = schema.ADDRESSBOOK_HOME_METADATA aor = schema.ADDRESSBOOK_OBJECT_REVISIONS yield Delete( - From=ahm, - Where=ahm.RESOURCE_ID == self._resourceID - ).on(self._txn) - - yield Delete( From=ab, Where=ab.ADDRESSBOOK_HOME_RESOURCE_ID == self._resourceID ).on(self._txn)