[CalendarServer-changes] [9257] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri May 18 15:39:24 PDT 2012
Revision: 9257
http://trac.macosforge.org/projects/calendarserver/changeset/9257
Author: sagen at apple.com
Date: 2012-05-18 15:39:24 -0700 (Fri, 18 May 2012)
Log Message:
-----------
Updates purge tool to handle remove attachments
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/purge.py
CalendarServer/trunk/calendarserver/tools/test/test_purge.py
CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/carddav/datastore/sql.py
Modified: CalendarServer/trunk/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/purge.py 2012-05-18 17:44:13 UTC (rev 9256)
+++ CalendarServer/trunk/calendarserver/tools/purge.py 2012-05-18 22:39:24 UTC (rev 9257)
@@ -198,7 +198,7 @@
def doWork(self):
rootResource = self.rootResource()
directory = rootResource.getDirectory()
- total = (yield purgeUIDs(directory, rootResource, self.uids,
+ total = (yield purgeUIDs(self._store, directory, rootResource, self.uids,
verbose=self.verbose, dryrun=self.dryrun,
completely=self.completely, doimplicit=self.doimplicit))
if self.verbose:
@@ -503,14 +503,14 @@
@inlineCallbacks
-def purgeUIDs(directory, root, uids, verbose=False, dryrun=False,
+def purgeUIDs(store, directory, root, uids, verbose=False, dryrun=False,
completely=False, doimplicit=True):
total = 0
allAssignments = { }
for uid in uids:
- count, allAssignments[uid] = (yield purgeUID(uid, directory, root,
+ count, allAssignments[uid] = (yield purgeUID(store, uid, directory, root,
verbose=verbose, dryrun=dryrun, completely=completely, doimplicit=doimplicit))
total += count
@@ -627,7 +627,7 @@
@inlineCallbacks
-def purgeUID(uid, directory, root, verbose=False, dryrun=False, proxies=True,
+def purgeUID(store, uid, directory, root, verbose=False, dryrun=False, proxies=True,
when=None, completely=False, doimplicit=True):
if when is None:
@@ -658,7 +658,10 @@
davxml.HRef.fromString("/principals/__uids__/%s/" % (uid,))
)
- calendarHome = yield principal.calendarHome(request)
+ # See if calendar home is provisioned
+ txn = store.newTransaction()
+ calHomeProvisioned = ((yield txn.calendarHomeWithUID(uid)) is not None)
+ (yield txn.commit())
# Anything in the past is left alone
whenString = when.getText()
@@ -679,102 +682,117 @@
perUserFilter = PerUserDataFilter(uid)
try:
- for collName in (yield calendarHome.listChildren()):
- collection = (yield calendarHome.getChild(collName))
- if collection.isCalendarCollection() or collName == "inbox":
+ if calHomeProvisioned:
+ calendarHome = yield principal.calendarHome(request)
+ for collName in (yield calendarHome.listChildren()):
+ collection = (yield calendarHome.getChild(collName))
+ if collection.isCalendarCollection() or collName == "inbox":
- childNames = []
+ childNames = []
- if completely:
- # all events
- for childName in (yield collection.listChildren()):
- childNames.append(childName)
- else:
- # events matching filter
- for childName, childUid, childType in (yield collection.index().indexedSearch(filter)):
- childNames.append(childName)
-
- for childName in childNames:
-
- childResource = (yield collection.getChild(childName))
if completely:
- action = CANCELEVENT_SHOULD_DELETE
+ # all events
+ for childName in (yield collection.listChildren()):
+ childNames.append(childName)
else:
- event = (yield childResource.iCalendar())
- event = perUserFilter.filter(event)
- action = cancelEvent(event, when, cua)
+ # events matching filter
+ for childName, childUid, childType in (yield collection.index().indexedSearch(filter)):
+ childNames.append(childName)
- uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName)
- request.path = uri
- if action == CANCELEVENT_MODIFIED:
- count += 1
- request._rememberResource(childResource, uri)
- storer = StoreCalendarObjectResource(
- request=request,
- destination=childResource,
- destination_uri=uri,
- destinationcal=True,
- destinationparent=collection,
- calendar=str(event),
- )
- if verbose:
- if dryrun:
- print "Would modify: %s" % (uri,)
- else:
- print "Modifying: %s" % (uri,)
- if not dryrun:
- result = (yield storer.run())
+ for childName in childNames:
- elif action == CANCELEVENT_SHOULD_DELETE:
- incrementCount = dryrun
- request._rememberResource(childResource, uri)
- if verbose:
- if dryrun:
- print "Would delete: %s" % (uri,)
- else:
- print "Deleting: %s" % (uri,)
- if not dryrun:
- retry = False
- try:
- result = (yield childResource.storeRemove(request, doimplicit, uri))
- if result != NO_CONTENT:
- print "Error deleting %s/%s/%s: %s" % (uid,
- collName, childName, result)
- retry = True
+ childResource = (yield collection.getChild(childName))
+ if completely:
+ action = CANCELEVENT_SHOULD_DELETE
+ else:
+ event = (yield childResource.iCalendar())
+ event = perUserFilter.filter(event)
+ action = cancelEvent(event, when, cua)
+
+ uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName)
+ request.path = uri
+ if action == CANCELEVENT_MODIFIED:
+ count += 1
+ request._rememberResource(childResource, uri)
+ storer = StoreCalendarObjectResource(
+ request=request,
+ destination=childResource,
+ destination_uri=uri,
+ destinationcal=True,
+ destinationparent=collection,
+ calendar=str(event),
+ )
+ if verbose:
+ if dryrun:
+ print "Would modify: %s" % (uri,)
else:
- incrementCount = True
+ print "Modifying: %s" % (uri,)
+ if not dryrun:
+ result = (yield storer.run())
- except Exception, e:
- print "Exception deleting %s/%s/%s: %s" % (uid,
- collName, childName, str(e))
- traceback.print_stack()
- retry = True
-
- if retry and doimplicit:
- # Try again with implicit scheduling off
- print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName)
+ elif action == CANCELEVENT_SHOULD_DELETE:
+ incrementCount = dryrun
+ request._rememberResource(childResource, uri)
+ if verbose:
+ if dryrun:
+ print "Would delete: %s" % (uri,)
+ else:
+ print "Deleting: %s" % (uri,)
+ if not dryrun:
+ retry = False
try:
- result = (yield childResource.storeRemove(request, False, uri))
+ result = (yield childResource.storeRemove(request, doimplicit, uri))
if result != NO_CONTENT:
print "Error deleting %s/%s/%s: %s" % (uid,
collName, childName, result)
+ retry = True
else:
incrementCount = True
+
except Exception, e:
- print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e))
+ print "Exception deleting %s/%s/%s: %s" % (uid,
+ collName, childName, str(e))
traceback.print_stack()
+ retry = True
- if incrementCount:
- count += 1
+ if retry and doimplicit:
+ # Try again with implicit scheduling off
+ print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName)
+ try:
+ result = (yield childResource.storeRemove(request, False, uri))
+ if result != NO_CONTENT:
+ print "Error deleting %s/%s/%s: %s" % (uid,
+ collName, childName, result)
+ else:
+ incrementCount = True
+ except Exception, e:
+ print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e))
+ traceback.print_stack()
+ if incrementCount:
+ count += 1
- txn = request._newStoreTransaction
+ txn = getattr(request, "_newStoreTransaction", None)
+ # Commit
+ if txn is not None:
+ (yield txn.commit())
+
+ except Exception, e:
+ # Abort
+ txn = getattr(request, "_newStoreTransaction", None)
+ if txn is not None:
+ (yield txn.abort())
+ raise e
+
+ try:
+ txn = store.newTransaction()
+
# Remove empty calendar collections (and calendar home if no more
# calendars)
- calHome = (yield txn.calendarHomeWithUID(uid))
- if calHome is not None:
- calendars = list((yield calHome.calendars()))
+ storeCalHome = (yield txn.calendarHomeWithUID(uid))
+ if storeCalHome is not None:
+ calendars = list((yield storeCalHome.calendars()))
remainingCalendars = len(calendars)
for calColl in calendars:
if len(list((yield calColl.calendarObjects()))) == 0:
@@ -786,7 +804,7 @@
else:
print "Deleting calendar: %s" % (calendarName,)
if not dryrun:
- (yield calHome.removeChildWithName(calendarName))
+ (yield storeCalHome.removeChildWithName(calendarName))
if not remainingCalendars:
if verbose:
@@ -795,13 +813,13 @@
else:
print "Deleting calendar home"
if not dryrun:
- (yield calHome.remove())
+ (yield storeCalHome.remove())
# Remove VCards
- abHome = (yield txn.addressbookHomeWithUID(uid))
- if abHome is not None:
- for abColl in list( (yield abHome.addressbooks()) ):
+ storeAbHome = (yield txn.addressbookHomeWithUID(uid))
+ if storeAbHome is not None:
+ for abColl in list( (yield storeAbHome.addressbooks()) ):
for card in list( (yield abColl.addressbookObjects()) ):
cardName = card.name()
if verbose:
@@ -821,7 +839,7 @@
print "Deleting addressbook: %s" % (abName,)
if not dryrun:
# Also remove the addressbook collection itself
- (yield abHome.removeChildWithName(abColl.name()))
+ (yield storeAbHome.removeChildWithName(abColl.name()))
if verbose:
if dryrun:
@@ -829,14 +847,13 @@
else:
print "Deleting addressbook home"
if not dryrun:
- (yield abHome.remove())
+ (yield storeAbHome.remove())
# Commit
(yield txn.commit())
except Exception, e:
# Abort
- txn = request._newStoreTransaction
(yield txn.abort())
raise e
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge.py 2012-05-18 17:44:13 UTC (rev 9256)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge.py 2012-05-18 22:39:24 UTC (rev 9257)
@@ -15,18 +15,26 @@
##
-from calendarserver.tools.purge import cancelEvent
+from calendarserver.tap.util import getRootResource
+from calendarserver.tools.purge import cancelEvent, purgeUID
from calendarserver.tools.purge import CANCELEVENT_MODIFIED, CANCELEVENT_SHOULD_DELETE
+from twistedcaldav.config import config
from twistedcaldav.ical import Component
from twistedcaldav.test.util import TestCase
from pycalendar.datetime import PyCalendarDateTime
from pycalendar.timezone import PyCalendarTimezone
+from twisted.trial import unittest
+from twisted.internet.defer import inlineCallbacks
+from txdav.common.datastore.test.util import buildStore, populateCalendarsFrom, CommonCommonTests
+from twext.web2.http_headers import MimeType
+import os
+
future = PyCalendarDateTime.getNowUTC()
future.offsetDay(1)
future = future.getText()
@@ -727,3 +735,144 @@
END:VEVENT
END:VCALENDAR
""".replace("\n", "\r\n")
+
+
+
+
+
+ATTACHMENT_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T195159Z
+UID:F2F14D94-B944-43D9-8F6F-97F95B2764CA
+DTEND;TZID=US/Pacific:20100304T141500
+TRANSP:OPAQUE
+SUMMARY:Attachment
+DTSTART;TZID=US/Pacific:20100304T120000
+DTSTAMP:20100303T195203Z
+SEQUENCE:2
+X-APPLE-DROPBOX:/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/dropbox/F2F14D94-B944-43D9-8F6F-97F95B2764CA.dropbox
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+
+
+
+class PurgePrincipalTests(CommonCommonTests, unittest.TestCase):
+ """
+ Tests for purging the data belonging to a given principal
+ """
+ uid = "6423F94A-6B76-4A3A-815B-D52CFD77935D"
+
+ metadata = {
+ "accessMode": "PUBLIC",
+ "isScheduleObject": True,
+ "scheduleTag": "abc",
+ "scheduleEtags": (),
+ "hasPrivateComment": False,
+ }
+
+ requirements = {
+ uid : {
+ "calendar1" : {
+ "attachment.ics" : (ATTACHMENT_ICS, metadata,),
+ }
+ },
+ }
+
+ @inlineCallbacks
+ def setUp(self):
+ yield super(PurgePrincipalTests, self).setUp()
+ self._sqlCalendarStore = yield buildStore(self, self.notifierFactory)
+ yield self.populate()
+
+ self.patch(config.DirectoryService.params, "xmlFile",
+ os.path.join(
+ os.path.dirname(__file__), "purge", "accounts.xml"
+ )
+ )
+ self.patch(config.ResourceService.params, "xmlFile",
+ os.path.join(
+ os.path.dirname(__file__), "purge", "resources.xml"
+ )
+ )
+ self.rootResource = getRootResource(config, self._sqlCalendarStore)
+ self.directory = self.rootResource.getDirectory()
+
+ # Add attachment to attachment.ics
+ txn = self._sqlCalendarStore.newTransaction()
+ home = (yield txn.calendarHomeWithUID(self.uid))
+ calendar = (yield home.calendarWithName("calendar1"))
+ event = (yield calendar.calendarObjectWithName("attachment.ics"))
+ attachment = (yield event.createAttachmentWithName("attachment.txt"))
+ t = attachment.store(MimeType("text", "x-fixture"))
+ t.write("attachment")
+ t.write(" text")
+ (yield t.loseConnection())
+ (yield txn.commit())
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateCalendarsFrom(self.requirements, self.storeUnderTest())
+ self.notifierFactory.reset()
+
+
+ def storeUnderTest(self):
+ """
+ Create and return a L{CalendarStore} for testing.
+ """
+ return self._sqlCalendarStore
+
+
+ @inlineCallbacks
+ def test_purgeUID(self):
+ """
+ Verify purgeUID removes homes, and doesn't provision homes that don't exist
+ """
+
+ # Now you see it
+ txn = self._sqlCalendarStore.newTransaction()
+ home = (yield txn.calendarHomeWithUID(self.uid))
+ self.assertNotEquals(home, None)
+ (yield txn.commit())
+
+ count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory,
+ self.rootResource, verbose=False, proxies=False, completely=True))
+ self.assertEquals(count, 1) # 1 event
+
+ # Now you don't
+ txn = self._sqlCalendarStore.newTransaction()
+ home = (yield txn.calendarHomeWithUID(self.uid))
+ self.assertEquals(home, None)
+ (yield txn.commit())
+
+ count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory,
+ self.rootResource, verbose=False, proxies=False, completely=True))
+ self.assertEquals(count, 0)
+
+ # And you still don't (making sure it's not provisioned)
+ txn = self._sqlCalendarStore.newTransaction()
+ home = (yield txn.calendarHomeWithUID(self.uid))
+ self.assertEquals(home, None)
+ (yield txn.commit())
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2012-05-18 17:44:13 UTC (rev 9256)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2012-05-18 22:39:24 UTC (rev 9257)
@@ -531,7 +531,7 @@
(yield txn.commit())
# Purge home1
- total, ignored = (yield purgeUID("home1", self.directory,
+ total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory,
self.rootResource, verbose=False, proxies=False,
when=PyCalendarDateTime(2010, 4, 1, 12, 0, 0, 0, PyCalendarTimezone(utc=True))))
@@ -568,7 +568,7 @@
(yield txn.commit())
# Purge home1 completely
- total, ignored = (yield purgeUID("home1", self.directory,
+ total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory,
self.rootResource, verbose=False, proxies=False, completely=True))
# 4 items deleted: 3 events and 1 vcard
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-05-18 17:44:13 UTC (rev 9256)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-05-18 22:39:24 UTC (rev 9257)
@@ -129,12 +129,27 @@
def remove(self):
ch = schema.CALENDAR_HOME
cb = schema.CALENDAR_BIND
- chm = schema.CALENDAR_HOME_METADATA
cor = schema.CALENDAR_OBJECT_REVISIONS
+ at = schema.ATTACHMENT
+ # delete attachments corresponding to this home, also removing from disk
+ rows = (yield Select(
+ [at.DROPBOX_ID, at.PATH, ],
+ From=at,
+ Where=(
+ at.CALENDAR_HOME_RESOURCE_ID == self._resourceID
+ ),
+ ).on(self._txn))
+ for dropboxID, path in rows:
+ attachment = Attachment._attachmentPathRoot(self._txn, dropboxID).child(path)
+ if attachment.exists():
+ self._txn.postCommit(attachment.remove)
+
yield Delete(
- From=chm,
- Where=chm.RESOURCE_ID == self._resourceID
+ From=at,
+ Where=(
+ at.CALENDAR_HOME_RESOURCE_ID == self._resourceID
+ ),
).on(self._txn)
yield Delete(
Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2012-05-18 17:44:13 UTC (rev 9256)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2012-05-18 22:39:24 UTC (rev 9257)
@@ -106,15 +106,9 @@
def remove(self):
ah = schema.ADDRESSBOOK_HOME
ab = schema.ADDRESSBOOK_BIND
- ahm = schema.ADDRESSBOOK_HOME_METADATA
aor = schema.ADDRESSBOOK_OBJECT_REVISIONS
yield Delete(
- From=ahm,
- Where=ahm.RESOURCE_ID == self._resourceID
- ).on(self._txn)
-
- yield Delete(
From=ab,
Where=ab.ADDRESSBOOK_HOME_RESOURCE_ID == self._resourceID
).on(self._txn)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120518/672da1cb/attachment-0001.html>
More information about the calendarserver-changes
mailing list