[CalendarServer-changes] [6743] CalendarServer/branches/users/sagen/purge_old_events
source_changes at macosforge.org
source_changes at macosforge.org
Mon Jan 17 11:51:52 PST 2011
Revision: 6743
http://trac.macosforge.org/projects/calendarserver/changeset/6743
Author: sagen at apple.com
Date: 2011-01-17 11:51:36 -0800 (Mon, 17 Jan 2011)
Log Message:
-----------
purge_old_events updated to new store
Modified Paths:
--------------
CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/purge.py
CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/icalendarstore.py
CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/file.py
CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/sql.py
Modified: CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/purge.py 2011-01-17 19:32:38 UTC (rev 6742)
+++ CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/purge.py 2011-01-17 19:51:36 UTC (rev 6743)
@@ -86,6 +86,10 @@
class PurgeOldEventsService(Service):
+ cutoff = None
+ dryrun = False
+ verbose = False
+
def __init__(self, store):
self._store = store
@@ -93,6 +97,18 @@
try:
rootResource = getRootResource(config, self._store)
directory = rootResource.getDirectory()
+ total = (yield purgeOldEvents(directory, rootResource, self.cutoff,
+ verbose=self.verbose, dryrun=self.dryrun))
+ if self.verbose:
+ amount = "%d event%s" % (total, "s" if total > 1 else "")
+ if self.dryrun:
+ print "Would have deleted %s" % (amount,)
+ else:
+ print "Deleted %s" % (amount,)
+ except Exception, e:
+ print "Error:", e
+ raise
+
finally:
reactor.stop()
@@ -197,13 +213,13 @@
usage_purge_events("Too many arguments: %s" % (args,))
cutoff = (date.today()-timedelta(days=days)).strftime("%Y%m%dT000000Z")
+ PurgeOldEventsService.cutoff = cutoff
+ PurgeOldEventsService.dryrun = dryrun
+ PurgeOldEventsService.verbose = verbose
shared_main(
configFileName,
PurgeOldEventsService,
- cutoff,
- verbose=verbose,
- dryrun=dryrun,
)
@@ -273,111 +289,57 @@
@inlineCallbacks
def purgeOldEvents(directory, root, date, verbose=False, dryrun=False):
- calendars = root.getChild("calendars")
- uidsFPath = calendars.fp.child("__uids__")
-
if dryrun:
print "Dry run"
- if verbose:
- print "Scanning calendar homes ...",
-
records = []
calendars = root.getChild("calendars")
- uidsFPath = calendars.fp.child("__uids__")
+ uids = calendars.getChild("__uids__")
- if uidsFPath.exists():
- for firstFPath in uidsFPath.children():
- if len(firstFPath.basename()) == 2:
- for secondFPath in firstFPath.children():
- if len(secondFPath.basename()) == 2:
- for homeFPath in secondFPath.children():
- uid = homeFPath.basename()
- record = directory.recordWithUID(uid)
- if record is not None:
- records.append(record)
-
if verbose:
- print "%d calendar homes found" % (len(records),)
+ print "Querying database for old events...",
- log.info("Purging events from %d calendar homes" % (len(records),))
+ oldEvents = (yield self._store.eventsOlderThan(date))
- filter = caldavxml.Filter(
- caldavxml.ComponentFilter(
- caldavxml.ComponentFilter(
- TimeRange(start=date,),
- name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
- ),
- name="VCALENDAR",
- )
- )
- filter = calendarqueryfilter.Filter(filter)
-
+ request = FakeRequest(root, None, None)
+ request.checkedSACL = True
eventCount = 0
- for record in records:
- # Get the calendar home
- principalCollection = directory.principalCollection
- principal = principalCollection.principalForRecord(record)
- calendarHome = yield principal.calendarHome()
-
+ for homeName, calendarName, eventName, maxDate in oldEvents:
+ eventCount += 1
if verbose:
- print "%s %-15s :" % (record.uid, record.shortNames[0]),
+ print "%s/%s/%s %s" % (homeName, calendarName, eventName, maxDate)
+ if not dryrun:
+ calendarHome = uids.getChild(homeName)
+ calendar = calendarHome.getChild(calendarName)
+ event = calendar.getChild(eventName)
+ uri = "/calendars/__uids__/%s/%s/%s" % (
+ homeName,
+ calendarName,
+ eventName
+ )
+ request.authnUser = request.authzUser = davxml.Principal(
+ davxml.HRef.fromString("/principals/__uids__/%s/" % (homeName,))
+ )
+ request.path = uri
+ request._rememberResource(event, uri)
+ if verbose:
+ print "Removing %s/%s/%s" % (homeName, calendarName, eventName)
+ result = (yield event.storeRemove(request, True, uri))
- homeEventCount = 0
- # For each collection in calendar home...
- for collName in calendarHome.listChildren():
- collection = calendarHome.getChild(collName)
- if collection.isCalendarCollection():
- # ...use their indexes to figure out which events to purge.
+ if verbose:
+ print "%d old events found" % (eventCount,)
- # First, get the list of all child resources...
- resources = set(collection.listChildren())
-
- # ...and ignore those that appear *after* the given cutoff
- for name, uid, type in collection.index().indexedSearch(filter):
- if isinstance(name, unicode):
- name = name.encode("utf-8")
- if name in resources:
- resources.remove(name)
-
- for name in resources:
- resource = collection.getChild(name)
- uri = "/calendars/__uids__/%s/%s/%s" % (
- record.uid,
- collName,
- name
- )
- try:
- if not dryrun:
- (yield deleteResource(root, collection, resource,
- uri, record.guid))
- eventCount += 1
- homeEventCount += 1
- except Exception, e:
- log.error("Failed to purge old event: %s (%s)" %
- (uri, e))
-
+ if not dryrun:
if verbose:
- print "%d events" % (homeEventCount,)
+ print "Committing changes"
+ txn = request._newStoreTransaction
+ (yield txn.commit())
returnValue(eventCount)
-def deleteResource(root, collection, resource, uri, guid, implicit=False):
- request = FakeRequest(root, "DELETE", uri)
- request.authnUser = request.authzUser = davxml.Principal(
- davxml.HRef.fromString("/principals/__uids__/%s/" % (guid,))
- )
- # TODO: this seems hacky, even for a stub request:
- request._rememberResource(resource, uri)
- # Cyrus says to use resource.storeRemove( ) -- see twistedcaldav/method/delete_common.py
- deleter = DeleteResource(request, resource, uri,
- collection, "infinity", allowImplicitSchedule=implicit)
- return deleter.run()
-
-
@inlineCallbacks
def purgeGUIDs(directory, root, guids, verbose=False, dryrun=False):
total = 0
Modified: CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/test/test_purge_old_events.py 2011-01-17 19:32:38 UTC (rev 6742)
+++ CalendarServer/branches/users/sagen/purge_old_events/calendarserver/tools/test/test_purge_old_events.py 2011-01-17 19:51:36 UTC (rev 6743)
@@ -33,8 +33,10 @@
from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
from txdav.base.propertystore.base import PropertyName
+import datetime
+
OLD_ICS = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 4.0.1//EN
@@ -83,17 +85,195 @@
END:VCALENDAR
""".replace("\n", "\r\n")
+ENDLESS_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;UNTIL=20061029T090000Z;BYMONTH=10;BYDAY=-1SU
+DTSTART:19621028T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;UNTIL=20060402T100000Z;BYMONTH=4;BYDAY=1SU
+DTSTART:19870405T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T194654Z
+UID:9FDE0E4C-1495-4CAF-863B-F7F0FB15FE8C
+DTEND;TZID=US/Pacific:20000308T151500
+RRULE:FREQ=YEARLY;INTERVAL=1
+TRANSP:OPAQUE
+SUMMARY:Ancient Repeating Endless
+DTSTART;TZID=US/Pacific:20000308T111500
+DTSTAMP:20100303T194710Z
+SEQUENCE:4
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+REPEATING_AWHILE_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;UNTIL=20061029T090000Z;BYMONTH=10;BYDAY=-1SU
+DTSTART:19621028T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;UNTIL=20060402T100000Z;BYMONTH=4;BYDAY=1SU
+DTSTART:19870405T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T194716Z
+UID:76236B32-2BC4-4D78-956B-8D42D4086200
+DTEND;TZID=US/Pacific:20000309T151500
+RRULE:FREQ=YEARLY;INTERVAL=1;COUNT=3
+TRANSP:OPAQUE
+SUMMARY:Ancient Repeat Awhile
+DTSTART;TZID=US/Pacific:20000309T111500
+DTSTAMP:20100303T194747Z
+SEQUENCE:6
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+STRADDLING_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T213643Z
+UID:1C219DAD-D374-4822-8C98-ADBA85E253AB
+DTEND;TZID=US/Pacific:20090508T121500
+RRULE:FREQ=MONTHLY;INTERVAL=1;UNTIL=20100509T065959Z
+TRANSP:OPAQUE
+SUMMARY:Straddling cut-off
+DTSTART;TZID=US/Pacific:20090508T111500
+DTSTAMP:20100303T213704Z
+SEQUENCE:5
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+RECENT_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T195159Z
+UID:F2F14D94-B944-43D9-8F6F-97F95B2764CA
+DTEND;TZID=US/Pacific:20100304T141500
+TRANSP:OPAQUE
+SUMMARY:Recent
+DTSTART;TZID=US/Pacific:20100304T120000
+DTSTAMP:20100303T195203Z
+SEQUENCE:2
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+
+
class PurgeOldEventsTests(CommonCommonTests, unittest.TestCase):
"""
- Calendar SQL storage tests.
+ Tests for deleting events older than a given date
"""
requirements = {
"home1" : {
"calendar1" : {
- "1.ics" : OLD_ICS,
+ "old.ics" : OLD_ICS,
+ "endless.ics" : ENDLESS_ICS,
}
+ },
+ "home2" : {
+ "calendar2" : {
+ "straddling.ics" : STRADDLING_ICS,
+ "recent.ics" : RECENT_ICS,
+ },
+ "calendar3" : {
+ "repeating_awhile.ics" : REPEATING_AWHILE_ICS,
+ }
}
}
@@ -117,6 +297,20 @@
return self._sqlCalendarStore
- def test_store(self):
- # Just a test trigger the populate( )
- print self._sqlCalendarStore
+ @inlineCallbacks
+ def test_eventsOlderThan(self):
+ cutoff = datetime.datetime(2010, 4, 1)
+ results = (yield self._sqlCalendarStore.eventsOlderThan(cutoff))
+ import types
+ self.assertEquals(types.GeneratorType, type(results))
+ results = list(results)
+ for result in results:
+ print result
+ self.assertEquals(set(results),
+ set([
+ ('home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'),
+ ('home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'),
+ ('home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'),
+ ])
+ )
+
Modified: CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/icalendarstore.py
===================================================================
--- CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/icalendarstore.py 2011-01-17 19:32:38 UTC (rev 6742)
+++ CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/icalendarstore.py 2011-01-17 19:51:36 UTC (rev 6743)
@@ -78,7 +78,19 @@
home} is an L{ICalendarHome} provider.
"""
+ def eventsOlderThan(self, cutoff):
+ """
+ Query the data store for all events which exist *completely* earlier
+ then the cutoff datetime
+ @param cutoff: Any events which have any instances more recent than
+ this are not returned. All others are returned.
+ @type cutoff: C{datetime.datetime}
+
+ @return: a deferred returning a generator of tuples of the form:
+ (calendar_home_name, calendar_name, event_name, latest_date_string).
+ """
+
#
# Interfaces
#
Modified: CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/file.py 2011-01-17 19:32:38 UTC (rev 6742)
+++ CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/file.py 2011-01-17 19:51:36 UTC (rev 6743)
@@ -145,6 +145,9 @@
def eachAddressbookHome(self):
return self._homesOfType(EADDRESSBOOKTYPE)
+ def eventsOlderThan(self, cutoff):
+ """ Not implemented for file """
+ return succeed([])
class CommonStoreTransaction(DataStoreTransaction):
Modified: CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/sql.py 2011-01-17 19:32:38 UTC (rev 6742)
+++ CalendarServer/branches/users/sagen/purge_old_events/txdav/common/datastore/sql.py 2011-01-17 19:51:36 UTC (rev 6743)
@@ -109,6 +109,53 @@
return []
+ def eventsOlderThan(self, cutoff):
+ """
+ All events which exist *completely* earlier than cutoff datetime
+
+ @param cutoff: Any events which have any instances more recent than
+ this are not returned. All others are returned.
+ @type cutoff: C{datetime.datetime}
+
+ @return: a deferred returning a generator of tuples of the form:
+ (calendar_home_name, calendar_name, event_name, latest_date_string).
+ """
+ txn = self.newTransaction(label="Finding old events")
+ d = txn.execSQL(
+
+ """
+ select
+ ch.OWNER_UID,
+ cb.CALENDAR_RESOURCE_NAME,
+ co.RESOURCE_NAME,
+ max(tr.END_DATE)
+ from
+ TIME_RANGE tr,
+ CALENDAR_BIND cb,
+ CALENDAR_OBJECT co,
+ CALENDAR_HOME ch
+ where
+ cb.BIND_MODE=%s AND
+ cb.CALENDAR_RESOURCE_ID=tr.CALENDAR_RESOURCE_ID AND
+ tr.CALENDAR_OBJECT_RESOURCE_ID=co.RESOURCE_ID AND
+ ch.RESOURCE_ID=cb.CALENDAR_HOME_RESOURCE_ID
+ group by
+ ch.OWNER_UID,
+ cb.CALENDAR_RESOURCE_NAME,
+ co.RESOURCE_NAME
+ having
+ max(tr.END_DATE) < %s
+ """, (_BIND_MODE_OWN, cutoff,)
+ )
+
+ def yieldResults(results):
+ for result in results:
+ yield tuple(result)
+
+ d.addCallback(yieldResults)
+ return d
+
+
def newTransaction(self, label="unlabeled", migrating=False):
"""
@see L{IDataStore.newTransaction}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110117/27a6115c/attachment-0001.html>
More information about the calendarserver-changes
mailing list