[CalendarServer-changes] [6796] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jan 20 13:29:04 PST 2011
Revision: 6796
http://trac.macosforge.org/projects/calendarserver/changeset/6796
Author: sagen at apple.com
Date: 2011-01-20 13:29:03 -0800 (Thu, 20 Jan 2011)
Log Message:
-----------
Purge-old-events utility now goes directly through data store API (no longer going through DAV resource APIs)
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/purge.py
CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/trunk/txdav/caldav/icalendarstore.py
CalendarServer/trunk/txdav/common/datastore/file.py
CalendarServer/trunk/txdav/common/datastore/sql.py
Modified: CalendarServer/trunk/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/purge.py 2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/calendarserver/tools/purge.py 2011-01-20 21:29:03 UTC (rev 6796)
@@ -51,6 +51,7 @@
print ""
print "options:"
print " -d --days <number>: specify how many days in the past to retain (default=365)"
+ print " -b --batch <number>: number of events to remove in each transaction (default=100)"
print " -f --config <path>: Specify caldavd.plist configuration path"
print " -h --help: print this help and exit"
print " -n --dry-run: only calculate how many events to purge"
@@ -87,6 +88,7 @@
class PurgeOldEventsService(Service):
cutoff = None
+ batchSize = None
dryrun = False
verbose = False
@@ -99,16 +101,8 @@
rootResource = getRootResource(config, self._store)
directory = rootResource.getDirectory()
total = (yield purgeOldEvents(self._store, directory, rootResource,
- self.cutoff, verbose=self.verbose, dryrun=self.dryrun))
- if self.verbose:
- if total:
- amount = "%d event%s" % (total, "s" if total > 1 else "")
- if self.dryrun:
- print "Would have deleted %s" % (amount,)
- else:
- print "Deleted %s" % (amount,)
- else:
- print "No old events found"
+ self.cutoff, self.batchSize, verbose=self.verbose,
+ dryrun=self.dryrun))
except Exception, e:
print "Error:", e
raise
@@ -171,8 +165,9 @@
try:
(optargs, args) = getopt(
- sys.argv[1:], "d:f:hnv", [
+ sys.argv[1:], "d:b:f:hnv", [
"days=",
+ "batch=",
"dry-run",
"config=",
"help",
@@ -187,6 +182,7 @@
#
configFileName = None
days = 365
+ batchSize = 100
dryrun = False
verbose = False
@@ -201,6 +197,13 @@
print "Invalid value for --days: %s" % (arg,)
usage_purge_events(e)
+ elif opt in ("-b", "--batch"):
+ try:
+ batchSize = int(arg)
+ except ValueError, e:
+ print "Invalid value for --batch: %s" % (arg,)
+ usage_purge_events(e)
+
elif opt in ("-v", "--verbose"):
verbose = True
@@ -216,8 +219,12 @@
if args:
usage_purge_events("Too many arguments: %s" % (args,))
+ if dryrun:
+ verbose = True
+
cutoff = (date.today()-timedelta(days=days)).strftime("%Y%m%dT000000Z")
PurgeOldEventsService.cutoff = cutoff
+ PurgeOldEventsService.batchSize = batchSize
PurgeOldEventsService.dryrun = dryrun
PurgeOldEventsService.verbose = verbose
@@ -291,14 +298,52 @@
@inlineCallbacks
-def purgeOldEvents(store, directory, root, date, verbose=False, dryrun=False):
+def purgeOldEvents(store, directory, root, date, batchSize, verbose=False,
+ dryrun=False):
if dryrun:
- print "Dry run"
+ if verbose:
+ print "(Dry run) Searching for old events..."
+ txn = store.newTransaction(label="Find old events")
+ oldEvents = (yield txn.eventsOlderThan(date))
+ eventCount = len(oldEvents)
+ if verbose:
+ if eventCount == 0:
+ print "No events are older than %s" % (date,)
+ elif eventCount == 1:
+ print "1 event is older than %s" % (date,)
+ else:
+ print "%d events are older than %s" % (eventCount, date)
+ returnValue(eventCount)
if verbose:
- print "Querying database for old events..."
+ print "Removing events older than %s..." % (date,)
+ numEventsRemoved = -1
+ totalRemoved = 0
+ while numEventsRemoved:
+ txn = store.newTransaction(label="Remove old events")
+ numEventsRemoved = (yield txn.removeOldEvents(date, batchSize=batchSize))
+ (yield txn.commit())
+ if numEventsRemoved:
+ totalRemoved += numEventsRemoved
+ if verbose:
+ print "%d," % (totalRemoved,),
+
+ if verbose:
+ print
+ if totalRemoved == 0:
+ print "No events were removed"
+ elif totalRemoved == 1:
+ print "1 event was removed in total"
+ else:
+ print "%d events were removed in total" % (totalRemoved,)
+
+ returnValue(totalRemoved)
+
+
+ """
+
oldEvents = (yield store.eventsOlderThan(date))
request = FakeRequest(root, None, None)
@@ -360,6 +405,7 @@
returnValue(eventCount)
+ """
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2011-01-20 21:29:03 UTC (rev 6796)
@@ -296,20 +296,53 @@
@inlineCallbacks
def test_eventsOlderThan(self):
cutoff = datetime.datetime(2010, 4, 1)
- results = (yield self._sqlCalendarStore.eventsOlderThan(cutoff))
- import types
- self.assertEquals(types.GeneratorType, type(results))
- results = list(results)
- self.assertEquals(set(results),
- set([
- ('home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'),
- ('home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'),
- ('home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'),
- ])
+ txn = self._sqlCalendarStore.newTransaction()
+
+ # Query for all old events
+ results = (yield txn.eventsOlderThan(cutoff))
+ self.assertEquals(results,
+ [
+ ['home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'],
+ ['home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'],
+ ['home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'],
+ ]
)
+ # Query for oldest event
+ results = (yield txn.eventsOlderThan(cutoff, batchSize=1))
+ self.assertEquals(results,
+ [
+ ['home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'],
+ ]
+ )
@inlineCallbacks
+ def test_removeOldEvents(self):
+ cutoff = datetime.datetime(2010, 4, 1)
+ txn = self._sqlCalendarStore.newTransaction()
+
+ # Remove oldest event
+ count = (yield txn.removeOldEvents(cutoff, batchSize=1))
+ self.assertEquals(count, 1)
+ results = (yield txn.eventsOlderThan(cutoff))
+ self.assertEquals(results,
+ [
+ ['home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'],
+ ['home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'],
+ ]
+ )
+
+ # Remove remaining oldest events
+ count = (yield txn.removeOldEvents(cutoff))
+ self.assertEquals(count, 2)
+ results = (yield txn.eventsOlderThan(cutoff))
+ self.assertEquals(results, [ ])
+
+ # Remove oldest events (none left)
+ count = (yield txn.removeOldEvents(cutoff))
+ self.assertEquals(count, 0)
+
+ @inlineCallbacks
def test_purgeOldEvents(self):
self.patch(config.DirectoryService.params, "xmlFile",
os.path.join(
@@ -324,10 +357,23 @@
self.patch(config.Memcached.Pools.Default, "ClientEnabled", False)
rootResource = getRootResource(config, self._sqlCalendarStore)
directory = rootResource.getDirectory()
+
+ # Dry run
total = (yield purgeOldEvents(self._sqlCalendarStore, directory,
- rootResource, datetime.datetime(2010, 4, 1)))
+ rootResource, datetime.datetime(2010, 4, 1), 2, dryrun=True,
+ verbose=False))
self.assertEquals(total, 3)
+ # Actually remove
+ total = (yield purgeOldEvents(self._sqlCalendarStore, directory,
+ rootResource, datetime.datetime(2010, 4, 1), 2, verbose=False))
+ self.assertEquals(total, 3)
+
+ # There should be no more left
+ total = (yield purgeOldEvents(self._sqlCalendarStore, directory,
+ rootResource, datetime.datetime(2010, 4, 1), 2, verbose=False))
+ self.assertEquals(total, 0)
+
@inlineCallbacks
def test_purgeGUID(self):
self.patch(config.DirectoryService.params, "xmlFile",
Modified: CalendarServer/trunk/txdav/caldav/icalendarstore.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/icalendarstore.py 2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/txdav/caldav/icalendarstore.py 2011-01-20 21:29:03 UTC (rev 6796)
@@ -78,19 +78,7 @@
home} is an L{ICalendarHome} provider.
"""
- def eventsOlderThan(self, cutoff):
- """
- Query the data store for all events which exist *completely* earlier
- then the cutoff datetime
- @param cutoff: Any events which have any instances more recent than
- this are not returned. All others are returned.
- @type cutoff: C{datetime.datetime}
-
- @return: a deferred returning a generator of tuples of the form:
- (calendar_home_name, calendar_name, event_name, latest_date_string).
- """
-
#
# Interfaces
#
Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py 2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/txdav/common/datastore/file.py 2011-01-20 21:29:03 UTC (rev 6796)
@@ -145,11 +145,7 @@
def eachAddressbookHome(self):
return self._homesOfType(EADDRESSBOOKTYPE)
- def eventsOlderThan(self, cutoff):
- """ Not implemented for file """
- return succeed([])
-
class CommonStoreTransaction(DataStoreTransaction):
"""
In-memory implementation of
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2011-01-20 21:29:03 UTC (rev 6796)
@@ -113,53 +113,7 @@
return []
- def eventsOlderThan(self, cutoff):
- """
- All events which exist *completely* earlier than cutoff datetime
- @param cutoff: Any events which have any instances more recent than
- this are not returned. All others are returned.
- @type cutoff: C{datetime.datetime}
-
- @return: a deferred returning a generator of tuples of the form:
- (calendar_home_name, calendar_name, event_name, latest_date_string).
- """
- txn = self.newTransaction(label="Finding old events")
- d = txn.execSQL(
-
- """
- select
- ch.OWNER_UID,
- cb.CALENDAR_RESOURCE_NAME,
- co.RESOURCE_NAME,
- max(tr.END_DATE)
- from
- TIME_RANGE tr,
- CALENDAR_BIND cb,
- CALENDAR_OBJECT co,
- CALENDAR_HOME ch
- where
- cb.BIND_MODE=%s AND
- cb.CALENDAR_RESOURCE_ID=tr.CALENDAR_RESOURCE_ID AND
- tr.CALENDAR_OBJECT_RESOURCE_ID=co.RESOURCE_ID AND
- ch.RESOURCE_ID=cb.CALENDAR_HOME_RESOURCE_ID
- group by
- ch.OWNER_UID,
- cb.CALENDAR_RESOURCE_NAME,
- co.RESOURCE_NAME
- having
- max(tr.END_DATE) < %s
- """, (_BIND_MODE_OWN, cutoff,)
- )
-
- def yieldResults(results):
- for result in results:
- yield tuple(result)
-
- d.addCallback(yieldResults)
- return d
-
-
def newTransaction(self, label="unlabeled", migrating=False):
"""
@see L{IDataStore.newTransaction}
@@ -282,7 +236,62 @@
return self._sqlTxn.abort()
+ def eventsOlderThan(self, cutoff, batchSize=None):
+ """
+ Return up to the oldest batchSize events which exist completely earlier
+ than "cutoff" (datetime)
+ """
+ query = """
+ select
+ ch.OWNER_UID,
+ cb.CALENDAR_RESOURCE_NAME,
+ co.RESOURCE_NAME,
+ max(tr.END_DATE)
+ from
+ TIME_RANGE tr,
+ CALENDAR_BIND cb,
+ CALENDAR_OBJECT co,
+ CALENDAR_HOME ch
+ where
+ cb.BIND_MODE=%s AND
+ cb.CALENDAR_RESOURCE_ID=tr.CALENDAR_RESOURCE_ID AND
+ tr.CALENDAR_OBJECT_RESOURCE_ID=co.RESOURCE_ID AND
+ ch.RESOURCE_ID=cb.CALENDAR_HOME_RESOURCE_ID
+ group by
+ ch.OWNER_UID,
+ cb.CALENDAR_RESOURCE_NAME,
+ co.RESOURCE_NAME
+ having
+ max(tr.END_DATE) < %s
+ order by max(tr.END_DATE)
+ """
+ args = [_BIND_MODE_OWN, cutoff]
+ if batchSize is not None:
+ query += "limit %s"
+ args.append(batchSize)
+
+ return self.execSQL(query, args)
+
+
+ @inlineCallbacks
+ def removeOldEvents(self, cutoff, batchSize=None):
+ """
+ Remove up to batchSize events older than "cutoff" and return how
+ many were removed.
+ """
+
+ results = (yield self.eventsOlderThan(cutoff, batchSize=batchSize))
+ count = 0
+ for uid, calendarName, eventName, maxDate in results:
+ home = (yield self.calendarHomeWithUID(uid))
+ calendar = (yield home.childWithName(calendarName))
+ (yield calendar.removeObjectResourceWithName(eventName))
+ count += 1
+ returnValue(count)
+
+
+
class CommonHome(LoggingMixIn):
# All these need to be initialized by derived classes for each store type
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110120/33f20a09/attachment-0001.html>
More information about the calendarserver-changes
mailing list