[CalendarServer-changes] [6796] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Thu Jan 20 13:29:04 PST 2011


Revision: 6796
          http://trac.macosforge.org/projects/calendarserver/changeset/6796
Author:   sagen at apple.com
Date:     2011-01-20 13:29:03 -0800 (Thu, 20 Jan 2011)
Log Message:
-----------
Purge-old-events utility now goes directly through data store API (no longer going through DAV resource APIs)

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tools/purge.py
    CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
    CalendarServer/trunk/txdav/caldav/icalendarstore.py
    CalendarServer/trunk/txdav/common/datastore/file.py
    CalendarServer/trunk/txdav/common/datastore/sql.py

Modified: CalendarServer/trunk/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/purge.py	2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/calendarserver/tools/purge.py	2011-01-20 21:29:03 UTC (rev 6796)
@@ -51,6 +51,7 @@
     print ""
     print "options:"
     print "  -d --days <number>: specify how many days in the past to retain (default=365)"
+    print "  -b --batch <number>: number of events to remove in each transaction (default=100)"
     print "  -f --config <path>: Specify caldavd.plist configuration path"
     print "  -h --help: print this help and exit"
     print "  -n --dry-run: only calculate how many events to purge"
@@ -87,6 +88,7 @@
 class PurgeOldEventsService(Service):
 
     cutoff = None
+    batchSize = None
     dryrun = False
     verbose = False
 
@@ -99,16 +101,8 @@
             rootResource = getRootResource(config, self._store)
             directory = rootResource.getDirectory()
             total = (yield purgeOldEvents(self._store, directory, rootResource,
-                self.cutoff, verbose=self.verbose, dryrun=self.dryrun))
-            if self.verbose:
-                if total:
-                    amount = "%d event%s" % (total, "s" if total > 1 else "")
-                    if self.dryrun:
-                        print "Would have deleted %s" % (amount,)
-                    else:
-                        print "Deleted %s" % (amount,)
-                else:
-                    print "No old events found"
+                self.cutoff, self.batchSize, verbose=self.verbose,
+                dryrun=self.dryrun))
         except Exception, e:
             print "Error:", e
             raise
@@ -171,8 +165,9 @@
 
     try:
         (optargs, args) = getopt(
-            sys.argv[1:], "d:f:hnv", [
+            sys.argv[1:], "d:b:f:hnv", [
                 "days=",
+                "batch=",
                 "dry-run",
                 "config=",
                 "help",
@@ -187,6 +182,7 @@
     #
     configFileName = None
     days = 365
+    batchSize = 100
     dryrun = False
     verbose = False
 
@@ -201,6 +197,13 @@
                 print "Invalid value for --days: %s" % (arg,)
                 usage_purge_events(e)
 
+        elif opt in ("-b", "--batch"):
+            try:
+                batchSize = int(arg)
+            except ValueError, e:
+                print "Invalid value for --batch: %s" % (arg,)
+                usage_purge_events(e)
+
         elif opt in ("-v", "--verbose"):
             verbose = True
 
@@ -216,8 +219,12 @@
     if args:
         usage_purge_events("Too many arguments: %s" % (args,))
 
+    if dryrun:
+        verbose = True
+
     cutoff = (date.today()-timedelta(days=days)).strftime("%Y%m%dT000000Z")
     PurgeOldEventsService.cutoff = cutoff
+    PurgeOldEventsService.batchSize = batchSize
     PurgeOldEventsService.dryrun = dryrun
     PurgeOldEventsService.verbose = verbose
 
@@ -291,14 +298,52 @@
 
 
 @inlineCallbacks
-def purgeOldEvents(store, directory, root, date, verbose=False, dryrun=False):
+def purgeOldEvents(store, directory, root, date, batchSize, verbose=False,
+    dryrun=False):
 
     if dryrun:
-        print "Dry run"
+        if verbose:
+            print "(Dry run) Searching for old events..."
+        txn = store.newTransaction(label="Find old events")
+        oldEvents = (yield txn.eventsOlderThan(date))
+        eventCount = len(oldEvents)
+        if verbose:
+            if eventCount == 0:
+                print "No events are older than %s" % (date,)
+            elif eventCount == 1:
+                print "1 event is older than %s" % (date,)
+            else:
+                print "%d events are older than %s" % (eventCount, date)
+        returnValue(eventCount)
 
     if verbose:
-        print "Querying database for old events..."
+        print "Removing events older than %s..." % (date,)
 
+    numEventsRemoved = -1
+    totalRemoved = 0
+    while numEventsRemoved:
+        txn = store.newTransaction(label="Remove old events")
+        numEventsRemoved = (yield txn.removeOldEvents(date, batchSize=batchSize))
+        (yield txn.commit())
+        if numEventsRemoved:
+            totalRemoved += numEventsRemoved
+            if verbose:
+                print "%d," % (totalRemoved,),
+
+    if verbose:
+        print
+        if totalRemoved == 0:
+            print "No events were removed"
+        elif totalRemoved == 1:
+            print "1 event was removed in total"
+        else:
+            print "%d events were removed in total" % (totalRemoved,)
+
+    returnValue(totalRemoved)
+
+
+    """
+
     oldEvents = (yield store.eventsOlderThan(date))
 
     request = FakeRequest(root, None, None)
@@ -360,6 +405,7 @@
 
     returnValue(eventCount)
 
+    """
 
 
 

Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py	2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py	2011-01-20 21:29:03 UTC (rev 6796)
@@ -296,20 +296,53 @@
     @inlineCallbacks
     def test_eventsOlderThan(self):
         cutoff = datetime.datetime(2010, 4, 1)
-        results = (yield self._sqlCalendarStore.eventsOlderThan(cutoff))
-        import types
-        self.assertEquals(types.GeneratorType, type(results))
-        results = list(results)
-        self.assertEquals(set(results),
-            set([
-                ('home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'),
-                ('home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'),
-                ('home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'),
-            ])
+        txn = self._sqlCalendarStore.newTransaction()
+
+        # Query for all old events
+        results = (yield txn.eventsOlderThan(cutoff))
+        self.assertEquals(results,
+            [
+                ['home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'],
+                ['home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'],
+                ['home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'],
+            ]
         )
 
+        # Query for oldest event
+        results = (yield txn.eventsOlderThan(cutoff, batchSize=1))
+        self.assertEquals(results,
+            [
+                ['home1', 'calendar1', 'old.ics', '2000-03-07 23:15:00'],
+            ]
+        )
 
     @inlineCallbacks
+    def test_removeOldEvents(self):
+        cutoff = datetime.datetime(2010, 4, 1)
+        txn = self._sqlCalendarStore.newTransaction()
+
+        # Remove oldest event
+        count = (yield txn.removeOldEvents(cutoff, batchSize=1))
+        self.assertEquals(count, 1)
+        results = (yield txn.eventsOlderThan(cutoff))
+        self.assertEquals(results,
+            [
+                ['home2', 'calendar3', 'repeating_awhile.ics', '2002-03-09 23:15:00'],
+                ['home2', 'calendar2', 'recent.ics', '2010-03-04 22:15:00'],
+            ]
+        )
+
+        # Remove remaining oldest events
+        count = (yield txn.removeOldEvents(cutoff))
+        self.assertEquals(count, 2)
+        results = (yield txn.eventsOlderThan(cutoff))
+        self.assertEquals(results, [ ])
+
+        # Remove oldest events (none left)
+        count = (yield txn.removeOldEvents(cutoff))
+        self.assertEquals(count, 0)
+
+    @inlineCallbacks
     def test_purgeOldEvents(self):
         self.patch(config.DirectoryService.params, "xmlFile",
             os.path.join(
@@ -324,10 +357,23 @@
         self.patch(config.Memcached.Pools.Default, "ClientEnabled", False)
         rootResource = getRootResource(config, self._sqlCalendarStore)
         directory = rootResource.getDirectory()
+
+        # Dry run
         total = (yield purgeOldEvents(self._sqlCalendarStore, directory,
-            rootResource, datetime.datetime(2010, 4, 1)))
+            rootResource, datetime.datetime(2010, 4, 1), 2, dryrun=True,
+            verbose=False))
         self.assertEquals(total, 3)
 
+        # Actually remove
+        total = (yield purgeOldEvents(self._sqlCalendarStore, directory,
+            rootResource, datetime.datetime(2010, 4, 1), 2, verbose=False))
+        self.assertEquals(total, 3)
+
+        # There should be no more left
+        total = (yield purgeOldEvents(self._sqlCalendarStore, directory,
+            rootResource, datetime.datetime(2010, 4, 1), 2, verbose=False))
+        self.assertEquals(total, 0)
+
     @inlineCallbacks
     def test_purgeGUID(self):
         self.patch(config.DirectoryService.params, "xmlFile",

Modified: CalendarServer/trunk/txdav/caldav/icalendarstore.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/icalendarstore.py	2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/txdav/caldav/icalendarstore.py	2011-01-20 21:29:03 UTC (rev 6796)
@@ -78,19 +78,7 @@
             home} is an L{ICalendarHome} provider.
         """
 
-    def eventsOlderThan(self, cutoff):
-        """
-        Query the data store for all events which exist *completely* earlier
-        then the cutoff datetime
 
-        @param cutoff: Any events which have any instances more recent than
-        this are not returned.  All others are returned.
-        @type cutoff: C{datetime.datetime}
-
-        @return: a deferred returning a generator of tuples of the form:
-        (calendar_home_name, calendar_name, event_name, latest_date_string).
-        """
-
 #
 # Interfaces
 #

Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py	2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/txdav/common/datastore/file.py	2011-01-20 21:29:03 UTC (rev 6796)
@@ -145,11 +145,7 @@
     def eachAddressbookHome(self):
         return self._homesOfType(EADDRESSBOOKTYPE)
 
-    def eventsOlderThan(self, cutoff):
-        """ Not implemented for file """
-        return succeed([])
 
-
 class CommonStoreTransaction(DataStoreTransaction):
     """
     In-memory implementation of

Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py	2011-01-20 17:55:07 UTC (rev 6795)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py	2011-01-20 21:29:03 UTC (rev 6796)
@@ -113,53 +113,7 @@
         return []
 
 
-    def eventsOlderThan(self, cutoff):
-        """
-        All events which exist *completely* earlier than cutoff datetime
 
-        @param cutoff: Any events which have any instances more recent than
-        this are not returned.  All others are returned.
-        @type cutoff: C{datetime.datetime}
-
-        @return: a deferred returning a generator of tuples of the form:
-        (calendar_home_name, calendar_name, event_name, latest_date_string).
-        """
-        txn = self.newTransaction(label="Finding old events")
-        d = txn.execSQL(
-
-            """
-            select
-                ch.OWNER_UID,
-                cb.CALENDAR_RESOURCE_NAME,
-                co.RESOURCE_NAME,
-                max(tr.END_DATE)
-            from
-                TIME_RANGE tr,
-                CALENDAR_BIND cb,
-                CALENDAR_OBJECT co,
-                CALENDAR_HOME ch
-            where
-                cb.BIND_MODE=%s AND
-                cb.CALENDAR_RESOURCE_ID=tr.CALENDAR_RESOURCE_ID AND
-                tr.CALENDAR_OBJECT_RESOURCE_ID=co.RESOURCE_ID AND
-                ch.RESOURCE_ID=cb.CALENDAR_HOME_RESOURCE_ID
-            group by
-                ch.OWNER_UID,
-                cb.CALENDAR_RESOURCE_NAME,
-                co.RESOURCE_NAME
-            having
-                max(tr.END_DATE) < %s
-            """, (_BIND_MODE_OWN, cutoff,)
-        )
-
-        def yieldResults(results):
-            for result in results:
-                yield tuple(result)
-
-        d.addCallback(yieldResults)
-        return d
-
-
     def newTransaction(self, label="unlabeled", migrating=False):
         """
         @see L{IDataStore.newTransaction}
@@ -282,7 +236,62 @@
         return self._sqlTxn.abort()
 
 
+    def eventsOlderThan(self, cutoff, batchSize=None):
+        """
+        Return up to the oldest batchSize events which exist completely earlier
+        than "cutoff" (datetime)
+        """
 
+        query = """
+            select
+                ch.OWNER_UID,
+                cb.CALENDAR_RESOURCE_NAME,
+                co.RESOURCE_NAME,
+                max(tr.END_DATE)
+            from
+                TIME_RANGE tr,
+                CALENDAR_BIND cb,
+                CALENDAR_OBJECT co,
+                CALENDAR_HOME ch
+            where
+                cb.BIND_MODE=%s AND
+                cb.CALENDAR_RESOURCE_ID=tr.CALENDAR_RESOURCE_ID AND
+                tr.CALENDAR_OBJECT_RESOURCE_ID=co.RESOURCE_ID AND
+                ch.RESOURCE_ID=cb.CALENDAR_HOME_RESOURCE_ID
+            group by
+                ch.OWNER_UID,
+                cb.CALENDAR_RESOURCE_NAME,
+                co.RESOURCE_NAME
+            having
+                max(tr.END_DATE) < %s
+            order by max(tr.END_DATE)
+            """
+        args = [_BIND_MODE_OWN, cutoff]
+        if batchSize is not None:
+            query += "limit %s"
+            args.append(batchSize)
+
+        return self.execSQL(query, args)
+
+
+    @inlineCallbacks
+    def removeOldEvents(self, cutoff, batchSize=None):
+        """
+        Remove up to batchSize events older than "cutoff" and return how
+        many were removed.
+        """
+
+        results = (yield self.eventsOlderThan(cutoff, batchSize=batchSize))
+        count = 0
+        for uid, calendarName, eventName, maxDate in results:
+            home = (yield self.calendarHomeWithUID(uid))
+            calendar = (yield home.childWithName(calendarName))
+            (yield calendar.removeObjectResourceWithName(eventName))
+            count += 1
+        returnValue(count)
+
+
+
 class CommonHome(LoggingMixIn):
 
     # All these need to be initialized by derived classes for each store type
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110120/33f20a09/attachment-0001.html>


More information about the calendarserver-changes mailing list