[CalendarServer-changes] [10192] CalendarServer/branches/release/CalendarServer-4.3-dev
source_changes at macosforge.org
source_changes at macosforge.org
Thu Dec 20 13:15:56 PST 2012
Revision: 10192
http://trac.calendarserver.org//changeset/10192
Author: cdaboo at apple.com
Date: 2012-12-20 13:15:56 -0800 (Thu, 20 Dec 2012)
Log Message:
-----------
Add option to purge "old" attachments, and purge by user GUID. Re-factored purgeXXX code. Fixed man page typos.
Modified Paths:
--------------
CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_attachments
CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_events
CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_principals
CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/purge.py
CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge.py
CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_principals.8
CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_push.8
CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_migrate_resources.8
CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_attachments.8
CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_events.8
CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_principals.8
CalendarServer/branches/release/CalendarServer-4.3-dev/twext/enterprise/dal/syntax.py
CalendarServer/branches/release/CalendarServer-4.3-dev/txdav/common/datastore/sql.py
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_attachments
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_attachments 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_attachments 2012-12-20 21:15:56 UTC (rev 10192)
@@ -29,5 +29,5 @@
except ImportError:
sys.exc_clear()
- from calendarserver.tools.purge import main_purge_orphaned_attachments
- main_purge_orphaned_attachments()
+ from calendarserver.tools.purge import PurgeAttachmentsService
+ PurgeAttachmentsService.main()
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_events
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_events 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_events 2012-12-20 21:15:56 UTC (rev 10192)
@@ -29,5 +29,5 @@
except ImportError:
sys.exc_clear()
- from calendarserver.tools.purge import main_purge_events
- main_purge_events()
+ from calendarserver.tools.purge import PurgeOldEventsService
+ PurgeOldEventsService.main()
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_principals
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_principals 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/bin/calendarserver_purge_principals 2012-12-20 21:15:56 UTC (rev 10192)
@@ -29,5 +29,5 @@
except ImportError:
sys.exc_clear()
- from calendarserver.tools.purge import main_purge_principals
- main_purge_principals()
+ from calendarserver.tools.purge import PurgePrincipalService
+ PurgePrincipalService.main()
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/purge.py 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/purge.py 2012-12-20 21:15:56 UTC (rev 10192)
@@ -16,22 +16,24 @@
# limitations under the License.
##
-import os
-import sys
-import traceback
+from calendarserver.tap.util import FakeRequest
+from calendarserver.tap.util import getRootResource
+from calendarserver.tools import tables
+from calendarserver.tools.cmdline import utilityMain
+from calendarserver.tools.principals import removeProxy
+
from errno import ENOENT, EACCES
from getopt import getopt, GetoptError
from pycalendar.datetime import PyCalendarDateTime
+from twext.python.log import Logger
+from twext.web2.responsecode import NO_CONTENT
+
from twisted.application.service import Service
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
-from twext.python.log import Logger
-from txdav.xml import element as davxml
-from twext.web2.responsecode import NO_CONTENT
-
from twistedcaldav import caldavxml
from twistedcaldav.caldavxml import TimeRange
from twistedcaldav.config import config, ConfigurationError
@@ -40,87 +42,24 @@
from twistedcaldav.method.put_common import StoreCalendarObjectResource
from twistedcaldav.query import calendarqueryfilter
-from calendarserver.tap.util import FakeRequest
-from calendarserver.tap.util import getRootResource
+from txdav.xml import element as davxml
-from calendarserver.tools.cmdline import utilityMain
-from calendarserver.tools.principals import removeProxy
+import collections
+import os
+import sys
+import traceback
log = Logger()
DEFAULT_BATCH_SIZE = 100
DEFAULT_RETAIN_DAYS = 365
-def usage_purge_events(e=None):
-
- name = os.path.basename(sys.argv[0])
- print "usage: %s [options]" % (name,)
- print ""
- print " Remove old events from the calendar server"
- print ""
- print "options:"
- print " -h --help: print this help and exit"
- print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -d --days <number>: specify how many days in the past to retain (default=%d)" % (DEFAULT_RETAIN_DAYS,)
- #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
- print " -n --dry-run: calculate how many events to purge, but do not purge data"
- print " -v --verbose: print progress information"
- print ""
-
- if e:
- sys.stderr.write("%s\n" % (e,))
- sys.exit(64)
- else:
- sys.exit(0)
-
-def usage_purge_orphaned_attachments(e=None):
-
- name = os.path.basename(sys.argv[0])
- print "usage: %s [options]" % (name,)
- print ""
- print " Remove orphaned attachments from the calendar server"
- print ""
- print "options:"
- print " -h --help: print this help and exit"
- print " -f --config <path>: Specify caldavd.plist configuration path"
- #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
- print " -n --dry-run: calculate how many attachments to purge, but do not purge data"
- print " -v --verbose: print progress information"
- print ""
-
- if e:
- sys.stderr.write("%s\n" % (e,))
- sys.exit(64)
- else:
- sys.exit(0)
-
-def usage_purge_principal(e=None):
-
- name = os.path.basename(sys.argv[0])
- print "usage: %s [options]" % (name,)
- print ""
- print " Remove a principal's events and contacts from the calendar server"
- print ""
- print "options:"
- print " -c --completely: By default, only future events are canceled; this option cancels all events"
- print " -h --help: print this help and exit"
- print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -n --dry-run: calculate how many events and contacts to purge, but do not purge data"
- print " -v --verbose: print progress information"
- print ""
-
- if e:
- sys.stderr.write("%s\n" % (e,))
- sys.exit(64)
- else:
- sys.exit(0)
-
-
class WorkerService(Service):
def __init__(self, store):
self._store = store
+
def rootResource(self):
try:
rootResource = getRootResource(config, self._store)
@@ -165,759 +104,990 @@
dryrun = False
verbose = False
- def doWork(self):
- rootResource = self.rootResource()
- directory = rootResource.getDirectory()
- return purgeOldEvents(self._store, directory, rootResource,
- self.cutoff, self.batchSize, verbose=self.verbose,
- dryrun=self.dryrun)
+ @classmethod
+ def usage(cls, e=None):
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print " Remove old events from the calendar server"
+ print ""
+ print "options:"
+ print " -h --help: print this help and exit"
+ print " -f --config <path>: Specify caldavd.plist configuration path"
+ print " -d --days <number>: specify how many days in the past to retain (default=%d)" % (DEFAULT_RETAIN_DAYS,)
+ #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ print " -n --dry-run: calculate how many events to purge, but do not purge data"
+ print " -v --verbose: print progress information"
+ print ""
+ if e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(64)
+ else:
+ sys.exit(0)
-class PurgeOrphanedAttachmentsService(WorkerService):
- batchSize = None
- dryrun = False
- verbose = False
+ @classmethod
+ def main(cls):
- def doWork(self):
- return purgeOrphanedAttachments(
- self._store, self.batchSize,
- verbose=self.verbose, dryrun=self.dryrun)
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "d:b:f:hnv", [
+ "days=",
+ "batch=",
+ "dry-run",
+ "config=",
+ "help",
+ "verbose",
+ ],
+ )
+ except GetoptError, e:
+ cls.usage(e)
+ #
+ # Get configuration
+ #
+ configFileName = None
+ days = DEFAULT_RETAIN_DAYS
+ batchSize = DEFAULT_BATCH_SIZE
+ dryrun = False
+ verbose = False
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ cls.usage()
-class PurgePrincipalService(WorkerService):
+ elif opt in ("-d", "--days"):
+ try:
+ days = int(arg)
+ except ValueError, e:
+ print "Invalid value for --days: %s" % (arg,)
+ cls.usage(e)
- uids = None
- dryrun = False
- verbose = False
- completely = False
+ elif opt in ("-b", "--batch"):
+ try:
+ batchSize = int(arg)
+ except ValueError, e:
+ print "Invalid value for --batch: %s" % (arg,)
+ cls.usage(e)
- @inlineCallbacks
- def doWork(self):
- rootResource = self.rootResource()
- directory = rootResource.getDirectory()
- total = (yield purgeUIDs(self._store, directory, rootResource, self.uids,
- verbose=self.verbose, dryrun=self.dryrun,
- completely=self.completely, doimplicit=self.doimplicit))
- if self.verbose:
- amount = "%d event%s" % (total, "s" if total > 1 else "")
- if self.dryrun:
- print "Would have modified or deleted %s" % (amount,)
- else:
- print "Modified or deleted %s" % (amount,)
+ elif opt in ("-v", "--verbose"):
+ verbose = True
+ elif opt in ("-n", "--dry-run"):
+ dryrun = True
+ elif opt in ("-f", "--config"):
+ configFileName = arg
+ else:
+ raise NotImplementedError(opt)
-def main_purge_events():
+ if args:
+ cls.usage("Too many arguments: %s" % (args,))
- try:
- (optargs, args) = getopt(
- sys.argv[1:], "d:b:f:hnv", [
- "days=",
- "batch=",
- "dry-run",
- "config=",
- "help",
- "verbose",
- ],
- )
- except GetoptError, e:
- usage_purge_events(e)
+ if dryrun:
+ verbose = True
- #
- # Get configuration
- #
- configFileName = None
- days = DEFAULT_RETAIN_DAYS
- batchSize = DEFAULT_BATCH_SIZE
- dryrun = False
- verbose = False
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-days)
+ cls.cutoff = cutoff
+ cls.batchSize = batchSize
+ cls.dryrun = dryrun
+ cls.verbose = verbose
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage_purge_events()
+ utilityMain(
+ configFileName,
+ cls,
+ )
- elif opt in ("-d", "--days"):
- try:
- days = int(arg)
- except ValueError, e:
- print "Invalid value for --days: %s" % (arg,)
- usage_purge_events(e)
- elif opt in ("-b", "--batch"):
- try:
- batchSize = int(arg)
- except ValueError, e:
- print "Invalid value for --batch: %s" % (arg,)
- usage_purge_events(e)
+ @classmethod
+ @inlineCallbacks
+ def purgeOldEvents(cls, store, cutoff, batchSize, verbose=False, dryrun=False):
- elif opt in ("-v", "--verbose"):
- verbose = True
+ service = cls(store)
+ service.cutoff = cutoff
+ service.batchSize = batchSize
+ service.dryrun = dryrun
+ service.verbose = verbose
+ result = (yield service.doWork())
+ returnValue(result)
- elif opt in ("-n", "--dry-run"):
- dryrun = True
- elif opt in ("-f", "--config"):
- configFileName = arg
+ @inlineCallbacks
+ def doWork(self):
- else:
- raise NotImplementedError(opt)
+ if self.dryrun:
+ if self.verbose:
+ print "(Dry run) Searching for old events..."
+ txn = self._store.newTransaction(label="Find old events")
+ oldEvents = (yield txn.eventsOlderThan(self.cutoff))
+ eventCount = len(oldEvents)
+ if self.verbose:
+ if eventCount == 0:
+ print "No events are older than %s" % (self.cutoff,)
+ elif eventCount == 1:
+ print "1 event is older than %s" % (self.cutoff,)
+ else:
+ print "%d events are older than %s" % (eventCount, self.cutoff)
+ returnValue(eventCount)
- if args:
- usage_purge_events("Too many arguments: %s" % (args,))
+ if self.verbose:
+ print "Removing events older than %s..." % (self.cutoff,)
- if dryrun:
- verbose = True
+ numEventsRemoved = -1
+ totalRemoved = 0
+ while numEventsRemoved:
+ txn = self._store.newTransaction(label="Remove old events")
+ numEventsRemoved = (yield txn.removeOldEvents(self.cutoff, batchSize=self.batchSize))
+ (yield txn.commit())
+ if numEventsRemoved:
+ totalRemoved += numEventsRemoved
+ if self.verbose:
+ print "%d," % (totalRemoved,),
- cutoff = PyCalendarDateTime.getToday()
- cutoff.setDateOnly(False)
- cutoff.offsetDay(-days)
- PurgeOldEventsService.cutoff = cutoff
- PurgeOldEventsService.batchSize = batchSize
- PurgeOldEventsService.dryrun = dryrun
- PurgeOldEventsService.verbose = verbose
+ if self.verbose:
+ print
+ if totalRemoved == 0:
+ print "No events were removed"
+ elif totalRemoved == 1:
+ print "1 event was removed in total"
+ else:
+ print "%d events were removed in total" % (totalRemoved,)
- utilityMain(
- configFileName,
- PurgeOldEventsService,
- )
+ returnValue(totalRemoved)
-def main_purge_orphaned_attachments():
- try:
- (optargs, args) = getopt(
- sys.argv[1:], "d:b:f:hnv", [
- "batch=",
- "dry-run",
- "config=",
- "help",
- "verbose",
- ],
- )
- except GetoptError, e:
- usage_purge_orphaned_attachments(e)
+class PurgeAttachmentsService(WorkerService):
- #
- # Get configuration
- #
- configFileName = None
- batchSize = DEFAULT_BATCH_SIZE
+ uuid = None
+ cutoff = None
+ batchSize = None
dryrun = False
verbose = False
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage_purge_orphaned_attachments()
+ @classmethod
+ def usage(cls, e=None):
- elif opt in ("-b", "--batch"):
- try:
- batchSize = int(arg)
- except ValueError, e:
- print "Invalid value for --batch: %s" % (arg,)
- usage_purge_orphaned_attachments(e)
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print " Remove old or orphaned attachments from the calendar server"
+ print ""
+ print "options:"
+ print " -h --help: print this help and exit"
+ print " -f --config <path>: Specify caldavd.plist configuration path"
+ print " -u --uuid <owner uid>: target a specific user UID"
+ #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ print " -d --days <number>: specify how many days in the past to retain (default=%d) zero means no removal of old attachments" % (DEFAULT_RETAIN_DAYS,)
+ print " -n --dry-run: calculate how many attachments to purge, but do not purge data"
+ print " -v --verbose: print progress information"
+ print ""
- elif opt in ("-v", "--verbose"):
- verbose = True
+ if e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(64)
+ else:
+ sys.exit(0)
- elif opt in ("-n", "--dry-run"):
- dryrun = True
- elif opt in ("-f", "--config"):
- configFileName = arg
+ @classmethod
+ def main(cls):
- else:
- raise NotImplementedError(opt)
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "d:b:f:hnu:v", [
+ "uuid=",
+ "days=",
+ "batch=",
+ "dry-run",
+ "config=",
+ "help",
+ "verbose",
+ ],
+ )
+ except GetoptError, e:
+ cls.usage(e)
- if args:
- usage_purge_orphaned_attachments("Too many arguments: %s" % (args,))
+ #
+ # Get configuration
+ #
+ configFileName = None
+ uuid = None
+ days = DEFAULT_RETAIN_DAYS
+ batchSize = DEFAULT_BATCH_SIZE
+ dryrun = False
+ verbose = False
- if dryrun:
- verbose = True
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ cls.usage()
- PurgeOrphanedAttachmentsService.batchSize = batchSize
- PurgeOrphanedAttachmentsService.dryrun = dryrun
- PurgeOrphanedAttachmentsService.verbose = verbose
+ elif opt in ("-u", "--uuid"):
+ uuid = arg
- utilityMain(
- configFileName,
- PurgeOrphanedAttachmentsService,
- )
+ elif opt in ("-d", "--days"):
+ try:
+ days = int(arg)
+ except ValueError, e:
+ print "Invalid value for --days: %s" % (arg,)
+ cls.usage(e)
+ elif opt in ("-b", "--batch"):
+ try:
+ batchSize = int(arg)
+ except ValueError, e:
+ print "Invalid value for --batch: %s" % (arg,)
+ cls.usage(e)
-def main_purge_principals():
+ elif opt in ("-v", "--verbose"):
+ verbose = True
- try:
- (optargs, args) = getopt(
- sys.argv[1:], "cf:hnv", [
- "completely",
- "dry-run",
- "config=",
- "help",
- "verbose",
- "noimplicit",
- ],
- )
- except GetoptError, e:
- usage_purge_principal(e)
+ elif opt in ("-n", "--dry-run"):
+ dryrun = True
- #
- # Get configuration
- #
- configFileName = None
- dryrun = False
- verbose = False
- completely = False
- doimplicit = True
+ elif opt in ("-f", "--config"):
+ configFileName = arg
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage_purge_principal()
+ else:
+ raise NotImplementedError(opt)
- elif opt in ("-c", "--completely"):
- completely = True
+ if args:
+ cls.usage("Too many arguments: %s" % (args,))
- elif opt in ("-v", "--verbose"):
+ if dryrun:
verbose = True
- elif opt in ("-n", "--dry-run"):
- dryrun = True
+ cls.uuid = uuid
+ if days > 0:
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-days)
+ cls.cutoff = cutoff
+ else:
+ cls.cutoff = None
+ cls.batchSize = batchSize
+ cls.dryrun = dryrun
+ cls.verbose = verbose
- elif opt in ("-f", "--config"):
- configFileName = arg
+ utilityMain(
+ configFileName,
+ cls,
+ )
- elif opt in ("--noimplicit"):
- doimplicit = False
+ @classmethod
+ @inlineCallbacks
+ def purgeOrphanedAttachments(cls, store, uuid, days, limit, dryrun, verbose):
+
+ service = cls(store)
+ service.uuid = uuid
+ if days > 0:
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-days)
+ service.cutoff = cutoff
else:
- raise NotImplementedError(opt)
+ service.cutoff = None
+ service.batchSize = limit
+ service.dryrun = dryrun
+ service.verbose = verbose
+ result = (yield service.doWork())
+ returnValue(result)
- # args is a list of uids
- PurgePrincipalService.uids = args
- PurgePrincipalService.completely = completely
- PurgePrincipalService.dryrun = dryrun
- PurgePrincipalService.verbose = verbose
- PurgePrincipalService.doimplicit = doimplicit
+ @inlineCallbacks
+ def doWork(self):
- utilityMain(
- configFileName,
- PurgePrincipalService
- )
+ if self.dryrun:
+ orphans = (yield self._orphansDryRun())
+ if self.cutoff is not None:
+ cutoffs = (yield self._cutoffDryRun())
+ else:
+ cutoffs = ()
+ returnValue(self._dryRunSummary(orphans, cutoffs))
+ else:
+ total = (yield self._orphansPurge())
+ if self.cutoff is not None:
+ total += (yield self._cutoffPurge())
+ returnValue(total)
- at inlineCallbacks
-def purgeOldEvents(store, directory, root, date, batchSize, verbose=False,
- dryrun=False):
- if dryrun:
- if verbose:
- print "(Dry run) Searching for old events..."
- txn = store.newTransaction(label="Find old events")
- oldEvents = (yield txn.eventsOlderThan(date))
- eventCount = len(oldEvents)
- if verbose:
- if eventCount == 0:
- print "No events are older than %s" % (date,)
- elif eventCount == 1:
- print "1 event is older than %s" % (date,)
- else:
- print "%d events are older than %s" % (eventCount, date)
- returnValue(eventCount)
+ @inlineCallbacks
+ def _orphansDryRun(self):
- if verbose:
- print "Removing events older than %s..." % (date,)
+ if self.verbose:
+ print "(Dry run) Searching for orphaned attachments..."
+ txn = self._store.newTransaction(label="Find orphaned attachments")
+ orphans = (yield txn.orphanedAttachments(self.uuid))
+ returnValue(orphans)
- numEventsRemoved = -1
- totalRemoved = 0
- while numEventsRemoved:
- txn = store.newTransaction(label="Remove old events")
- numEventsRemoved = (yield txn.removeOldEvents(date, batchSize=batchSize))
- (yield txn.commit())
- if numEventsRemoved:
- totalRemoved += numEventsRemoved
- if verbose:
- print "%d," % (totalRemoved,),
- if verbose:
- print
- if totalRemoved == 0:
- print "No events were removed"
- elif totalRemoved == 1:
- print "1 event was removed in total"
- else:
- print "%d events were removed in total" % (totalRemoved,)
+ @inlineCallbacks
+ def _cutoffDryRun(self):
- returnValue(totalRemoved)
+ if self.verbose:
+ print "(Dry run) Searching for old attachments..."
+ txn = self._store.newTransaction(label="Find old attachments")
+ cutoffs = (yield txn.oldAttachments(self.cutoff, self.uuid))
+ yield txn.commit()
+ returnValue(cutoffs)
- at inlineCallbacks
-def purgeOrphanedAttachments(store, batchSize, verbose=False, dryrun=False):
+ def _dryRunSummary(self, orphans, cutoffs):
- if dryrun:
- if verbose:
- print "(Dry run) Searching for orphaned attachments..."
- txn = store.newTransaction(label="Find orphaned attachments")
- orphans = (yield txn.orphanedAttachments())
- orphanCount = len(orphans)
- if verbose:
- if orphanCount == 0:
- print "No orphaned attachments"
- elif orphanCount == 1:
- print "1 orphaned attachment"
- else:
- print "%d orphaned attachments" % (orphanCount,)
- returnValue(orphanCount)
+ if self.verbose:
+ byuser = {}
+ ByUserData = collections.namedtuple('ByUserData', ['quota', 'orphanSize', 'orphanCount', 'cutoffSize', 'cutoffCount'])
+ for user, quota, size, count in orphans:
+ byuser[user] = ByUserData(quota=quota, orphanSize=size, orphanCount=count, cutoffSize=0, cutoffCount=0)
+ for user, quota, size, count in cutoffs:
+ if user in byuser:
+ byuser[user] = byuser[user]._replace(cutoffSize=size, cutoffCount=count)
+ else:
+ byuser[user] = ByUserData(quota=quota, orphanSize=0, orphanCount=0, cutoffSize=size, cutoffCount=count)
- if verbose:
- print "Removing orphaned attachments..."
+ # Print table of results
+ table = tables.Table()
+ table.addHeader(("User", "Current Quota", "Orphan Size", "Orphan Count", "Old Size", "Old Count", "Total Size", "Total Count"))
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
- numOrphansRemoved = -1
- totalRemoved = 0
- while numOrphansRemoved:
- txn = store.newTransaction(label="Remove orphaned attachments")
- numOrphansRemoved = (yield txn.removeOrphanedAttachments(batchSize=batchSize))
- (yield txn.commit())
- if numOrphansRemoved:
- totalRemoved += numOrphansRemoved
- if verbose:
- print "%d," % (totalRemoved,),
+ totalOrphans = 0
+ totalCutoffs = 0
+ for user, data in sorted(byuser.items(), key=lambda x: x[0]):
+ table.addRow((
+ user,
+ data.quota,
+ data.orphanSize,
+ data.orphanCount,
+ data.cutoffSize,
+ data.cutoffCount,
+ data.orphanSize + data.cutoffSize,
+ data.orphanCount + data.cutoffCount,
+ ))
+ totalOrphans += data.orphanCount
+ totalCutoffs += data.cutoffCount
+ total = totalOrphans + totalCutoffs
+ table.addFooter(("Total:", "", "", totalOrphans, "", totalCutoffs, "", total))
- if verbose:
- print
- if totalRemoved == 0:
- print "No orphaned attachments were removed"
- elif totalRemoved == 1:
- print "1 orphaned attachment was removed in total"
+ print "\n"
+ print "Orphaned/Old Attachments by User:\n"
+ table.printTable()
else:
- print "%d orphaned attachments were removed in total" % (totalRemoved,)
+ total = sum([x[3] for x in orphans]) + sum([x[3] for x in cutoffs])
- returnValue(totalRemoved)
+ return total
+ @inlineCallbacks
+ def _orphansPurge(self):
+ if self.verbose:
+ print "Removing orphaned attachments..."
+ numOrphansRemoved = -1
+ totalRemoved = 0
+ while numOrphansRemoved:
+ txn = self._store.newTransaction(label="Remove orphaned attachments")
+ numOrphansRemoved = (yield txn.removeOrphanedAttachments(self.uuid, batchSize=self.batchSize))
+ yield txn.commit()
+ if numOrphansRemoved:
+ totalRemoved += numOrphansRemoved
+ if self.verbose:
+ print "%d," % (totalRemoved,),
- at inlineCallbacks
-def purgeUIDs(store, directory, root, uids, verbose=False, dryrun=False,
- completely=False, doimplicit=True):
- total = 0
+ if self.verbose:
+ print
+ if totalRemoved == 0:
+ print "No orphaned attachments were removed"
+ elif totalRemoved == 1:
+ print "1 orphaned attachment was removed in total"
+ else:
+ print "%d orphaned attachments were removed in total" % (totalRemoved,)
- allAssignments = { }
+ returnValue(totalRemoved)
- for uid in uids:
- count, allAssignments[uid] = (yield purgeUID(store, uid, directory, root,
- verbose=verbose, dryrun=dryrun, completely=completely, doimplicit=doimplicit))
- total += count
- # TODO: figure out what to do with the purged proxy assignments...
- # ...print to stdout?
- # ...save in a file?
+ @inlineCallbacks
+ def _cutoffPurge(self):
- returnValue(total)
+ if self.verbose:
+ print "Removing old attachments..."
+ numOldRemoved = -1
+ totalRemoved = 0
+ while numOldRemoved:
+ txn = self._store.newTransaction(label="Remove old attachments")
+ numOldRemoved = (yield txn.removeOldAttachments(self.cutoff, self.uuid, batchSize=self.batchSize))
+ yield txn.commit()
+ if numOldRemoved:
+ totalRemoved += numOldRemoved
+ if self.verbose:
+ print "%d," % (totalRemoved,),
-CANCELEVENT_SKIPPED = 1
-CANCELEVENT_MODIFIED = 2
-CANCELEVENT_NOT_MODIFIED = 3
-CANCELEVENT_SHOULD_DELETE = 4
+ if self.verbose:
+ print
+ if totalRemoved == 0:
+ print "No old attachments were removed"
+ elif totalRemoved == 1:
+ print "1 old attachment was removed in total"
+ else:
+ print "%d old attachments were removed in total" % (totalRemoved,)
-def cancelEvent(event, when, cua):
- """
- Modify a VEVENT such that all future occurrences are removed
+ returnValue(totalRemoved)
- @param event: the event to modify
- @type event: L{twistedcaldav.ical.Component}
- @param when: the cutoff date (anything after which is removed)
- @type when: PyCalendarDateTime
- @param cua: Calendar User Address of principal being purged, to compare
- to see if it's the organizer of the event or just an attendee
- @type cua: string
+class PurgePrincipalService(WorkerService):
- Assumes that event does not occur entirely in the past.
+ root = None
+ directory = None
+ uids = None
+ dryrun = False
+ verbose = False
+ completely = False
+ doimplicit = True
+ proxies = True
+ when = None
- @return: one of the 4 constants above to indicate what action to take
- """
+ @classmethod
+ def usage(cls, e=None):
- whenDate = when.duplicate()
- whenDate.setDateOnly(True)
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print " Remove a principal's events and contacts from the calendar server"
+ print ""
+ print "options:"
+ print " -c --completely: By default, only future events are canceled; this option cancels all events"
+ print " -h --help: print this help and exit"
+ print " -f --config <path>: Specify caldavd.plist configuration path"
+ print " -n --dry-run: calculate how many events and contacts to purge, but do not purge data"
+ print " -v --verbose: print progress information"
+ print ""
- # Only process VEVENT
- if event.mainType() != "VEVENT":
- return CANCELEVENT_SKIPPED
+ if e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(64)
+ else:
+ sys.exit(0)
- main = event.masterComponent()
- if main is None:
- # No master component, so this is an attendee being invited to one or
- # more occurrences
- main = event.mainComponent(allow_multiple=True)
- # Anything completely in the future is deleted
- dtstart = main.getStartDateUTC()
- isDateTime = not dtstart.isDateOnly()
- if dtstart > when:
- return CANCELEVENT_SHOULD_DELETE
+ @classmethod
+ def main(cls):
- organizer = main.getOrganizer()
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "cf:hnv", [
+ "completely",
+ "dry-run",
+ "config=",
+ "help",
+ "verbose",
+ "noimplicit",
+ ],
+ )
+ except GetoptError, e:
+ cls.usage(e)
- # Non-meetings are deleted
- if organizer is None:
- return CANCELEVENT_SHOULD_DELETE
+ #
+ # Get configuration
+ #
+ configFileName = None
+ dryrun = False
+ verbose = False
+ completely = False
+ doimplicit = True
- # Meetings which cua is merely an attendee are deleted (thus implicitly
- # declined)
- # FIXME: I think we want to decline anything after the cut-off, not delete
- # the whole event.
- if organizer != cua:
- return CANCELEVENT_SHOULD_DELETE
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ cls.usage()
- dirty = False
+ elif opt in ("-c", "--completely"):
+ completely = True
- # Set the UNTIL on RRULE to cease at the cutoff
- if main.hasProperty("RRULE"):
- for rrule in main.properties("RRULE"):
- rrule = rrule.value()
- if rrule.getUseCount():
- rrule.setUseCount(False)
+ elif opt in ("-v", "--verbose"):
+ verbose = True
- rrule.setUseUntil(True)
- if isDateTime:
- rrule.setUntil(when)
+ elif opt in ("-n", "--dry-run"):
+ dryrun = True
+
+ elif opt in ("-f", "--config"):
+ configFileName = arg
+
+ elif opt in ("--noimplicit"):
+ doimplicit = False
+
else:
- rrule.setUntil(whenDate)
- dirty = True
+ raise NotImplementedError(opt)
- # Remove any EXDATEs and RDATEs beyond the cutoff
- for dateType in ("EXDATE", "RDATE"):
- if main.hasProperty(dateType):
- for exdate_rdate in main.properties(dateType):
- newValues = []
- for value in exdate_rdate.value():
- if value.getValue() < when:
- newValues.append(value)
- else:
- exdate_rdate.value().remove(value)
- dirty = True
- if not newValues:
- main.removeProperty(exdate_rdate)
- dirty = True
+ # args is a list of uids
+ cls.uids = args
+ cls.completely = completely
+ cls.dryrun = dryrun
+ cls.verbose = verbose
+ cls.doimplicit = doimplicit
+ utilityMain(
+ configFileName,
+ cls
+ )
- # Remove any overridden components beyond the cutoff
- for component in tuple(event.subcomponents()):
- if component.name() == "VEVENT":
- dtstart = component.getStartDateUTC()
- remove = False
- if dtstart > when:
- remove = True
- if remove:
- event.removeComponent(component)
- dirty = True
- if dirty:
- return CANCELEVENT_MODIFIED
- else:
- return CANCELEVENT_NOT_MODIFIED
+ @classmethod
+ @inlineCallbacks
+ def purgeUIDs(cls, store, directory, root, uids, verbose=False, dryrun=False,
+ completely=False, doimplicit=True, proxies=True, when=None):
+ service = cls(store)
+ service.root = root
+ service.directory = directory
+ service.uids = uids
+ service.verbose = verbose
+ service.dryrun = dryrun
+ service.completely = completely
+ service.doimplicit = doimplicit
+ service.proxies = proxies
+ service.when = when
+ result = (yield service.doWork())
+ returnValue(result)
- at inlineCallbacks
-def purgeUID(store, uid, directory, root, verbose=False, dryrun=False, proxies=True,
- when=None, completely=False, doimplicit=True):
- if when is None:
- when = PyCalendarDateTime.getNowUTC()
+ @inlineCallbacks
+ def doWork(self):
- # Does the record exist?
- record = directory.recordWithUID(uid)
- if record is None:
- # The user has already been removed from the directory service. We
- # need to fashion a temporary, fake record
+ if self.root is None:
+ self.root = self.getRootResource()
+ if self.directory is None:
+ self.directory = self.root.getDirectory()
- # FIXME: probaby want a more elegant way to accomplish this,
- # since it requires the aggregate directory to examine these first:
- record = DirectoryRecord(directory, "users", uid, shortNames=(uid,),
- enabledForCalendaring=True)
- record.enabled = True
- directory._tmpRecords["shortNames"][uid] = record
- directory._tmpRecords["uids"][uid] = record
+ total = 0
- cua = "urn:uuid:%s" % (uid,)
+ allAssignments = {}
- principalCollection = directory.principalCollection
- principal = principalCollection.principalForRecord(record)
+ for uid in self.uids:
+ count, allAssignments[uid] = (yield self._purgeUID(uid))
+ total += count
- request = FakeRequest(root, None, None)
- request.checkedSACL = True
- request.authnUser = request.authzUser = davxml.Principal(
- davxml.HRef.fromString("/principals/__uids__/%s/" % (uid,))
- )
+ if self.verbose:
+ amount = "%d event%s" % (total, "s" if total > 1 else "")
+ if self.dryrun:
+ print "Would have modified or deleted %s" % (amount,)
+ else:
+ print "Modified or deleted %s" % (amount,)
- # See if calendar home is provisioned
- txn = store.newTransaction()
- storeCalHome = (yield txn.calendarHomeWithUID(uid))
- calHomeProvisioned = storeCalHome is not None
+ returnValue((total, allAssignments,))
- # If in "completely" mode, unshare collections, remove notifications
- if calHomeProvisioned and completely:
- # Process shared-to-me calendars
- names = list((yield storeCalHome.listSharedChildren()))
- for name in names:
- if verbose:
- if dryrun:
- print "Would unshare: %s" % (name,)
- else:
- print "Unsharing: %s" % (name,)
- if not dryrun:
- child = (yield storeCalHome.sharedChildWithName(name))
- (yield child.unshare())
+ @inlineCallbacks
+ def _purgeUID(self, uid):
- # Process shared calendars
- children = list((yield storeCalHome.children()))
- for child in children:
- if verbose:
- if dryrun:
- print "Would unshare: %s" % (child.name(),)
- else:
- print "Unsharing: %s" % (child.name(),)
- if not dryrun:
- (yield child.unshare())
+ if self.when is None:
+ self.when = PyCalendarDateTime.getNowUTC()
- if not dryrun:
- (yield storeCalHome.removeUnacceptedShares())
- (yield storeCalHome.removeInvites())
- notificationHome = (yield txn.notificationsWithUID(uid))
- if notificationHome is not None:
- (yield notificationHome.remove())
+ # Does the record exist?
+ record = self.directory.recordWithUID(uid)
+ if record is None:
+ # The user has already been removed from the directory service. We
+ # need to fashion a temporary, fake record
- (yield txn.commit())
+ # FIXME: probably want a more elegant way to accomplish this,
+ # since it requires the aggregate directory to examine these first:
+ record = DirectoryRecord(self.directory, "users", uid, shortNames=(uid,), enabledForCalendaring=True)
+ record.enabled = True
+ self.directory._tmpRecords["shortNames"][uid] = record
+ self.directory._tmpRecords["uids"][uid] = record
- # Anything in the past is left alone
- whenString = when.getText()
- filter = caldavxml.Filter(
- caldavxml.ComponentFilter(
- caldavxml.ComponentFilter(
- TimeRange(start=whenString,),
- name=("VEVENT",),
- ),
- name="VCALENDAR",
- )
- )
- filter = calendarqueryfilter.Filter(filter)
+ cua = "urn:uuid:%s" % (uid,)
- count = 0
- assignments = []
+ principalCollection = self.directory.principalCollection
+ principal = principalCollection.principalForRecord(record)
- perUserFilter = PerUserDataFilter(uid)
+ request = FakeRequest(self.root, None, None)
+ request.checkedSACL = True
+ request.authnUser = request.authzUser = davxml.Principal(
+ davxml.HRef.fromString("/principals/__uids__/%s/" % (uid,))
+ )
- try:
- if calHomeProvisioned:
- calendarHome = yield principal.calendarHome(request)
- for collName in (yield calendarHome.listChildren()):
- collection = (yield calendarHome.getChild(collName))
+ # See if calendar home is provisioned
+ txn = self._store.newTransaction()
+ storeCalHome = (yield txn.calendarHomeWithUID(uid))
+ calHomeProvisioned = storeCalHome is not None
- if collection.isCalendarCollection() or collName == "inbox":
- childNames = []
+ # If in "completely" mode, unshare collections, remove notifications
+ if calHomeProvisioned and self.completely:
- if completely:
- # all events
- for childName in (yield collection.listChildren()):
- childNames.append(childName)
+ # Process shared-to-me calendars
+ names = list((yield storeCalHome.listSharedChildren()))
+ for name in names:
+ if self.verbose:
+ if self.dryrun:
+ print "Would unshare: %s" % (name,)
else:
- # events matching filter
- for childName, childUid, childType in (yield collection.index().indexedSearch(filter)):
- childNames.append(childName)
+ print "Unsharing: %s" % (name,)
+ if not self.dryrun:
+ child = (yield storeCalHome.sharedChildWithName(name))
+ (yield child.unshare())
- for childName in childNames:
+ # Process shared calendars
+ children = list((yield storeCalHome.children()))
+ for child in children:
+ if self.verbose:
+ if self.dryrun:
+ print "Would unshare: %s" % (child.name(),)
+ else:
+ print "Unsharing: %s" % (child.name(),)
+ if not self.dryrun:
+ (yield child.unshare())
- childResource = (yield collection.getChild(childName))
- if completely:
- action = CANCELEVENT_SHOULD_DELETE
+ if not self.dryrun:
+ (yield storeCalHome.removeUnacceptedShares())
+ (yield storeCalHome.removeInvites())
+ notificationHome = (yield txn.notificationsWithUID(uid))
+ if notificationHome is not None:
+ (yield notificationHome.remove())
+
+ (yield txn.commit())
+
+ # Anything in the past is left alone
+ whenString = self.when.getText()
+ filter = caldavxml.Filter(
+ caldavxml.ComponentFilter(
+ caldavxml.ComponentFilter(
+ TimeRange(start=whenString,),
+ name=("VEVENT",),
+ ),
+ name="VCALENDAR",
+ )
+ )
+ filter = calendarqueryfilter.Filter(filter)
+
+ count = 0
+ assignments = []
+
+ perUserFilter = PerUserDataFilter(uid)
+
+ try:
+ if calHomeProvisioned:
+ calendarHome = yield principal.calendarHome(request)
+ for collName in (yield calendarHome.listChildren()):
+ collection = (yield calendarHome.getChild(collName))
+
+ if collection.isCalendarCollection() or collName == "inbox":
+ childNames = []
+
+ if self.completely:
+ # all events
+ for childName in (yield collection.listChildren()):
+ childNames.append(childName)
else:
- event = (yield childResource.iCalendar())
- event = perUserFilter.filter(event)
- action = cancelEvent(event, when, cua)
+ # events matching filter
+ for childName, _ignore_childUid, _ignore_childType in (yield collection.index().indexedSearch(filter)):
+ childNames.append(childName)
- uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName)
- request.path = uri
- if action == CANCELEVENT_MODIFIED:
- count += 1
- request._rememberResource(childResource, uri)
- storer = StoreCalendarObjectResource(
- request=request,
- destination=childResource,
- destination_uri=uri,
- destinationcal=True,
- destinationparent=collection,
- calendar=str(event),
- )
- if verbose:
- if dryrun:
- print "Would modify: %s" % (uri,)
- else:
- print "Modifying: %s" % (uri,)
- if not dryrun:
- result = (yield storer.run())
+ for childName in childNames:
- elif action == CANCELEVENT_SHOULD_DELETE:
- incrementCount = dryrun
- request._rememberResource(childResource, uri)
- if verbose:
- if dryrun:
- print "Would delete: %s" % (uri,)
- else:
- print "Deleting: %s" % (uri,)
- if not dryrun:
- retry = False
- try:
- result = (yield childResource.storeRemove(request, doimplicit, uri))
- if result != NO_CONTENT:
- print "Error deleting %s/%s/%s: %s" % (uid,
- collName, childName, result)
- retry = True
+ childResource = (yield collection.getChild(childName))
+ if self.completely:
+ action = self.CANCELEVENT_SHOULD_DELETE
+ else:
+ event = (yield childResource.iCalendar())
+ event = perUserFilter.filter(event)
+ action = self._cancelEvent(event, self.when, cua)
+
+ uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName)
+ request.path = uri
+ if action == self.CANCELEVENT_MODIFIED:
+ count += 1
+ request._rememberResource(childResource, uri)
+ storer = StoreCalendarObjectResource(
+ request=request,
+ destination=childResource,
+ destination_uri=uri,
+ destinationcal=True,
+ destinationparent=collection,
+ calendar=str(event),
+ )
+ if self.verbose:
+ if self.dryrun:
+ print "Would modify: %s" % (uri,)
else:
- incrementCount = True
+ print "Modifying: %s" % (uri,)
+ if not self.dryrun:
+ result = (yield storer.run())
- except Exception, e:
- print "Exception deleting %s/%s/%s: %s" % (uid,
- collName, childName, str(e))
- traceback.print_stack()
- retry = True
-
- if retry and doimplicit:
- # Try again with implicit scheduling off
- print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName)
+ elif action == self.CANCELEVENT_SHOULD_DELETE:
+ incrementCount = self.dryrun
+ request._rememberResource(childResource, uri)
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete: %s" % (uri,)
+ else:
+ print "Deleting: %s" % (uri,)
+ if not self.dryrun:
+ retry = False
try:
- result = (yield childResource.storeRemove(request, False, uri))
+ result = (yield childResource.storeRemove(request, self.doimplicit, uri))
if result != NO_CONTENT:
print "Error deleting %s/%s/%s: %s" % (uid,
collName, childName, result)
+ retry = True
else:
incrementCount = True
+
except Exception, e:
- print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e))
+ print "Exception deleting %s/%s/%s: %s" % (uid,
+ collName, childName, str(e))
traceback.print_stack()
+ retry = True
- if incrementCount:
- count += 1
+ if retry and self.doimplicit:
+ # Try again with implicit scheduling off
+ print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName)
+ try:
+ result = (yield childResource.storeRemove(request, False, uri))
+ if result != NO_CONTENT:
+ print "Error deleting %s/%s/%s: %s" % (uid,
+ collName, childName, result)
+ else:
+ incrementCount = True
+ except Exception, e:
+ print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e))
+ traceback.print_stack()
+ if incrementCount:
+ count += 1
- txn = getattr(request, "_newStoreTransaction", None)
- # Commit
- if txn is not None:
- (yield txn.commit())
+ txn = getattr(request, "_newStoreTransaction", None)
+ # Commit
+ if txn is not None:
+ (yield txn.commit())
- except Exception, e:
- # Abort
- txn = getattr(request, "_newStoreTransaction", None)
- if txn is not None:
- (yield txn.abort())
- raise e
+ except Exception, e:
+ # Abort
+ txn = getattr(request, "_newStoreTransaction", None)
+ if txn is not None:
+ (yield txn.abort())
+ raise e
- try:
- txn = store.newTransaction()
+ try:
+ txn = self._store.newTransaction()
- # Remove empty calendar collections (and calendar home if no more
- # calendars)
- storeCalHome = (yield txn.calendarHomeWithUID(uid))
- if storeCalHome is not None:
- calendars = list((yield storeCalHome.calendars()))
- remainingCalendars = len(calendars)
- for calColl in calendars:
- if len(list((yield calColl.calendarObjects()))) == 0:
- remainingCalendars -= 1
- calendarName = calColl.name()
- if verbose:
- if dryrun:
- print "Would delete calendar: %s" % (calendarName,)
+ # Remove empty calendar collections (and calendar home if no more
+ # calendars)
+ storeCalHome = (yield txn.calendarHomeWithUID(uid))
+ if storeCalHome is not None:
+ calendars = list((yield storeCalHome.calendars()))
+ remainingCalendars = len(calendars)
+ for calColl in calendars:
+ if len(list((yield calColl.calendarObjects()))) == 0:
+ remainingCalendars -= 1
+ calendarName = calColl.name()
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete calendar: %s" % (calendarName,)
+ else:
+ print "Deleting calendar: %s" % (calendarName,)
+ if not self.dryrun:
+ (yield storeCalHome.removeChildWithName(calendarName))
+
+ if not remainingCalendars:
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete calendar home"
else:
- print "Deleting calendar: %s" % (calendarName,)
- if not dryrun:
- (yield storeCalHome.removeChildWithName(calendarName))
+ print "Deleting calendar home"
+ if not self.dryrun:
+ (yield storeCalHome.remove())
- if not remainingCalendars:
- if verbose:
- if dryrun:
- print "Would delete calendar home"
+ # Remove VCards
+ storeAbHome = (yield txn.addressbookHomeWithUID(uid))
+ if storeAbHome is not None:
+ for abColl in list((yield storeAbHome.addressbooks())):
+ for card in list((yield abColl.addressbookObjects())):
+ cardName = card.name()
+ if self.verbose:
+ uri = "/addressbooks/__uids__/%s/%s/%s" % (uid, abColl.name(), cardName)
+ if self.dryrun:
+ print "Would delete: %s" % (uri,)
+ else:
+ print "Deleting: %s" % (uri,)
+ if not self.dryrun:
+ (yield abColl.removeObjectResourceWithName(cardName))
+ count += 1
+ if self.verbose:
+ abName = abColl.name()
+ if self.dryrun:
+ print "Would delete addressbook: %s" % (abName,)
+ else:
+ print "Deleting addressbook: %s" % (abName,)
+ if not self.dryrun:
+ # Also remove the addressbook collection itself
+ (yield storeAbHome.removeChildWithName(abColl.name()))
+
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete addressbook home"
else:
- print "Deleting calendar home"
- if not dryrun:
- (yield storeCalHome.remove())
+ print "Deleting addressbook home"
+ if not self.dryrun:
+ (yield storeAbHome.remove())
+ # Commit
+ (yield txn.commit())
- # Remove VCards
- storeAbHome = (yield txn.addressbookHomeWithUID(uid))
- if storeAbHome is not None:
- for abColl in list( (yield storeAbHome.addressbooks()) ):
- for card in list( (yield abColl.addressbookObjects()) ):
- cardName = card.name()
- if verbose:
- uri = "/addressbooks/__uids__/%s/%s/%s" % (uid, abColl.name(), cardName)
- if dryrun:
- print "Would delete: %s" % (uri,)
- else:
- print "Deleting: %s" % (uri,)
- if not dryrun:
- (yield abColl.removeObjectResourceWithName(cardName))
- count += 1
- if verbose:
- abName = abColl.name()
- if dryrun:
- print "Would delete addressbook: %s" % (abName,)
- else:
- print "Deleting addressbook: %s" % (abName,)
- if not dryrun:
- # Also remove the addressbook collection itself
- (yield storeAbHome.removeChildWithName(abColl.name()))
+ except Exception, e:
+ # Abort
+ (yield txn.abort())
+ raise e
- if verbose:
- if dryrun:
- print "Would delete addressbook home"
- else:
- print "Deleting addressbook home"
- if not dryrun:
- (yield storeAbHome.remove())
+ if self.proxies and not self.dryrun:
+ if self.verbose:
+ print "Deleting any proxy assignments"
+ assignments = (yield self._purgeProxyAssignments(principal))
- # Commit
- (yield txn.commit())
+ returnValue((count, assignments))
- except Exception, e:
- # Abort
- (yield txn.abort())
- raise e
+ CANCELEVENT_SKIPPED = 1
+ CANCELEVENT_MODIFIED = 2
+ CANCELEVENT_NOT_MODIFIED = 3
+ CANCELEVENT_SHOULD_DELETE = 4
- if proxies and not dryrun:
- if verbose:
- print "Deleting any proxy assignments"
- assignments = (yield purgeProxyAssignments(principal))
+ @classmethod
+ def _cancelEvent(cls, event, when, cua):
+ """
+ Modify a VEVENT such that all future occurrences are removed
- returnValue((count, assignments))
+ @param event: the event to modify
+ @type event: L{twistedcaldav.ical.Component}
+ @param when: the cutoff date (anything after which is removed)
+ @type when: PyCalendarDateTime
- at inlineCallbacks
-def purgeProxyAssignments(principal):
+ @param cua: Calendar User Address of principal being purged, to compare
+ to see if it's the organizer of the event or just an attendee
+ @type cua: string
- assignments = []
+ Assumes that event does not occur entirely in the past.
- for proxyType in ("read", "write"):
+ @return: one of the 4 constants above to indicate what action to take
+ """
- proxyFor = (yield principal.proxyFor(proxyType == "write"))
- for other in proxyFor:
- assignments.append((principal.record.uid, proxyType, other.record.uid))
- (yield removeProxy(other, principal))
+ whenDate = when.duplicate()
+ whenDate.setDateOnly(True)
- subPrincipal = principal.getChild("calendar-proxy-" + proxyType)
- proxies = (yield subPrincipal.readProperty(davxml.GroupMemberSet, None))
- for other in proxies.children:
- assignments.append((str(other).split("/")[3], proxyType, principal.record.uid))
+ # Only process VEVENT
+ if event.mainType() != "VEVENT":
+ return cls.CANCELEVENT_SKIPPED
- (yield subPrincipal.writeProperty(davxml.GroupMemberSet(), None))
+ main = event.masterComponent()
+ if main is None:
+ # No master component, so this is an attendee being invited to one or
+ # more occurrences
+ main = event.mainComponent(allow_multiple=True)
- returnValue(assignments)
+ # Anything completely in the future is deleted
+ dtstart = main.getStartDateUTC()
+ isDateTime = not dtstart.isDateOnly()
+ if dtstart > when:
+ return cls.CANCELEVENT_SHOULD_DELETE
+ organizer = main.getOrganizer()
+
+ # Non-meetings are deleted
+ if organizer is None:
+ return cls.CANCELEVENT_SHOULD_DELETE
+
+ # Meetings which cua is merely an attendee are deleted (thus implicitly
+ # declined)
+ # FIXME: I think we want to decline anything after the cut-off, not delete
+ # the whole event.
+ if organizer != cua:
+ return cls.CANCELEVENT_SHOULD_DELETE
+
+ dirty = False
+
+ # Set the UNTIL on RRULE to cease at the cutoff
+ if main.hasProperty("RRULE"):
+ for rrule in main.properties("RRULE"):
+ rrule = rrule.value()
+ if rrule.getUseCount():
+ rrule.setUseCount(False)
+
+ rrule.setUseUntil(True)
+ if isDateTime:
+ rrule.setUntil(when)
+ else:
+ rrule.setUntil(whenDate)
+ dirty = True
+
+ # Remove any EXDATEs and RDATEs beyond the cutoff
+ for dateType in ("EXDATE", "RDATE"):
+ if main.hasProperty(dateType):
+ for exdate_rdate in main.properties(dateType):
+ newValues = []
+ for value in exdate_rdate.value():
+ if value.getValue() < when:
+ newValues.append(value)
+ else:
+ exdate_rdate.value().remove(value)
+ dirty = True
+ if not newValues:
+ main.removeProperty(exdate_rdate)
+ dirty = True
+
+ # Remove any overridden components beyond the cutoff
+ for component in tuple(event.subcomponents()):
+ if component.name() == "VEVENT":
+ dtstart = component.getStartDateUTC()
+ remove = False
+ if dtstart > when:
+ remove = True
+ if remove:
+ event.removeComponent(component)
+ dirty = True
+
+ if dirty:
+ return cls.CANCELEVENT_MODIFIED
+ else:
+ return cls.CANCELEVENT_NOT_MODIFIED
+
+
+ @classmethod
+ @inlineCallbacks
+ def _purgeProxyAssignments(cls, principal):
+
+ assignments = []
+
+ for proxyType in ("read", "write"):
+
+ proxyFor = (yield principal.proxyFor(proxyType == "write"))
+ for other in proxyFor:
+ assignments.append((principal.record.uid, proxyType, other.record.uid))
+ (yield removeProxy(other, principal))
+
+ subPrincipal = principal.getChild("calendar-proxy-" + proxyType)
+ proxies = (yield subPrincipal.readProperty(davxml.GroupMemberSet, None))
+ for other in proxies.children:
+ assignments.append((str(other).split("/")[3], proxyType, principal.record.uid))
+
+ (yield subPrincipal.writeProperty(davxml.GroupMemberSet(), None))
+
+ returnValue(assignments)
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge.py 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge.py 2012-12-20 21:15:56 UTC (rev 10192)
@@ -16,8 +16,7 @@
from calendarserver.tap.util import getRootResource
-from calendarserver.tools.purge import cancelEvent, purgeUID
-from calendarserver.tools.purge import CANCELEVENT_MODIFIED, CANCELEVENT_SHOULD_DELETE
+from calendarserver.tools.purge import PurgePrincipalService
from twistedcaldav.config import config
from twistedcaldav.ical import Component
@@ -233,55 +232,61 @@
def test_cancelRepeating(self):
# A repeating event where purged CUA is organizer
event = Component.fromString(REPEATING_1_ICS_BEFORE)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_MODIFIED)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_MODIFIED)
self.assertEquals(str(event), REPEATING_1_ICS_AFTER)
+
def test_cancelAllDayRepeating(self):
# A repeating All Day event where purged CUA is organizer
event = Component.fromString(REPEATING_2_ICS_BEFORE)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_MODIFIED)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_MODIFIED)
self.assertEquals(str(event), REPEATING_2_ICS_AFTER)
+
def test_cancelFutureEvent(self):
# A future event
event = Component.fromString(FUTURE_EVENT_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelNonMeeting(self):
# A repeating non-meeting event
event = Component.fromString(REPEATING_NON_MEETING_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelAsAttendee(self):
# A repeating meeting event where purged CUA is an attendee
event = Component.fromString(REPEATING_ATTENDEE_MEETING_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelAsAttendeeOccurrence(self):
# A repeating meeting occurrence with no master, where purged CUA is
# an attendee
event = Component.fromString(INVITED_TO_OCCURRENCE_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:9DC04A71-E6DD-11DF-9492-0800200C9A66")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelAsAttendeeMultipleOccurrences(self):
# Multiple meeting occurrences with no master, where purged CUA is
# an attendee
event = Component.fromString(INVITED_TO_MULTIPLE_OCCURRENCES_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:9DC04A71-E6DD-11DF-9492-0800200C9A66")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
# This event begins on Nov 30, 2010, has two EXDATES (Dec 3 and 9), and has two
# overridden instances (Dec 4 and 11). The Dec 11 one will be removed since
@@ -725,7 +730,6 @@
DTEND;TZID=America/Los_Angeles:20111105T170000
TRANSP:OPAQUE
ORGANIZER;CN="Amanda Test":urn:uuid:9DC04A70-E6DD-11DF-9492-0800200C9A66
-
UID:44A391CF-52F5-46B4-B35A-E000E3002084
DTSTAMP:20111102T162426Z
SEQUENCE:5
@@ -738,9 +742,6 @@
""".replace("\n", "\r\n")
-
-
-
ATTACHMENT_ICS = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 4.0.1//EN
@@ -778,7 +779,6 @@
-
class PurgePrincipalTests(CommonCommonTests, unittest.TestCase):
"""
Tests for purging the data belonging to a given principal
@@ -869,9 +869,9 @@
@inlineCallbacks
- def test_purgeUID(self):
+ def test_purgeUIDs(self):
"""
- Verify purgeUID removes homes, and doesn't provision homes that don't exist
+ Verify purgeUIDs removes homes, and doesn't provision homes that don't exist
"""
# Now you see it
@@ -880,8 +880,8 @@
self.assertNotEquals(home, None)
(yield txn.commit())
- count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory,
- self.rootResource, verbose=False, proxies=False, completely=True))
+ count, ignored = (yield PurgePrincipalService.purgeUIDs(self.storeUnderTest(), self.directory,
+ self.rootResource, (self.uid,), verbose=False, proxies=False, completely=True))
self.assertEquals(count, 1) # 1 event
# Now you don't
@@ -893,8 +893,8 @@
self.assertEquals((yield home2.sharedChildWithName(self.sharedName)), None)
(yield txn.commit())
- count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory,
- self.rootResource, verbose=False, proxies=False, completely=True))
+ count, ignored = (yield PurgePrincipalService.purgeUIDs(self.storeUnderTest(), self.directory,
+ self.rootResource, (self.uid,), verbose=False, proxies=False, completely=True))
self.assertEquals(count, 0)
# And you still don't (making sure it's not provisioned)
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge_old_events.py 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/calendarserver/tools/test/test_purge_old_events.py 2012-12-20 21:15:56 UTC (rev 10192)
@@ -19,7 +19,8 @@
"""
from calendarserver.tap.util import getRootResource
-from calendarserver.tools.purge import purgeOldEvents, purgeUID, purgeOrphanedAttachments
+from calendarserver.tools.purge import PurgeOldEventsService, PurgeAttachmentsService, \
+ PurgePrincipalService
from pycalendar.datetime import PyCalendarDateTime
from pycalendar.timezone import PyCalendarTimezone
@@ -31,6 +32,7 @@
from twisted.trial import unittest
from twistedcaldav.config import config
+from twistedcaldav.ical import Component
from twistedcaldav.vcard import Component as VCardComponent
from txdav.common.datastore.sql_tables import schema
@@ -139,6 +141,106 @@
END:VCALENDAR
""".replace("\n", "\r\n") % {"year": now - 5}
+OLD_ATTACHMENT2_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;UNTIL=20061029T090000Z;BYMONTH=10;BYDAY=-1SU
+DTSTART:19621028T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;UNTIL=20060402T100000Z;BYMONTH=4;BYDAY=1SU
+DTSTART:19870405T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T181216Z
+UID:57A5D1F6-9A57-4F74-9520-25C617F54B88-2
+TRANSP:OPAQUE
+SUMMARY:Ancient event with attachment #2
+DTSTART;TZID=US/Pacific:%(year)s0408T111500
+DTEND;TZID=US/Pacific:%(year)s0408T151500
+DTSTAMP:20100303T181220Z
+X-APPLE-DROPBOX:/calendars/__uids__/user01/dropbox/57A5D1F6-9A57-4F74-95
+ 20-25C617F54B88-2.dropbox
+SEQUENCE:2
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n") % {"year": now - 5}
+
+CURRENT_ATTACHMENT3_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;UNTIL=20061029T090000Z;BYMONTH=10;BYDAY=-1SU
+DTSTART:19621028T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;UNTIL=20060402T100000Z;BYMONTH=4;BYDAY=1SU
+DTSTART:19870405T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T181216Z
+UID:57A5D1F6-9A57-4F74-9520-25C617F54B88-3
+TRANSP:OPAQUE
+SUMMARY:Ancient event with attachment #3
+DTSTART;TZID=US/Pacific:%(year)s0408T111500
+DTEND;TZID=US/Pacific:%(year)s0408T151500
+DTSTAMP:20100303T181220Z
+X-APPLE-DROPBOX:/calendars/__uids__/user01/dropbox/57A5D1F6-9A57-4F74-95
+ 20-25C617F54B88-2.dropbox
+SEQUENCE:2
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n") % {"year": now + 1}
+
ENDLESS_ICS = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 4.0.1//EN
@@ -432,7 +534,7 @@
@inlineCallbacks
- def _addAttachment(self, orphan=False):
+ def _addAttachment(self, orphan=False, old2=False, current3=False):
txn = self._sqlCalendarStore.newTransaction()
@@ -440,7 +542,7 @@
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar1"))
event = (yield calendar.calendarObjectWithName("oldattachment.ics"))
- attachment = (yield event.createAttachmentWithName("oldattachment.ics"))
+ attachment = (yield event.createAttachmentWithName("oldattachment.txt"))
t = attachment.store(MimeType("text", "x-fixture"))
t.write("old attachment")
t.write(" text")
@@ -454,6 +556,26 @@
Where=co.RESOURCE_ID == event._resourceID,
).on(txn)
+ if old2:
+ event = (yield calendar.createCalendarObjectWithName(
+ "oldattachment2.ics", Component.fromString(OLD_ATTACHMENT2_ICS)
+ ))
+ attachment = (yield event.createAttachmentWithName("oldattachment2.txt"))
+ t = attachment.store(MimeType("text", "x-fixture"))
+ t.write("old attachment #2")
+ t.write(" text")
+ (yield t.loseConnection())
+
+ if current3:
+ event = (yield calendar.createCalendarObjectWithName(
+ "currentattachment3.ics", Component.fromString(CURRENT_ATTACHMENT3_ICS)
+ ))
+ attachment = (yield event.createAttachmentWithName("currentattachment3.txt"))
+ t = attachment.store(MimeType("text", "x-fixture"))
+ t.write("current attachment #3")
+ t.write(" text")
+ (yield t.loseConnection())
+
(yield txn.commit())
returnValue(attachment)
@@ -507,19 +629,31 @@
def test_purgeOldEvents(self):
# Dry run
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, dryrun=True,
- verbose=False))
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ dryrun=True,
+ verbose=False
+ ))
self.assertEquals(total, 4)
# Actually remove
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, verbose=False))
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ verbose=False
+ ))
self.assertEquals(total, 4)
# There should be no more left
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, verbose=False))
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ verbose=False
+ ))
self.assertEquals(total, 0)
@@ -543,8 +677,8 @@
(yield txn.commit())
# Purge home1
- total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory,
- self.rootResource, verbose=False, proxies=False,
+ total, ignored = (yield PurgePrincipalService.purgeUIDs(self._sqlCalendarStore, self.directory,
+ self.rootResource, ("home1",), verbose=False, proxies=False,
when=PyCalendarDateTime(now, 4, 1, 12, 0, 0, 0, PyCalendarTimezone(utc=True))))
# 2 items deleted: 1 event and 1 vcard
@@ -580,8 +714,8 @@
(yield txn.commit())
# Purge home1 completely
- total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory,
- self.rootResource, verbose=False, proxies=False, completely=True))
+ total, ignored = (yield PurgePrincipalService.purgeUIDs(self._sqlCalendarStore, self.directory,
+ self.rootResource, ("home1",), verbose=False, proxies=False, completely=True))
# 4 items deleted: 3 events and 1 vcard
self.assertEquals(total, 4)
@@ -595,49 +729,373 @@
@inlineCallbacks
- def test_purgeOrphanedAttachments(self):
+ def test_purgeOrphanedAttachmentsWithoutCutoffWithPurgeOld(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertEqual(quota, 0)
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
- (yield self._addAttachment(orphan=True))
+ (yield self._addAttachment(orphan=True, current3=True))
(yield self.commit())
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
# Remove old events first
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, verbose=False))
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ verbose=False
+ ))
self.assertEquals(total, 4)
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
# Dry run
- total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
- dryrun=True, verbose=False))
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=True, verbose=False))
self.assertEquals(total, 1)
(yield self.commit())
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 == quota3)
# Actually remove
- total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
- dryrun=False, verbose=False))
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
self.assertEquals(total, 1)
(yield self.commit())
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quotaAfter = (yield home.quotaUsedBytes())
- self.assertEqual(quotaAfter, 0)
+ quota5 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota5 < quota4)
# There should be no more left
- total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
- dryrun=False, verbose=False))
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithoutCutoff(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithoutCutoffWithMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home1", 0, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home1", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home1", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithoutCutoffWithoutMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home2", 0, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 0)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home2", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 == quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home2", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithCutoffOld(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 2)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 2)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithCutoffOldWithMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home1", 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 2)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home1", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 2)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home1", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithCutoffOldWithoutMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home2", 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 0)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home2", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 == quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, "home2", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithCutoffCurrent(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, current3=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeOrphanedAttachmentsWithCutoffCurrentOld(self):
+ """
+ L{PurgeAttachmentsService.purgeOrphanedAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment(orphan=True, old2=True, current3=True))
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 1)
+ (yield self.commit())
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeOrphanedAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_principals.8
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_principals.8 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_principals.8 2012-12-20 21:15:56 UTC (rev 10192)
@@ -50,7 +50,7 @@
resources.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Nm
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_push.8
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_push.8 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_manage_push.8 2012-12-20 21:15:56 UTC (rev 10192)
@@ -31,7 +31,7 @@
currently subscribed to via APNS.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read data that belongs to the server.
.Nm
takes a list of userids as arguments and then displays the resources
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_migrate_resources.8
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_migrate_resources.8 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_migrate_resources.8 2012-12-20 21:15:56 UTC (rev 10192)
@@ -31,7 +31,7 @@
OpenDirectory into the calendar server's internal directory.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_attachments.8
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_attachments.8 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_attachments.8 2012-12-20 21:15:56 UTC (rev 10192)
@@ -23,16 +23,19 @@
.Sh SYNOPSIS
.Nm
.Op Fl -config Ar file
+.Op Fl -uuid Ar guid
+.Op Fl -days Ar NUMBER
.Op Fl -dry-run
.Op Fl -verbose
.Op Fl -help
.Sh DESCRIPTION
.Nm
is a tool for removing attachments that are no longer referenced by
-any calendar events.
+any calendar events, or only referenced by events older than a specified
+cut-off.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
@@ -42,6 +45,10 @@
.It Fl f, -config Ar FILE
Use the Calendar Server configuration specified in the given file.
Defaults to /etc/caldavd/caldavd.plist.
+.It Fl u, -uuid Ar GUID
+Target a specific user via their GUID.
+.It Fl d, -days Ar NUMBER
+Specify how many days in the past to retain. Defaults to 365 days.
.It Fl n, -dry-run
Calculate and display how many orphaned attachments would be removed,
but don't actually remove them.
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_events.8
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_events.8 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_events.8 2012-12-20 21:15:56 UTC (rev 10192)
@@ -36,7 +36,7 @@
removed.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_principals.8
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_principals.8 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/doc/calendarserver_purge_principals.8 2012-12-20 21:15:56 UTC (rev 10192)
@@ -36,7 +36,7 @@
events in the past are retained, but any ongoing events are canceled.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/twext/enterprise/dal/syntax.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/twext/enterprise/dal/syntax.py 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/twext/enterprise/dal/syntax.py 2012-12-20 21:15:56 UTC (rev 10192)
@@ -41,6 +41,8 @@
except ImportError:
cx_Oracle = None
+
+
class DALError(Exception):
"""
Base class for exceptions raised by this module. This can be raised
@@ -327,8 +329,8 @@
# 0'.)
__add__ = comparison("+")
__sub__ = comparison("-")
- __div__= comparison("/")
- __mul__= comparison("*")
+ __div__ = comparison("/")
+ __mul__ = comparison("*")
def __nonzero__(self):
@@ -365,6 +367,8 @@
def Contains(self, other):
return CompoundComparison(self, "like", CompoundComparison(Constant('%'), '||', CompoundComparison(Constant(other), '||', Constant('%'))))
+
+
class FunctionInvocation(ExpressionSyntax):
def __init__(self, function, *args):
self.function = function
@@ -453,6 +457,7 @@
Count = Function("count")
+Sum = Function("sum")
Max = Function("max")
Len = Function("character_length", "length")
Upper = Function("upper")
@@ -780,23 +785,28 @@
return self.model.table.name + '.' + name
+
class ResultAliasSyntax(ExpressionSyntax):
def __init__(self, expression, alias=None):
self.expression = expression
self.alias = alias
+
def aliasName(self, queryGenerator):
if self.alias is None:
self.alias = queryGenerator.nextGeneratedID()
return self.alias
+
def columnReference(self):
return AliasReferenceSyntax(self)
+
def allColumns(self):
return self.expression.allColumns()
+
def subSQL(self, queryGenerator, allTables):
result = SQLFragment()
result.append(self.expression.subSQL(queryGenerator, allTables))
@@ -804,18 +814,22 @@
return result
+
class AliasReferenceSyntax(ExpressionSyntax):
def __init__(self, resultAlias):
self.resultAlias = resultAlias
+
def allColumns(self):
return self.resultAlias.allColumns()
+
def subSQL(self, queryGenerator, allTables):
return SQLFragment(self.resultAlias.aliasName(queryGenerator))
+
class AliasedColumnSyntax(ColumnSyntax):
"""
An L{AliasedColumnSyntax} is like a L{ColumnSyntax}, but it generates SQL
@@ -898,9 +912,9 @@
def subSQL(self, queryGenerator, allTables):
- if ( queryGenerator.dialect == ORACLE_DIALECT
+ if (queryGenerator.dialect == ORACLE_DIALECT
and isinstance(self.b, Constant) and self.b.value == ''
- and self.op in ('=', '!=') ):
+ and self.op in ('=', '!=')):
return NullComparison(self.a, self.op).subSQL(queryGenerator, allTables)
stmt = SQLFragment()
result = self._subexpression(self.a, queryGenerator, allTables)
@@ -948,6 +962,7 @@
def __init__(self):
self.name = "*"
+
def allColumns(self):
return []
@@ -1024,6 +1039,7 @@
return self.columns
+
class SetExpression(object):
"""
A UNION, INTERSECT, or EXCEPT construct used inside a SELECT.
@@ -1052,6 +1068,7 @@
if self.optype not in (None, SetExpression.OPTYPE_ALL, SetExpression.OPTYPE_DISTINCT,):
raise DALError("Must have either 'all' or 'distinct' in a set expression")
+
def subSQL(self, queryGenerator, allTables):
result = SQLFragment()
for select in self.selects:
@@ -1063,9 +1080,12 @@
result.append(select.subSQL(queryGenerator, allTables))
return result
+
def allColumns(self):
return []
+
+
class Union(SetExpression):
"""
A UNION construct used inside a SELECT.
@@ -1073,6 +1093,8 @@
def setOpSQL(self, queryGenerator):
return SQLFragment(" UNION ")
+
+
class Intersect(SetExpression):
"""
An INTERSECT construct used inside a SELECT.
@@ -1080,6 +1102,8 @@
def setOpSQL(self, queryGenerator):
return SQLFragment(" INTERSECT ")
+
+
class Except(SetExpression):
"""
An EXCEPT construct used inside a SELECT.
@@ -1092,6 +1116,8 @@
else:
raise NotImplementedError("Unsupported dialect")
+
+
class Select(_Statement):
"""
'select' statement.
@@ -1131,6 +1157,7 @@
if self.From.As is None:
self.From.As = ""
+
def __eq__(self, other):
"""
Create a comparison.
@@ -1238,6 +1265,7 @@
for column in self.columns.columns:
yield column
+
def tables(self):
"""
Determine the tables used by the result columns.
@@ -1255,6 +1283,7 @@
return [TableSyntax(table) for table in tables]
+
def _commaJoined(stmts):
first = True
cstatement = SQLFragment()
@@ -1663,6 +1692,7 @@
return SQLFragment('savepoint %s' % (self.name,))
+
class RollbackToSavepoint(_LockingStatement):
"""
An SQL 'rollback to savepoint' statement.
@@ -1676,6 +1706,7 @@
return SQLFragment('rollback to savepoint %s' % (self.name,))
+
class ReleaseSavepoint(_LockingStatement):
"""
An SQL 'release savepoint' statement.
@@ -1821,4 +1852,3 @@
# (Although this is a special keyword in a CREATE statement, in an INSERT it
# behaves like an expression to the best of my knowledge.)
default = NamedValue('default')
-
Modified: CalendarServer/branches/release/CalendarServer-4.3-dev/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.3-dev/txdav/common/datastore/sql.py 2012-12-19 15:07:20 UTC (rev 10191)
+++ CalendarServer/branches/release/CalendarServer-4.3-dev/txdav/common/datastore/sql.py 2012-12-20 21:15:56 UTC (rev 10192)
@@ -70,7 +70,7 @@
from twext.enterprise.dal.syntax import \
Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction, \
- Select, Update, ColumnSyntax, TableSyntax, Upper, Count, ALL_COLUMNS
+ Select, Update, ColumnSyntax, TableSyntax, Upper, Count, ALL_COLUMNS, Sum
from twistedcaldav.config import config
@@ -781,7 +781,7 @@
Execute some SQL (delegate to L{IAsyncTransaction}).
"""
if self._stats:
- statsContext = self._stats.startStatement(a[0], a[1])
+ statsContext = self._stats.startStatement(a[0], a[1] if len(a) > 1 else ())
self.currentStatement = a[0]
if self._store.logTransactionWaits and a[0].split(" ", 1)[0].lower() in ("insert", "update", "delete",):
self.iudCount += 1
@@ -790,7 +790,7 @@
a = ("-- Label: %s\n" % (self._label.replace("%", "%%"),) + a[0],) + a[1:]
if self._store.logSQL:
log.error("SQL: %r %r" % (a, kw,))
- results = ()
+ results = None
try:
results = (yield self._sqlTxn.execSQL(*a, **kw))
finally:
@@ -921,41 +921,72 @@
returnValue(count)
- def _orphanedBase(limited): #@NoSelf
+ def _orphanedSummary(self, uuid, limited):
at = schema.ATTACHMENT
co = schema.CALENDAR_OBJECT
+ ch = schema.CALENDAR_HOME
+ chm = schema.CALENDAR_HOME_METADATA
+
kwds = {}
if limited:
kwds["Limit"] = Parameter('batchSize')
+
+ where = co.DROPBOX_ID == None
+ if uuid:
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
return Select(
- [at.DROPBOX_ID, at.PATH],
- From=at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer"),
- Where=co.DROPBOX_ID == None,
+ [ch.OWNER_UID, chm.QUOTA_USED_BYTES, Sum(at.SIZE), Count(at.DROPBOX_ID)],
+ From=at.join(
+ co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer").join(
+ ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID).join(
+ chm, ch.RESOURCE_ID == chm.RESOURCE_ID
+ ),
+ Where=where,
+ GroupBy=(ch.OWNER_UID, chm.QUOTA_USED_BYTES),
**kwds
)
- _orphanedLimited = _orphanedBase(True)
- _orphanedUnlimited = _orphanedBase(False)
- del _orphanedBase
-
- def orphanedAttachments(self, batchSize=None):
+ def orphanedAttachments(self, uuid=None, batchSize=None):
"""
Find attachments no longer referenced by any events.
- Returns a deferred to a list of (dropbox_id, path) tuples.
+ Returns a deferred to a list of (calendar_home_owner_uid, quota used, total orphan size, total orphan count) tuples.
"""
+ kwds = {}
+ if uuid:
+ kwds["uuid"] = uuid
if batchSize is not None:
- kwds = {'batchSize': batchSize}
- query = self._orphanedLimited
- else:
- kwds = {}
- query = self._orphanedUnlimited
- return query.on(self, **kwds)
+ kwds["batchSize"] = batchSize
+ return self._orphanedSummary(uuid, batchSize is not None).on(self, **kwds)
+ def _orphanedBase(self, uuid, limited):
+ ch = schema.CALENDAR_HOME
+ at = schema.ATTACHMENT
+ co = schema.CALENDAR_OBJECT
+
+ kwds = {}
+ if limited:
+ kwds["Limit"] = Parameter('batchSize')
+
+ sfrom = at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer")
+ where = co.DROPBOX_ID == None
+ if uuid:
+ sfrom = sfrom.join(ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID)
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
+ return Select(
+ [at.DROPBOX_ID, at.PATH],
+ From=sfrom,
+ Where=where,
+ **kwds
+ )
+
+
@inlineCallbacks
- def removeOrphanedAttachments(self, batchSize=None):
+ def removeOrphanedAttachments(self, uuid=None, batchSize=None):
"""
Remove attachments that no longer have any references to them
"""
@@ -963,16 +994,125 @@
# TODO: see if there is a better way to import Attachment
from txdav.caldav.datastore.sql import Attachment
- results = (yield self.orphanedAttachments(batchSize=batchSize))
+ kwds = {}
+ if uuid:
+ kwds["uuid"] = uuid
+ if batchSize is not None:
+ kwds["batchSize"] = batchSize
+ results = (yield self._orphanedBase(uuid, batchSize is not None).on(self, **kwds))
+
count = 0
for dropboxID, path in results:
attachment = (yield Attachment.loadWithName(self, dropboxID, path))
- (yield attachment.remove())
+ yield attachment.remove()
count += 1
returnValue(count)
+ def _oldAttachmentsSummaryBase(self, uuid, limited):
+ ch = schema.CALENDAR_HOME
+ chm = schema.CALENDAR_HOME_METADATA
+ co = schema.CALENDAR_OBJECT
+ tr = schema.TIME_RANGE
+ at = schema.ATTACHMENT
+ kwds = {}
+ if limited:
+ kwds["Limit"] = Parameter('batchSize')
+
+ where = co.DROPBOX_ID == Select(
+ [at.DROPBOX_ID],
+ From=at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "inner").join(
+ tr, co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID
+ ),
+ GroupBy=(at.DROPBOX_ID,),
+ Having=Max(tr.END_DATE) < Parameter("CutOff"),
+ )
+
+ if uuid:
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
+ return Select(
+ [ch.OWNER_UID, chm.QUOTA_USED_BYTES, Sum(at.SIZE), Count(at.DROPBOX_ID)],
+ From=at.join(
+ co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer").join(
+ ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID).join(
+ chm, ch.RESOURCE_ID == chm.RESOURCE_ID
+ ),
+ Where=where,
+ GroupBy=(ch.OWNER_UID, chm.QUOTA_USED_BYTES),
+ **kwds
+ )
+
+
+ def oldAttachments(self, cutoff, uuid, batchSize=None):
+ """
+ Find attachments attached to only events whose last instance is older than the specified cut-off.
+
+ Returns a deferred to a list of (calendar_home_owner_uid, quota used, total old size, total old count) tuples.
+ """
+ kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ if uuid:
+ kwds["uuid"] = uuid
+ if batchSize is not None:
+ kwds["batchSize"] = batchSize
+ return self._oldAttachmentsSummaryBase(uuid, batchSize is not None).on(self, **kwds)
+
+
+ def _oldAttachmentsBase(self, uuid, limited):
+ ch = schema.CALENDAR_HOME
+ co = schema.CALENDAR_OBJECT
+ tr = schema.TIME_RANGE
+ at = schema.ATTACHMENT
+
+ kwds = {}
+ if limited:
+ kwds["Limit"] = Parameter('batchSize')
+
+ sfrom = at.join(
+ co, at.DROPBOX_ID == co.DROPBOX_ID, "inner").join(
+ tr, co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID
+ )
+ where = None
+ if uuid:
+ sfrom = sfrom.join(ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID)
+ where = (ch.OWNER_UID == Parameter('uuid'))
+
+ return Select(
+ [at.DROPBOX_ID, at.PATH, ],
+ From=sfrom,
+ Where=where,
+ GroupBy=(at.DROPBOX_ID, at.PATH,),
+ Having=Max(tr.END_DATE) < Parameter("CutOff"),
+ **kwds
+ )
+
+
+ @inlineCallbacks
+ def removeOldAttachments(self, cutoff, uuid, batchSize=None):
+ """
+ Remove attachments attached to events in the past.
+ """
+
+ # TODO: see if there is a better way to import Attachment
+ from txdav.caldav.datastore.sql import Attachment
+
+ kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ if uuid:
+ kwds["uuid"] = uuid
+ if batchSize is not None:
+ kwds["batchSize"] = batchSize
+ results = (yield self._oldAttachmentsBase(uuid, batchSize is not None).on(self, **kwds))
+
+ count = 0
+ for dropboxID, path in results:
+ attachment = (yield Attachment.loadWithName(self, dropboxID, path))
+ yield attachment.remove()
+ count += 1
+ returnValue(count)
+
+
+
class _EmptyCacher(object):
def set(self, key, value):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121220/891f9732/attachment-0001.html>
More information about the calendarserver-changes
mailing list