[CalendarServer-changes] [10203] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jan 4 11:27:30 PST 2013
Revision: 10203
http://trac.calendarserver.org//changeset/10203
Author: cdaboo at apple.com
Date: 2013-01-04 11:27:30 -0800 (Fri, 04 Jan 2013)
Log Message:
-----------
Purge tools can now purge a specific user or attachments on "old" events.
Modified Paths:
--------------
CalendarServer/trunk/bin/_calendarserver_preamble.py
CalendarServer/trunk/bin/calendarserver_purge_attachments
CalendarServer/trunk/bin/calendarserver_purge_events
CalendarServer/trunk/bin/calendarserver_purge_principals
CalendarServer/trunk/calendarserver/tools/gateway.py
CalendarServer/trunk/calendarserver/tools/purge.py
CalendarServer/trunk/calendarserver/tools/shell/cmd.py
CalendarServer/trunk/calendarserver/tools/test/test_purge.py
CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
CalendarServer/trunk/doc/calendarserver_manage_principals.8
CalendarServer/trunk/doc/calendarserver_manage_push.8
CalendarServer/trunk/doc/calendarserver_migrate_resources.8
CalendarServer/trunk/doc/calendarserver_purge_attachments.8
CalendarServer/trunk/doc/calendarserver_purge_events.8
CalendarServer/trunk/doc/calendarserver_purge_principals.8
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/sql.py
Property Changed:
----------------
CalendarServer/trunk/
Property changes on: CalendarServer/trunk
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
Modified: CalendarServer/trunk/bin/_calendarserver_preamble.py
===================================================================
--- CalendarServer/trunk/bin/_calendarserver_preamble.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/bin/_calendarserver_preamble.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -42,10 +42,13 @@
return
child = Popen((run, "-e"), stdout=PIPE)
- stdout, stderr = child.communicate()
+ stdout, _ignore_stderr = child.communicate()
stdout = stdout.rstrip("\n")
- evars = eval(stdout)
+ try:
+ evars = eval(stdout)
+ except SyntaxError:
+ return
os.environ.update(evars)
# PYTHONPATH needs special treatment, because Python has already processed
@@ -69,4 +72,3 @@
bootstrapFromRun()
-
Modified: CalendarServer/trunk/bin/calendarserver_purge_attachments
===================================================================
--- CalendarServer/trunk/bin/calendarserver_purge_attachments 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/bin/calendarserver_purge_attachments 2013-01-04 19:27:30 UTC (rev 10203)
@@ -29,5 +29,5 @@
except ImportError:
sys.exc_clear()
- from calendarserver.tools.purge import main_purge_orphaned_attachments
- main_purge_orphaned_attachments()
+ from calendarserver.tools.purge import PurgeAttachmentsService
+ PurgeAttachmentsService.main()
Modified: CalendarServer/trunk/bin/calendarserver_purge_events
===================================================================
--- CalendarServer/trunk/bin/calendarserver_purge_events 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/bin/calendarserver_purge_events 2013-01-04 19:27:30 UTC (rev 10203)
@@ -29,5 +29,5 @@
except ImportError:
sys.exc_clear()
- from calendarserver.tools.purge import main_purge_events
- main_purge_events()
+ from calendarserver.tools.purge import PurgeOldEventsService
+ PurgeOldEventsService.main()
Modified: CalendarServer/trunk/bin/calendarserver_purge_principals
===================================================================
--- CalendarServer/trunk/bin/calendarserver_purge_principals 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/bin/calendarserver_purge_principals 2013-01-04 19:27:30 UTC (rev 10203)
@@ -29,5 +29,5 @@
except ImportError:
sys.exc_clear()
- from calendarserver.tools.purge import main_purge_principals
- main_purge_principals()
+ from calendarserver.tools.purge import PurgePrincipalService
+ PurgePrincipalService.main()
Modified: CalendarServer/trunk/calendarserver/tools/gateway.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/gateway.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/calendarserver/tools/gateway.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -31,7 +31,7 @@
principalForPrincipalID, proxySubprincipal, addProxy, removeProxy,
getProxies, setProxies, ProxyError, ProxyWarning, updateRecord
)
-from calendarserver.tools.purge import WorkerService, purgeOldEvents, DEFAULT_BATCH_SIZE, DEFAULT_RETAIN_DAYS
+from calendarserver.tools.purge import WorkerService, PurgeOldEventsService, DEFAULT_BATCH_SIZE, DEFAULT_RETAIN_DAYS
from calendarserver.tools.cmdline import utilityMain
from twext.python.log import StandardIOObserver
@@ -57,6 +57,7 @@
sys.exit(0)
+
class RunnerService(WorkerService):
"""
A wrapper around Runner which uses utilityMain to get the store
@@ -73,13 +74,14 @@
directory = rootResource.getDirectory()
runner = Runner(rootResource, directory, self._store, self.commands)
if runner.validate():
- yield runner.run( )
+ yield runner.run()
+
def main():
try:
- (optargs, args) = getopt(
+ (optargs, _ignore_args) = getopt(
sys.argv[1:], "hef:", [
"help",
"error",
@@ -108,7 +110,6 @@
else:
raise NotImplementedError(opt)
-
#
# Read commands from stdin
#
@@ -158,10 +159,11 @@
self.store = store
self.commands = commands
+
def validate(self):
# Make sure commands are valid
for command in self.commands:
- if not command.has_key('command'):
+ if 'command' not in command:
respondWithError("'command' missing from plist")
return False
commandName = command['command']
@@ -171,6 +173,7 @@
return False
return True
+
@inlineCallbacks
def run(self):
try:
@@ -188,14 +191,16 @@
# Locations
+
def command_getLocationList(self, command):
respondWithRecordsOfType(self.dir, command, "locations")
+
@inlineCallbacks
def command_createLocation(self, command):
kwargs = {}
for key, info in attrMap.iteritems():
- if command.has_key(key):
+ if key in command:
kwargs[info['attr']] = command[key]
try:
@@ -211,6 +216,7 @@
respondWithRecordsOfType(self.dir, command, "locations")
+
@inlineCallbacks
def command_getLocationAttributes(self, command):
guid = command['GeneratedUID']
@@ -243,7 +249,7 @@
kwargs = {}
for key, info in attrMap.iteritems():
- if command.has_key(key):
+ if key in command:
kwargs[info['attr']] = command[key]
try:
record = (yield updateRecord(False, self.dir, "locations", **kwargs))
@@ -258,10 +264,11 @@
yield self.command_getLocationAttributes(command)
+
def command_deleteLocation(self, command):
kwargs = {}
for key, info in attrMap.iteritems():
- if command.has_key(key):
+ if key in command:
kwargs[info['attr']] = command[key]
try:
self.dir.destroyRecord("locations", **kwargs)
@@ -272,14 +279,16 @@
# Resources
+
def command_getResourceList(self, command):
respondWithRecordsOfType(self.dir, command, "resources")
+
@inlineCallbacks
def command_createResource(self, command):
kwargs = {}
for key, info in attrMap.iteritems():
- if command.has_key(key):
+ if key in command:
kwargs[info['attr']] = command[key]
try:
@@ -295,6 +304,7 @@
respondWithRecordsOfType(self.dir, command, "resources")
+
@inlineCallbacks
def command_setResourceAttributes(self, command):
@@ -307,7 +317,7 @@
kwargs = {}
for key, info in attrMap.iteritems():
- if command.has_key(key):
+ if key in command:
kwargs[info['attr']] = command[key]
try:
record = (yield updateRecord(False, self.dir, "resources", **kwargs))
@@ -322,10 +332,11 @@
yield self.command_getResourceAttributes(command)
+
def command_deleteResource(self, command):
kwargs = {}
for key, info in attrMap.iteritems():
- if command.has_key(key):
+ if key in command:
kwargs[info['attr']] = command[key]
try:
self.dir.destroyRecord("resources", **kwargs)
@@ -336,6 +347,7 @@
# Proxies
+
@inlineCallbacks
def command_listWriteProxies(self, command):
principal = principalForPrincipalID(command['Principal'], directory=self.dir)
@@ -344,6 +356,7 @@
return
(yield respondWithProxies(self.dir, command, principal, "write"))
+
@inlineCallbacks
def command_addWriteProxy(self, command):
principal = principalForPrincipalID(command['Principal'],
@@ -365,6 +378,7 @@
pass
(yield respondWithProxies(self.dir, command, principal, "write"))
+
@inlineCallbacks
def command_removeWriteProxy(self, command):
principal = principalForPrincipalID(command['Principal'], directory=self.dir)
@@ -384,6 +398,7 @@
pass
(yield respondWithProxies(self.dir, command, principal, "write"))
+
@inlineCallbacks
def command_listReadProxies(self, command):
principal = principalForPrincipalID(command['Principal'], directory=self.dir)
@@ -392,6 +407,7 @@
return
(yield respondWithProxies(self.dir, command, principal, "read"))
+
@inlineCallbacks
def command_addReadProxy(self, command):
principal = principalForPrincipalID(command['Principal'], directory=self.dir)
@@ -411,6 +427,7 @@
pass
(yield respondWithProxies(self.dir, command, principal, "read"))
+
@inlineCallbacks
def command_removeReadProxy(self, command):
principal = principalForPrincipalID(command['Principal'], directory=self.dir)
@@ -444,10 +461,11 @@
cutoff = PyCalendarDateTime.getToday()
cutoff.setDateOnly(False)
cutoff.offsetDay(-retainDays)
- eventCount = (yield purgeOldEvents(self.store, self.dir, self.root, cutoff, DEFAULT_BATCH_SIZE))
+ eventCount = (yield PurgeOldEventsService.purgeOldEvents(self.store, cutoff, DEFAULT_BATCH_SIZE))
respond(command, {'EventsRemoved' : eventCount, "RetainDays" : retainDays})
+
@inlineCallbacks
def respondWithProxies(directory, command, principal, proxyType):
proxies = []
@@ -464,6 +482,7 @@
})
+
def recordToDict(record):
recordDict = {}
for key, info in attrMap.iteritems():
@@ -479,6 +498,8 @@
pass
return recordDict
+
+
def respondWithRecordsOfType(directory, command, recordType):
result = []
for record in directory.listRecords(recordType):
@@ -486,11 +507,15 @@
result.append(recordDict)
respond(command, result)
+
+
def respond(command, result):
- sys.stdout.write(writePlistToString( { 'command' : command['command'], 'result' : result } ) )
+ sys.stdout.write(writePlistToString({'command' : command['command'], 'result' : result}))
+
+
def respondWithError(msg, status=1):
- sys.stdout.write(writePlistToString( { 'error' : msg, } ) )
+ sys.stdout.write(writePlistToString({'error' : msg, }))
"""
try:
reactor.stop()
Modified: CalendarServer/trunk/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/purge.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/calendarserver/tools/purge.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -16,21 +16,24 @@
# limitations under the License.
##
-import os
-import sys
+from calendarserver.tap.util import FakeRequest
+from calendarserver.tap.util import getRootResource
+from calendarserver.tools import tables
+from calendarserver.tools.cmdline import utilityMain
+from calendarserver.tools.principals import removeProxy
+
from errno import ENOENT, EACCES
from getopt import getopt, GetoptError
from pycalendar.datetime import PyCalendarDateTime
+from twext.python.log import Logger
+from twext.web2.responsecode import NO_CONTENT
+
from twisted.application.service import Service
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
-from twext.python.log import Logger
-from txdav.xml import element as davxml
-from twext.web2.responsecode import NO_CONTENT
-
from twistedcaldav import caldavxml
from twistedcaldav.caldavxml import TimeRange
from twistedcaldav.config import config, ConfigurationError
@@ -39,90 +42,17 @@
from twistedcaldav.method.put_common import StoreCalendarObjectResource
from twistedcaldav.query import calendarqueryfilter
-from calendarserver.tap.util import FakeRequest
-from calendarserver.tap.util import getRootResource
+from txdav.xml import element as davxml
-from calendarserver.tools.cmdline import utilityMain
-from calendarserver.tools.principals import removeProxy
-from calendarserver.tools import tables
-
import collections
+import os
+import sys
log = Logger()
DEFAULT_BATCH_SIZE = 100
DEFAULT_RETAIN_DAYS = 365
-def usage_purge_events(e=None):
-
- name = os.path.basename(sys.argv[0])
- print "usage: %s [options]" % (name,)
- print ""
- print " Remove old events from the calendar server"
- print ""
- print "options:"
- print " -h --help: print this help and exit"
- print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -d --days <number>: specify how many days in the past to retain (default=%d)" % (DEFAULT_RETAIN_DAYS,)
- #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
- print " -n --dry-run: calculate how many events to purge, but do not purge data"
- print " -v --verbose: print progress information"
- print ""
-
- if e:
- sys.stderr.write("%s\n" % (e,))
- sys.exit(64)
- else:
- sys.exit(0)
-
-
-
-def usage_purge_orphaned_attachments(e=None):
-
- name = os.path.basename(sys.argv[0])
- print "usage: %s [options]" % (name,)
- print ""
- print " Remove orphaned attachments from the calendar server"
- print ""
- print "options:"
- print " -h --help: print this help and exit"
- print " -f --config <path>: Specify caldavd.plist configuration path"
- #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
- print " -n --dry-run: calculate how many attachments to purge, but do not purge data"
- print " -v --verbose: print progress information"
- print ""
-
- if e:
- sys.stderr.write("%s\n" % (e,))
- sys.exit(64)
- else:
- sys.exit(0)
-
-
-
-def usage_purge_principal(e=None):
-
- name = os.path.basename(sys.argv[0])
- print "usage: %s [options]" % (name,)
- print ""
- print " Remove a principal's events and contacts from the calendar server"
- print ""
- print "options:"
- print " -c --completely: By default, only future events are canceled; this option cancels all events"
- print " -h --help: print this help and exit"
- print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -n --dry-run: calculate how many events and contacts to purge, but do not purge data"
- print " -v --verbose: print progress information"
- print ""
-
- if e:
- sys.stderr.write("%s\n" % (e,))
- sys.exit(64)
- else:
- sys.exit(0)
-
-
-
class WorkerService(Service):
def __init__(self, store):
@@ -173,767 +103,1035 @@
dryrun = False
verbose = False
- def doWork(self):
- rootResource = self.rootResource()
- directory = rootResource.getDirectory()
- return purgeOldEvents(self._store, directory, rootResource,
- self.cutoff, self.batchSize, verbose=self.verbose,
- dryrun=self.dryrun)
+ @classmethod
+ def usage(cls, e=None):
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print " Remove old events from the calendar server"
+ print ""
+ print "options:"
+ print " -h --help: print this help and exit"
+ print " -f --config <path>: Specify caldavd.plist configuration path"
+ print " -d --days <number>: specify how many days in the past to retain (default=%d)" % (DEFAULT_RETAIN_DAYS,)
+ #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ print " -n --dry-run: calculate how many events to purge, but do not purge data"
+ print " -v --verbose: print progress information"
+ print ""
+ if e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(64)
+ else:
+ sys.exit(0)
-class PurgeOrphanedAttachmentsService(WorkerService):
- batchSize = None
- dryrun = False
- verbose = False
+ @classmethod
+ def main(cls):
- def doWork(self):
- return purgeOrphanedAttachments(
- self._store, self.batchSize,
- verbose=self.verbose, dryrun=self.dryrun)
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "d:b:f:hnv", [
+ "days=",
+ "batch=",
+ "dry-run",
+ "config=",
+ "help",
+ "verbose",
+ ],
+ )
+ except GetoptError, e:
+ cls.usage(e)
+ #
+ # Get configuration
+ #
+ configFileName = None
+ days = DEFAULT_RETAIN_DAYS
+ batchSize = DEFAULT_BATCH_SIZE
+ dryrun = False
+ verbose = False
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ cls.usage()
-class PurgePrincipalService(WorkerService):
+ elif opt in ("-d", "--days"):
+ try:
+ days = int(arg)
+ except ValueError, e:
+ print "Invalid value for --days: %s" % (arg,)
+ cls.usage(e)
- uids = None
- dryrun = False
- verbose = False
- completely = False
+ elif opt in ("-b", "--batch"):
+ try:
+ batchSize = int(arg)
+ except ValueError, e:
+ print "Invalid value for --batch: %s" % (arg,)
+ cls.usage(e)
- @inlineCallbacks
- def doWork(self):
- rootResource = self.rootResource()
- directory = rootResource.getDirectory()
- total = (yield purgeUIDs(self._store, directory, rootResource, self.uids,
- verbose=self.verbose, dryrun=self.dryrun,
- completely=self.completely, doimplicit=self.doimplicit))
- if self.verbose:
- amount = "%d event%s" % (total, "s" if total > 1 else "")
- if self.dryrun:
- print "Would have modified or deleted %s" % (amount,)
- else:
- print "Modified or deleted %s" % (amount,)
+ elif opt in ("-v", "--verbose"):
+ verbose = True
+ elif opt in ("-n", "--dry-run"):
+ dryrun = True
+ elif opt in ("-f", "--config"):
+ configFileName = arg
-def main_purge_events():
+ else:
+ raise NotImplementedError(opt)
- try:
- (optargs, args) = getopt(
- sys.argv[1:], "d:b:f:hnv", [
- "days=",
- "batch=",
- "dry-run",
- "config=",
- "help",
- "verbose",
- ],
- )
- except GetoptError, e:
- usage_purge_events(e)
+ if args:
+ cls.usage("Too many arguments: %s" % (args,))
- #
- # Get configuration
- #
- configFileName = None
- days = DEFAULT_RETAIN_DAYS
- batchSize = DEFAULT_BATCH_SIZE
- dryrun = False
- verbose = False
+ if dryrun:
+ verbose = True
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage_purge_events()
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-days)
+ cls.cutoff = cutoff
+ cls.batchSize = batchSize
+ cls.dryrun = dryrun
+ cls.verbose = verbose
- elif opt in ("-d", "--days"):
- try:
- days = int(arg)
- except ValueError, e:
- print "Invalid value for --days: %s" % (arg,)
- usage_purge_events(e)
+ utilityMain(
+ configFileName,
+ cls,
+ )
- elif opt in ("-b", "--batch"):
- try:
- batchSize = int(arg)
- except ValueError, e:
- print "Invalid value for --batch: %s" % (arg,)
- usage_purge_events(e)
- elif opt in ("-v", "--verbose"):
- verbose = True
+ @classmethod
+ @inlineCallbacks
+ def purgeOldEvents(cls, store, cutoff, batchSize, verbose=False, dryrun=False):
- elif opt in ("-n", "--dry-run"):
- dryrun = True
+ service = cls(store)
+ service.cutoff = cutoff
+ service.batchSize = batchSize
+ service.dryrun = dryrun
+ service.verbose = verbose
+ result = (yield service.doWork())
+ returnValue(result)
- elif opt in ("-f", "--config"):
- configFileName = arg
- else:
- raise NotImplementedError(opt)
+ @inlineCallbacks
+ def doWork(self):
- if args:
- usage_purge_events("Too many arguments: %s" % (args,))
+ if self.dryrun:
+ if self.verbose:
+ print "(Dry run) Searching for old events..."
+ txn = self._store.newTransaction(label="Find old events")
+ oldEvents = (yield txn.eventsOlderThan(self.cutoff))
+ eventCount = len(oldEvents)
+ if self.verbose:
+ if eventCount == 0:
+ print "No events are older than %s" % (self.cutoff,)
+ elif eventCount == 1:
+ print "1 event is older than %s" % (self.cutoff,)
+ else:
+ print "%d events are older than %s" % (eventCount, self.cutoff)
+ returnValue(eventCount)
- if dryrun:
- verbose = True
+ if self.verbose:
+ print "Removing events older than %s..." % (self.cutoff,)
- cutoff = PyCalendarDateTime.getToday()
- cutoff.setDateOnly(False)
- cutoff.offsetDay(-days)
- PurgeOldEventsService.cutoff = cutoff
- PurgeOldEventsService.batchSize = batchSize
- PurgeOldEventsService.dryrun = dryrun
- PurgeOldEventsService.verbose = verbose
+ numEventsRemoved = -1
+ totalRemoved = 0
+ while numEventsRemoved:
+ txn = self._store.newTransaction(label="Remove old events")
+ numEventsRemoved = (yield txn.removeOldEvents(self.cutoff, batchSize=self.batchSize))
+ (yield txn.commit())
+ if numEventsRemoved:
+ totalRemoved += numEventsRemoved
+ if self.verbose:
+ print "%d," % (totalRemoved,),
- utilityMain(
- configFileName,
- PurgeOldEventsService,
- )
+ if self.verbose:
+ print
+ if totalRemoved == 0:
+ print "No events were removed"
+ elif totalRemoved == 1:
+ print "1 event was removed in total"
+ else:
+ print "%d events were removed in total" % (totalRemoved,)
+ returnValue(totalRemoved)
-def main_purge_orphaned_attachments():
- try:
- (optargs, args) = getopt(
- sys.argv[1:], "d:b:f:hnv", [
- "batch=",
- "dry-run",
- "config=",
- "help",
- "verbose",
- ],
- )
- except GetoptError, e:
- usage_purge_orphaned_attachments(e)
+class PurgeAttachmentsService(WorkerService):
- #
- # Get configuration
- #
- configFileName = None
- batchSize = DEFAULT_BATCH_SIZE
+ uuid = None
+ cutoff = None
+ batchSize = None
dryrun = False
verbose = False
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage_purge_orphaned_attachments()
+ @classmethod
+ def usage(cls, e=None):
- elif opt in ("-b", "--batch"):
- try:
- batchSize = int(arg)
- except ValueError, e:
- print "Invalid value for --batch: %s" % (arg,)
- usage_purge_orphaned_attachments(e)
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print " Remove old or orphaned attachments from the calendar server"
+ print ""
+ print "options:"
+ print " -h --help: print this help and exit"
+ print " -f --config <path>: Specify caldavd.plist configuration path"
+ print " -u --uuid <owner uid>: target a specific user UID"
+ #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ print " -d --days <number>: specify how many days in the past to retain (default=%d) zero means no removal of old attachments" % (DEFAULT_RETAIN_DAYS,)
+ print " -n --dry-run: calculate how many attachments to purge, but do not purge data"
+ print " -v --verbose: print progress information"
+ print ""
- elif opt in ("-v", "--verbose"):
- verbose = True
+ if e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(64)
+ else:
+ sys.exit(0)
- elif opt in ("-n", "--dry-run"):
- dryrun = True
- elif opt in ("-f", "--config"):
- configFileName = arg
+ @classmethod
+ def main(cls):
- else:
- raise NotImplementedError(opt)
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "d:b:f:hnu:v", [
+ "uuid=",
+ "days=",
+ "batch=",
+ "dry-run",
+ "config=",
+ "help",
+ "verbose",
+ ],
+ )
+ except GetoptError, e:
+ cls.usage(e)
- if args:
- usage_purge_orphaned_attachments("Too many arguments: %s" % (args,))
+ #
+ # Get configuration
+ #
+ configFileName = None
+ uuid = None
+ days = DEFAULT_RETAIN_DAYS
+ batchSize = DEFAULT_BATCH_SIZE
+ dryrun = False
+ verbose = False
- if dryrun:
- verbose = True
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ cls.usage()
- PurgeOrphanedAttachmentsService.batchSize = batchSize
- PurgeOrphanedAttachmentsService.dryrun = dryrun
- PurgeOrphanedAttachmentsService.verbose = verbose
+ elif opt in ("-u", "--uuid"):
+ uuid = arg
- utilityMain(
- configFileName,
- PurgeOrphanedAttachmentsService,
- )
+ elif opt in ("-d", "--days"):
+ try:
+ days = int(arg)
+ except ValueError, e:
+ print "Invalid value for --days: %s" % (arg,)
+ cls.usage(e)
+ elif opt in ("-b", "--batch"):
+ try:
+ batchSize = int(arg)
+ except ValueError, e:
+ print "Invalid value for --batch: %s" % (arg,)
+ cls.usage(e)
+ elif opt in ("-v", "--verbose"):
+ verbose = True
-def main_purge_principals():
+ elif opt in ("-n", "--dry-run"):
+ dryrun = True
- try:
- (optargs, args) = getopt(
- sys.argv[1:], "cf:hnv", [
- "completely",
- "dry-run",
- "config=",
- "help",
- "verbose",
- "noimplicit",
- ],
- )
- except GetoptError, e:
- usage_purge_principal(e)
+ elif opt in ("-f", "--config"):
+ configFileName = arg
- #
- # Get configuration
- #
- configFileName = None
- dryrun = False
- verbose = False
- completely = False
- doimplicit = True
+ else:
+ raise NotImplementedError(opt)
- for opt, arg in optargs:
- if opt in ("-h", "--help"):
- usage_purge_principal()
+ if args:
+ cls.usage("Too many arguments: %s" % (args,))
- elif opt in ("-c", "--completely"):
- completely = True
-
- elif opt in ("-v", "--verbose"):
+ if dryrun:
verbose = True
- elif opt in ("-n", "--dry-run"):
- dryrun = True
+ cls.uuid = uuid
+ if days > 0:
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-days)
+ cls.cutoff = cutoff
+ else:
+ cls.cutoff = None
+ cls.batchSize = batchSize
+ cls.dryrun = dryrun
+ cls.verbose = verbose
- elif opt in ("-f", "--config"):
- configFileName = arg
+ utilityMain(
+ configFileName,
+ cls,
+ )
- elif opt in ("--noimplicit"):
- doimplicit = False
+ @classmethod
+ @inlineCallbacks
+ def purgeAttachments(cls, store, uuid, days, limit, dryrun, verbose):
+
+ service = cls(store)
+ service.uuid = uuid
+ if days > 0:
+ cutoff = PyCalendarDateTime.getToday()
+ cutoff.setDateOnly(False)
+ cutoff.offsetDay(-days)
+ service.cutoff = cutoff
else:
- raise NotImplementedError(opt)
+ service.cutoff = None
+ service.batchSize = limit
+ service.dryrun = dryrun
+ service.verbose = verbose
+ result = (yield service.doWork())
+ returnValue(result)
- # args is a list of uids
- PurgePrincipalService.uids = args
- PurgePrincipalService.completely = completely
- PurgePrincipalService.dryrun = dryrun
- PurgePrincipalService.verbose = verbose
- PurgePrincipalService.doimplicit = doimplicit
- utilityMain(
- configFileName,
- PurgePrincipalService
- )
+ @inlineCallbacks
+ def doWork(self):
+ if self.dryrun:
+ orphans = (yield self._orphansDryRun())
+ if self.cutoff is not None:
+ dropbox = (yield self._dropboxDryRun())
+ managed = (yield self._managedDryRun())
+ else:
+ dropbox = ()
+ managed = ()
+ returnValue(self._dryRunSummary(orphans, dropbox, managed))
+ else:
+ total = (yield self._orphansPurge())
+ if self.cutoff is not None:
+ total += (yield self._dropboxPurge())
+ total += (yield self._managedPurge())
+ returnValue(total)
- at inlineCallbacks
-def purgeOldEvents(store, directory, root, date, batchSize, verbose=False,
- dryrun=False):
- if dryrun:
- if verbose:
- print "(Dry run) Searching for old events..."
- txn = store.newTransaction(label="Find old events")
- oldEvents = (yield txn.eventsOlderThan(date))
- eventCount = len(oldEvents)
- if verbose:
- if eventCount == 0:
- print "No events are older than %s" % (date,)
- elif eventCount == 1:
- print "1 event is older than %s" % (date,)
- else:
- print "%d events are older than %s" % (eventCount, date)
- returnValue(eventCount)
+ @inlineCallbacks
+ def _orphansDryRun(self):
- if verbose:
- print "Removing events older than %s..." % (date,)
+ if self.verbose:
+ print "(Dry run) Searching for orphaned attachments..."
+ txn = self._store.newTransaction(label="Find orphaned attachments")
+ orphans = (yield txn.orphanedAttachments(self.uuid))
+ returnValue(orphans)
- numEventsRemoved = -1
- totalRemoved = 0
- while numEventsRemoved:
- txn = store.newTransaction(label="Remove old events")
- numEventsRemoved = (yield txn.removeOldEvents(date, batchSize=batchSize))
- (yield txn.commit())
- if numEventsRemoved:
- totalRemoved += numEventsRemoved
- if verbose:
- print "%d," % (totalRemoved,),
- if verbose:
- print
- if totalRemoved == 0:
- print "No events were removed"
- elif totalRemoved == 1:
- print "1 event was removed in total"
- else:
- print "%d events were removed in total" % (totalRemoved,)
+ @inlineCallbacks
+ def _dropboxDryRun(self):
- returnValue(totalRemoved)
+ if self.verbose:
+ print "(Dry run) Searching for old dropbox attachments..."
+ txn = self._store.newTransaction(label="Find old dropbox attachments")
+ cutoffs = (yield txn.oldDropboxAttachments(self.cutoff, self.uuid))
+ yield txn.commit()
+ returnValue(cutoffs)
- at inlineCallbacks
-def purgeOrphanedAttachments(store, batchSize, verbose=False, dryrun=False):
+ @inlineCallbacks
+ def _managedDryRun(self):
- if dryrun:
- if verbose:
- print "(Dry run) Searching for orphaned attachments..."
- txn = store.newTransaction(label="Find orphaned attachments")
- orphans = (yield txn.orphanedAttachments())
- if verbose:
- # Print aggregate details by user
- byuser = collections.defaultdict(int)
- for owner_uid, _ignore_dropbox_id, _ignore_path, size in orphans:
- byuser[owner_uid] += size
+ if self.verbose:
+ print "(Dry run) Searching for old managed attachments..."
+ txn = self._store.newTransaction(label="Find old managed attachments")
+ cutoffs = (yield txn.oldManagedAttachments(self.cutoff, self.uuid))
+ yield txn.commit()
+ returnValue(cutoffs)
+
+
+ def _dryRunSummary(self, orphans, dropbox, managed):
+
+ if self.verbose:
+ byuser = {}
+ ByUserData = collections.namedtuple(
+ 'ByUserData',
+ ['quota', 'orphanSize', 'orphanCount', 'dropboxSize', 'dropboxCount', 'managedSize', 'managedCount']
+ )
+ for user, quota, size, count in orphans:
+ byuser[user] = ByUserData(quota=quota, orphanSize=size, orphanCount=count, dropboxSize=0, dropboxCount=0, managedSize=0, managedCount=0)
+ for user, quota, size, count in dropbox:
+ if user in byuser:
+ byuser[user] = byuser[user]._replace(dropboxSize=size, dropboxCount=count)
+ else:
+ byuser[user] = ByUserData(quota=quota, orphanSize=0, orphanCount=0, dropboxSize=size, dropboxCount=count, managedSize=0, managedCount=0)
+ for user, quota, size, count in managed:
+ if user in byuser:
+ byuser[user] = byuser[user]._replace(managedSize=size, managedCount=count)
+ else:
+ byuser[user] = ByUserData(quota=quota, orphanSize=0, orphanCount=0, dropboxSize=0, dropboxCount=0, managedSize=size, managedCount=count)
+
# Print table of results
table = tables.Table()
- table.addHeader(("User", "Current Quota", "Orphaned Size", "Orphaned Count"))
+ table.addHeader(("User", "Current Quota", "Orphan Size", "Orphan Count", "Dropbox Size", "Dropbox Count", "Managed Size", "Managed Count", "Total Size", "Total Count"))
table.setDefaultColumnFormats(
(
tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
)
)
- total = 0
- for user, quota, size, count in sorted(orphans):
- table.addRow((
- user,
- quota,
- size,
- count,
- ))
- total += count
- table.addFooter(("Total:", "", "", total))
+ totals = [0] * 8
+ for user, data in sorted(byuser.items(), key=lambda x: x[0]):
+ cols = (
+ data.orphanSize,
+ data.orphanCount,
+ data.dropboxSize,
+ data.dropboxCount,
+ data.managedSize,
+ data.managedCount,
+ data.orphanSize + data.dropboxSize + data.managedSize,
+ data.orphanCount + data.dropboxCount + data.managedCount,
+ )
+ table.addRow((user, data.quota,) + cols)
+ for ctr, value in enumerate(cols):
+ totals[ctr] += value
+ table.addFooter(("Total:", "",) + tuple(totals))
+ total = totals[7]
+
print "\n"
- print "Orphaned Attachments by User:\n"
+ print "Orphaned/Old Attachments by User:\n"
table.printTable()
else:
- total = sum([x[3] for x in orphans])
+ total = sum([x[3] for x in orphans]) + sum([x[3] for x in dropbox]) + sum([x[3] for x in managed])
- returnValue(total)
+ return total
- if verbose:
- print "Removing orphaned attachments..."
- numOrphansRemoved = -1
- totalRemoved = 0
- while numOrphansRemoved:
- txn = store.newTransaction(label="Remove orphaned attachments")
- numOrphansRemoved = (yield txn.removeOrphanedAttachments(batchSize=batchSize))
- (yield txn.commit())
- if numOrphansRemoved:
- totalRemoved += numOrphansRemoved
- if verbose:
- print "%d," % (totalRemoved,),
+ @inlineCallbacks
+ def _orphansPurge(self):
- if verbose:
- print
- if totalRemoved == 0:
- print "No orphaned attachments were removed"
- elif totalRemoved == 1:
- print "1 orphaned attachment was removed in total"
- else:
- print "%d orphaned attachments were removed in total" % (totalRemoved,)
+ if self.verbose:
+ print "Removing orphaned attachments...",
- returnValue(totalRemoved)
+ numOrphansRemoved = -1
+ totalRemoved = 0
+ while numOrphansRemoved:
+ txn = self._store.newTransaction(label="Remove orphaned attachments")
+ numOrphansRemoved = (yield txn.removeOrphanedAttachments(self.uuid, batchSize=self.batchSize))
+ yield txn.commit()
+ if numOrphansRemoved:
+ totalRemoved += numOrphansRemoved
+ if self.verbose:
+ print " %d," % (totalRemoved,),
+ elif self.verbose:
+ print
+ if self.verbose:
+ if totalRemoved == 0:
+ print "No orphaned attachments were removed"
+ elif totalRemoved == 1:
+ print "1 orphaned attachment was removed in total"
+ else:
+ print "%d orphaned attachments were removed in total" % (totalRemoved,)
+ print
+ returnValue(totalRemoved)
- at inlineCallbacks
-def purgeUIDs(store, directory, root, uids, verbose=False, dryrun=False,
- completely=False, doimplicit=True):
- total = 0
- allAssignments = {}
+ @inlineCallbacks
+ def _dropboxPurge(self):
- for uid in uids:
- count, allAssignments[uid] = (yield purgeUID(store, uid, directory, root,
- verbose=verbose, dryrun=dryrun, completely=completely, doimplicit=doimplicit))
- total += count
+ if self.verbose:
+ print "Removing old dropbox attachments...",
- # TODO: figure out what to do with the purged proxy assignments...
- # ...print to stdout?
- # ...save in a file?
+ numOldRemoved = -1
+ totalRemoved = 0
+ while numOldRemoved:
+ txn = self._store.newTransaction(label="Remove old dropbox attachments")
+ numOldRemoved = (yield txn.removeOldDropboxAttachments(self.cutoff, self.uuid, batchSize=self.batchSize))
+ yield txn.commit()
+ if numOldRemoved:
+ totalRemoved += numOldRemoved
+ if self.verbose:
+ print " %d," % (totalRemoved,),
+ elif self.verbose:
+ print
- returnValue(total)
+ if self.verbose:
+ if totalRemoved == 0:
+ print "No old dropbox attachments were removed"
+ elif totalRemoved == 1:
+ print "1 old dropbox attachment was removed in total"
+ else:
+ print "%d old dropbox attachments were removed in total" % (totalRemoved,)
+ print
+ returnValue(totalRemoved)
-CANCELEVENT_SKIPPED = 1
-CANCELEVENT_MODIFIED = 2
-CANCELEVENT_NOT_MODIFIED = 3
-CANCELEVENT_SHOULD_DELETE = 4
-def cancelEvent(event, when, cua):
- """
- Modify a VEVENT such that all future occurrences are removed
+ @inlineCallbacks
+ def _managedPurge(self):
- @param event: the event to modify
- @type event: L{twistedcaldav.ical.Component}
+ if self.verbose:
+ print "Removing old managed attachments...",
- @param when: the cutoff date (anything after which is removed)
- @type when: PyCalendarDateTime
+ numOldRemoved = -1
+ totalRemoved = 0
+ while numOldRemoved:
+ txn = self._store.newTransaction(label="Remove old managed attachments")
+ numOldRemoved = (yield txn.removeOldManagedAttachments(self.cutoff, self.uuid, batchSize=self.batchSize))
+ yield txn.commit()
+ if numOldRemoved:
+ totalRemoved += numOldRemoved
+ if self.verbose:
+ print " %d," % (totalRemoved,),
+ elif self.verbose:
+ print
- @param cua: Calendar User Address of principal being purged, to compare
- to see if it's the organizer of the event or just an attendee
- @type cua: string
+ if self.verbose:
+ if totalRemoved == 0:
+ print "No old managed attachments were removed"
+ elif totalRemoved == 1:
+ print "1 old managed attachment was removed in total"
+ else:
+ print "%d old managed attachments were removed in total" % (totalRemoved,)
+ print
- Assumes that event does not occur entirely in the past.
+ returnValue(totalRemoved)
- @return: one of the 4 constants above to indicate what action to take
- """
- whenDate = when.duplicate()
- whenDate.setDateOnly(True)
- # Only process VEVENT
- if event.mainType() != "VEVENT":
- return CANCELEVENT_SKIPPED
+class PurgePrincipalService(WorkerService):
- main = event.masterComponent()
- if main is None:
- # No master component, so this is an attendee being invited to one or
- # more occurrences
- main = event.mainComponent(allow_multiple=True)
+ root = None
+ directory = None
+ uids = None
+ dryrun = False
+ verbose = False
+ completely = False
+ doimplicit = True
+ proxies = True
+ when = None
- # Anything completely in the future is deleted
- dtstart = main.getStartDateUTC()
- isDateTime = not dtstart.isDateOnly()
- if dtstart > when:
- return CANCELEVENT_SHOULD_DELETE
+ @classmethod
+ def usage(cls, e=None):
- organizer = main.getOrganizer()
+ name = os.path.basename(sys.argv[0])
+ print "usage: %s [options]" % (name,)
+ print ""
+ print " Remove a principal's events and contacts from the calendar server"
+ print ""
+ print "options:"
+ print " -c --completely: By default, only future events are canceled; this option cancels all events"
+ print " -h --help: print this help and exit"
+ print " -f --config <path>: Specify caldavd.plist configuration path"
+ print " -n --dry-run: calculate how many events and contacts to purge, but do not purge data"
+ print " -v --verbose: print progress information"
+ print ""
- # Non-meetings are deleted
- if organizer is None:
- return CANCELEVENT_SHOULD_DELETE
+ if e:
+ sys.stderr.write("%s\n" % (e,))
+ sys.exit(64)
+ else:
+ sys.exit(0)
- # Meetings which cua is merely an attendee are deleted (thus implicitly
- # declined)
- # FIXME: I think we want to decline anything after the cut-off, not delete
- # the whole event.
- if organizer != cua:
- return CANCELEVENT_SHOULD_DELETE
- dirty = False
+ @classmethod
+ def main(cls):
- # Set the UNTIL on RRULE to cease at the cutoff
- if main.hasProperty("RRULE"):
- for rrule in main.properties("RRULE"):
- rrule = rrule.value()
- if rrule.getUseCount():
- rrule.setUseCount(False)
+ try:
+ (optargs, args) = getopt(
+ sys.argv[1:], "cf:hnv", [
+ "completely",
+ "dry-run",
+ "config=",
+ "help",
+ "verbose",
+ "noimplicit",
+ ],
+ )
+ except GetoptError, e:
+ cls.usage(e)
- rrule.setUseUntil(True)
- if isDateTime:
- rrule.setUntil(when)
+ #
+ # Get configuration
+ #
+ configFileName = None
+ dryrun = False
+ verbose = False
+ completely = False
+ doimplicit = True
+
+ for opt, arg in optargs:
+ if opt in ("-h", "--help"):
+ cls.usage()
+
+ elif opt in ("-c", "--completely"):
+ completely = True
+
+ elif opt in ("-v", "--verbose"):
+ verbose = True
+
+ elif opt in ("-n", "--dry-run"):
+ dryrun = True
+
+ elif opt in ("-f", "--config"):
+ configFileName = arg
+
+ elif opt in ("--noimplicit"):
+ doimplicit = False
+
else:
- rrule.setUntil(whenDate)
- dirty = True
+ raise NotImplementedError(opt)
- # Remove any EXDATEs and RDATEs beyond the cutoff
- for dateType in ("EXDATE", "RDATE"):
- if main.hasProperty(dateType):
- for exdate_rdate in main.properties(dateType):
- newValues = []
- for value in exdate_rdate.value():
- if value.getValue() < when:
- newValues.append(value)
- else:
- exdate_rdate.value().remove(value)
- dirty = True
- if not newValues:
- main.removeProperty(exdate_rdate)
- dirty = True
+ # args is a list of uids
+ cls.uids = args
+ cls.completely = completely
+ cls.dryrun = dryrun
+ cls.verbose = verbose
+ cls.doimplicit = doimplicit
- # Remove any overridden components beyond the cutoff
- for component in tuple(event.subcomponents()):
- if component.name() == "VEVENT":
- dtstart = component.getStartDateUTC()
- remove = False
- if dtstart > when:
- remove = True
- if remove:
- event.removeComponent(component)
- dirty = True
+ utilityMain(
+ configFileName,
+ cls
+ )
- if dirty:
- return CANCELEVENT_MODIFIED
- else:
- return CANCELEVENT_NOT_MODIFIED
+ @classmethod
+ @inlineCallbacks
+ def purgeUIDs(cls, store, directory, root, uids, verbose=False, dryrun=False,
+ completely=False, doimplicit=True, proxies=True, when=None):
+ service = cls(store)
+ service.root = root
+ service.directory = directory
+ service.uids = uids
+ service.verbose = verbose
+ service.dryrun = dryrun
+ service.completely = completely
+ service.doimplicit = doimplicit
+ service.proxies = proxies
+ service.when = when
+ result = (yield service.doWork())
+ returnValue(result)
- at inlineCallbacks
-def purgeUID(store, uid, directory, root, verbose=False, dryrun=False, proxies=True,
- when=None, completely=False, doimplicit=True):
- if when is None:
- when = PyCalendarDateTime.getNowUTC()
+ @inlineCallbacks
+ def doWork(self):
- # Does the record exist?
- record = directory.recordWithUID(uid)
- if record is None:
- # The user has already been removed from the directory service. We
- # need to fashion a temporary, fake record
+ if self.root is None:
+ self.root = self.getRootResource()
+ if self.directory is None:
+ self.directory = self.root.getDirectory()
- # FIXME: probaby want a more elegant way to accomplish this,
- # since it requires the aggregate directory to examine these first:
- record = DirectoryRecord(directory, "users", uid, shortNames=(uid,),
- enabledForCalendaring=True)
- record.enabled = True
- directory._tmpRecords["shortNames"][uid] = record
- directory._tmpRecords["uids"][uid] = record
+ total = 0
- cua = "urn:uuid:%s" % (uid,)
+ allAssignments = {}
- principalCollection = directory.principalCollection
- principal = principalCollection.principalForRecord(record)
+ for uid in self.uids:
+ count, allAssignments[uid] = (yield self._purgeUID(uid))
+ total += count
- request = FakeRequest(root, None, None)
- request.checkedSACL = True
- request.authnUser = request.authzUser = davxml.Principal(
- davxml.HRef.fromString("/principals/__uids__/%s/" % (uid,))
- )
+ if self.verbose:
+ amount = "%d event%s" % (total, "s" if total > 1 else "")
+ if self.dryrun:
+ print "Would have modified or deleted %s" % (amount,)
+ else:
+ print "Modified or deleted %s" % (amount,)
- # See if calendar home is provisioned
- txn = store.newTransaction()
- storeCalHome = (yield txn.calendarHomeWithUID(uid))
- calHomeProvisioned = storeCalHome is not None
+ returnValue((total, allAssignments,))
- # If in "completely" mode, unshare collections, remove notifications
- if calHomeProvisioned and completely:
- # Process shared and shared-to-me calendars
- children = list((yield storeCalHome.children()))
- for child in children:
- if verbose:
- if dryrun:
- print "Would unshare: %s" % (child.name(),)
- else:
- print "Unsharing: %s" % (child.name(),)
- if not dryrun:
- (yield child.unshare())
+ @inlineCallbacks
+ def _purgeUID(self, uid):
- if not dryrun:
- (yield storeCalHome.removeUnacceptedShares())
- notificationHome = (yield txn.notificationsWithUID(uid))
- if notificationHome is not None:
- (yield notificationHome.remove())
+ if self.when is None:
+ self.when = PyCalendarDateTime.getNowUTC()
- (yield txn.commit())
+ # Does the record exist?
+ record = self.directory.recordWithUID(uid)
+ if record is None:
+ # The user has already been removed from the directory service. We
+ # need to fashion a temporary, fake record
- # Anything in the past is left alone
- whenString = when.getText()
- filter = caldavxml.Filter(
- caldavxml.ComponentFilter(
- caldavxml.ComponentFilter(
- TimeRange(start=whenString,),
- name=("VEVENT",),
- ),
- name="VCALENDAR",
- )
- )
- filter = calendarqueryfilter.Filter(filter)
+ # FIXME: probably want a more elegant way to accomplish this,
+ # since it requires the aggregate directory to examine these first:
+ record = DirectoryRecord(self.directory, "users", uid, shortNames=(uid,), enabledForCalendaring=True)
+ record.enabled = True
+ self.directory._tmpRecords["shortNames"][uid] = record
+ self.directory._tmpRecords["uids"][uid] = record
- count = 0
- assignments = []
+ cua = "urn:uuid:%s" % (uid,)
- perUserFilter = PerUserDataFilter(uid)
+ principalCollection = self.directory.principalCollection
+ principal = principalCollection.principalForRecord(record)
- try:
- if calHomeProvisioned:
- calendarHome = yield principal.calendarHome(request)
- for collName in (yield calendarHome.listChildren()):
- collection = (yield calendarHome.getChild(collName))
+ request = FakeRequest(self.root, None, None)
+ request.checkedSACL = True
+ request.authnUser = request.authzUser = davxml.Principal(
+ davxml.HRef.fromString("/principals/__uids__/%s/" % (uid,))
+ )
- if collection.isCalendarCollection() or collName == "inbox":
- childNames = []
+ # See if calendar home is provisioned
+ txn = self._store.newTransaction()
+ storeCalHome = (yield txn.calendarHomeWithUID(uid))
+ calHomeProvisioned = storeCalHome is not None
- if completely:
- # all events
- for childName in (yield collection.listChildren()):
- childNames.append(childName)
+ # If in "completely" mode, unshare collections, remove notifications
+ if calHomeProvisioned and self.completely:
+
+ # Process shared and shared-to-me calendars
+ children = list((yield storeCalHome.children()))
+ for child in children:
+ if self.verbose:
+ if self.dryrun:
+ print "Would unshare: %s" % (child.name(),)
else:
- # events matching filter
- for childName, _ignore_childUid, _ignore_childType in (yield collection.index().indexedSearch(filter)):
- childNames.append(childName)
+ print "Unsharing: %s" % (child.name(),)
+ if not self.dryrun:
+ (yield child.unshare())
- for childName in childNames:
+ if not self.dryrun:
+ (yield storeCalHome.removeUnacceptedShares())
+ notificationHome = (yield txn.notificationsWithUID(uid))
+ if notificationHome is not None:
+ (yield notificationHome.remove())
- childResource = (yield collection.getChild(childName))
- if completely:
- action = CANCELEVENT_SHOULD_DELETE
+ (yield txn.commit())
+
+ # Anything in the past is left alone
+ whenString = self.when.getText()
+ filter = caldavxml.Filter(
+ caldavxml.ComponentFilter(
+ caldavxml.ComponentFilter(
+ TimeRange(start=whenString,),
+ name=("VEVENT",),
+ ),
+ name="VCALENDAR",
+ )
+ )
+ filter = calendarqueryfilter.Filter(filter)
+
+ count = 0
+ assignments = []
+
+ perUserFilter = PerUserDataFilter(uid)
+
+ try:
+ if calHomeProvisioned:
+ calendarHome = yield principal.calendarHome(request)
+ for collName in (yield calendarHome.listChildren()):
+ collection = (yield calendarHome.getChild(collName))
+
+ if collection.isCalendarCollection() or collName == "inbox":
+ childNames = []
+
+ if self.completely:
+ # all events
+ for childName in (yield collection.listChildren()):
+ childNames.append(childName)
else:
- event = (yield childResource.iCalendar())
- event = perUserFilter.filter(event)
- action = cancelEvent(event, when, cua)
+ # events matching filter
+ for childName, _ignore_childUid, _ignore_childType in (yield collection.index().indexedSearch(filter)):
+ childNames.append(childName)
- uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName)
- request.path = uri
- if action == CANCELEVENT_MODIFIED:
- count += 1
- request._rememberResource(childResource, uri)
- storer = StoreCalendarObjectResource(
- request=request,
- destination=childResource,
- destination_uri=uri,
- destinationcal=True,
- destinationparent=collection,
- calendar=str(event),
- )
- if verbose:
- if dryrun:
- print "Would modify: %s" % (uri,)
- else:
- print "Modifying: %s" % (uri,)
- if not dryrun:
- result = (yield storer.run())
+ for childName in childNames:
- elif action == CANCELEVENT_SHOULD_DELETE:
- incrementCount = dryrun
- request._rememberResource(childResource, uri)
- if verbose:
- if dryrun:
- print "Would delete: %s" % (uri,)
- else:
- print "Deleting: %s" % (uri,)
- if not dryrun:
- retry = False
- try:
- result = (yield childResource.storeRemove(request, doimplicit, uri))
- if result != NO_CONTENT:
- print "Error deleting %s/%s/%s: %s" % (uid,
- collName, childName, result)
- retry = True
+ childResource = (yield collection.getChild(childName))
+ if self.completely:
+ action = self.CANCELEVENT_SHOULD_DELETE
+ else:
+ event = (yield childResource.iCalendar())
+ event = perUserFilter.filter(event)
+ action = self._cancelEvent(event, self.when, cua)
+
+ uri = "/calendars/__uids__/%s/%s/%s" % (uid, collName, childName)
+ request.path = uri
+ if action == self.CANCELEVENT_MODIFIED:
+ count += 1
+ request._rememberResource(childResource, uri)
+ storer = StoreCalendarObjectResource(
+ request=request,
+ destination=childResource,
+ destination_uri=uri,
+ destinationcal=True,
+ destinationparent=collection,
+ calendar=str(event),
+ )
+ if self.verbose:
+ if self.dryrun:
+ print "Would modify: %s" % (uri,)
else:
- incrementCount = True
+ print "Modifying: %s" % (uri,)
+ if not self.dryrun:
+ result = (yield storer.run())
- except Exception, e:
- print "Exception deleting %s/%s/%s: %s" % (uid,
- collName, childName, str(e))
- retry = True
-
- if retry and doimplicit:
- # Try again with implicit scheduling off
- print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName)
+ elif action == self.CANCELEVENT_SHOULD_DELETE:
+ incrementCount = self.dryrun
+ request._rememberResource(childResource, uri)
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete: %s" % (uri,)
+ else:
+ print "Deleting: %s" % (uri,)
+ if not self.dryrun:
+ retry = False
try:
- result = (yield childResource.storeRemove(request, False, uri))
+ result = (yield childResource.storeRemove(request, self.doimplicit, uri))
if result != NO_CONTENT:
print "Error deleting %s/%s/%s: %s" % (uid,
collName, childName, result)
+ retry = True
else:
incrementCount = True
+
except Exception, e:
- print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e))
+ print "Exception deleting %s/%s/%s: %s" % (uid,
+ collName, childName, str(e))
+ retry = True
- if incrementCount:
- count += 1
+ if retry and self.doimplicit:
+ # Try again with implicit scheduling off
+ print "Retrying deletion of %s/%s/%s with implicit scheduling turned off" % (uid, collName, childName)
+ try:
+ result = (yield childResource.storeRemove(request, False, uri))
+ if result != NO_CONTENT:
+ print "Error deleting %s/%s/%s: %s" % (uid,
+ collName, childName, result)
+ else:
+ incrementCount = True
+ except Exception, e:
+ print "Still couldn't delete %s/%s/%s even with implicit scheduling turned off: %s" % (uid, collName, childName, str(e))
- txn = getattr(request, "_newStoreTransaction", None)
- # Commit
- if txn is not None:
- (yield txn.commit())
+ if incrementCount:
+ count += 1
- except Exception, e:
- # Abort
- txn = getattr(request, "_newStoreTransaction", None)
- if txn is not None:
- (yield txn.abort())
- raise e
+ txn = getattr(request, "_newStoreTransaction", None)
+ # Commit
+ if txn is not None:
+ (yield txn.commit())
- try:
- txn = store.newTransaction()
+ except Exception, e:
+ # Abort
+ txn = getattr(request, "_newStoreTransaction", None)
+ if txn is not None:
+ (yield txn.abort())
+ raise e
- # Remove empty calendar collections (and calendar home if no more
- # calendars)
- storeCalHome = (yield txn.calendarHomeWithUID(uid))
- if storeCalHome is not None:
- calendars = list((yield storeCalHome.calendars()))
- remainingCalendars = len(calendars)
- for calColl in calendars:
- if len(list((yield calColl.calendarObjects()))) == 0:
- remainingCalendars -= 1
- calendarName = calColl.name()
- if verbose:
- if dryrun:
- print "Would delete calendar: %s" % (calendarName,)
+ try:
+ txn = self._store.newTransaction()
+
+ # Remove empty calendar collections (and calendar home if no more
+ # calendars)
+ storeCalHome = (yield txn.calendarHomeWithUID(uid))
+ if storeCalHome is not None:
+ calendars = list((yield storeCalHome.calendars()))
+ remainingCalendars = len(calendars)
+ for calColl in calendars:
+ if len(list((yield calColl.calendarObjects()))) == 0:
+ remainingCalendars -= 1
+ calendarName = calColl.name()
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete calendar: %s" % (calendarName,)
+ else:
+ print "Deleting calendar: %s" % (calendarName,)
+ if not self.dryrun:
+ (yield storeCalHome.removeChildWithName(calendarName))
+
+ if not remainingCalendars:
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete calendar home"
else:
- print "Deleting calendar: %s" % (calendarName,)
- if not dryrun:
- (yield storeCalHome.removeChildWithName(calendarName))
+ print "Deleting calendar home"
+ if not self.dryrun:
+ (yield storeCalHome.remove())
- if not remainingCalendars:
- if verbose:
- if dryrun:
- print "Would delete calendar home"
- else:
- print "Deleting calendar home"
- if not dryrun:
- (yield storeCalHome.remove())
+ # Remove VCards
+ storeAbHome = (yield txn.addressbookHomeWithUID(uid))
+ if storeAbHome is not None:
+ for abColl in list((yield storeAbHome.addressbooks())):
+ for card in list((yield abColl.addressbookObjects())):
+ cardName = card.name()
+ if self.verbose:
+ uri = "/addressbooks/__uids__/%s/%s/%s" % (uid, abColl.name(), cardName)
+ if self.dryrun:
+ print "Would delete: %s" % (uri,)
+ else:
+ print "Deleting: %s" % (uri,)
+ if not self.dryrun:
+ (yield abColl.removeObjectResourceWithName(cardName))
+ count += 1
+ if self.verbose:
+ abName = abColl.name()
+ if self.dryrun:
+ print "Would delete addressbook: %s" % (abName,)
+ else:
+ print "Deleting addressbook: %s" % (abName,)
+ if not self.dryrun:
+ # Also remove the addressbook collection itself
+ (yield storeAbHome.removeChildWithName(abColl.name()))
- # Remove VCards
- storeAbHome = (yield txn.addressbookHomeWithUID(uid))
- if storeAbHome is not None:
- for abColl in list((yield storeAbHome.addressbooks())):
- for card in list((yield abColl.addressbookObjects())):
- cardName = card.name()
- if verbose:
- uri = "/addressbooks/__uids__/%s/%s/%s" % (uid, abColl.name(), cardName)
- if dryrun:
- print "Would delete: %s" % (uri,)
- else:
- print "Deleting: %s" % (uri,)
- if not dryrun:
- (yield abColl.removeObjectResourceWithName(cardName))
- count += 1
- if verbose:
- abName = abColl.name()
- if dryrun:
- print "Would delete addressbook: %s" % (abName,)
+ if self.verbose:
+ if self.dryrun:
+ print "Would delete addressbook home"
else:
- print "Deleting addressbook: %s" % (abName,)
- if not dryrun:
- # Also remove the addressbook collection itself
- (yield storeAbHome.removeChildWithName(abColl.name()))
+ print "Deleting addressbook home"
+ if not self.dryrun:
+ (yield storeAbHome.remove())
- if verbose:
- if dryrun:
- print "Would delete addressbook home"
- else:
- print "Deleting addressbook home"
- if not dryrun:
- (yield storeAbHome.remove())
+ # Commit
+ (yield txn.commit())
- # Commit
- (yield txn.commit())
+ except Exception, e:
+ # Abort
+ (yield txn.abort())
+ raise e
- except Exception, e:
- # Abort
- (yield txn.abort())
- raise e
+ if self.proxies and not self.dryrun:
+ if self.verbose:
+ print "Deleting any proxy assignments"
+ assignments = (yield self._purgeProxyAssignments(principal))
- if proxies and not dryrun:
- if verbose:
- print "Deleting any proxy assignments"
- assignments = (yield purgeProxyAssignments(principal))
+ returnValue((count, assignments))
- returnValue((count, assignments))
+ CANCELEVENT_SKIPPED = 1
+ CANCELEVENT_MODIFIED = 2
+ CANCELEVENT_NOT_MODIFIED = 3
+ CANCELEVENT_SHOULD_DELETE = 4
+ @classmethod
+ def _cancelEvent(cls, event, when, cua):
+ """
+ Modify a VEVENT such that all future occurrences are removed
+ @param event: the event to modify
+ @type event: L{twistedcaldav.ical.Component}
- at inlineCallbacks
-def purgeProxyAssignments(principal):
+ @param when: the cutoff date (anything after which is removed)
+ @type when: PyCalendarDateTime
- assignments = []
+ @param cua: Calendar User Address of principal being purged, to compare
+ to see if it's the organizer of the event or just an attendee
+ @type cua: string
- for proxyType in ("read", "write"):
+ Assumes that event does not occur entirely in the past.
- proxyFor = (yield principal.proxyFor(proxyType == "write"))
- for other in proxyFor:
- assignments.append((principal.record.uid, proxyType, other.record.uid))
- (yield removeProxy(other, principal))
+ @return: one of the 4 constants above to indicate what action to take
+ """
- subPrincipal = principal.getChild("calendar-proxy-" + proxyType)
- proxies = (yield subPrincipal.readProperty(davxml.GroupMemberSet, None))
- for other in proxies.children:
- assignments.append((str(other).split("/")[3], proxyType, principal.record.uid))
+ whenDate = when.duplicate()
+ whenDate.setDateOnly(True)
- (yield subPrincipal.writeProperty(davxml.GroupMemberSet(), None))
+ # Only process VEVENT
+ if event.mainType() != "VEVENT":
+ return cls.CANCELEVENT_SKIPPED
- returnValue(assignments)
+ main = event.masterComponent()
+ if main is None:
+ # No master component, so this is an attendee being invited to one or
+ # more occurrences
+ main = event.mainComponent(allow_multiple=True)
+
+ # Anything completely in the future is deleted
+ dtstart = main.getStartDateUTC()
+ isDateTime = not dtstart.isDateOnly()
+ if dtstart > when:
+ return cls.CANCELEVENT_SHOULD_DELETE
+
+ organizer = main.getOrganizer()
+
+ # Non-meetings are deleted
+ if organizer is None:
+ return cls.CANCELEVENT_SHOULD_DELETE
+
+ # Meetings which cua is merely an attendee are deleted (thus implicitly
+ # declined)
+ # FIXME: I think we want to decline anything after the cut-off, not delete
+ # the whole event.
+ if organizer != cua:
+ return cls.CANCELEVENT_SHOULD_DELETE
+
+ dirty = False
+
+ # Set the UNTIL on RRULE to cease at the cutoff
+ if main.hasProperty("RRULE"):
+ for rrule in main.properties("RRULE"):
+ rrule = rrule.value()
+ if rrule.getUseCount():
+ rrule.setUseCount(False)
+
+ rrule.setUseUntil(True)
+ if isDateTime:
+ rrule.setUntil(when)
+ else:
+ rrule.setUntil(whenDate)
+ dirty = True
+
+ # Remove any EXDATEs and RDATEs beyond the cutoff
+ for dateType in ("EXDATE", "RDATE"):
+ if main.hasProperty(dateType):
+ for exdate_rdate in main.properties(dateType):
+ newValues = []
+ for value in exdate_rdate.value():
+ if value.getValue() < when:
+ newValues.append(value)
+ else:
+ exdate_rdate.value().remove(value)
+ dirty = True
+ if not newValues:
+ main.removeProperty(exdate_rdate)
+ dirty = True
+
+ # Remove any overridden components beyond the cutoff
+ for component in tuple(event.subcomponents()):
+ if component.name() == "VEVENT":
+ dtstart = component.getStartDateUTC()
+ remove = False
+ if dtstart > when:
+ remove = True
+ if remove:
+ event.removeComponent(component)
+ dirty = True
+
+ if dirty:
+ return cls.CANCELEVENT_MODIFIED
+ else:
+ return cls.CANCELEVENT_NOT_MODIFIED
+
+
+ @classmethod
+ @inlineCallbacks
+ def _purgeProxyAssignments(cls, principal):
+
+ assignments = []
+
+ for proxyType in ("read", "write"):
+
+ proxyFor = (yield principal.proxyFor(proxyType == "write"))
+ for other in proxyFor:
+ assignments.append((principal.record.uid, proxyType, other.record.uid))
+ (yield removeProxy(other, principal))
+
+ subPrincipal = principal.getChild("calendar-proxy-" + proxyType)
+ proxies = (yield subPrincipal.readProperty(davxml.GroupMemberSet, None))
+ for other in proxies.children:
+ assignments.append((str(other).split("/")[3], proxyType, principal.record.uid))
+
+ (yield subPrincipal.writeProperty(davxml.GroupMemberSet(), None))
+
+ returnValue(assignments)
Modified: CalendarServer/trunk/calendarserver/tools/shell/cmd.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/shell/cmd.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/calendarserver/tools/shell/cmd.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -38,7 +38,7 @@
from calendarserver.version import version
from calendarserver.tap.util import getRootResource
from calendarserver.tools.tables import Table
-from calendarserver.tools.purge import purgeUID
+from calendarserver.tools.purge import PurgePrincipalService
from calendarserver.tools.shell.vfs import Folder, RootFolder
from calendarserver.tools.shell.directory import findRecords, summarizeRecords, recordInfo
@@ -49,6 +49,7 @@
"""
+
class UnknownArguments(UsageError):
"""
Unknown arguments.
@@ -58,6 +59,7 @@
self.arguments = arguments
+
class InsufficientArguments(UsageError):
"""
Insufficient arguments.
@@ -66,6 +68,7 @@
UsageError.__init__(self, "Insufficient arguments.")
+
class CommandsBase(object):
"""
Base class for commands.
@@ -78,6 +81,7 @@
self.wd = RootFolder(protocol.service)
+
@property
def terminal(self):
return self.protocol.terminal
@@ -86,6 +90,7 @@
# Utilities
#
+
def documentationForCommand(self, command):
"""
@return: the documentation for the given C{command} as a
@@ -131,6 +136,7 @@
else:
return succeed(None)
+
@inlineCallbacks
def getTargets(self, tokens, wdFallback=False):
"""
@@ -154,6 +160,7 @@
else:
returnValue(())
+
def directoryRecordWithID(self, id):
"""
Obtains a directory record corresponding to the given C{id}.
@@ -177,6 +184,7 @@
return record
+
def commands(self, showHidden=False):
"""
@return: an iterable of C{(name, method)} tuples, where
@@ -189,6 +197,7 @@
if showHidden or not hasattr(m, "hidden"):
yield (attr[4:], m)
+
@staticmethod
def complete(word, items):
"""
@@ -207,6 +216,7 @@
if item.startswith(word):
yield item[len(word):]
+
def complete_commands(self, word):
"""
@return: an iterable of command name completions.
@@ -225,6 +235,7 @@
return completions
+
@inlineCallbacks
def complete_files(self, tokens, filter=None):
"""
@@ -243,7 +254,7 @@
word = token
else:
base = (yield self.wd.locate(token[:i].split("/")))
- word = token[i+1:]
+ word = token[i + 1:]
else:
base = self.wd
@@ -261,6 +272,7 @@
returnValue(self.complete(word, files))
+
class Commands(CommandsBase):
"""
Data store commands.
@@ -324,6 +336,7 @@
for info in sorted(result):
self.terminal.write(format % (info))
+
def complete_help(self, tokens):
if len(tokens) == 0:
return (name for name, method in self.commands())
@@ -456,7 +469,7 @@
if not isinstance(wd, Folder):
raise NotFoundError("Not a folder: %s" % (wd,))
- #log.msg("wd -> %s" % (wd,))
+ #log.msg("wd -> %s" % (wd,))
self.wd = wd
@@ -464,7 +477,7 @@
def complete_cd(self, tokens):
returnValue((yield self.complete_files(
tokens,
- filter = lambda item: True #issubclass(item[0], Folder)
+ filter=lambda item: True #issubclass(item[0], Folder)
)))
@@ -547,7 +560,7 @@
implicit = True
- for option, value in options:
+ for option, _ignore_value in options:
if option == "--no-implicit":
# Not in docstring; this is really dangerous.
implicit = False
@@ -634,7 +647,7 @@
usage: purge_principals principal_id [principal_id ...]
"""
- dryRun = True
+ dryRun = True
completely = False
doimplicit = True
@@ -664,12 +677,15 @@
total = 0
for record in records:
- count, assignments = (yield purgeUID(
- record.uid, directory, rootResource,
- verbose = False,
- dryrun = dryRun,
- completely = completely,
- doimplicit = doimplicit,
+ count, _ignore_assignments = (yield PurgePrincipalService.purgeUIDs(
+ self.protocol.service.store,
+ directory,
+ rootResource,
+ (record.uid,),
+ verbose=False,
+ dryrun=dryRun,
+ completely=completely,
+ doimplicit=doimplicit,
))
total += count
@@ -700,7 +716,7 @@
if len(tokens) < 3:
raise InsufficientArguments()
- mode = tokens.pop(0)
+ mode = tokens.pop(0)
principalID = tokens.pop(0)
record = self.directoryRecordWithID(principalID)
@@ -717,7 +733,7 @@
else:
raise UsageError("Unknown mode: %s" % (mode,))
- for target in targets:
+ for _ignore_target in targets:
raise NotImplementedError()
cmd_share.hidden = "incomplete"
@@ -741,9 +757,9 @@
from twext.enterprise.dal import syntax
localVariables = dict(
- self = self,
- store = self.protocol.service.store,
- schema = schema,
+ self=self,
+ store=self.protocol.service.store,
+ schema=schema,
)
# FIXME: Use syntax.__all__, which needs to be defined
@@ -752,6 +768,7 @@
localVariables[key] = value
class Handler(object):
+
def addOutput(innerSelf, bytes, async=False):
"""
This is a delegate method, called by ManholeInterpreter.
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -16,8 +16,7 @@
from calendarserver.tap.util import getRootResource
-from calendarserver.tools.purge import cancelEvent, purgeUID
-from calendarserver.tools.purge import CANCELEVENT_MODIFIED, CANCELEVENT_SHOULD_DELETE
+from calendarserver.tools.purge import PurgePrincipalService
from twistedcaldav.config import config
from twistedcaldav.ical import Component
@@ -233,55 +232,61 @@
def test_cancelRepeating(self):
# A repeating event where purged CUA is organizer
event = Component.fromString(REPEATING_1_ICS_BEFORE)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_MODIFIED)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_MODIFIED)
self.assertEquals(str(event), REPEATING_1_ICS_AFTER)
+
def test_cancelAllDayRepeating(self):
# A repeating All Day event where purged CUA is organizer
event = Component.fromString(REPEATING_2_ICS_BEFORE)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_MODIFIED)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_MODIFIED)
self.assertEquals(str(event), REPEATING_2_ICS_AFTER)
+
def test_cancelFutureEvent(self):
# A future event
event = Component.fromString(FUTURE_EVENT_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelNonMeeting(self):
# A repeating non-meeting event
event = Component.fromString(REPEATING_NON_MEETING_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelAsAttendee(self):
# A repeating meeting event where purged CUA is an attendee
event = Component.fromString(REPEATING_ATTENDEE_MEETING_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:0F168477-CF3D-45D3-AE60-9875EA02C4D1")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelAsAttendeeOccurrence(self):
# A repeating meeting occurrence with no master, where purged CUA is
# an attendee
event = Component.fromString(INVITED_TO_OCCURRENCE_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:9DC04A71-E6DD-11DF-9492-0800200C9A66")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
+
def test_cancelAsAttendeeMultipleOccurrences(self):
# Multiple meeting occurrences with no master, where purged CUA is
# an attendee
event = Component.fromString(INVITED_TO_MULTIPLE_OCCURRENCES_ICS)
- action = cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
+ action = PurgePrincipalService._cancelEvent(event, PyCalendarDateTime(2010, 12, 6, 12, 0, 0, PyCalendarTimezone(utc=True)),
"urn:uuid:9DC04A71-E6DD-11DF-9492-0800200C9A66")
- self.assertEquals(action, CANCELEVENT_SHOULD_DELETE)
+ self.assertEquals(action, PurgePrincipalService.CANCELEVENT_SHOULD_DELETE)
# This event begins on Nov 30, 2010, has two EXDATES (Dec 3 and 9), and has two
# overridden instances (Dec 4 and 11). The Dec 11 one will be removed since
@@ -725,7 +730,6 @@
DTEND;TZID=America/Los_Angeles:20111105T170000
TRANSP:OPAQUE
ORGANIZER;CN="Amanda Test":urn:uuid:9DC04A70-E6DD-11DF-9492-0800200C9A66
-
UID:44A391CF-52F5-46B4-B35A-E000E3002084
DTSTAMP:20111102T162426Z
SEQUENCE:5
@@ -738,9 +742,6 @@
""".replace("\n", "\r\n")
-
-
-
ATTACHMENT_ICS = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 4.0.1//EN
@@ -778,7 +779,6 @@
-
class PurgePrincipalTests(CommonCommonTests, unittest.TestCase):
"""
Tests for purging the data belonging to a given principal
@@ -869,9 +869,9 @@
@inlineCallbacks
- def test_purgeUID(self):
+ def test_purgeUIDs(self):
"""
- Verify purgeUID removes homes, and doesn't provision homes that don't exist
+ Verify purgeUIDs removes homes, and doesn't provision homes that don't exist
"""
# Now you see it
@@ -880,8 +880,8 @@
self.assertNotEquals(home, None)
(yield txn.commit())
- count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory,
- self.rootResource, verbose=False, proxies=False, completely=True))
+ count, ignored = (yield PurgePrincipalService.purgeUIDs(self.storeUnderTest(), self.directory,
+ self.rootResource, (self.uid,), verbose=False, proxies=False, completely=True))
self.assertEquals(count, 1) # 1 event
# Now you don't
@@ -893,8 +893,8 @@
self.assertEquals((yield home2.childWithName(self.sharedName)), None)
(yield txn.commit())
- count, ignored = (yield purgeUID(self.storeUnderTest(), self.uid, self.directory,
- self.rootResource, verbose=False, proxies=False, completely=True))
+ count, ignored = (yield PurgePrincipalService.purgeUIDs(self.storeUnderTest(), self.directory,
+ self.rootResource, (self.uid,), verbose=False, proxies=False, completely=True))
self.assertEquals(count, 0)
# And you still don't (making sure it's not provisioned)
Modified: CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/calendarserver/tools/test/test_purge_old_events.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -19,12 +19,13 @@
"""
from calendarserver.tap.util import getRootResource
-from calendarserver.tools.purge import purgeOldEvents, purgeUID, purgeOrphanedAttachments
+from calendarserver.tools.purge import PurgeOldEventsService, PurgeAttachmentsService, \
+ PurgePrincipalService
from pycalendar.datetime import PyCalendarDateTime
from pycalendar.timezone import PyCalendarTimezone
-from twext.enterprise.dal.syntax import Update
+from twext.enterprise.dal.syntax import Update, Delete
from twext.web2.http_headers import MimeType
from twisted.internet.defer import inlineCallbacks, returnValue
@@ -89,7 +90,7 @@
END:VCALENDAR
""".replace("\n", "\r\n") % {"year": now - 5}
-OLD_ATTACHMENT_ICS = """BEGIN:VCALENDAR
+ATTACHMENT_ICS = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 4.0.1//EN
CALSCALE:GREGORIAN
@@ -126,19 +127,66 @@
END:VTIMEZONE
BEGIN:VEVENT
CREATED:20100303T181216Z
-UID:57A5D1F6-9A57-4F74-9520-25C617F54B88
+UID:57A5D1F6-9A57-4F74-9520-25C617F54B88-%(uid)s
TRANSP:OPAQUE
SUMMARY:Ancient event with attachment
DTSTART;TZID=US/Pacific:%(year)s0308T111500
DTEND;TZID=US/Pacific:%(year)s0308T151500
DTSTAMP:20100303T181220Z
-X-APPLE-DROPBOX:/calendars/__uids__/user01/dropbox/57A5D1F6-9A57-4F74-95
- 20-25C617F54B88.dropbox
+X-APPLE-DROPBOX:/calendars/__uids__/user01/dropbox/%(dropboxid)s.dropbox
SEQUENCE:2
END:VEVENT
END:VCALENDAR
-""".replace("\n", "\r\n") % {"year": now - 5}
+""".replace("\n", "\r\n")
+MATTACHMENT_ICS = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;UNTIL=20061029T090000Z;BYMONTH=10;BYDAY=-1SU
+DTSTART:19621028T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;UNTIL=20060402T100000Z;BYMONTH=4;BYDAY=1SU
+DTSTART:19870405T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100303T181216Z
+UID:57A5D1F6-9A57-4F74-9520-25C617F54B88-%(uid)s
+TRANSP:OPAQUE
+SUMMARY:Ancient event with attachment
+DTSTART;TZID=US/Pacific:%(year)s0308T111500
+DTEND;TZID=US/Pacific:%(year)s0308T151500
+DTSTAMP:20100303T181220Z
+SEQUENCE:2
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
ENDLESS_ICS = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//iCal 4.0.1//EN
@@ -335,13 +383,26 @@
"calendar1" : {
"old.ics" : (OLD_ICS, metadata,),
"endless.ics" : (ENDLESS_ICS, metadata,),
- "oldattachment.ics" : (OLD_ATTACHMENT_ICS, metadata,),
+ "oldattachment1.ics" : (ATTACHMENT_ICS % {"year": now - 5, "uid": "1.1", "dropboxid": "1.1"}, metadata,),
+ "oldattachment2.ics" : (ATTACHMENT_ICS % {"year": now - 5, "uid": "1.2", "dropboxid": "1.2"}, metadata,),
+ "currentattachment3.ics" : (ATTACHMENT_ICS % {"year": now + 1, "uid": "1.3", "dropboxid": "1.3"}, metadata,),
+ "oldmattachment1.ics" : (MATTACHMENT_ICS % {"year": now - 5, "uid": "1.1m"}, metadata,),
+ "oldmattachment2.ics" : (MATTACHMENT_ICS % {"year": now - 5, "uid": "1.2m"}, metadata,),
+ "currentmattachment3.ics" : (MATTACHMENT_ICS % {"year": now + 1, "uid": "1.3m"}, metadata,),
}
},
"home2" : {
"calendar2" : {
"straddling.ics" : (STRADDLING_ICS, metadata,),
"recent.ics" : (RECENT_ICS, metadata,),
+ "oldattachment1.ics" : (ATTACHMENT_ICS % {"year": now - 5, "uid": "2.1", "dropboxid": "2.1"}, metadata,),
+ "currentattachment2.ics" : (ATTACHMENT_ICS % {"year": now + 1, "uid": "2.2", "dropboxid": "2.1"}, metadata,),
+ "oldattachment3.ics" : (ATTACHMENT_ICS % {"year": now - 5, "uid": "2.3", "dropboxid": "2.2"}, metadata,),
+ "oldattachment4.ics" : (ATTACHMENT_ICS % {"year": now - 5, "uid": "2.4", "dropboxid": "2.2"}, metadata,),
+ "oldmattachment1.ics" : (MATTACHMENT_ICS % {"year": now - 5, "uid": "2.1"}, metadata,),
+ "currentmattachment2.ics" : (MATTACHMENT_ICS % {"year": now + 1, "uid": "2.2"}, metadata,),
+ "oldmattachment3.ics" : (MATTACHMENT_ICS % {"year": now - 5, "uid": "2.3"}, metadata,),
+ "oldmattachment4.ics" : (MATTACHMENT_ICS % {"year": now - 5, "uid": "2.4"}, metadata,),
},
"calendar3" : {
"repeating_awhile.ics" : (REPEATING_AWHILE_ICS, metadata,),
@@ -377,7 +438,15 @@
yield populateCalendarsFrom(self.requirements, self.storeUnderTest())
self.notifierFactory.reset()
+ txn = self._sqlCalendarStore.newTransaction()
+ Delete(
+ From=schema.ATTACHMENT,
+ Where=None
+ ).on(txn)
+ (yield txn.commit())
+
+
def storeUnderTest(self):
"""
Create and return a L{CalendarStore} for testing.
@@ -395,14 +464,23 @@
self.assertEquals(sorted(results),
sorted([
['home1', 'calendar1', 'old.ics', '1901-01-01 01:00:00'],
- ['home1', 'calendar1', 'oldattachment.ics', '1901-01-01 01:00:00'],
+ ['home1', 'calendar1', 'oldattachment1.ics', '1901-01-01 01:00:00'],
+ ['home1', 'calendar1', 'oldattachment2.ics', '1901-01-01 01:00:00'],
+ ['home1', 'calendar1', 'oldmattachment1.ics', '1901-01-01 01:00:00'],
+ ['home1', 'calendar1', 'oldmattachment2.ics', '1901-01-01 01:00:00'],
['home2', 'calendar3', 'repeating_awhile.ics', '1901-01-01 01:00:00'],
['home2', 'calendar2', 'recent.ics', '%s-03-04 22:15:00' % (now,)],
+ ['home2', 'calendar2', 'oldattachment1.ics', '1901-01-01 01:00:00'],
+ ['home2', 'calendar2', 'oldattachment3.ics', '1901-01-01 01:00:00'],
+ ['home2', 'calendar2', 'oldattachment4.ics', '1901-01-01 01:00:00'],
+ ['home2', 'calendar2', 'oldmattachment1.ics', '1901-01-01 01:00:00'],
+ ['home2', 'calendar2', 'oldmattachment3.ics', '1901-01-01 01:00:00'],
+ ['home2', 'calendar2', 'oldmattachment4.ics', '1901-01-01 01:00:00'],
])
)
# Query for oldest event - actually with limited time caching, the oldest event
- # cannot be precisely know, all we get back is the first one in the sorted list
+ # cannot be precisely known, all we get back is the first one in the sorted list
# where each has the 1901 "dummy" time stamp to indicate a partial cache
results = (yield txn.eventsOlderThan(cutoff, batchSize=1))
self.assertEquals(len(results), 1)
@@ -418,11 +496,11 @@
count = (yield txn.removeOldEvents(cutoff, batchSize=1))
self.assertEquals(count, 1)
results = (yield txn.eventsOlderThan(cutoff))
- self.assertEquals(len(results), 3)
+ self.assertEquals(len(results), 12)
# Remove remaining oldest events
count = (yield txn.removeOldEvents(cutoff))
- self.assertEquals(count, 3)
+ self.assertEquals(count, 12)
results = (yield txn.eventsOlderThan(cutoff))
self.assertEquals(results, [])
@@ -432,30 +510,60 @@
@inlineCallbacks
- def _addAttachment(self, orphan=False):
+ def _addAttachment(self, home, calendar, event, name):
txn = self._sqlCalendarStore.newTransaction()
# Create an event with an attachment
- home = (yield txn.calendarHomeWithUID("home1"))
- calendar = (yield home.calendarWithName("calendar1"))
- event = (yield calendar.calendarObjectWithName("oldattachment.ics"))
- attachment = (yield event.createAttachmentWithName("oldattachment.ics"))
+ home = (yield txn.calendarHomeWithUID(home))
+ calendar = (yield home.calendarWithName(calendar))
+ event = (yield calendar.calendarObjectWithName(event))
+ attachment = (yield event.createAttachmentWithName(name))
t = attachment.store(MimeType("text", "x-fixture"))
- t.write("old attachment")
- t.write(" text")
+ t.write("%s/%s/%s/%s" % (home, calendar, event, name,))
+ t.write(" attachment")
(yield t.loseConnection())
- if orphan:
- # Reset dropbox id in calendar_object
- co = schema.CALENDAR_OBJECT
- Update(
- {co.DROPBOX_ID: None, },
- Where=co.RESOURCE_ID == event._resourceID,
- ).on(txn)
+ (yield txn.commit())
+ returnValue(attachment)
+
+
+ @inlineCallbacks
+ def _orphanAttachment(self, home, calendar, event):
+
+ txn = self._sqlCalendarStore.newTransaction()
+
+ # Reset dropbox id in calendar_object
+ home = (yield txn.calendarHomeWithUID(home))
+ calendar = (yield home.calendarWithName(calendar))
+ event = (yield calendar.calendarObjectWithName(event))
+ co = schema.CALENDAR_OBJECT
+ Update(
+ {co.DROPBOX_ID: None, },
+ Where=co.RESOURCE_ID == event._resourceID,
+ ).on(txn)
+
(yield txn.commit())
+
+ @inlineCallbacks
+ def _addManagedAttachment(self, home, calendar, event, name):
+
+ txn = self._sqlCalendarStore.newTransaction()
+
+ # Create an event with an attachment
+ home = (yield txn.calendarHomeWithUID(home))
+ calendar = (yield home.calendarWithName(calendar))
+ event = (yield calendar.calendarObjectWithName(event))
+ attachment = (yield event.createManagedAttachment())
+ t = attachment.store(MimeType("text", "x-fixture"), name)
+ t.write("%s/%s/%s/%s" % (home, calendar, event, name,))
+ t.write(" managed attachment")
+ (yield t.loseConnection())
+
+ (yield txn.commit())
+
returnValue(attachment)
@@ -463,63 +571,102 @@
def test_removeOrphanedAttachments(self):
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertEqual(quota, 0)
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
- attachment = (yield self._addAttachment())
+ attachment = (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
attachmentPath = attachment._path.path
self.assertTrue(os.path.exists(attachmentPath))
- (yield self.commit())
+ mattachment1 = (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ mattachment2 = (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+
+ mattachmentPath1 = mattachment1._path.path
+ self.assertTrue(os.path.exists(mattachmentPath1))
+ mattachmentPath2 = mattachment2._path.path
+ self.assertTrue(os.path.exists(mattachmentPath2))
+
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
orphans = (yield self.transactionUnderTest().orphanedAttachments())
self.assertEquals(len(orphans), 0)
count = (yield self.transactionUnderTest().removeOrphanedAttachments(batchSize=100))
self.assertEquals(count, 0)
- (yield self.commit())
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
quota = (yield home.quotaUsedBytes())
+ (yield self.commit())
self.assertNotEqual(quota, 0)
- # File still exists
+ # Files still exist
self.assertTrue(os.path.exists(attachmentPath))
+ self.assertTrue(os.path.exists(mattachmentPath1))
+ self.assertTrue(os.path.exists(mattachmentPath2))
# Delete all old events (including the event containing the attachment)
cutoff = PyCalendarDateTime(now, 4, 1, 0, 0, 0)
count = (yield self.transactionUnderTest().removeOldEvents(cutoff))
+
+ # See which events have gone and which exist
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ calendar = (yield home.calendarWithName("calendar1"))
+ self.assertNotEqual((yield calendar.calendarObjectWithName("endless.ics")), None)
+ self.assertNotEqual((yield calendar.calendarObjectWithName("currentmattachment3.ics")), None)
+ self.assertEqual((yield calendar.calendarObjectWithName("old.ics")), None)
+ self.assertEqual((yield calendar.calendarObjectWithName("oldattachment1.ics")), None)
+ self.assertEqual((yield calendar.calendarObjectWithName("oldmattachment1.ics")), None)
(yield self.commit())
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertEqual(quota, 0)
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 < quota2)
+ self.assertNotEqual(quota3, 0)
# Just look for orphaned attachments - none left
orphans = (yield self.transactionUnderTest().orphanedAttachments())
self.assertEquals(len(orphans), 0)
+ # Files
+ self.assertFalse(os.path.exists(attachmentPath))
+ self.assertFalse(os.path.exists(mattachmentPath1))
+ self.assertTrue(os.path.exists(mattachmentPath2))
+
@inlineCallbacks
def test_purgeOldEvents(self):
# Dry run
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, dryrun=True,
- verbose=False))
- self.assertEquals(total, 4)
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ dryrun=True,
+ verbose=False
+ ))
+ self.assertEquals(total, 13)
# Actually remove
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, verbose=False))
- self.assertEquals(total, 4)
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ verbose=False
+ ))
+ self.assertEquals(total, 13)
# There should be no more left
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, verbose=False))
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ verbose=False
+ ))
self.assertEquals(total, 0)
@@ -534,21 +681,21 @@
VCardComponent.fromString(VCARD_1)))
self.assertEquals(len((yield abColl.addressbookObjects())), 1)
- # Verify there are 3 events in calendar1
+ # Verify there are 8 events in calendar1
calHome = (yield txn.calendarHomeWithUID("home1"))
calColl = (yield calHome.calendarWithName("calendar1"))
- self.assertEquals(len((yield calColl.calendarObjects())), 3)
+ self.assertEquals(len((yield calColl.calendarObjects())), 8)
# Make the newly created objects available to the purgeUID transaction
(yield txn.commit())
# Purge home1
- total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory,
- self.rootResource, verbose=False, proxies=False,
+ total, ignored = (yield PurgePrincipalService.purgeUIDs(self._sqlCalendarStore, self.directory,
+ self.rootResource, ("home1",), verbose=False, proxies=False,
when=PyCalendarDateTime(now, 4, 1, 12, 0, 0, 0, PyCalendarTimezone(utc=True))))
- # 2 items deleted: 1 event and 1 vcard
- self.assertEquals(total, 2)
+ # 4 items deleted: 3 events and 1 vcard
+ self.assertEquals(total, 4)
txn = self._sqlCalendarStore.newTransaction()
# adressbook home is deleted since it's now empty
@@ -557,7 +704,7 @@
calHome = (yield txn.calendarHomeWithUID("home1"))
calColl = (yield calHome.calendarWithName("calendar1"))
- self.assertEquals(len((yield calColl.calendarObjects())), 2)
+ self.assertEquals(len((yield calColl.calendarObjects())), 5)
@inlineCallbacks
@@ -571,20 +718,20 @@
VCardComponent.fromString(VCARD_1)))
self.assertEquals(len((yield abColl.addressbookObjects())), 1)
- # Verify there are 3 events in calendar1
+ # Verify there are 8 events in calendar1
calHome = (yield txn.calendarHomeWithUID("home1"))
calColl = (yield calHome.calendarWithName("calendar1"))
- self.assertEquals(len((yield calColl.calendarObjects())), 3)
+ self.assertEquals(len((yield calColl.calendarObjects())), 8)
# Make the newly created objects available to the purgeUID transaction
(yield txn.commit())
# Purge home1 completely
- total, ignored = (yield purgeUID(self._sqlCalendarStore, "home1", self.directory,
- self.rootResource, verbose=False, proxies=False, completely=True))
+ total, ignored = (yield PurgePrincipalService.purgeUIDs(self._sqlCalendarStore, self.directory,
+ self.rootResource, ("home1",), verbose=False, proxies=False, completely=True))
- # 4 items deleted: 3 events and 1 vcard
- self.assertEquals(total, 4)
+ # 9 items deleted: 8 events and 1 vcard
+ self.assertEquals(total, 9)
# Homes have been deleted as well
txn = self._sqlCalendarStore.newTransaction()
@@ -595,49 +742,430 @@
@inlineCallbacks
- def test_purgeOrphanedAttachments(self):
+ def test_purgeAttachmentsWithoutCutoffWithPurgeOld(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertEqual(quota, 0)
-
- (yield self._addAttachment(orphan=True))
+ quota1 = (yield home.quotaUsedBytes())
(yield self.commit())
+ self.assertEqual(quota1, 0)
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
# Remove old events first
- total = (yield purgeOldEvents(self._sqlCalendarStore, self.directory,
- self.rootResource, PyCalendarDateTime(now, 4, 1, 0, 0, 0), 2, verbose=False))
- self.assertEquals(total, 4)
+ total = (yield PurgeOldEventsService.purgeOldEvents(
+ self._sqlCalendarStore,
+ PyCalendarDateTime(now, 4, 1, 0, 0, 0),
+ 2,
+ verbose=False
+ ))
+ self.assertEquals(total, 13)
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 < quota2)
# Dry run
- total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
- dryrun=True, verbose=False))
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=True, verbose=False))
self.assertEquals(total, 1)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
(yield self.commit())
+ self.assertTrue(quota4 == quota3)
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 1)
+
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quota = (yield home.quotaUsedBytes())
- self.assertNotEqual(quota, 0)
+ quota5 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota5 < quota4)
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeAttachmentsWithoutCutoff(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "currentattachment2.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 3)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 == quota2)
+
# Actually remove
- total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
- dryrun=False, verbose=False))
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 3)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeAttachmentsWithoutCutoffWithMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "currentattachment2.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home1", 0, 2, dryrun=True, verbose=False))
self.assertEquals(total, 1)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
(yield self.commit())
+ self.assertTrue(quota3 == quota2)
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home1", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 1)
+
home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
- quotaAfter = (yield home.quotaUsedBytes())
- self.assertEqual(quotaAfter, 0)
+ quota4 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota4 < quota3)
# There should be no more left
- total = (yield purgeOrphanedAttachments(self._sqlCalendarStore, 2,
- dryrun=False, verbose=False))
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home1", 0, 2, dryrun=False, verbose=False))
self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeAttachmentsWithoutCutoffWithoutMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home2", 0, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 0)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home2", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota4 == quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home2", 0, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeAttachmentsWithCutoffOld(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "currentattachment2.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 13)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 13)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, None, 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeAttachmentsWithCutoffOldWithMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "currentattachment2.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home1", 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 6)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home1", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 6)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota4 < quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home1", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
+
+
+ @inlineCallbacks
+ def test_purgeAttachmentsWithCutoffOldWithoutMatchingUUID(self):
+ """
+ L{PurgeAttachmentsService.purgeAttachments} purges only orphaned attachments, not current ones.
+ """
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota1 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertEqual(quota1, 0)
+
+ (yield self._addAttachment("home1", "calendar1", "oldattachment1.ics", "att1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.1"))
+ (yield self._addAttachment("home1", "calendar1", "oldattachment2.ics", "att2.2"))
+ (yield self._addAttachment("home1", "calendar1", "currentattachment3.ics", "att3"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment1.ics", "att4"))
+ (yield self._addAttachment("home2", "calendar2", "currentattachment2.ics", "att5"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment3.ics", "att6"))
+ (yield self._addAttachment("home2", "calendar2", "oldattachment4.ics", "att7"))
+ (yield self._orphanAttachment("home1", "calendar1", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "oldattachment1.ics"))
+ (yield self._orphanAttachment("home2", "calendar2", "currentattachment2.ics"))
+
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment1.ics", "matt1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.1"))
+ (yield self._addManagedAttachment("home1", "calendar1", "oldmattachment2.ics", "matt2.2"))
+ (yield self._addManagedAttachment("home1", "calendar1", "currentmattachment3.ics", "matt3"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment1.ics", "matt4"))
+ (yield self._addManagedAttachment("home2", "calendar2", "currentmattachment2.ics", "matt5"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment3.ics", "matt6"))
+ (yield self._addManagedAttachment("home2", "calendar2", "oldmattachment4.ics", "matt7"))
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota2 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota2 > quota1)
+
+ # Dry run
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home2", 14, 2, dryrun=True, verbose=False))
+ self.assertEquals(total, 7)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota3 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota3 == quota2)
+
+ # Actually remove
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home2", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 7)
+
+ home = (yield self.transactionUnderTest().calendarHomeWithUID("home1"))
+ quota4 = (yield home.quotaUsedBytes())
+ (yield self.commit())
+ self.assertTrue(quota4 == quota3)
+
+ # There should be no more left
+ total = (yield PurgeAttachmentsService.purgeAttachments(self._sqlCalendarStore, "home2", 14, 2, dryrun=False, verbose=False))
+ self.assertEquals(total, 0)
Modified: CalendarServer/trunk/doc/calendarserver_manage_principals.8
===================================================================
--- CalendarServer/trunk/doc/calendarserver_manage_principals.8 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/doc/calendarserver_manage_principals.8 2013-01-04 19:27:30 UTC (rev 10203)
@@ -52,7 +52,7 @@
resources.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Nm
Modified: CalendarServer/trunk/doc/calendarserver_manage_push.8
===================================================================
--- CalendarServer/trunk/doc/calendarserver_manage_push.8 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/doc/calendarserver_manage_push.8 2013-01-04 19:27:30 UTC (rev 10203)
@@ -31,7 +31,7 @@
currently subscribed to via APNS.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read data that belongs to the server.
.Nm
takes a list of userids as arguments and then displays the resources
Modified: CalendarServer/trunk/doc/calendarserver_migrate_resources.8
===================================================================
--- CalendarServer/trunk/doc/calendarserver_migrate_resources.8 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/doc/calendarserver_migrate_resources.8 2013-01-04 19:27:30 UTC (rev 10203)
@@ -31,7 +31,7 @@
OpenDirectory into the calendar server's internal directory.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
Modified: CalendarServer/trunk/doc/calendarserver_purge_attachments.8
===================================================================
--- CalendarServer/trunk/doc/calendarserver_purge_attachments.8 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/doc/calendarserver_purge_attachments.8 2013-01-04 19:27:30 UTC (rev 10203)
@@ -23,16 +23,19 @@
.Sh SYNOPSIS
.Nm
.Op Fl -config Ar file
+.Op Fl -uuid Ar guid
+.Op Fl -days Ar NUMBER
.Op Fl -dry-run
.Op Fl -verbose
.Op Fl -help
.Sh DESCRIPTION
.Nm
is a tool for removing attachments that are no longer referenced by
-any calendar events.
+any calendar events, or only referenced by events older than a specified
+cut-off.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
@@ -42,6 +45,10 @@
.It Fl f, -config Ar FILE
Use the Calendar Server configuration specified in the given file.
Defaults to /etc/caldavd/caldavd.plist.
+.It Fl u, -uuid Ar GUID
+Target a specific user via their GUID.
+.It Fl d, -days Ar NUMBER
+Specify how many days in the past to retain. Defaults to 365 days.
.It Fl n, -dry-run
Calculate and display how many orphaned attachments would be removed,
but don't actually remove them.
Modified: CalendarServer/trunk/doc/calendarserver_purge_events.8
===================================================================
--- CalendarServer/trunk/doc/calendarserver_purge_events.8 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/doc/calendarserver_purge_events.8 2013-01-04 19:27:30 UTC (rev 10203)
@@ -36,7 +36,7 @@
removed.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
Modified: CalendarServer/trunk/doc/calendarserver_purge_principals.8
===================================================================
--- CalendarServer/trunk/doc/calendarserver_purge_principals.8 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/doc/calendarserver_purge_principals.8 2013-01-04 19:27:30 UTC (rev 10203)
@@ -36,7 +36,7 @@
events in the past are retained, but any ongoing events are canceled.
.Pp
.Nm
-should be run as a user with the same priviledges as the Calendar
+should be run as a user with the same privileges as the Calendar
Server itself, as it needs to read and write data that belongs to the
server.
.Sh OPTIONS
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -1861,7 +1861,7 @@
def isManaged(self):
- return not self._dropboxID
+ return self._dropboxID == "."
def name(self):
@@ -2138,7 +2138,7 @@
att = schema.ATTACHMENT
rows = (yield Insert({
att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
- att.DROPBOX_ID : None,
+ att.DROPBOX_ID : ".",
att.CONTENT_TYPE : "",
att.SIZE : 0,
att.MD5 : "",
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2013-01-04 04:40:40 UTC (rev 10202)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2013-01-04 19:27:30 UTC (rev 10203)
@@ -883,14 +883,14 @@
return self._sqlTxn.abort()
- def _oldEventsBase(limited): #@NoSelf
+ def _oldEventsBase(self, limit):
ch = schema.CALENDAR_HOME
co = schema.CALENDAR_OBJECT
cb = schema.CALENDAR_BIND
tr = schema.TIME_RANGE
kwds = {}
- if limited:
- kwds["Limit"] = Parameter("batchSize")
+ if limit:
+ kwds["Limit"] = limit
return Select(
[
ch.OWNER_UID,
@@ -915,11 +915,7 @@
**kwds
)
- _oldEventsLimited = _oldEventsBase(True)
- _oldEventsUnlimited = _oldEventsBase(False)
- del _oldEventsBase
-
def eventsOlderThan(self, cutoff, batchSize=None):
"""
Return up to the oldest batchSize events which exist completely earlier
@@ -937,12 +933,7 @@
raise ValueError("Cannot query events older than %s" % (truncateLowerLimit.getText(),))
kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
- if batchSize is not None:
- kwds["batchSize"] = batchSize
- query = self._oldEventsLimited
- else:
- query = self._oldEventsUnlimited
- return query.on(self, **kwds)
+ return self._oldEventsBase(batchSize).on(self, **kwds)
@inlineCallbacks
@@ -969,14 +960,29 @@
returnValue(count)
- def _orphanedSummary(limited): #@NoSelf
- at = schema.ATTACHMENT
- co = schema.CALENDAR_OBJECT
+ def orphanedAttachments(self, uuid=None, batchSize=None):
+ """
+ Find attachments no longer referenced by any events.
+
+ Returns a deferred to a list of (calendar_home_owner_uid, quota used, total orphan size, total orphan count) tuples.
+ """
+ kwds = {}
+ if uuid:
+ kwds["uuid"] = uuid
+
+ options = {}
+ if batchSize:
+ options["Limit"] = batchSize
+
ch = schema.CALENDAR_HOME
chm = schema.CALENDAR_HOME_METADATA
- kwds = {}
- if limited:
- kwds["Limit"] = Parameter('batchSize')
+ co = schema.CALENDAR_OBJECT
+ at = schema.ATTACHMENT
+
+ where = (co.DROPBOX_ID == None).And(at.DROPBOX_ID != ".")
+ if uuid:
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
return Select(
[ch.OWNER_UID, chm.QUOTA_USED_BYTES, Sum(at.SIZE), Count(at.DROPBOX_ID)],
From=at.join(
@@ -984,64 +990,133 @@
ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID).join(
chm, ch.RESOURCE_ID == chm.RESOURCE_ID
),
- Where=co.DROPBOX_ID == None,
+ Where=where,
GroupBy=(ch.OWNER_UID, chm.QUOTA_USED_BYTES),
- **kwds
- )
+ **options
+ ).on(self, **kwds)
- _orphanedSummaryLimited = _orphanedSummary(True)
- _orphanedSummaryUnlimited = _orphanedSummary(False)
- del _orphanedSummary
- def orphanedAttachments(self, batchSize=None):
+ @inlineCallbacks
+ def removeOrphanedAttachments(self, uuid=None, batchSize=None):
"""
- Find attachments no longer referenced by any events.
+ Remove attachments that no longer have any references to them
+ """
- Returns a deferred to a list of (calendar_home_owner_uid, dropbox_id, path, size) tuples.
+ # TODO: see if there is a better way to import Attachment
+ from txdav.caldav.datastore.sql import DropBoxAttachment
+
+ kwds = {}
+ if uuid:
+ kwds["uuid"] = uuid
+
+ options = {}
+ if batchSize:
+ options["Limit"] = batchSize
+
+ ch = schema.CALENDAR_HOME
+ co = schema.CALENDAR_OBJECT
+ at = schema.ATTACHMENT
+
+ sfrom = at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer")
+ where = (co.DROPBOX_ID == None).And(at.DROPBOX_ID != ".")
+ if uuid:
+ sfrom = sfrom.join(ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID)
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
+ results = (yield Select(
+ [at.DROPBOX_ID, at.PATH],
+ From=sfrom,
+ Where=where,
+ **options
+ ).on(self, **kwds))
+
+ count = 0
+ for dropboxID, path in results:
+ attachment = (yield DropBoxAttachment.load(self, dropboxID, path))
+ yield attachment.remove()
+ count += 1
+ returnValue(count)
+
+
+ def oldDropboxAttachments(self, cutoff, uuid):
"""
- if batchSize is not None:
- kwds = {'batchSize': batchSize}
- query = self._orphanedSummaryLimited
- else:
- kwds = {}
- query = self._orphanedSummaryUnlimited
- return query.on(self, **kwds)
+ Find managed attachments attached to only events whose last instance is older than the specified cut-off.
+ Returns a deferred to a list of (calendar_home_owner_uid, quota used, total old size, total old count) tuples.
+ """
+ kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ if uuid:
+ kwds["uuid"] = uuid
- def _orphanedBase(limited): #@NoSelf
+ ch = schema.CALENDAR_HOME
+ chm = schema.CALENDAR_HOME_METADATA
+ co = schema.CALENDAR_OBJECT
+ tr = schema.TIME_RANGE
at = schema.ATTACHMENT
- co = schema.CALENDAR_OBJECT
- kwds = {}
- if limited:
- kwds["Limit"] = Parameter('batchSize')
+
+ where = at.DROPBOX_ID.In(Select(
+ [at.DROPBOX_ID],
+ From=at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "inner").join(
+ tr, co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID
+ ),
+ GroupBy=(at.DROPBOX_ID,),
+ Having=Max(tr.END_DATE) < Parameter("CutOff"),
+ ))
+
+ if uuid:
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
return Select(
- [at.DROPBOX_ID, at.PATH],
- From=at.join(co, at.DROPBOX_ID == co.DROPBOX_ID, "left outer"),
- Where=co.DROPBOX_ID == None,
- **kwds
- )
+ [ch.OWNER_UID, chm.QUOTA_USED_BYTES, Sum(at.SIZE), Count(at.DROPBOX_ID)],
+ From=at.join(
+ ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID).join(
+ chm, ch.RESOURCE_ID == chm.RESOURCE_ID
+ ),
+ Where=where,
+ GroupBy=(ch.OWNER_UID, chm.QUOTA_USED_BYTES),
+ ).on(self, **kwds)
- _orphanedLimited = _orphanedBase(True)
- _orphanedUnlimited = _orphanedBase(False)
- del _orphanedBase
-
@inlineCallbacks
- def removeOrphanedAttachments(self, batchSize=None):
+ def removeOldDropboxAttachments(self, cutoff, uuid, batchSize=None):
"""
- Remove attachments that no longer have any references to them
+ Remove dropbox attachments attached to events in the past.
"""
# TODO: see if there is a better way to import Attachment
from txdav.caldav.datastore.sql import DropBoxAttachment
- if batchSize is not None:
- kwds = {'batchSize': batchSize}
- query = self._orphanedLimited
- else:
- kwds = {}
- query = self._orphanedUnlimited
- results = (yield query.on(self, **kwds))
+ kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ if uuid:
+ kwds["uuid"] = uuid
+
+ options = {}
+ if batchSize:
+ options["Limit"] = batchSize
+
+ ch = schema.CALENDAR_HOME
+ co = schema.CALENDAR_OBJECT
+ tr = schema.TIME_RANGE
+ at = schema.ATTACHMENT
+
+ sfrom = at.join(
+ co, at.DROPBOX_ID == co.DROPBOX_ID, "inner").join(
+ tr, co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID
+ )
+ where = None
+ if uuid:
+ sfrom = sfrom.join(ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID)
+ where = (ch.OWNER_UID == Parameter('uuid'))
+
+ results = (yield Select(
+ [at.DROPBOX_ID, at.PATH, ],
+ From=sfrom,
+ Where=where,
+ GroupBy=(at.DROPBOX_ID, at.PATH,),
+ Having=Max(tr.END_DATE) < Parameter("CutOff"),
+ **options
+ ).on(self, **kwds))
+
count = 0
for dropboxID, path in results:
attachment = (yield DropBoxAttachment.load(self, dropboxID, path))
@@ -1050,7 +1125,95 @@
returnValue(count)
+ def oldManagedAttachments(self, cutoff, uuid):
+ """
+ Find managed attachments attached to only events whose last instance is older than the specified cut-off.
+ Returns a deferred to a list of (calendar_home_owner_uid, quota used, total old size, total old count) tuples.
+ """
+ kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ if uuid:
+ kwds["uuid"] = uuid
+
+ ch = schema.CALENDAR_HOME
+ chm = schema.CALENDAR_HOME_METADATA
+ tr = schema.TIME_RANGE
+ at = schema.ATTACHMENT
+ atco = schema.ATTACHMENT_CALENDAR_OBJECT
+
+ where = at.ATTACHMENT_ID.In(Select(
+ [at.ATTACHMENT_ID],
+ From=at.join(
+ atco, at.ATTACHMENT_ID == atco.ATTACHMENT_ID, "inner").join(
+ tr, atco.CALENDAR_OBJECT_RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID
+ ),
+ GroupBy=(at.ATTACHMENT_ID,),
+ Having=Max(tr.END_DATE) < Parameter("CutOff"),
+ ))
+
+ if uuid:
+ where = where.And(ch.OWNER_UID == Parameter('uuid'))
+
+ return Select(
+ [ch.OWNER_UID, chm.QUOTA_USED_BYTES, Sum(at.SIZE), Count(at.ATTACHMENT_ID)],
+ From=at.join(
+ ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID).join(
+ chm, ch.RESOURCE_ID == chm.RESOURCE_ID
+ ),
+ Where=where,
+ GroupBy=(ch.OWNER_UID, chm.QUOTA_USED_BYTES),
+ ).on(self, **kwds)
+
+
+ @inlineCallbacks
+ def removeOldManagedAttachments(self, cutoff, uuid, batchSize=None):
+ """
+ Remove attachments attached to events in the past.
+ """
+
+ # TODO: see if there is a better way to import Attachment
+ from txdav.caldav.datastore.sql import ManagedAttachment
+
+ kwds = {"CutOff": pyCalendarTodatetime(cutoff)}
+ if uuid:
+ kwds["uuid"] = uuid
+
+ options = {}
+ if batchSize:
+ options["Limit"] = batchSize
+
+ ch = schema.CALENDAR_HOME
+ tr = schema.TIME_RANGE
+ at = schema.ATTACHMENT
+ atco = schema.ATTACHMENT_CALENDAR_OBJECT
+
+ sfrom = atco.join(
+ at, atco.ATTACHMENT_ID == at.ATTACHMENT_ID, "inner").join(
+ tr, atco.CALENDAR_OBJECT_RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID
+ )
+ where = None
+ if uuid:
+ sfrom = sfrom.join(ch, at.CALENDAR_HOME_RESOURCE_ID == ch.RESOURCE_ID)
+ where = (ch.OWNER_UID == Parameter('uuid'))
+
+ results = (yield Select(
+ [atco.ATTACHMENT_ID, atco.MANAGED_ID, ],
+ From=sfrom,
+ Where=where,
+ GroupBy=(atco.ATTACHMENT_ID, atco.MANAGED_ID,),
+ Having=Max(tr.END_DATE) < Parameter("CutOff"),
+ **options
+ ).on(self, **kwds))
+
+ count = 0
+ for _ignore, managedID in results:
+ attachment = (yield ManagedAttachment.load(self, managedID))
+ yield attachment.remove()
+ count += 1
+ returnValue(count)
+
+
+
class _EmptyCacher(object):
def set(self, key, value):
@@ -1819,6 +1982,7 @@
).on(self._txn, **kwds)
+
class _SharedSyncLogic(object):
"""
Logic for maintaining sync-token shared between notification collections and
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130104/6e37e944/attachment-0001.html>
More information about the calendarserver-changes
mailing list