[CalendarServer-changes] [8810] CalendarServer/trunk/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Fri Mar 2 14:03:12 PST 2012
Revision: 8810
http://trac.macosforge.org/projects/calendarserver/changeset/8810
Author: sagen at apple.com
Date: 2012-03-02 14:03:12 -0800 (Fri, 02 Mar 2012)
Log Message:
-----------
When migrating data from filesystem to DB, skip any inbox items older than a cutoff (60 days by default)
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
CalendarServer/trunk/twistedcaldav/test/util.py
CalendarServer/trunk/twistedcaldav/upgrade.py
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2012-03-02 22:02:27 UTC (rev 8809)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2012-03-02 22:03:12 UTC (rev 8810)
@@ -915,6 +915,10 @@
# use to carry out work.
"UtilityServiceClass": "",
+ # Inbox items created more than MigratedInboxDaysCutoff days in the past are removed
+ # during migration
+ "MigratedInboxDaysCutoff": 60,
+
"Includes": [], # Other plists to parse after this one
}
Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2012-03-02 22:02:27 UTC (rev 8809)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2012-03-02 22:03:12 UTC (rev 8810)
@@ -1225,7 +1225,111 @@
(yield self.verifyDirectoryComparison(before, after))
+
@inlineCallbacks
+ def test_calendarsUpgradeWithInboxItems(self):
+ """
+ Verify that inbox items older than 60 days are deleted
+ """
+
+ self.setUpXMLDirectory()
+
+ before = {
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "inbox" :
+ {
+ db_basename : {
+ "@contents": "",
+ },
+ "@xattrs" :
+ {
+ # Zlib compressed XML
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ "oldinboxitem" : {
+ "@contents": "",
+ "@timestamp": 1, # really old file
+ },
+ "newinboxitem" : {
+ "@contents": "",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ NEWPROXYFILE :
+ {
+ "@contents" : "",
+ }
+ }
+
+ after = {
+ ".calendarserver_version" :
+ {
+ "@contents" : "2",
+ },
+ "inboxitems.txt" :
+ {
+ "@contents" : None, # ignore contents, the paths inside are random test directory paths
+ },
+ "calendars" :
+ {
+ "__uids__" :
+ {
+ "64" :
+ {
+ "23" :
+ {
+ "6423F94A-6B76-4A3A-815B-D52CFD77935D" :
+ {
+ "inbox" :
+ {
+ db_basename : {
+ "@contents": "",
+ },
+ "@xattrs" :
+ {
+ freeBusyAttr : zlib.compress("<?xml version='1.0' encoding='UTF-8'?>\r\n<calendar-free-busy-set xmlns='urn:ietf:params:xml:ns:caldav'>\r\n <href xmlns='DAV:'>/calendars/__uids__/6423F94A-6B76-4A3A-815B-D52CFD77935D/calendar/</href>\r\n</calendar-free-busy-set>\r\n"),
+ },
+ "newinboxitem" : {
+ "@contents": "",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ NEWPROXYFILE :
+ {
+ "@contents" : None,
+ },
+ MailGatewayTokensDatabase.dbFilename :
+ {
+ "@contents" : None,
+ },
+ "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
+ {
+ "@contents" : None,
+ "@optional" : True,
+ },
+ }
+
+ (yield self.verifyDirectoryComparison(before, after))
+
+
+ @inlineCallbacks
def test_calendarsUpgradeWithError(self):
"""
Verify that a problem with one resource doesn't stop the process, but
Modified: CalendarServer/trunk/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/util.py 2012-03-02 22:02:27 UTC (rev 8809)
+++ CalendarServer/trunk/twistedcaldav/test/util.py 2012-03-02 22:03:12 UTC (rev 8810)
@@ -182,6 +182,7 @@
with open(childPath, "w") as child:
child.write(childStructure["@contents"])
+
else:
# This is a directory
os.mkdir(childPath)
@@ -195,6 +196,11 @@
except IOError:
pass
+ # Set access and modified times
+ if childStructure.has_key("@timestamp"):
+ timestamp = childStructure["@timestamp"]
+ os.utime(childPath, (timestamp, timestamp))
+
createChildren(root, structure)
return root
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2012-03-02 22:02:27 UTC (rev 8809)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2012-03-02 22:03:12 UTC (rev 8810)
@@ -17,7 +17,7 @@
from __future__ import with_statement
-import xattr, os, zlib, hashlib, datetime, pwd, grp, shutil, errno, operator
+import xattr, os, zlib, hashlib, datetime, pwd, grp, shutil, errno, operator, time
from zlib import compress
from cPickle import loads as unpickle, UnpicklingError
@@ -485,6 +485,8 @@
# list of pending inbox items
total = 0
inboxItems = set()
+ # Remove any inbox items created more than MigratedInboxDaysCutoff days in the past
+ cutoffTimestamp = time.time() - (config.MigratedInboxDaysCutoff * 24 * 60 * 60)
for first in os.listdir(uidHomes):
if len(first) == 2:
firstPath = os.path.join(uidHomes, first)
@@ -501,7 +503,12 @@
if os.path.exists(inboxPath):
for inboxItem in os.listdir(inboxPath):
if not inboxItem.startswith("."):
- inboxItems.add(os.path.join(inboxPath, inboxItem))
+ itemPath = os.path.join(inboxPath, inboxItem)
+ timestamp = os.path.getmtime(itemPath)
+ if timestamp < cutoffTimestamp:
+ os.remove(itemPath)
+ else:
+ inboxItems.add(itemPath)
if inboxItems:
inboxItemsFile = os.path.join(config.DataRoot, INBOX_ITEMS)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120302/8d82af24/attachment-0001.html>
More information about the calendarserver-changes
mailing list