[CalendarServer-changes] [11334] CalendarServer/trunk/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Jun 12 13:36:25 PDT 2013
Revision: 11334
http://trac.calendarserver.org//changeset/11334
Author: cdaboo at apple.com
Date: 2013-06-12 13:36:24 -0700 (Wed, 12 Jun 2013)
Log Message:
-----------
Fix bug where attempt to migrate orphaned dropbox attachments results in an endless loop.
Modified Paths:
--------------
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2013-06-12 17:44:53 UTC (rev 11333)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2013-06-12 20:36:24 UTC (rev 11334)
@@ -171,13 +171,12 @@
# Count number to process so we can display progress
rows = (yield Select(
- (Count(at.DROPBOX_ID),),
+ (at.DROPBOX_ID,),
From=at,
Where=at.DROPBOX_ID != ".",
- GroupBy=at.DROPBOX_ID,
- Limit=batchSize,
+ Distinct=True,
).on(txn))
- total = rows[0][0]
+ total = len(rows)
count = 0
log.warn("%d dropbox ids to migrate" % (total,))
except RuntimeError, e:
@@ -247,6 +246,13 @@
log.debug(" processing attachment object: %s" % (name,))
attachment = (yield DropBoxAttachment.load(txn, dropbox_id, name))
+ # Check for orphans
+ if len(cobjs) == 0:
+ # Just remove the attachment
+ log.warn("Orphaned dropbox id removed: %s" % (attachment._path,))
+ yield attachment.remove()
+ continue
+
# Find owner objects and group all by UID
owners = []
cobj_by_UID = collections.defaultdict(list)
@@ -277,7 +283,10 @@
else:
# TODO: look for cobjs that were not changed and remove their ATTACH properties.
# These could happen if the owner object no longer exists.
- pass
+ # For now just remove the attachment
+ log.warn("Unowned dropbox id removed: %s" % (attachment._path,))
+ yield attachment.remove()
+ continue
log.debug(" finished dropbox id: %s" % (dropbox_id,))
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py 2013-06-12 17:44:53 UTC (rev 11333)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py 2013-06-12 20:36:24 UTC (rev 11334)
@@ -14,26 +14,29 @@
# limitations under the License.
##
-from twext.enterprise.dal.syntax import Delete
+from twext.enterprise.dal.syntax import Delete, Insert, Select, Count
from twisted.internet.defer import inlineCallbacks, succeed, returnValue
-from twisted.trial.unittest import TestCase
from twistedcaldav.config import config
from txdav.caldav.datastore.sql import CalendarStoreFeatures
+from txdav.caldav.datastore.test.util import CommonStoreTests
from txdav.common.datastore.sql_tables import schema
from txdav.common.datastore.test.util import theStoreBuilder, \
StubNotifierFactory
from txdav.common.datastore.upgrade.sql.others import attachment_migration
from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseOtherStep
+import hashlib
+import os
+
"""
Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
"""
-class AttachmentMigrationTests(TestCase):
+class AttachmentMigrationModeTests(CommonStoreTests):
"""
Tests for L{UpgradeDatabaseSchemaStep}.
"""
@@ -175,3 +178,77 @@
managed = (yield txn.calendarserverValue("MANAGED-ATTACHMENTS", raiseIfMissing=False))
yield txn.commit()
self.assertEqual(managed, None)
+
+
+
+class AttachmentMigrationTests(CommonStoreTests):
+ """
+ Tests for L{UpgradeDatabaseSchemaStep}.
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+ self.patch(config, "EnableManagedAttachments", True)
+
+ yield super(AttachmentMigrationTests, self).setUp()
+
+ self._sqlCalendarStore.enableManagedAttachments = True
+
+ txn = self.transactionUnderTest()
+ cs = schema.CALENDARSERVER
+ yield Delete(
+ From=cs,
+ Where=cs.NAME == "MANAGED-ATTACHMENTS"
+ ).on(txn)
+ yield self.commit()
+
+
+ @inlineCallbacks
+ def test_upgradeOrphanedAttachment(self):
+ """
+ Test L{attachment_migration.doUpgrade} when an orphaned attachment is present.
+ """
+
+ def _hasDropboxAttachments(_self, txn):
+ return succeed(True)
+ self.patch(CalendarStoreFeatures, "hasDropboxAttachments", _hasDropboxAttachments)
+
+ # Create orphaned attachment
+ dropboxID = "ABCD.dropbox"
+ attachmentName = "test.txt"
+ home = yield self.homeUnderTest(name="user01")
+ at = schema.ATTACHMENT
+ yield Insert(
+ {
+ at.CALENDAR_HOME_RESOURCE_ID: home._resourceID,
+ at.DROPBOX_ID: dropboxID,
+ at.CONTENT_TYPE: "text/plain",
+ at.SIZE: 10,
+ at.MD5: "abcd",
+ at.PATH: attachmentName,
+ }
+ ).on(self.transactionUnderTest())
+ yield self.commit()
+
+ hasheduid = hashlib.md5(dropboxID).hexdigest()
+ fp = self._sqlCalendarStore.attachmentsPath.child(hasheduid[0:2]).child(hasheduid[2:4]).child(hasheduid)
+ fp.makedirs()
+ fp = fp.child(attachmentName)
+ fp.setContent("1234567890")
+
+ self.assertTrue(os.path.exists(fp.path))
+
+ upgrader = UpgradeDatabaseOtherStep(self._sqlCalendarStore)
+ yield attachment_migration.doUpgrade(upgrader)
+
+ txn = upgrader.sqlStore.newTransaction()
+ managed = (yield txn.calendarserverValue("MANAGED-ATTACHMENTS", raiseIfMissing=False))
+ count = (yield Select(
+ [Count(at.DROPBOX_ID), ],
+ From=at,
+ ).on(txn))[0][0]
+ yield txn.commit()
+ self.assertEqual(count, 0)
+ self.assertNotEqual(managed, None)
+
+ self.assertFalse(os.path.exists(fp.path))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130612/3acb8682/attachment.html>
More information about the calendarserver-changes
mailing list