[CalendarServer-changes] [11334] CalendarServer/trunk/txdav

source_changes at macosforge.org source_changes at macosforge.org
Wed Jun 12 13:36:25 PDT 2013


Revision: 11334
          http://trac.calendarserver.org//changeset/11334
Author:   cdaboo at apple.com
Date:     2013-06-12 13:36:24 -0700 (Wed, 12 Jun 2013)
Log Message:
-----------
Fix bug where attempt to migrate orphaned dropbox attachments results in an endless loop.

Modified Paths:
--------------
    CalendarServer/trunk/txdav/caldav/datastore/sql.py
    CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py

Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py	2013-06-12 17:44:53 UTC (rev 11333)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py	2013-06-12 20:36:24 UTC (rev 11334)
@@ -171,13 +171,12 @@
 
             # Count number to process so we can display progress
             rows = (yield Select(
-                (Count(at.DROPBOX_ID),),
+                (at.DROPBOX_ID,),
                 From=at,
                 Where=at.DROPBOX_ID != ".",
-                GroupBy=at.DROPBOX_ID,
-                Limit=batchSize,
+                Distinct=True,
             ).on(txn))
-            total = rows[0][0]
+            total = len(rows)
             count = 0
             log.warn("%d dropbox ids to migrate" % (total,))
         except RuntimeError, e:
@@ -247,6 +246,13 @@
             log.debug("  processing attachment object: %s" % (name,))
             attachment = (yield DropBoxAttachment.load(txn, dropbox_id, name))
 
+            # Check for orphans
+            if len(cobjs) == 0:
+                # Just remove the attachment
+                log.warn("Orphaned dropbox id removed: %s" % (attachment._path,))
+                yield attachment.remove()
+                continue
+
             # Find owner objects and group all by UID
             owners = []
             cobj_by_UID = collections.defaultdict(list)
@@ -277,7 +283,10 @@
             else:
                 # TODO: look for cobjs that were not changed and remove their ATTACH properties.
                 # These could happen if the owner object no longer exists.
-                pass
+                # For now just remove the attachment
+                log.warn("Unowned dropbox id removed: %s" % (attachment._path,))
+                yield attachment.remove()
+                continue
 
         log.debug("  finished dropbox id: %s" % (dropbox_id,))
 

Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py	2013-06-12 17:44:53 UTC (rev 11333)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/others/test/test_attachment_migration.py	2013-06-12 20:36:24 UTC (rev 11334)
@@ -14,26 +14,29 @@
 # limitations under the License.
 ##
 
-from twext.enterprise.dal.syntax import Delete
+from twext.enterprise.dal.syntax import Delete, Insert, Select, Count
 
 from twisted.internet.defer import inlineCallbacks, succeed, returnValue
-from twisted.trial.unittest import TestCase
 
 from twistedcaldav.config import config
 
 from txdav.caldav.datastore.sql import CalendarStoreFeatures
+from txdav.caldav.datastore.test.util import CommonStoreTests
 from txdav.common.datastore.sql_tables import schema
 from txdav.common.datastore.test.util import theStoreBuilder, \
     StubNotifierFactory
 from txdav.common.datastore.upgrade.sql.others import attachment_migration
 from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseOtherStep
 
+import hashlib
+import os
+
 """
 Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
 """
 
 
-class AttachmentMigrationTests(TestCase):
+class AttachmentMigrationModeTests(CommonStoreTests):
     """
     Tests for L{UpgradeDatabaseSchemaStep}.
     """
@@ -175,3 +178,77 @@
         managed = (yield txn.calendarserverValue("MANAGED-ATTACHMENTS", raiseIfMissing=False))
         yield txn.commit()
         self.assertEqual(managed, None)
+
+
+
+class AttachmentMigrationTests(CommonStoreTests):
+    """
+    Tests for L{UpgradeDatabaseSchemaStep}.
+    """
+
+    @inlineCallbacks
+    def setUp(self):
+        self.patch(config, "EnableManagedAttachments", True)
+
+        yield super(AttachmentMigrationTests, self).setUp()
+
+        self._sqlCalendarStore.enableManagedAttachments = True
+
+        txn = self.transactionUnderTest()
+        cs = schema.CALENDARSERVER
+        yield Delete(
+            From=cs,
+            Where=cs.NAME == "MANAGED-ATTACHMENTS"
+        ).on(txn)
+        yield self.commit()
+
+
+    @inlineCallbacks
+    def test_upgradeOrphanedAttachment(self):
+        """
+        Test L{attachment_migration.doUpgrade} when an orphaned attachment is present.
+        """
+
+        def _hasDropboxAttachments(_self, txn):
+            return succeed(True)
+        self.patch(CalendarStoreFeatures, "hasDropboxAttachments", _hasDropboxAttachments)
+
+        # Create orphaned attachment
+        dropboxID = "ABCD.dropbox"
+        attachmentName = "test.txt"
+        home = yield self.homeUnderTest(name="user01")
+        at = schema.ATTACHMENT
+        yield Insert(
+            {
+                at.CALENDAR_HOME_RESOURCE_ID: home._resourceID,
+                at.DROPBOX_ID: dropboxID,
+                at.CONTENT_TYPE: "text/plain",
+                at.SIZE: 10,
+                at.MD5: "abcd",
+                at.PATH: attachmentName,
+            }
+        ).on(self.transactionUnderTest())
+        yield self.commit()
+
+        hasheduid = hashlib.md5(dropboxID).hexdigest()
+        fp = self._sqlCalendarStore.attachmentsPath.child(hasheduid[0:2]).child(hasheduid[2:4]).child(hasheduid)
+        fp.makedirs()
+        fp = fp.child(attachmentName)
+        fp.setContent("1234567890")
+
+        self.assertTrue(os.path.exists(fp.path))
+
+        upgrader = UpgradeDatabaseOtherStep(self._sqlCalendarStore)
+        yield attachment_migration.doUpgrade(upgrader)
+
+        txn = upgrader.sqlStore.newTransaction()
+        managed = (yield txn.calendarserverValue("MANAGED-ATTACHMENTS", raiseIfMissing=False))
+        count = (yield Select(
+            [Count(at.DROPBOX_ID), ],
+            From=at,
+        ).on(txn))[0][0]
+        yield txn.commit()
+        self.assertEqual(count, 0)
+        self.assertNotEqual(managed, None)
+
+        self.assertFalse(os.path.exists(fp.path))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130612/3acb8682/attachment.html>


More information about the calendarserver-changes mailing list