<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14403] CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14403">14403</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-02-11 12:34:55 -0800 (Wed, 11 Feb 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Checkpoint: attachment sync'ing during migration.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingattachmentspy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingconduitpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingrequestpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingresourcepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtesttest_conduitpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentoracledialectsql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_51_to_52sql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_51_to_52sql">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -638,6 +638,24 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+ def getAllAttachments(self):
+ """
+ Return all the L{Attachment} objects associated with this calendar home.
+ Needed during migration.
+ """
+ attachments = yield Attachment.loadAllAttachments(self)
+ returnValue(attachments)
+
+
+ def getAttachmentByID(self, id):
+ """
+ Return a specific attachment associated with this calendar home.
+ Needed during migration only.
+ """
+ return Attachment.loadAttachmentByID(self, id)
+
+
+ @inlineCallbacks
</ins><span class="cx"> def getAllDropboxIDs(self):
</span><span class="cx"> co = schema.CALENDAR_OBJECT
</span><span class="cx"> cb = schema.CALENDAR_BIND
</span><span class="lines">@@ -4646,8 +4664,10 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def attachments(self):
</span><span class="cx"> if self._dropboxID:
</span><del>- rows = yield self._attachmentsQuery.on(self._txn,
- dropboxID=self._dropboxID)
</del><ins>+ rows = yield self._attachmentsQuery.on(
+ self._txn,
+ dropboxID=self._dropboxID,
+ )
</ins><span class="cx"> result = []
</span><span class="cx"> for row in rows:
</span><span class="cx"> result.append((yield self.attachmentWithName(row[0])))
</span><span class="lines">@@ -4920,7 +4940,7 @@
</span><span class="cx">
</span><span class="cx"> _TEMPORARY_UPLOADS_DIRECTORY = "Temporary"
</span><span class="cx">
</span><del>- def __init__(self, attachment, contentType, dispositionName, creating=False):
</del><ins>+ def __init__(self, attachment, contentType, dispositionName, creating=False, migrating=False):
</ins><span class="cx"> super(AttachmentStorageTransport, self).__init__(
</span><span class="cx"> attachment, contentType, dispositionName)
</span><span class="cx">
</span><span class="lines">@@ -4930,6 +4950,7 @@
</span><span class="cx"> self._path = CachingFilePath(fileName)
</span><span class="cx"> self._hash = hashlib.md5()
</span><span class="cx"> self._creating = creating
</span><ins>+ self._migrating = migrating
</ins><span class="cx">
</span><span class="cx"> self._txn.postAbort(self.aborted)
</span><span class="cx">
</span><span class="lines">@@ -4970,6 +4991,10 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def loseConnection(self):
</span><ins>+ """
+ Note that when self._migrating is set we only care about the data and don't need to
+ do any quota checks/adjustments.
+ """
</ins><span class="cx">
</span><span class="cx"> # FIXME: this should be synchronously accessible; IAttachment should
</span><span class="cx"> # have a method for getting its parent just as CalendarObject/Calendar
</span><span class="lines">@@ -4986,20 +5011,21 @@
</span><span class="cx"> self._file.close()
</span><span class="cx">
</span><span class="cx"> # Check max size for attachment
</span><del>- if newSize > config.MaximumAttachmentSize:
</del><ins>+ if not self._migrating and newSize > config.MaximumAttachmentSize:
</ins><span class="cx"> self._path.remove()
</span><span class="cx"> if self._creating:
</span><span class="cx"> yield self._attachment._internalRemove()
</span><span class="cx"> raise AttachmentSizeTooLarge()
</span><span class="cx">
</span><span class="cx"> # Check overall user quota
</span><del>- allowed = home.quotaAllowedBytes()
- if allowed is not None and allowed < ((yield home.quotaUsedBytes())
- + (newSize - oldSize)):
- self._path.remove()
- if self._creating:
- yield self._attachment._internalRemove()
- raise QuotaExceeded()
</del><ins>+ if not self._migrating:
+ allowed = home.quotaAllowedBytes()
+ if allowed is not None and allowed < ((yield home.quotaUsedBytes())
+ + (newSize - oldSize)):
+ self._path.remove()
+ if self._creating:
+ yield self._attachment._internalRemove()
+ raise QuotaExceeded()
</ins><span class="cx">
</span><span class="cx"> self._path.moveTo(self._attachment._path)
</span><span class="cx">
</span><span class="lines">@@ -5010,7 +5036,7 @@
</span><span class="cx"> newSize
</span><span class="cx"> )
</span><span class="cx">
</span><del>- if home:
</del><ins>+ if not self._migrating and home:
</ins><span class="cx"> # Adjust quota
</span><span class="cx"> yield home.adjustQuotaUsedBytes(self._attachment.size() - oldSize)
</span><span class="cx">
</span><span class="lines">@@ -5028,6 +5054,153 @@
</span><span class="cx">
</span><span class="cx"> implements(IAttachment)
</span><span class="cx">
</span><ins>+ _attachmentSchema = schema.ATTACHMENT
+
+ @classmethod
+ def makeClass(cls, txn, attachmentData):
+ """
+ Given the various database rows, build the actual class.
+
+ @param parent: the parent collection object
+ @type parent: L{CommonHomeChild}
+
+ @param objectData: the standard set of object columns
+ @type objectData: C{list}
+
+ @param propstore: a property store to use, or C{None} to load it
+ automatically
+ @type propstore: L{PropertyStore}
+
+ @return: the constructed child class
+ @rtype: L{CommonHomeChild}
+ """
+
+ att = schema.ATTACHMENT
+ dropbox_id = attachmentData[cls._allColumns().index(att.DROPBOX_ID)]
+ c = ManagedAttachment if dropbox_id == "." else DropBoxAttachment
+ child = c(
+ txn,
+ attachmentData[cls._allColumns().index(att.ATTACHMENT_ID)],
+ attachmentData[cls._allColumns().index(att.DROPBOX_ID)],
+ attachmentData[cls._allColumns().index(att.PATH)],
+ )
+
+ for attr, value in zip(child._rowAttributes(), attachmentData):
+ setattr(child, attr, value)
+ child._contentType = MimeType.fromString(child._contentType)
+
+ return child
+
+
+ @classmethod
+ def _allColumns(cls):
+ """
+ Full set of columns in the object table that need to be loaded to
+ initialize the object resource state.
+ """
+ att = cls._attachmentSchema
+ return [
+ att.ATTACHMENT_ID,
+ att.DROPBOX_ID,
+ att.CALENDAR_HOME_RESOURCE_ID,
+ att.CONTENT_TYPE,
+ att.SIZE,
+ att.MD5,
+ att.CREATED,
+ att.MODIFIED,
+ att.PATH,
+ ]
+
+
+ @classmethod
+ def _rowAttributes(cls):
+ """
+ Object attributes used to store the column values from L{_allColumns}. This used to create
+ a mapping when serializing the object for cross-pod requests.
+ """
+ return (
+ "_attachmentID",
+ "_dropboxID",
+ "_ownerHomeID",
+ "_contentType",
+ "_size",
+ "_md5",
+ "_created",
+ "_modified",
+ "_name",
+ )
+
+
+ @classmethod
+ @inlineCallbacks
+ def loadAllAttachments(cls, home):
+ """
+ Load all attachments assigned to the specified home collection. This should only be
+ used when sync'ing an entire home's set of attachments.
+ """
+
+ results = []
+
+ # Load from the main table first
+ att = cls._attachmentSchema
+ dataRows = yield Select(
+ cls._allColumns(),
+ From=att,
+ Where=att.CALENDAR_HOME_RESOURCE_ID == home.id(),
+ ).on(home._txn)
+
+ # Create the actual objects
+ for row in dataRows:
+ child = cls.makeClass(home._txn, row)
+ results.append(child)
+
+ returnValue(results)
+
+
+ @classmethod
+ @inlineCallbacks
+ def loadAttachmentByID(cls, home, id):
+ """
+ Load one attachments assigned to the specified home collection. This should only be
+ used when sync'ing an entire home's set of attachments.
+ """
+
+ # Load from the main table first
+ att = cls._attachmentSchema
+ rows = yield Select(
+ cls._allColumns(),
+ From=att,
+ Where=(att.CALENDAR_HOME_RESOURCE_ID == home.id()).And(
+ att.ATTACHMENT_ID == id),
+ ).on(home._txn)
+
+ # Create the actual object
+ returnValue(cls.makeClass(home._txn, rows[0]) if len(rows) == 1 else None)
+
+
+ def externalize(self):
+ """
+ Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
+ and reconstituted at the other end. Note that the other end may have a different schema so
+ the attributes may not match exactly and will need to be processed accordingly.
+ """
+ result = dict([(attr[1:], getattr(self, attr, None)) for attr in self._rowAttributes()])
+ result["contentType"] = generateContentType(result["contentType"])
+ return result
+
+
+ @classmethod
+ def internalize(cls, txn, mapping):
+ """
+ Given a mapping generated by L{externalize}, convert the values into an array of database
+ like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
+ Note that there may be a schema mismatch with the external data, so treat missing items as
+ C{None} and ignore extra items.
+ """
+
+ return cls.makeClass(txn, [mapping.get(row[1:]) for row in cls._rowAttributes()])
+
+
</ins><span class="cx"> def __init__(self, txn, a_id, dropboxID, name, ownerHomeID=None, justCreated=False):
</span><span class="cx"> self._txn = txn
</span><span class="cx"> self._attachmentID = a_id
</span><span class="lines">@@ -5061,23 +5234,13 @@
</span><span class="cx"> @return: C{True} if this attachment exists, C{False} otherwise.
</span><span class="cx"> """
</span><span class="cx"> att = schema.ATTACHMENT
</span><del>- if self._dropboxID:
</del><ins>+ if self._dropboxID and self._dropboxID != ".":
</ins><span class="cx"> where = (att.DROPBOX_ID == self._dropboxID).And(
</span><span class="cx"> att.PATH == self._name)
</span><span class="cx"> else:
</span><span class="cx"> where = (att.ATTACHMENT_ID == self._attachmentID)
</span><span class="cx"> rows = (yield Select(
</span><del>- [
- att.ATTACHMENT_ID,
- att.DROPBOX_ID,
- att.CALENDAR_HOME_RESOURCE_ID,
- att.CONTENT_TYPE,
- att.SIZE,
- att.MD5,
- att.CREATED,
- att.MODIFIED,
- att.PATH,
- ],
</del><ins>+ self._allColumns(),
</ins><span class="cx"> From=att,
</span><span class="cx"> Where=where
</span><span class="cx"> ).on(self._txn))
</span><span class="lines">@@ -5085,20 +5248,28 @@
</span><span class="cx"> if not rows:
</span><span class="cx"> returnValue(None)
</span><span class="cx">
</span><del>- row_iter = iter(rows[0])
- self._attachmentID = row_iter.next()
- self._dropboxID = row_iter.next()
- self._ownerHomeID = row_iter.next()
- self._contentType = MimeType.fromString(row_iter.next())
- self._size = row_iter.next()
- self._md5 = row_iter.next()
- self._created = sqltime(row_iter.next())
- self._modified = sqltime(row_iter.next())
- self._name = row_iter.next()
</del><ins>+ for attr, value in zip(self._rowAttributes(), rows[0]):
+ setattr(self, attr, value)
+ self._contentType = MimeType.fromString(self._contentType)
</ins><span class="cx">
</span><span class="cx"> returnValue(self)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def copyRemote(self, remote):
+ """
+ Copy properties from a remote (external) attachment that is being migrated.
+
+ @param remote: the external attachment
+ @type remote: L{Attachment}
+ """
+ yield self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
+
+
+ def id(self):
+ return self._attachmentID
+
+
</ins><span class="cx"> def dropboxID(self):
</span><span class="cx"> return self._dropboxID
</span><span class="cx">
</span><span class="lines">@@ -5115,10 +5286,10 @@
</span><span class="cx"> pass # stub
</span><span class="cx">
</span><span class="cx">
</span><del>- def store(self, contentType, dispositionName=None):
</del><ins>+ def store(self, contentType, dispositionName=None, migrating=False):
</ins><span class="cx"> if not self._name:
</span><span class="cx"> self._name = dispositionName
</span><del>- return AttachmentStorageTransport(self, contentType, dispositionName, self._justCreated)
</del><ins>+ return AttachmentStorageTransport(self, contentType, dispositionName, self._justCreated, migrating=migrating)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def retrieve(self, protocol):
</span><span class="lines">@@ -5135,17 +5306,19 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def remove(self):
</del><ins>+ def remove(self, adjustQuota=True):
</ins><span class="cx"> oldSize = self._size
</span><span class="cx"> self._txn.postCommit(self.removePaths)
</span><span class="cx"> yield self._internalRemove()
</span><ins>+
</ins><span class="cx"> # Adjust quota
</span><del>- home = (yield self._txn.calendarHomeWithResourceID(self._ownerHomeID))
- if home:
- yield home.adjustQuotaUsedBytes(-oldSize)
</del><ins>+ if adjustQuota:
+ home = (yield self._txn.calendarHomeWithResourceID(self._ownerHomeID))
+ if home:
+ yield home.adjustQuotaUsedBytes(-oldSize)
</ins><span class="cx">
</span><del>- # Send change notification to home
- yield home.notifyChanged()
</del><ins>+ # Send change notification to home
+ yield home.notifyChanged()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def removePaths(self):
</span><span class="lines">@@ -5200,7 +5373,7 @@
</span><span class="cx"> ).on(txn))
</span><span class="cx">
</span><span class="cx"> for attachmentID, dropboxID in rows:
</span><del>- if dropboxID:
</del><ins>+ if dropboxID != ".":
</ins><span class="cx"> attachment = DropBoxAttachment(txn, attachmentID, None, None)
</span><span class="cx"> else:
</span><span class="cx"> attachment = ManagedAttachment(txn, attachmentID, None, None)
</span><span class="lines">@@ -5473,7 +5646,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def create(cls, txn, managedID, ownerHomeID, referencedBy):
</span><span class="cx"> """
</span><del>- Create a new Attachment object.
</del><ins>+ Create a new Attachment object and reference it.
</ins><span class="cx">
</span><span class="cx"> @param txn: The transaction to use
</span><span class="cx"> @type txn: L{CommonStoreTransaction}
</span><span class="lines">@@ -5504,7 +5677,8 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def update(cls, txn, oldManagedID, ownerHomeID, referencedBy, oldAttachmentID):
</span><span class="cx"> """
</span><del>- Create a new Attachment object.
</del><ins>+ Update an Attachment object. This creates a new one and adjusts the reference to the old
+ one to point to the new one. If the old one is no longer referenced at all, it is deleted.
</ins><span class="cx">
</span><span class="cx"> @param txn: The transaction to use
</span><span class="cx"> @type txn: L{CommonStoreTransaction}
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -22,7 +22,8 @@
</span><span class="cx">
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx">
</span><del>-from txdav.caldav.datastore.sql import CalendarHome, Calendar, CalendarObject
</del><ins>+from txdav.caldav.datastore.sql import CalendarHome, Calendar, CalendarObject, \
+ Attachment
</ins><span class="cx"> from txdav.caldav.icalendarstore import ComponentUpdateState, ComponentRemoveState
</span><span class="cx"> from txdav.common.datastore.sql_external import CommonHomeExternal, CommonHomeChildExternal, \
</span><span class="cx"> CommonObjectResourceExternal
</span><span class="lines">@@ -61,6 +62,30 @@
</span><span class="cx"> raise AssertionError("CommonHomeExternal: not supported")
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def getAllAttachments(self):
+ """
+ Return all the L{Attachment} objects associated with this calendar home.
+ Needed during migration.
+ """
+ raw_results = yield self._txn.store().conduit.send_get_all_attachments(self)
+
+ results = []
+ for attachment in raw_results:
+ results.append(Attachment.internalize(self._txn, attachment))
+ returnValue(results)
+
+
+ @inlineCallbacks
+ def readAttachmentData(self, remote_id, attachment):
+ """
+ Read the data associated with an attachment associated with this calendar home.
+ Needed during migration only.
+ """
+ stream = attachment.store(attachment.contentType(), attachment.name(), migrating=True)
+ yield self._txn.store().conduit.send_get_attachment_data(self, remote_id, stream)
+
+
</ins><span class="cx"> def getAllDropboxIDs(self):
</span><span class="cx"> """
</span><span class="cx"> No children.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingattachmentspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -15,6 +15,8 @@
</span><span class="cx"> ##
</span><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><ins>+from txdav.caldav.icalendarstore import InvalidAttachmentOperation
+from txweb2.http_headers import generateContentType
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> class AttachmentsConduitMixin(object):
</span><span class="lines">@@ -150,3 +152,73 @@
</span><span class="cx"> request["rids"],
</span><span class="cx"> request["managedID"],
</span><span class="cx"> )
</span><ins>+
+
+ @inlineCallbacks
+ def send_get_all_attachments(self, home):
+ """
+ Managed attachment removeAttachment call.
+
+ @param home: the home whose attachments are being requested
+ @type home: L{CalendarHome}
+ """
+
+ actionName = "get-all-attachments"
+ txn, request, server = yield self._getRequestForStoreObject(actionName, home, False)
+
+ response = yield self.sendRequestToServer(txn, server, request)
+ returnValue(response)
+
+
+ @inlineCallbacks
+ def recv_get_all_attachments(self, txn, request):
+ """
+ Process an getAllAttachments cross-pod request. Request arguments as per L{send_get_all_attachments}.
+
+ @param request: request arguments
+ @type request: C{dict}
+ """
+
+ home, _ignore = yield self._getStoreObjectForRequest(txn, request)
+ attachments = yield home.getAllAttachments()
+ returnValue([attachment.externalize() for attachment in attachments])
+
+
+ @inlineCallbacks
+ def send_get_attachment_data(self, home, attachment_id, stream):
+ """
+ Managed attachment readAttachmentData call. We are using streams on the sender and the receiver
+ side to avoid reading the whole attachment into memory.
+
+ @param home: the home whose attachment is being read
+ @type home: L{CalendarHome}
+ @param attachment_id: attachment-id to get
+ @type attachment_id: C{str}
+ @param stream: attachment data stream to write to
+ @type stream: L{IStream}
+ """
+
+ actionName = "get-attachment-data"
+ txn, request, server = yield self._getRequestForStoreObject(actionName, home, False)
+ request["attachmentID"] = attachment_id
+
+ response = yield self.sendRequestToServer(txn, server, request, writeStream=stream)
+ returnValue(response)
+
+
+ @inlineCallbacks
+ def recv_get_attachment_data(self, txn, request, stream):
+ """
+ Process an getAttachmentData cross-pod request. Request arguments as per L{send_get_attachment_data}.
+
+ @param request: request arguments
+ @type request: C{dict}
+ """
+
+ home, _ignore = yield self._getStoreObjectForRequest(txn, request)
+ attachment = yield home.getAttachmentByID(request["attachmentID"])
+ if attachment is None:
+ raise InvalidAttachmentOperation("Attachment is missing: {}".format(request["attachmentID"]))
+
+ attachment.retrieve(stream)
+ returnValue((generateContentType(attachment.contentType()), attachment.name(),))
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingconduitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -72,6 +72,7 @@
</span><span class="cx"> @param store: the L{CommonDataStore} in use.
</span><span class="cx"> """
</span><span class="cx"> self.store = store
</span><ins>+ self.streamingActions = ("get-attachment-data",)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -108,9 +109,9 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def sendRequestToServer(self, txn, server, data, stream=None, streamType=None):
</del><ins>+ def sendRequestToServer(self, txn, server, data, stream=None, streamType=None, writeStream=None):
</ins><span class="cx">
</span><del>- request = self.conduitRequestClass(server, data, stream, streamType)
</del><ins>+ request = self.conduitRequestClass(server, data, stream, streamType, writeStream)
</ins><span class="cx"> try:
</span><span class="cx"> response = (yield request.doRequest(txn))
</span><span class="cx"> except Exception as e:
</span><span class="lines">@@ -123,6 +124,24 @@
</span><span class="cx"> returnValue(response.get("value"))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def isStreamAction(self, data):
+ """
+ Check to see if this is a request that will return a data stream rather than a JSON response.
+ e.g., this is used to retrieve attachment data on another pod.
+
+ @param data: the JSON data to process
+ @type data: C{dict}
+ """
+ # Must have a dict with an "action" key
+ try:
+ action = data["action"]
+ except (KeyError, TypeError) as e:
+ log.error("JSON data must have an object as its root with an 'action' attribute: {ex}\n{json}", ex=e, json=data)
+ return False
+
+ return action in self.streamingActions
+
+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def processRequest(self, data):
</span><span class="cx"> """
</span><span class="lines">@@ -171,3 +190,48 @@
</span><span class="cx"> yield txn.commit()
</span><span class="cx">
</span><span class="cx"> returnValue(result)
</span><ins>+
+
+ @inlineCallbacks
+ def processRequestStream(self, data, stream):
+ """
+ Process the request.
+
+ @param data: the JSON data to process
+ @type data: C{dict}
+
+ @return: a L{tuple} of content-type and name, if successful, else a L{dict} for a JSON result
+ @rtype: L{tuple} of (L{str}, L{str}), or L{dict}
+ """
+ # Must have a dict with an "action" key
+ try:
+ action = data["action"]
+ except (KeyError, TypeError) as e:
+ log.error("JSON data must have an object as its root with an 'action' attribute: {ex}\n{json}", ex=e, json=data)
+ raise FailedCrossPodRequestError("JSON data must have an object as its root with an 'action' attribute: {}\n{}".format(e, data,))
+
+ method = "recv_{}".format(action.replace("-", "_"))
+ if not hasattr(self, method):
+ log.error("Unsupported action: {action}", action=action)
+ raise FailedCrossPodRequestError("Unsupported action: {}".format(action))
+
+ # Need a transaction to work with
+ txn = self.store.newTransaction(repr("Conduit request"))
+
+ # Do the actual request processing
+ try:
+ result = (yield getattr(self, method)(txn, data, stream))
+ except Exception as e:
+ # Send the exception over to the other side
+ yield txn.abort()
+ log.error("Failed action: {action}, {ex}", action=action, ex=e)
+ result = {
+ "result": "exception",
+ "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
+ "details": str(e),
+ }
+
+ else:
+ yield txn.commit()
+
+ returnValue(result)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -23,6 +23,7 @@
</span><span class="cx"> from twisted.internet.defer import returnValue, inlineCallbacks
</span><span class="cx"> from twisted.python.failure import Failure
</span><span class="cx"> from txdav.caldav.icalendarstore import ComponentUpdateState
</span><ins>+from txdav.caldav.datastore.sql import ManagedAttachment
</ins><span class="cx"> from txdav.common.datastore.sql_tables import schema
</span><span class="cx"> from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
</span><span class="cx">
</span><span class="lines">@@ -159,7 +160,7 @@
</span><span class="cx"> yield self.syncCalendarHomeMetaData()
</span><span class="cx">
</span><span class="cx"> # TODO: sync attachments
</span><del>- pass
</del><ins>+ yield self.syncAttachments()
</ins><span class="cx">
</span><span class="cx"> # TODO: group attendee/sharee reconcile
</span><span class="cx"> pass
</span><span class="lines">@@ -172,6 +173,9 @@
</span><span class="cx"> rows, recalculate quota etc.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ # TODO: link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
+ pass
+
</ins><span class="cx"> # TODO: Re-write attachment URIs - not sure if we need this as reverse proxy may take care of it
</span><span class="cx"> pass
</span><span class="cx">
</span><span class="lines">@@ -184,7 +188,10 @@
</span><span class="cx"> # TODO: notifications
</span><span class="cx"> pass
</span><span class="cx">
</span><ins>+ # TODO: work items
+ pass
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def disableRemoteHome(self):
</span><span class="cx"> """
</span><span class="lines">@@ -320,11 +327,11 @@
</span><span class="cx"> """
</span><span class="cx"> Get local synchronization state for the home being migrated.
</span><span class="cx"> """
</span><del>- cms = schema.CALENDAR_MIGRATION_STATE
</del><ins>+ cm = schema.CALENDAR_MIGRATION
</ins><span class="cx"> rows = yield Select(
</span><del>- columns=(cms.REMOTE_RESOURCE_ID, cms.CALENDAR_RESOURCE_ID, cms.LAST_SYNC_TOKEN,),
- From=cms,
- Where=(cms.CALENDAR_HOME_RESOURCE_ID == self.homeId)
</del><ins>+ columns=(cm.REMOTE_RESOURCE_ID, cm.CALENDAR_RESOURCE_ID, cm.LAST_SYNC_TOKEN,),
+ From=cm,
+ Where=(cm.CALENDAR_HOME_RESOURCE_ID == self.homeId)
</ins><span class="cx"> ).on(txn)
</span><span class="cx"> returnValue(dict([(remote_id, self.CalendarSyncState(local_id, sync,)) for remote_id, local_id, sync in rows]))
</span><span class="cx">
</span><span class="lines">@@ -335,7 +342,7 @@
</span><span class="cx"> """
</span><span class="cx"> Get local synchronization state for the home being migrated.
</span><span class="cx"> """
</span><del>- cms = schema.CALENDAR_MIGRATION_STATE
</del><ins>+ cm = schema.CALENDAR_MIGRATION
</ins><span class="cx">
</span><span class="cx"> old_details = yield self.getSyncState(txn=txn)
</span><span class="cx">
</span><span class="lines">@@ -343,9 +350,9 @@
</span><span class="cx"> missing = set(old_details.keys()) - set(details.keys())
</span><span class="cx"> if missing:
</span><span class="cx"> yield Delete(
</span><del>- From=cms,
- Where=(cms.CALENDAR_HOME_RESOURCE_ID == self.homeId).And(
- cms.REMOTE_RESOURCE_ID.In(Parameter("missing", len(missing)))
</del><ins>+ From=cm,
+ Where=(cm.CALENDAR_HOME_RESOURCE_ID == self.homeId).And(
+ cm.REMOTE_RESOURCE_ID.In(Parameter("missing", len(missing)))
</ins><span class="cx"> )
</span><span class="cx"> ).on(txn, missing=missing)
</span><span class="cx">
</span><span class="lines">@@ -353,10 +360,10 @@
</span><span class="cx"> insert = set(details.keys()) - set(old_details.keys())
</span><span class="cx"> for key in insert:
</span><span class="cx"> yield Insert({
</span><del>- cms.CALENDAR_HOME_RESOURCE_ID: self.homeId,
- cms.REMOTE_RESOURCE_ID: key,
- cms.CALENDAR_RESOURCE_ID: details[key].localID,
- cms.LAST_SYNC_TOKEN: details[key].lastSyncToken,
</del><ins>+ cm.CALENDAR_HOME_RESOURCE_ID: self.homeId,
+ cm.REMOTE_RESOURCE_ID: key,
+ cm.CALENDAR_RESOURCE_ID: details[key].localID,
+ cm.LAST_SYNC_TOKEN: details[key].lastSyncToken,
</ins><span class="cx"> }).on(txn)
</span><span class="cx">
</span><span class="cx"> # Update existing ones
</span><span class="lines">@@ -364,11 +371,11 @@
</span><span class="cx"> for key in updates:
</span><span class="cx"> yield Update(
</span><span class="cx"> {
</span><del>- cms.CALENDAR_RESOURCE_ID: details[key].localID,
- cms.LAST_SYNC_TOKEN: details[key].lastSyncToken,
</del><ins>+ cm.CALENDAR_RESOURCE_ID: details[key].localID,
+ cm.LAST_SYNC_TOKEN: details[key].lastSyncToken,
</ins><span class="cx"> },
</span><del>- Where=(cms.CALENDAR_HOME_RESOURCE_ID == self.homeId).And(
- cms.REMOTE_RESOURCE_ID == key
</del><ins>+ Where=(cm.CALENDAR_HOME_RESOURCE_ID == self.homeId).And(
+ cm.REMOTE_RESOURCE_ID == key
</ins><span class="cx"> )
</span><span class="cx"> ).on(txn)
</span><span class="cx">
</span><span class="lines">@@ -612,6 +619,7 @@
</span><span class="cx"> # matches the remote one (which should help reduce the need for a client to resync
</span><span class="cx"> # the data when moved from one pod to the other).
</span><span class="cx"> txn._migrating = True
</span><ins>+ com = schema.CALENDAR_OBJECT_MIGRATION
</ins><span class="cx"> for obj_name in remote_objects.keys():
</span><span class="cx"> remote_object = remote_objects[obj_name]
</span><span class="cx"> remote_data = yield remote_object.component()
</span><span class="lines">@@ -623,9 +631,126 @@
</span><span class="cx"> else:
</span><span class="cx"> local_object = yield local_calendar._createCalendarObjectWithNameInternal(obj_name, remote_data, internal_state=ComponentUpdateState.RAW)
</span><span class="cx">
</span><ins>+ # Maintain the mapping from the remote to local id. Note that this mapping never changes as the ids on both
+ # sides are immutable - though it may get deleted if the local object is removed during sync (via a cascade).
+ yield Insert(
+ {
+ com.CALENDAR_HOME_RESOURCE_ID: self.homeId,
+ com.REMOTE_RESOURCE_ID: remote_object.id(),
+ com.LOCAL_RESOURCE_ID: local_object.id()
+ }
+ ).on(txn)
+
</ins><span class="cx"> # Sync meta-data such as schedule object, schedule tags, access mode etc
</span><span class="cx"> yield local_object.copyMetadata(remote_object)
</span><span class="cx">
</span><span class="cx"> # Purge the ones that remain
</span><span class="cx"> for local_object in local_objects.values():
</span><span class="cx"> yield local_object.purge()
</span><ins>+
+
+ @inlineCallbacks
+ def syncAttachments(self):
+ """
+ Sync attachments (both metadata and actual attachment data) for the home being migrated.
+ """
+
+ # Two steps - sync the table first in one txn, then sync each attachment's data
+ changed_ids, removed_ids = yield self.syncAttachmentTable()
+
+ for local_id in changed_ids:
+ yield self.syncAttachmentData(local_id)
+
+ returnValue((changed_ids, removed_ids,))
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def syncAttachmentTable(self, txn):
+ """
+ Sync the ATTACHMENT table data for the home being migrated. Return the list of local attachment ids that
+ now need there attachment data sync'd from the server.
+ """
+
+ remote_home = yield self._remoteHome(txn)
+ rattachments = yield remote_home.getAllAttachments()
+ rmap = dict([(attachment.id(), attachment) for attachment in rattachments])
+
+ local_home = yield txn.calendarHomeWithUID(self.migratingUid())
+ lattachments = yield local_home.getAllAttachments()
+ lmap = dict([(attachment.id(), attachment) for attachment in lattachments])
+
+ # Figure out the differences
+ am = schema.ATTACHMENT_MIGRATION
+ rows = yield Select(
+ [am.REMOTE_RESOURCE_ID, am.LOCAL_RESOURCE_ID],
+ From=am,
+ Where=(am.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+ ).on(txn)
+ mapping = dict(rows)
+
+ # Removed - remove attachment and migration state
+ removed = set(mapping.keys()) - set(rmap.keys())
+ for remove_id in removed:
+ local_id = mapping[remove_id]
+ att = yield ManagedAttachment.load(txn, None, None, attachmentID=local_id)
+ if att:
+ yield att.remove(adjustQuota=False)
+ yield Delete(
+ From=am,
+ Where=(am.LOCAL_RESOURCE_ID == local_id),
+ ).on(txn)
+
+ # Track which ones need attachment data sync'd over
+ data_ids = set()
+
+ # Added - add new attachment and migration state
+ added = set(rmap.keys()) - set(mapping.keys())
+ for added_id in added:
+ attachment = yield ManagedAttachment._create(txn, None, self.homeId)
+ yield Insert(
+ {
+ am.CALENDAR_HOME_RESOURCE_ID: self.homeId,
+ am.REMOTE_RESOURCE_ID: added_id,
+ am.LOCAL_RESOURCE_ID: attachment.id(),
+ }
+ ).on(txn)
+ data_ids.add(attachment.id())
+
+ # Possible updates - check for md5 change and sync
+ updates = set(mapping.keys()) & set(rmap.keys())
+ for updated_id in updates:
+ local_id = mapping[updated_id]
+ if rmap[updated_id].md5() != lmap[local_id].md5():
+ yield lmap[local_id].copyRemote(rmap[updated_id])
+ data_ids.add(local_id)
+
+ returnValue((data_ids, removed,))
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def syncAttachmentData(self, txn, local_id):
+ """
+ Sync the attachment data for the home being migrated.
+ """
+
+ remote_home = yield self._remoteHome(txn)
+ local_home = yield txn.calendarHomeWithUID(self.migratingUid())
+ attachment = yield local_home.getAttachmentByID(local_id)
+ if attachment is None:
+ returnValue(None)
+
+ am = schema.ATTACHMENT_MIGRATION
+ rows = yield Select(
+ [am.LOCAL_RESOURCE_ID, am.REMOTE_RESOURCE_ID],
+ From=am,
+ Where=(am.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+ ).on(txn)
+ mapping = dict(rows)
+ remote_id = mapping.get(local_id)
+ if remote_id is None:
+ returnValue(None)
+
+ # Read the data from the conduit
+ yield remote_home.readAttachmentData(remote_id, attachment)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -14,11 +14,15 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><ins>+from pycalendar.datetime import DateTime
+from twext.enterprise.dal.syntax import Select
</ins><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><ins>+from twistedcaldav.ical import Component, normalize_iCalStr
+from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
</ins><span class="cx"> from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
</span><del>-from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
-from pycalendar.datetime import DateTime
-from twistedcaldav.ical import Component, normalize_iCalStr
</del><ins>+from txdav.common.datastore.sql_tables import schema
+from txweb2.http_headers import MimeType
+from txweb2.stream import MemoryStream
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> class TestConduitAPI(MultiStoreConduitTest):
</span><span class="lines">@@ -246,10 +250,11 @@
</span><span class="cx">
</span><span class="cx"> home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
</span><span class="cx"> calendar0 = yield home0.childWithName("calendar")
</span><del>- yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
- yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
- yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
</del><ins>+ o1 = yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ o2 = yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
+ o3 = yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
</ins><span class="cx"> remote_id = calendar0.id()
</span><ins>+ mapping0 = dict([(o.name(), o.id()) for o in (o1, o2, o3)])
</ins><span class="cx"> yield self.commitTransaction(0)
</span><span class="cx">
</span><span class="cx"> syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
</span><span class="lines">@@ -273,12 +278,26 @@
</span><span class="cx"> self.assertEqual(len(local_sync_state), 1)
</span><span class="cx"> self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _checkCalendarObjectMigrationState(home, mapping1):
+ com = schema.CALENDAR_OBJECT_MIGRATION
+ mappings = yield Select(
+ columns=[com.REMOTE_RESOURCE_ID, com.LOCAL_RESOURCE_ID],
+ From=com,
+ Where=(com.CALENDAR_HOME_RESOURCE_ID == home.id())
+ ).on(self.theTransactionUnderTest(1))
+ expected_mappings = dict([(mapping0[name], mapping1[name]) for name in mapping0.keys()])
+ self.assertEqual(dict(mappings), expected_mappings)
+
+
</ins><span class="cx"> # Local calendar exists
</span><span class="cx"> home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
</span><span class="cx"> calendar1 = yield home1.childWithName("calendar")
</span><span class="cx"> self.assertTrue(calendar1 is not None)
</span><del>- children = yield calendar1.listObjectResources()
- self.assertEqual(set(children), set(("1.ics", "2.ics", "3.ics",)))
</del><ins>+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "2.ics", "3.ics",)))
+ mapping1 = dict([(o.name(), o.id()) for o in children])
+ yield _checkCalendarObjectMigrationState(home1, mapping1)
</ins><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="cx"> # Change one resource
</span><span class="lines">@@ -307,6 +326,7 @@
</span><span class="cx"> txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="2.ics"
</span><span class="cx"> )
</span><span class="cx"> yield object0.remove()
</span><ins>+ del mapping0["2.ics"]
</ins><span class="cx"> yield self.commitTransaction(0)
</span><span class="cx">
</span><span class="cx"> remote_sync_state = yield syncer.getCalendarSyncList()
</span><span class="lines">@@ -317,13 +337,16 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), name="calendar")
</span><del>- children = yield calendar1.listObjectResources()
- self.assertEqual(set(children), set(("1.ics", "3.ics",)))
</del><ins>+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "3.ics",)))
+ mapping1 = dict([(o.name(), o.id()) for o in children])
+ yield _checkCalendarObjectMigrationState(home1, mapping1)
</ins><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="cx"> # Add one resource
</span><span class="cx"> calendar0 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user01", name="calendar")
</span><del>- yield calendar0.createCalendarObjectWithName("4.ics", Component.fromString(self.caldata4))
</del><ins>+ o4 = yield calendar0.createCalendarObjectWithName("4.ics", Component.fromString(self.caldata4))
+ mapping0[o4.name()] = o4.id()
</ins><span class="cx"> yield self.commitTransaction(0)
</span><span class="cx">
</span><span class="cx"> remote_sync_state = yield syncer.getCalendarSyncList()
</span><span class="lines">@@ -334,8 +357,10 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(1), home=syncer.migratingUid(), name="calendar")
</span><del>- children = yield calendar1.listObjectResources()
- self.assertEqual(set(children), set(("1.ics", "3.ics", "4.ics",)))
</del><ins>+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "3.ics", "4.ics")))
+ mapping1 = dict([(o.name(), o.id()) for o in children])
+ yield _checkCalendarObjectMigrationState(home1, mapping1)
</ins><span class="cx"> yield self.commitTransaction(1)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -400,3 +425,166 @@
</span><span class="cx"> self.assertTrue("new-calendar" not in details1.values())
</span><span class="cx"> self.assertEqual(set(details1.values()), set(details0.values()))
</span><span class="cx"> yield self.commitTransaction(1)
</span><ins>+
+
+ @inlineCallbacks
+ def test_sync_attachments_add_remove(self):
+ """
+ Test that L{syncAttachments} syncs attachment data, then an update to the data,
+ and finally a removal of the data.
+ """
+
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
+ yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
+ remote_id = calendar0.id()
+ mapping0 = dict()
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ syncer.homeId = yield syncer.prepareCalendarHome()
+
+ # Trigger sync of the one calendar
+ local_sync_state = {}
+ remote_sync_state = yield syncer.getCalendarSyncList()
+ yield syncer.syncCalendar(
+ remote_id,
+ local_sync_state,
+ remote_sync_state,
+ )
+ self.assertEqual(len(local_sync_state), 1)
+ self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set())
+ self.assertEqual(removed, set())
+
+ @inlineCallbacks
+ def _checkAttachmentObjectMigrationState(home, mapping1):
+ am = schema.ATTACHMENT_MIGRATION
+ mappings = yield Select(
+ columns=[am.REMOTE_RESOURCE_ID, am.LOCAL_RESOURCE_ID],
+ From=am,
+ Where=(am.CALENDAR_HOME_RESOURCE_ID == home.id())
+ ).on(self.theTransactionUnderTest(1))
+ expected_mappings = dict([(mapping0[name], mapping1[name]) for name in mapping0.keys()])
+ self.assertEqual(dict(mappings), expected_mappings)
+
+
+ # Local calendar exists
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is not None)
+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "2.ics", "3.ics",)))
+
+ attachments = yield home1.getAllAttachments()
+ mapping1 = dict([(o.md5(), o.id()) for o in attachments])
+ yield _checkAttachmentObjectMigrationState(home1, mapping1)
+ yield self.commitTransaction(1)
+
+ # Add one attachment
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1."))
+ id0_1 = attachment.id()
+ md50_1 = attachment.md5()
+ managedid0_1 = attachment.managedID()
+ mapping0[md50_1] = id0_1
+ yield self.commitTransaction(0)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set((id0_1,)))
+ self.assertEqual(removed, set())
+
+ # Validate changes
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ attachments = yield home1.getAllAttachments()
+ mapping1 = dict([(o.md5(), o.id()) for o in attachments])
+ yield _checkAttachmentObjectMigrationState(home1, mapping1)
+
+ # Add another attachment
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="2.ics")
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test2.txt", MemoryStream("Here is some text #2."))
+ id0_2 = attachment.id()
+ md50_2 = attachment.md5()
+ mapping0[md50_2] = id0_2
+ yield self.commitTransaction(0)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set((id0_2,)))
+ self.assertEqual(removed, set())
+
+ # Validate changes
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ attachments = yield home1.getAllAttachments()
+ mapping1 = dict([(o.md5(), o.id()) for o in attachments])
+ yield _checkAttachmentObjectMigrationState(home1, mapping1)
+
+ # Change original attachment (this is actually a remove and a create all in one)
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ attachment, _ignore_location = yield object1.updateAttachment(managedid0_1, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1 - changed."))
+ del mapping0[md50_1]
+ id0_1_changed = attachment.id()
+ md50_1_changed = attachment.md5()
+ managedid0_1_changed = attachment.managedID()
+ mapping0[md50_1_changed] = id0_1_changed
+ yield self.commitTransaction(0)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set((id0_1_changed,)))
+ self.assertEqual(removed, set((id0_1,)))
+
+ # Validate changes
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ attachments = yield home1.getAllAttachments()
+ mapping1 = dict([(o.md5(), o.id()) for o in attachments])
+ yield _checkAttachmentObjectMigrationState(home1, mapping1)
+
+ # Add original to a different resource
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ component = yield object1.componentForUser()
+ attach = component.mainComponent().getProperty("ATTACH")
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="3.ics")
+ component = yield object1.componentForUser()
+ attach = component.mainComponent().addProperty(attach)
+ yield object1.setComponent(component)
+ yield self.commitTransaction(0)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set())
+ self.assertEqual(removed, set())
+
+ # Validate changes
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ attachments = yield home1.getAllAttachments()
+ mapping1 = dict([(o.md5(), o.id()) for o in attachments])
+ yield _checkAttachmentObjectMigrationState(home1, mapping1)
+
+ # Change original attachment in original resource (this creates a new one and does not remove the old)
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ attachment, _ignore_location = yield object1.updateAttachment(managedid0_1_changed, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1 - changed again."))
+ id0_1_changed_again = attachment.id()
+ md50_1_changed_again = attachment.md5()
+ mapping0[md50_1_changed_again] = id0_1_changed_again
+ yield self.commitTransaction(0)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set((id0_1_changed_again,)))
+ self.assertEqual(removed, set())
+
+ # Validate changes
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ attachments = yield home1.getAllAttachments()
+ mapping1 = dict([(o.md5(), o.id()) for o in attachments])
+ yield _checkAttachmentObjectMigrationState(home1, mapping1)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingrequestpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -23,7 +23,7 @@
</span><span class="cx"> from txweb2.client.http import HTTPClientProtocol, ClientRequest
</span><span class="cx"> from txweb2.dav.util import allDataFromStream
</span><span class="cx"> from txweb2.http_headers import Headers, MimeType
</span><del>-from txweb2.stream import MemoryStream
</del><ins>+from txweb2.stream import MemoryStream, readStream
</ins><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx"> from twisted.internet.protocol import Factory
</span><span class="lines">@@ -50,11 +50,12 @@
</span><span class="cx"> case the JSON data is sent in an HTTP header.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def __init__(self, server, data, stream=None, stream_type=None):
</del><ins>+ def __init__(self, server, data, stream=None, stream_type=None, writeStream=None):
</ins><span class="cx"> self.server = server
</span><span class="cx"> self.data = json.dumps(data)
</span><span class="cx"> self.stream = stream
</span><span class="cx"> self.streamType = stream_type
</span><ins>+ self.writeStream = writeStream
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -72,7 +73,27 @@
</span><span class="cx"> self.loggedResponse = yield self.logResponse(response)
</span><span class="cx"> emitAccounting("xPod", "", self.loggedRequest + "\n" + self.loggedResponse, "POST")
</span><span class="cx">
</span><del>- if response.code in (responsecode.OK, responsecode.BAD_REQUEST,):
</del><ins>+ if response.code == responsecode.OK:
+ if self.writeStream is None:
+ data = (yield allDataFromStream(response.stream))
+ data = json.loads(data)
+ else:
+ yield readStream(response.stream, self.writeStream.write)
+ yield self.writeStream.loseConnection()
+ content_type = response.headers.getHeader("content-type")
+ if content_type is None:
+ content_type = MimeType("application", "octet-stream")
+ content_disposition = response.headers.getHeader("content-disposition")
+ if content_disposition is None or "filename" not in content_disposition.params:
+ filename = ""
+ else:
+ filename = content_disposition.params["filename"]
+ response = {
+ "result": "ok",
+ "content-type": content_type,
+ "name": filename,
+ }
+ elif response.code == responsecode.BAD_REQUEST:
</ins><span class="cx"> data = (yield allDataFromStream(response.stream))
</span><span class="cx"> data = json.loads(data)
</span><span class="cx"> else:
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -18,9 +18,11 @@
</span><span class="cx"> from txweb2.dav.noneprops import NonePropertyStore
</span><span class="cx"> from txweb2.dav.util import allDataFromStream
</span><span class="cx"> from txweb2.http import Response, HTTPError, StatusResponse, JSONResponse
</span><del>-from txweb2.http_headers import MimeType
</del><ins>+from txweb2.http_headers import MimeType, MimeDisposition
+from txweb2.stream import ProducerStream
</ins><span class="cx">
</span><span class="cx"> from twisted.internet.defer import succeed, returnValue, inlineCallbacks
</span><ins>+from twisted.internet.protocol import Protocol
</ins><span class="cx">
</span><span class="cx"> from twistedcaldav.extensions import DAVResource, \
</span><span class="cx"> DAVResourceWithoutChildrenMixin
</span><span class="lines">@@ -154,19 +156,54 @@
</span><span class="cx"> request.extendedLogItems = {}
</span><span class="cx"> request.extendedLogItems["xpod"] = j["action"] if "action" in j else "unknown"
</span><span class="cx">
</span><del>- # Get the conduit to process the data
- try:
- result = yield self.store.conduit.processRequest(j)
- code = responsecode.OK if result["result"] == "ok" else responsecode.BAD_REQUEST
- except Exception as e:
- # Send the exception over to the other side
- result = {
- "result": "exception",
- "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
- "request": str(e),
- }
- code = responsecode.BAD_REQUEST
</del><ins>+ # Look for a streaming action which needs special handling
+ if self.store.conduit.isStreamAction(j):
+ # Get the conduit to process the data stream
+ try:
</ins><span class="cx">
</span><ins>+ stream = ProducerStream()
+ class StreamProtocol(Protocol):
+ def connectionMade(self):
+ stream.registerProducer(self.transport, False)
+ def dataReceived(self, data):
+ stream.write(data)
+ def connectionLost(self, reason):
+ stream.finish()
+
+ result = yield self.store.conduit.processRequestStream(j, StreamProtocol())
+
+ try:
+ ct, name = result
+ except ValueError:
+ code = responsecode.BAD_REQUEST
+ else:
+ headers = {"content-type": ct}
+ headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
+ returnValue(Response(responsecode.OK, headers, stream))
+
+ except Exception as e:
+ # Send the exception over to the other side
+ result = {
+ "result": "exception",
+ "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
+ "details": str(e),
+ }
+ code = responsecode.BAD_REQUEST
+
+ else:
+ # Get the conduit to process the data
+ try:
+ result = yield self.store.conduit.processRequest(j)
+ code = responsecode.OK if result["result"] == "ok" else responsecode.BAD_REQUEST
+ except Exception as e:
+ # Send the exception over to the other side
+ result = {
+ "result": "exception",
+ "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
+ "details": str(e),
+ }
+ code = responsecode.BAD_REQUEST
+
</ins><span class="cx"> response = JSONResponse(code, result)
</span><span class="cx"> returnValue(response)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtesttest_conduitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -1056,3 +1056,54 @@
</span><span class="cx"> attachment = yield ManagedAttachment.load(self.theTransactionUnderTest(0), resourceID, managedID)
</span><span class="cx"> self.assertTrue(attachment is None)
</span><span class="cx"> yield self.commitTransaction(0)
</span><ins>+
+
+ @inlineCallbacks
+ def test_get_all_attachments(self):
+ """
+ Test that action=get-all-attachments works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user01", name="calendar")
+ yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ yield self.commitTransaction(0)
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ yield self.commitTransaction(0)
+
+ shared_object = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ attachments = yield shared_object.ownerHome().getAllAttachments()
+ self.assertEqual(len(attachments), 1)
+ self.assertTrue(isinstance(attachments[0], ManagedAttachment))
+ self.assertEqual(attachments[0].contentType(), MimeType.fromString("text/plain"))
+ self.assertEqual(attachments[0].name(), "test.txt")
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_get_attachment_data(self):
+ """
+ Test that action=get-all-attachments works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user01", name="calendar")
+ yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ yield self.commitTransaction(0)
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ remote_id = attachment.id()
+ yield self.commitTransaction(0)
+
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser01")
+ shared_object = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ attachment = yield ManagedAttachment._create(self.theTransactionUnderTest(1), None, home1.id())
+ attachment._contentType = MimeType.fromString("text/plain")
+ attachment._name = "test.txt"
+ yield shared_object.ownerHome().readAttachmentData(remote_id, attachment)
+ yield self.commitTransaction(1)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -15,6 +15,7 @@
</span><span class="cx"> ##
</span><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><ins>+from twisted.internet.protocol import Protocol
</ins><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.scheduling.ischedule.localservers import (
</span><span class="cx"> Server, ServersDB
</span><span class="lines">@@ -26,6 +27,7 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> import txweb2.dav.test.util
</span><ins>+from txweb2.stream import ProducerStream, readStream
</ins><span class="cx">
</span><span class="cx"> from twext.enterprise.ienterprise import AlreadyFinishedError
</span><span class="cx">
</span><span class="lines">@@ -54,12 +56,13 @@
</span><span class="cx"> cls.storeMap[server.details()] = store
</span><span class="cx">
</span><span class="cx">
</span><del>- def __init__(self, server, data, stream=None, stream_type=None):
</del><ins>+ def __init__(self, server, data, stream=None, stream_type=None, writeStream=None):
</ins><span class="cx">
</span><span class="cx"> self.server = server
</span><span class="cx"> self.data = json.dumps(data)
</span><span class="cx"> self.stream = stream
</span><span class="cx"> self.streamType = stream_type
</span><ins>+ self.writeStream = writeStream
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -68,7 +71,20 @@
</span><span class="cx"> # Generate an HTTP client request
</span><span class="cx"> try:
</span><span class="cx"> response = (yield self._processRequest())
</span><del>- response = json.loads(response)
</del><ins>+ if self.writeStream is None:
+ response = json.loads(response)
+ else:
+ try:
+ ct, name, stream = response
+ response = {
+ "result": "ok",
+ "content-type": ct,
+ "name": name,
+ }
+ yield readStream(stream, self.writeStream.write)
+ yield self.writeStream.loseConnection()
+ except ValueError:
+ pass
</ins><span class="cx"> except Exception as e:
</span><span class="cx"> raise ValueError("Failed cross-pod request: {}".format(e))
</span><span class="cx">
</span><span class="lines">@@ -90,13 +106,32 @@
</span><span class="cx"> j["stream"] = self.stream
</span><span class="cx"> j["streamType"] = self.streamType
</span><span class="cx"> try:
</span><del>- result = yield store.conduit.processRequest(j)
</del><ins>+ if store.conduit.isStreamAction(j):
+ stream = ProducerStream()
+ class StreamProtocol(Protocol):
+ def connectionMade(self):
+ stream.registerProducer(self.transport, False)
+ def dataReceived(self, data):
+ stream.write(data)
+ def connectionLost(self, reason):
+ stream.finish()
+
+ result = yield store.conduit.processRequestStream(j, StreamProtocol())
+
+ try:
+ ct, name = result
+ except ValueError:
+ pass
+ else:
+ returnValue((ct, name, stream,))
+ else:
+ result = yield store.conduit.processRequest(j)
</ins><span class="cx"> except Exception as e:
</span><span class="cx"> # Send the exception over to the other side
</span><span class="cx"> result = {
</span><span class="cx"> "result": "exception",
</span><span class="cx"> "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
</span><del>- "request": str(e),
</del><ins>+ "details": str(e),
</ins><span class="cx"> }
</span><span class="cx"> result = json.dumps(result)
</span><span class="cx"> returnValue(result)
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentoracledialectsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -69,7 +69,7 @@
</span><span class="cx"> "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
</span><span class="cx"> );
</span><span class="cx">
</span><del>-create table CALENDAR_MIGRATION_STATE (
</del><ins>+create table CALENDAR_MIGRATION (
</ins><span class="cx"> "CALENDAR_HOME_RESOURCE_ID" integer references CALENDAR_HOME on delete cascade,
</span><span class="cx"> "REMOTE_RESOURCE_ID" integer not null,
</span><span class="cx"> "CALENDAR_RESOURCE_ID" integer references CALENDAR on delete cascade,
</span><span class="lines">@@ -217,6 +217,13 @@
</span><span class="cx"> primary key ("TIME_RANGE_INSTANCE_ID", "USER_ID")
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create table CALENDAR_OBJECT_MIGRATION (
+ "CALENDAR_HOME_RESOURCE_ID" integer references CALENDAR_HOME on delete cascade,
+ "REMOTE_RESOURCE_ID" integer not null,
+ "LOCAL_RESOURCE_ID" integer references CALENDAR_OBJECT on delete cascade,
+ primary key ("CALENDAR_HOME_RESOURCE_ID", "REMOTE_RESOURCE_ID")
+);
+
</ins><span class="cx"> create table ATTACHMENT (
</span><span class="cx"> "ATTACHMENT_ID" integer primary key,
</span><span class="cx"> "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
</span><span class="lines">@@ -237,6 +244,13 @@
</span><span class="cx"> unique ("MANAGED_ID", "CALENDAR_OBJECT_RESOURCE_ID")
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create table ATTACHMENT_MIGRATION (
+ "CALENDAR_HOME_RESOURCE_ID" integer references CALENDAR_HOME on delete cascade,
+ "REMOTE_RESOURCE_ID" integer not null,
+ "LOCAL_RESOURCE_ID" integer references ATTACHMENT on delete cascade,
+ primary key ("CALENDAR_HOME_RESOURCE_ID", "REMOTE_RESOURCE_ID")
+);
+
</ins><span class="cx"> create table RESOURCE_PROPERTY (
</span><span class="cx"> "RESOURCE_ID" integer not null,
</span><span class="cx"> "NAME" nvarchar2(255),
</span><span class="lines">@@ -633,7 +647,7 @@
</span><span class="cx"> DEFAULT_POLLS
</span><span class="cx"> );
</span><span class="cx">
</span><del>-create index CALENDAR_MIGRATION_ST_57f40e9a on CALENDAR_MIGRATION_STATE (
</del><ins>+create index CALENDAR_MIGRATION_CA_cc68f4ec on CALENDAR_MIGRATION (
</ins><span class="cx"> CALENDAR_RESOURCE_ID
</span><span class="cx"> );
</span><span class="cx">
</span><span class="lines">@@ -672,6 +686,15 @@
</span><span class="cx"> CALENDAR_OBJECT_RESOURCE_ID
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create index CALENDAR_OBJECT_MIGRA_0502cbef on CALENDAR_OBJECT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID,
+ LOCAL_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_MIGRA_3577efd9 on CALENDAR_OBJECT_MIGRATION (
+ LOCAL_RESOURCE_ID
+);
+
</ins><span class="cx"> create index ATTACHMENT_CALENDAR_H_0078845c on ATTACHMENT (
</span><span class="cx"> CALENDAR_HOME_RESOURCE_ID
</span><span class="cx"> );
</span><span class="lines">@@ -684,6 +707,15 @@
</span><span class="cx"> CALENDAR_OBJECT_RESOURCE_ID
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create index ATTACHMENT_MIGRATION__804bf85e on ATTACHMENT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID,
+ LOCAL_RESOURCE_ID
+);
+
+create index ATTACHMENT_MIGRATION__816947fe on ATTACHMENT_MIGRATION (
+ LOCAL_RESOURCE_ID
+);
+
</ins><span class="cx"> create index SHARED_ADDRESSBOOK_BI_e9a2e6d4 on SHARED_ADDRESSBOOK_BIND (
</span><span class="cx"> OWNER_HOME_RESOURCE_ID
</span><span class="cx"> );
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/current.sql 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -140,7 +140,7 @@
</span><span class="cx"> -- Calendar Migration --
</span><span class="cx"> ------------------------
</span><span class="cx">
</span><del>-create table CALENDAR_MIGRATION_STATE (
</del><ins>+create table CALENDAR_MIGRATION (
</ins><span class="cx"> CALENDAR_HOME_RESOURCE_ID integer references CALENDAR_HOME on delete cascade,
</span><span class="cx"> REMOTE_RESOURCE_ID integer not null,
</span><span class="cx"> CALENDAR_RESOURCE_ID integer references CALENDAR on delete cascade,
</span><span class="lines">@@ -149,8 +149,8 @@
</span><span class="cx"> primary key (CALENDAR_HOME_RESOURCE_ID, REMOTE_RESOURCE_ID) -- implicit index
</span><span class="cx"> );
</span><span class="cx">
</span><del>-create index CALENDAR_MIGRATION_STATE_CALENDAR_RESOURCE_ID on
- CALENDAR_MIGRATION_STATE(CALENDAR_RESOURCE_ID);
</del><ins>+create index CALENDAR_MIGRATION_CALENDAR_RESOURCE_ID on
+ CALENDAR_MIGRATION(CALENDAR_RESOURCE_ID);
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> ---------------------------
</span><span class="lines">@@ -381,6 +381,24 @@
</span><span class="cx"> );
</span><span class="cx">
</span><span class="cx">
</span><ins>+-------------------------------
+-- Calendar Object Migration --
+-------------------------------
+
+create table CALENDAR_OBJECT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID integer references CALENDAR_HOME on delete cascade,
+ REMOTE_RESOURCE_ID integer not null,
+ LOCAL_RESOURCE_ID integer references CALENDAR_OBJECT on delete cascade,
+
+ primary key (CALENDAR_HOME_RESOURCE_ID, REMOTE_RESOURCE_ID) -- implicit index
+);
+
+create index CALENDAR_OBJECT_MIGRATION_HOME_LOCAL on
+ CALENDAR_OBJECT_MIGRATION(CALENDAR_HOME_RESOURCE_ID, LOCAL_RESOURCE_ID);
+create index CALENDAR_OBJECT_MIGRATION_LOCAL_RESOURCE_ID on
+ CALENDAR_OBJECT_MIGRATION(LOCAL_RESOURCE_ID);
+
+
</ins><span class="cx"> ----------------
</span><span class="cx"> -- Attachment --
</span><span class="cx"> ----------------
</span><span class="lines">@@ -418,6 +436,24 @@
</span><span class="cx"> create index ATTACHMENT_CALENDAR_OBJECT_CALENDAR_OBJECT_RESOURCE_ID on
</span><span class="cx"> ATTACHMENT_CALENDAR_OBJECT(CALENDAR_OBJECT_RESOURCE_ID);
</span><span class="cx">
</span><ins>+-----------------------------------
+-- Calendar Attachment Migration --
+-----------------------------------
+
+create table ATTACHMENT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID integer references CALENDAR_HOME on delete cascade,
+ REMOTE_RESOURCE_ID integer not null,
+ LOCAL_RESOURCE_ID integer references ATTACHMENT on delete cascade,
+
+ primary key (CALENDAR_HOME_RESOURCE_ID, REMOTE_RESOURCE_ID) -- implicit index
+);
+
+create index ATTACHMENT_MIGRATION_HOME_LOCAL on
+ ATTACHMENT_MIGRATION(CALENDAR_HOME_RESOURCE_ID, LOCAL_RESOURCE_ID);
+create index ATTACHMENT_MIGRATION_LOCAL_RESOURCE_ID on
+ ATTACHMENT_MIGRATION(LOCAL_RESOURCE_ID);
+
+
</ins><span class="cx"> -----------------------
</span><span class="cx"> -- Resource Property --
</span><span class="cx"> -----------------------
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_51_to_52sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_51_to_52.sql 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -22,7 +22,7 @@
</span><span class="cx"> insert into HOME_STATUS (DESCRIPTION, ID) values ('migrating', 3);
</span><span class="cx">
</span><span class="cx"> -- New table
</span><del>-create table CALENDAR_MIGRATION_STATE (
</del><ins>+create table CALENDAR_MIGRATION (
</ins><span class="cx"> "CALENDAR_HOME_RESOURCE_ID" integer references CALENDAR_HOME on delete cascade,
</span><span class="cx"> "REMOTE_RESOURCE_ID" integer not null,
</span><span class="cx"> "CALENDAR_RESOURCE_ID" integer references CALENDAR on delete cascade,
</span><span class="lines">@@ -30,10 +30,42 @@
</span><span class="cx"> primary key ("CALENDAR_HOME_RESOURCE_ID", "REMOTE_RESOURCE_ID")
</span><span class="cx"> );
</span><span class="cx">
</span><del>-create index CALENDAR_MIGRATION_ST_57f40e9a on CALENDAR_MIGRATION_STATE (
</del><ins>+create index CALENDAR_MIGRATION_CA_cc68f4ec on CALENDAR_MIGRATION (
</ins><span class="cx"> CALENDAR_RESOURCE_ID
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+-- New table
+create table CALENDAR_OBJECT_MIGRATION (
+ "CALENDAR_HOME_RESOURCE_ID" integer references CALENDAR_HOME on delete cascade,
+ "REMOTE_RESOURCE_ID" integer not null,
+ "LOCAL_RESOURCE_ID" integer references CALENDAR_OBJECT on delete cascade,
+ primary key ("CALENDAR_HOME_RESOURCE_ID", "REMOTE_RESOURCE_ID")
+);
</ins><span class="cx">
</span><ins>+create index CALENDAR_OBJECT_MIGRA_0502cbef on CALENDAR_OBJECT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID,
+ LOCAL_RESOURCE_ID
+);
+create index CALENDAR_OBJECT_MIGRA_3577efd9 on CALENDAR_OBJECT_MIGRATION (
+ LOCAL_RESOURCE_ID
+);
+
+-- New table
+create table ATTACHMENT_MIGRATION (
+ "CALENDAR_HOME_RESOURCE_ID" integer references CALENDAR_HOME on delete cascade,
+ "REMOTE_RESOURCE_ID" integer not null,
+ "LOCAL_RESOURCE_ID" integer references ATTACHMENT on delete cascade,
+ primary key ("CALENDAR_HOME_RESOURCE_ID", "REMOTE_RESOURCE_ID")
+);
+
+create index ATTACHMENT_MIGRATION__804bf85e on ATTACHMENT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID,
+ LOCAL_RESOURCE_ID
+);
+create index ATTACHMENT_MIGRATION__816947fe on ATTACHMENT_MIGRATION (
+ LOCAL_RESOURCE_ID
+);
+
+
</ins><span class="cx"> -- update the version
</span><span class="cx"> update CALENDARSERVER set VALUE = '52' where NAME = 'VERSION';
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_51_to_52sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql (14402 => 14403)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql 2015-02-11 19:55:49 UTC (rev 14402)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_51_to_52.sql 2015-02-11 20:34:55 UTC (rev 14403)
</span><span class="lines">@@ -22,7 +22,7 @@
</span><span class="cx"> insert into HOME_STATUS values (3, 'migrating');
</span><span class="cx">
</span><span class="cx"> -- New table
</span><del>-create table CALENDAR_MIGRATION_STATE (
</del><ins>+create table CALENDAR_MIGRATION (
</ins><span class="cx"> CALENDAR_HOME_RESOURCE_ID integer references CALENDAR_HOME on delete cascade,
</span><span class="cx"> REMOTE_RESOURCE_ID integer not null,
</span><span class="cx"> CALENDAR_RESOURCE_ID integer references CALENDAR on delete cascade,
</span><span class="lines">@@ -31,9 +31,39 @@
</span><span class="cx"> primary key (CALENDAR_HOME_RESOURCE_ID, REMOTE_RESOURCE_ID) -- implicit index
</span><span class="cx"> );
</span><span class="cx">
</span><del>-create index CALENDAR_MIGRATION_STATE_CALENDAR_RESOURCE_ID on
- CALENDAR_MIGRATION_STATE(CALENDAR_RESOURCE_ID);
</del><ins>+create index CALENDAR_MIGRATION_CALENDAR_RESOURCE_ID on
+ CALENDAR_MIGRATION(CALENDAR_RESOURCE_ID);
</ins><span class="cx">
</span><ins>+
+-- New table
+create table CALENDAR_OBJECT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID integer references CALENDAR_HOME on delete cascade,
+ REMOTE_RESOURCE_ID integer not null,
+ LOCAL_RESOURCE_ID integer references CALENDAR_OBJECT on delete cascade,
+
+ primary key (CALENDAR_HOME_RESOURCE_ID, REMOTE_RESOURCE_ID) -- implicit index
+);
</ins><span class="cx">
</span><ins>+create index CALENDAR_OBJECT_MIGRATION_HOME_LOCAL on
+ CALENDAR_OBJECT_MIGRATION(CALENDAR_HOME_RESOURCE_ID, LOCAL_RESOURCE_ID);
+create index CALENDAR_OBJECT_MIGRATION_LOCAL_RESOURCE_ID on
+ CALENDAR_OBJECT_MIGRATION(LOCAL_RESOURCE_ID);
+
+
+-- New table
+create table ATTACHMENT_MIGRATION (
+ CALENDAR_HOME_RESOURCE_ID integer references CALENDAR_HOME on delete cascade,
+ REMOTE_RESOURCE_ID integer not null,
+ LOCAL_RESOURCE_ID integer references ATTACHMENT on delete cascade,
+
+ primary key (CALENDAR_HOME_RESOURCE_ID, REMOTE_RESOURCE_ID) -- implicit index
+);
+
+create index ATTACHMENT_MIGRATION_HOME_LOCAL on
+ ATTACHMENT_MIGRATION(CALENDAR_HOME_RESOURCE_ID, LOCAL_RESOURCE_ID);
+create index ATTACHMENT_MIGRATION_LOCAL_RESOURCE_ID on
+ ATTACHMENT_MIGRATION(LOCAL_RESOURCE_ID);
+
+
</ins><span class="cx"> -- update the version
</span><span class="cx"> update CALENDARSERVER set VALUE = '52' where NAME = 'VERSION';
</span></span></pre>
</div>
</div>
</body>
</html>