<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14412] CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14412">14412</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-02-13 13:21:26 -0800 (Fri, 13 Feb 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Checkpoint: migration final sync attachment links.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoreutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingattachmentspy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingconduitpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingrequestpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingresourcepy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtesttest_conduitpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py</a></li>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingutilpy">CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -143,7 +143,7 @@
</span><span class="cx"> @type txn: L{txdav.common.datastore.sql.CommonStoreTransaction}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- at = schema.ATTACHMENT
</del><ins>+ at = Attachment._attachmentSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> (at.DROPBOX_ID,),
</span><span class="cx"> From=at,
</span><span class="lines">@@ -175,8 +175,8 @@
</span><span class="cx"> txn = self._store.newTransaction("CalendarStoreFeatures.upgradeToManagedAttachments - preliminary work")
</span><span class="cx"> try:
</span><span class="cx"> # Clear out unused CALENDAR_OBJECT.DROPBOX_IDs
</span><del>- co = schema.CALENDAR_OBJECT
- at = schema.ATTACHMENT
</del><ins>+ co = CalendarObject._objectSchema
+ at = Attachment._attachmentSchema
</ins><span class="cx"> yield Update(
</span><span class="cx"> {co.DROPBOX_ID: None},
</span><span class="cx"> Where=co.RESOURCE_ID.In(Select(
</span><span class="lines">@@ -249,7 +249,7 @@
</span><span class="cx"> log.debug(" {0} affected calendar objects".format(len(cobjs),))
</span><span class="cx">
</span><span class="cx"> # Get names of each matching attachment
</span><del>- at = schema.ATTACHMENT
</del><ins>+ at = Attachment._attachmentSchema
</ins><span class="cx"> names = (yield Select(
</span><span class="cx"> (at.PATH,),
</span><span class="cx"> From=at,
</span><span class="lines">@@ -318,8 +318,8 @@
</span><span class="cx"> @type dropbox_id: C{str}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
</del><ins>+ co = CalendarObject._objectSchema
+ cb = Calendar._bindSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> (cb.CALENDAR_HOME_RESOURCE_ID, co.CALENDAR_RESOURCE_ID, co.RESOURCE_ID,),
</span><span class="cx"> From=co.join(cb, co.CALENDAR_RESOURCE_ID == cb.CALENDAR_RESOURCE_ID),
</span><span class="lines">@@ -511,7 +511,7 @@
</span><span class="cx"> assert self._txn._migrating
</span><span class="cx">
</span><span class="cx"> # Simple attributes that can be copied over as-is
</span><del>- chm = schema.CALENDAR_HOME_METADATA
</del><ins>+ chm = self._homeMetaDataSchema
</ins><span class="cx"> values = {
</span><span class="cx"> chm.ALARM_VEVENT_TIMED : other._alarm_vevent_timed,
</span><span class="cx"> chm.ALARM_VEVENT_ALLDAY : other._alarm_vevent_allday,
</span><span class="lines">@@ -617,8 +617,8 @@
</span><span class="cx"> """
</span><span class="cx"> Implement lookup via queries.
</span><span class="cx"> """
</span><del>- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
</del><ins>+ co = self._objectSchema
+ cb = self._bindSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [co.PARENT_RESOURCE_ID,
</span><span class="cx"> co.RESOURCE_ID],
</span><span class="lines">@@ -637,16 +637,22 @@
</span><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="cx">
</span><del>- @inlineCallbacks
</del><span class="cx"> def getAllAttachments(self):
</span><span class="cx"> """
</span><span class="cx"> Return all the L{Attachment} objects associated with this calendar home.
</span><span class="cx"> Needed during migration.
</span><span class="cx"> """
</span><del>- attachments = yield Attachment.loadAllAttachments(self)
- returnValue(attachments)
</del><ins>+ return Attachment.loadAllAttachments(self)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ def getAttachmentLinks(self):
+ """
+ Read the attachment<->calendar object mapping data associated with this calendar home.
+ Needed during migration only.
+ """
+ return AttachmentLink.linksForHome(self)
+
+
</ins><span class="cx"> def getAttachmentByID(self, id):
</span><span class="cx"> """
</span><span class="cx"> Return a specific attachment associated with this calendar home.
</span><span class="lines">@@ -657,8 +663,8 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def getAllDropboxIDs(self):
</span><del>- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
</del><ins>+ co = self._objectSchema
+ cb = self._bindSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [co.DROPBOX_ID],
</span><span class="cx"> From=co.join(cb, co.PARENT_RESOURCE_ID == cb.RESOURCE_ID),
</span><span class="lines">@@ -671,7 +677,7 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def getAllAttachmentNames(self):
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = Attachment._attachmentSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [att.DROPBOX_ID],
</span><span class="cx"> From=att,
</span><span class="lines">@@ -683,8 +689,8 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def getAllManagedIDs(self):
</span><del>- at = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ at = Attachment._attachmentSchema
+ attco = Attachment._attachmentLinkSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [attco.MANAGED_ID, ],
</span><span class="cx"> From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
</span><span class="lines">@@ -1565,7 +1571,7 @@
</span><span class="cx"> """
</span><span class="cx"> Query to find resources that need to be re-expanded
</span><span class="cx"> """
</span><del>- co = schema.CALENDAR_OBJECT
</del><ins>+ co = cls._objectSchema
</ins><span class="cx"> return Select(
</span><span class="cx"> [co.RESOURCE_NAME],
</span><span class="cx"> From=co,
</span><span class="lines">@@ -3598,7 +3604,7 @@
</span><span class="cx"> recurrenceLowerLimit = None
</span><span class="cx"> recurrenceLimit = DateTime(1900, 1, 1, 0, 0, 0, tzid=Timezone(utc=True))
</span><span class="cx">
</span><del>- co = schema.CALENDAR_OBJECT
</del><ins>+ co = self._objectSchema
</ins><span class="cx"> tr = schema.TIME_RANGE
</span><span class="cx">
</span><span class="cx"> # Do not update if reCreate (re-indexing - we don't want to re-write data
</span><span class="lines">@@ -3809,7 +3815,7 @@
</span><span class="cx"> """
</span><span class="cx"> assert self._txn._migrating
</span><span class="cx">
</span><del>- co = schema.CALENDAR_OBJECT
</del><ins>+ co = self._objectSchema
</ins><span class="cx"> values = {
</span><span class="cx"> co.ATTACHMENTS_MODE : other._attachment,
</span><span class="cx"> co.DROPBOX_ID : other._dropboxID,
</span><span class="lines">@@ -4058,7 +4064,7 @@
</span><span class="cx"> """
</span><span class="cx"> DAL query to load RECURRANCE_MIN, RECURRANCE_MAX via an object's resource ID.
</span><span class="cx"> """
</span><del>- co = schema.CALENDAR_OBJECT
</del><ins>+ co = cls._objectSchema
</ins><span class="cx"> return Select(
</span><span class="cx"> [co.RECURRANCE_MIN, co.RECURRANCE_MAX, ],
</span><span class="cx"> From=co,
</span><span class="lines">@@ -4593,8 +4599,8 @@
</span><span class="cx"> Get a list of managed attachments where the names returned are for the last path segment
</span><span class="cx"> of the attachment URI.
</span><span class="cx"> """
</span><del>- at = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ at = Attachment._attachmentSchema
+ attco = Attachment._attachmentLinkSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [attco.MANAGED_ID, at.PATH, ],
</span><span class="cx"> From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
</span><span class="lines">@@ -4610,8 +4616,8 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Scan all the associated attachments for the one that matches
</span><del>- at = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ at = Attachment._attachmentSchema
+ attco = Attachment._attachmentLinkSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [attco.MANAGED_ID, at.PATH, ],
</span><span class="cx"> From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
</span><span class="lines">@@ -5050,32 +5056,136 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+class AttachmentLink(object):
+ """
+ A binding between an L{Attachment} and an L{CalendarObject}.
+ """
+
+ _attachmentSchema = schema.ATTACHMENT
+ _attachmentLinkSchema = schema.ATTACHMENT_CALENDAR_OBJECT
+
+ @classmethod
+ def makeClass(cls, txn, linkData):
+ """
+ Given the various database rows, build the actual class.
+
+ @param objectData: the standard set of object columns
+ @type objectData: C{list}
+
+ @return: the constructed child class
+ @rtype: L{CommonHomeChild}
+ """
+
+ child = cls(txn)
+ for attr, value in zip(child._rowAttributes(), linkData):
+ setattr(child, attr, value)
+ return child
+
+
+ @classmethod
+ def _allColumns(cls):
+ """
+ Full set of columns in the object table that need to be loaded to
+ initialize the object resource state.
+ """
+ aco = cls._attachmentLinkSchema
+ return [
+ aco.ATTACHMENT_ID,
+ aco.MANAGED_ID,
+ aco.CALENDAR_OBJECT_RESOURCE_ID,
+ ]
+
+
+ @classmethod
+ def _rowAttributes(cls):
+ """
+ Object attributes used to store the column values from L{_allColumns}. This used to create
+ a mapping when serializing the object for cross-pod requests.
+ """
+ return (
+ "_attachmentID",
+ "_managedID",
+ "_calendarObjectID",
+ )
+
+
+ @classmethod
+ @inlineCallbacks
+ def linksForHome(cls, home):
+ """
+ Load all attachment<->calendar object mappings for the specified home collection.
+ """
+
+ # Load from the main table first
+ att = cls._attachmentSchema
+ attco = cls._attachmentLinkSchema
+ dataRows = yield Select(
+ cls._allColumns(),
+ From=attco.join(att, on=(attco.ATTACHMENT_ID == att.ATTACHMENT_ID)),
+ Where=att.CALENDAR_HOME_RESOURCE_ID == home.id(),
+ ).on(home._txn)
+
+ # Create the actual objects
+ returnValue([cls.makeClass(home._txn, row) for row in dataRows])
+
+
+ def __init__(self, txn):
+ self._txn = txn
+ for attr in self._rowAttributes():
+ setattr(self, attr, None)
+
+
+ def externalize(self):
+ """
+ Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
+ and reconstituted at the other end. Note that the other end may have a different schema so
+ the attributes may not match exactly and will need to be processed accordingly.
+ """
+ return dict([(attr[1:], getattr(self, attr, None)) for attr in self._rowAttributes()])
+
+
+ @classmethod
+ def internalize(cls, txn, mapping):
+ """
+ Given a mapping generated by L{externalize}, convert the values into an array of database
+ like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
+ Note that there may be a schema mismatch with the external data, so treat missing items as
+ C{None} and ignore extra items.
+ """
+
+ return cls.makeClass(txn, [mapping.get(row[1:]) for row in cls._rowAttributes()])
+
+
+ def insert(self):
+ """
+ Insert the object.
+ """
+
+ row = dict([(column, getattr(self, attr)) for column, attr in itertools.izip(self._allColumns(), self._rowAttributes())])
+ return Insert(row).on(self._txn)
+
+
+
</ins><span class="cx"> class Attachment(object):
</span><span class="cx">
</span><span class="cx"> implements(IAttachment)
</span><span class="cx">
</span><span class="cx"> _attachmentSchema = schema.ATTACHMENT
</span><ins>+ _attachmentLinkSchema = schema.ATTACHMENT_CALENDAR_OBJECT
</ins><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="cx"> def makeClass(cls, txn, attachmentData):
</span><span class="cx"> """
</span><span class="cx"> Given the various database rows, build the actual class.
</span><span class="cx">
</span><del>- @param parent: the parent collection object
- @type parent: L{CommonHomeChild}
</del><ins>+ @param attachmentData: the standard set of attachment columns
+ @type attachmentData: C{list}
</ins><span class="cx">
</span><del>- @param objectData: the standard set of object columns
- @type objectData: C{list}
-
- @param propstore: a property store to use, or C{None} to load it
- automatically
- @type propstore: L{PropertyStore}
-
</del><span class="cx"> @return: the constructed child class
</span><del>- @rtype: L{CommonHomeChild}
</del><ins>+ @rtype: L{Attachment}
</ins><span class="cx"> """
</span><span class="cx">
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = cls._attachmentSchema
</ins><span class="cx"> dropbox_id = attachmentData[cls._allColumns().index(att.DROPBOX_ID)]
</span><span class="cx"> c = ManagedAttachment if dropbox_id == "." else DropBoxAttachment
</span><span class="cx"> child = c(
</span><span class="lines">@@ -5233,7 +5343,7 @@
</span><span class="cx">
</span><span class="cx"> @return: C{True} if this attachment exists, C{False} otherwise.
</span><span class="cx"> """
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = self._attachmentSchema
</ins><span class="cx"> if self._dropboxID and self._dropboxID != ".":
</span><span class="cx"> where = (att.DROPBOX_ID == self._dropboxID).And(
</span><span class="cx"> att.PATH == self._name)
</span><span class="lines">@@ -5255,7 +5365,6 @@
</span><span class="cx"> returnValue(self)
</span><span class="cx">
</span><span class="cx">
</span><del>- @inlineCallbacks
</del><span class="cx"> def copyRemote(self, remote):
</span><span class="cx"> """
</span><span class="cx"> Copy properties from a remote (external) attachment that is being migrated.
</span><span class="lines">@@ -5263,7 +5372,7 @@
</span><span class="cx"> @param remote: the external attachment
</span><span class="cx"> @type remote: L{Attachment}
</span><span class="cx"> """
</span><del>- yield self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
</del><ins>+ return self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def id(self):
</span><span class="lines">@@ -5361,8 +5470,8 @@
</span><span class="cx">
</span><span class="cx"> TODO: this needs to be transactional wrt the actual file deletes.
</span><span class="cx"> """
</span><del>- att = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ att = cls._attachmentSchema
+ attco = cls._attachmentLinkSchema
</ins><span class="cx">
</span><span class="cx"> rows = (yield Select(
</span><span class="cx"> [att.ATTACHMENT_ID, att.DROPBOX_ID, ],
</span><span class="lines">@@ -5448,7 +5557,7 @@
</span><span class="cx"> raise AttachmentDropboxNotAllowed
</span><span class="cx">
</span><span class="cx"> # Now create the DB entry
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = cls._attachmentSchema
</ins><span class="cx"> rows = (yield Insert({
</span><span class="cx"> att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
</span><span class="cx"> att.DROPBOX_ID : dropboxID,
</span><span class="lines">@@ -5500,7 +5609,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # See if any other resources still reference this dropbox ID
</span><del>- co = schema.CALENDAR_OBJECT
</del><ins>+ co = CalendarObject._objectSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [co.RESOURCE_ID, ],
</span><span class="cx"> From=co,
</span><span class="lines">@@ -5510,7 +5619,7 @@
</span><span class="cx">
</span><span class="cx"> if not rows:
</span><span class="cx"> # Find each attachment with matching dropbox ID
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = cls._attachmentSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [att.PATH],
</span><span class="cx"> From=att,
</span><span class="lines">@@ -5532,7 +5641,7 @@
</span><span class="cx"> self._md5 = md5
</span><span class="cx"> self._size = size
</span><span class="cx">
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = self._attachmentSchema
</ins><span class="cx"> self._created, self._modified = map(
</span><span class="cx"> sqltime,
</span><span class="cx"> (yield Update(
</span><span class="lines">@@ -5559,7 +5668,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Change the DROPBOX_ID to a single "." to indicate a managed attachment.
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = self._attachmentSchema
</ins><span class="cx"> (yield Update(
</span><span class="cx"> {att.DROPBOX_ID : ".", },
</span><span class="cx"> Where=(att.ATTACHMENT_ID == self._attachmentID),
</span><span class="lines">@@ -5613,7 +5722,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Now create the DB entry
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = cls._attachmentSchema
</ins><span class="cx"> rows = (yield Insert({
</span><span class="cx"> att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
</span><span class="cx"> att.DROPBOX_ID : ".",
</span><span class="lines">@@ -5663,7 +5772,7 @@
</span><span class="cx"> attachment._objectResourceID = referencedBy
</span><span class="cx">
</span><span class="cx"> # Create the attachment<->calendar object relationship for managed attachments
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = cls._attachmentLinkSchema
</ins><span class="cx"> yield Insert({
</span><span class="cx"> attco.ATTACHMENT_ID : attachment._attachmentID,
</span><span class="cx"> attco.MANAGED_ID : attachment._managedID,
</span><span class="lines">@@ -5698,7 +5807,7 @@
</span><span class="cx"> attachment._objectResourceID = referencedBy
</span><span class="cx">
</span><span class="cx"> # Update the attachment<->calendar object relationship for managed attachments
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = cls._attachmentLinkSchema
</ins><span class="cx"> yield Update(
</span><span class="cx"> {
</span><span class="cx"> attco.ATTACHMENT_ID : attachment._attachmentID,
</span><span class="lines">@@ -5732,7 +5841,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> if managedID:
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = cls._attachmentLinkSchema
</ins><span class="cx"> where = (attco.MANAGED_ID == managedID)
</span><span class="cx"> if referencedID is not None:
</span><span class="cx"> where = where.And(attco.CALENDAR_OBJECT_RESOURCE_ID == referencedID)
</span><span class="lines">@@ -5760,7 +5869,7 @@
</span><span class="cx"> """
</span><span class="cx"> Find all the calendar object resourceIds referenced by this supplied managed-id.
</span><span class="cx"> """
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = cls._attachmentLinkSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [attco.CALENDAR_OBJECT_RESOURCE_ID, ],
</span><span class="cx"> From=attco,
</span><span class="lines">@@ -5776,9 +5885,9 @@
</span><span class="cx"> """
</span><span class="cx"> Return the "owner" home and referencing resource is, and UID for a managed-id.
</span><span class="cx"> """
</span><del>- att = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
- co = schema.CALENDAR_OBJECT
</del><ins>+ att = cls._attachmentSchema
+ attco = cls._attachmentLinkSchema
+ co = CalendarObject._objectSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [
</span><span class="cx"> att.CALENDAR_HOME_RESOURCE_ID,
</span><span class="lines">@@ -5801,7 +5910,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Find all reference attachment-ids and dereference
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = cls._attachmentLinkSchema
</ins><span class="cx"> rows = (yield Select(
</span><span class="cx"> [attco.MANAGED_ID, ],
</span><span class="cx"> From=attco,
</span><span class="lines">@@ -5821,7 +5930,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # Find the associated attachment-id and insert new reference
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = cls._attachmentLinkSchema
</ins><span class="cx"> aid = (yield Select(
</span><span class="cx"> [attco.ATTACHMENT_ID, ],
</span><span class="cx"> From=attco,
</span><span class="lines">@@ -5897,7 +6006,7 @@
</span><span class="cx"> self._name = dispositionName
</span><span class="cx"> self._md5 = md5
</span><span class="cx"> self._size = size
</span><del>- att = schema.ATTACHMENT
</del><ins>+ att = self._attachmentSchema
</ins><span class="cx"> self._created, self._modified = map(
</span><span class="cx"> sqltime,
</span><span class="cx"> (yield Update(
</span><span class="lines">@@ -5926,7 +6035,7 @@
</span><span class="cx"> @rtype: L{ManagedAttachment}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = self._attachmentLinkSchema
</ins><span class="cx"> yield Insert({
</span><span class="cx"> attco.ATTACHMENT_ID : self._attachmentID,
</span><span class="cx"> attco.MANAGED_ID : self._managedID,
</span><span class="lines">@@ -5941,7 +6050,7 @@
</span><span class="cx"> def removeFromResource(self, resourceID):
</span><span class="cx">
</span><span class="cx"> # Delete the reference
</span><del>- attco = schema.ATTACHMENT_CALENDAR_OBJECT
</del><ins>+ attco = self._attachmentLinkSchema
</ins><span class="cx"> yield Delete(
</span><span class="cx"> From=attco,
</span><span class="cx"> Where=(attco.ATTACHMENT_ID == self._attachmentID).And(
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoresql_externalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -23,7 +23,7 @@
</span><span class="cx"> from twext.python.log import Logger
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.sql import CalendarHome, Calendar, CalendarObject, \
</span><del>- Attachment
</del><ins>+ Attachment, AttachmentLink
</ins><span class="cx"> from txdav.caldav.icalendarstore import ComponentUpdateState, ComponentRemoveState
</span><span class="cx"> from txdav.common.datastore.sql_external import CommonHomeExternal, CommonHomeChildExternal, \
</span><span class="cx"> CommonObjectResourceExternal
</span><span class="lines">@@ -68,14 +68,10 @@
</span><span class="cx"> Return all the L{Attachment} objects associated with this calendar home.
</span><span class="cx"> Needed during migration.
</span><span class="cx"> """
</span><del>- raw_results = yield self._txn.store().conduit.send_get_all_attachments(self)
</del><ins>+ raw_results = yield self._txn.store().conduit.send_home_get_all_attachments(self)
+ returnValue([Attachment.internalize(self._txn, attachment) for attachment in raw_results])
</ins><span class="cx">
</span><del>- results = []
- for attachment in raw_results:
- results.append(Attachment.internalize(self._txn, attachment))
- returnValue(results)
</del><span class="cx">
</span><del>-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def readAttachmentData(self, remote_id, attachment):
</span><span class="cx"> """
</span><span class="lines">@@ -86,6 +82,16 @@
</span><span class="cx"> yield self._txn.store().conduit.send_get_attachment_data(self, remote_id, stream)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
+ def getAttachmentLinks(self):
+ """
+ Read the attachment<->calendar object mapping data associated with this calendar home.
+ Needed during migration only.
+ """
+ raw_results = yield self._txn.store().conduit.send_home_get_attachment_links(self)
+ returnValue([AttachmentLink.internalize(self._txn, attachment) for attachment in raw_results])
+
+
</ins><span class="cx"> def getAllDropboxIDs(self):
</span><span class="cx"> """
</span><span class="cx"> No children.
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcaldavdatastoreutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -522,6 +522,11 @@
</span><span class="cx"> self._contentType = http_headers.MimeType.fromString(getType(self._attachment.name(), self.contentTypes))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def resetDetails(self, contentType, dispositionName):
+ self._contentType = contentType
+ self._dispositionName = dispositionName
+
+
</ins><span class="cx"> def write(self, data):
</span><span class="cx"> """
</span><span class="cx"> Children must override this to actually write the data, but should
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingattachmentspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -16,6 +16,7 @@
</span><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx"> from txdav.caldav.icalendarstore import InvalidAttachmentOperation
</span><ins>+from txdav.common.datastore.podding.util import UtilityConduitMixin
</ins><span class="cx"> from txweb2.http_headers import generateContentType
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -155,36 +156,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def send_get_all_attachments(self, home):
- """
- Managed attachment removeAttachment call.
-
- @param home: the home whose attachments are being requested
- @type home: L{CalendarHome}
- """
-
- actionName = "get-all-attachments"
- txn, request, server = yield self._getRequestForStoreObject(actionName, home, False)
-
- response = yield self.sendRequestToServer(txn, server, request)
- returnValue(response)
-
-
- @inlineCallbacks
- def recv_get_all_attachments(self, txn, request):
- """
- Process an getAllAttachments cross-pod request. Request arguments as per L{send_get_all_attachments}.
-
- @param request: request arguments
- @type request: C{dict}
- """
-
- home, _ignore = yield self._getStoreObjectForRequest(txn, request)
- attachments = yield home.getAllAttachments()
- returnValue([attachment.externalize() for attachment in attachments])
-
-
- @inlineCallbacks
</del><span class="cx"> def send_get_attachment_data(self, home, attachment_id, stream):
</span><span class="cx"> """
</span><span class="cx"> Managed attachment readAttachmentData call. We are using streams on the sender and the receiver
</span><span class="lines">@@ -222,3 +193,8 @@
</span><span class="cx">
</span><span class="cx"> attachment.retrieve(stream)
</span><span class="cx"> returnValue((generateContentType(attachment.contentType()), attachment.name(),))
</span><ins>+
+
+# Calls on L{CommonHome} objects
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_all_attachments", "getAllAttachments", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_attachment_links", "getAttachmentLinks", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingconduitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -20,9 +20,10 @@
</span><span class="cx"> from txdav.common.datastore.podding.attachments import AttachmentsConduitMixin
</span><span class="cx"> from txdav.common.datastore.podding.base import FailedCrossPodRequestError
</span><span class="cx"> from txdav.common.datastore.podding.directory import DirectoryPoddingConduitMixin
</span><del>-from txdav.common.datastore.podding.store_api import StoreAPIConduitMixin
</del><span class="cx"> from txdav.common.datastore.podding.request import ConduitRequest
</span><span class="cx"> from txdav.common.datastore.podding.sharing_invites import SharingInvitesConduitMixin
</span><ins>+from txdav.common.datastore.podding.store_api import StoreAPIConduitMixin
+from txdav.common.datastore.podding.util import UtilityConduitMixin
</ins><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx"> from twisted.python.reflect import namedClass
</span><span class="lines">@@ -32,6 +33,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> class PoddingConduit(
</span><ins>+ UtilityConduitMixin,
</ins><span class="cx"> StoreAPIConduitMixin,
</span><span class="cx"> AttachmentsConduitMixin,
</span><span class="cx"> SharingInvitesConduitMixin,
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationhome_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -162,10 +162,7 @@
</span><span class="cx"> # TODO: sync attachments
</span><span class="cx"> yield self.syncAttachments()
</span><span class="cx">
</span><del>- # TODO: group attendee/sharee reconcile
- pass
</del><span class="cx">
</span><del>-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def finalSync(self):
</span><span class="cx"> """
</span><span class="lines">@@ -174,14 +171,20 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> # TODO: link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
</span><del>- pass
</del><ins>+ yield self.linkAttachments()
</ins><span class="cx">
</span><span class="cx"> # TODO: Re-write attachment URIs - not sure if we need this as reverse proxy may take care of it
</span><span class="cx"> pass
</span><span class="cx">
</span><ins>+ # TODO: group attendee reconcile
+ pass
+
</ins><span class="cx"> # TODO: shared collections reconcile
</span><span class="cx"> pass
</span><span class="cx">
</span><ins>+ # TODO: group sharee reconcile
+ pass
+
</ins><span class="cx"> # TODO: delegates reconcile
</span><span class="cx"> pass
</span><span class="cx">
</span><span class="lines">@@ -273,7 +276,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.sql_external import CalendarHomeExternal
</span><del>- resourceID = yield txn.store().conduit.send_home_resource_id(self, self.record)
</del><ins>+ resourceID = yield txn.store().conduit.send_home_resource_id(txn, self.record)
</ins><span class="cx"> home = CalendarHomeExternal(txn, self.record.uid, resourceID) if resourceID is not None else None
</span><span class="cx"> if home:
</span><span class="cx"> home._childClass = home._childClass._externalClass
</span><span class="lines">@@ -754,3 +757,82 @@
</span><span class="cx">
</span><span class="cx"> # Read the data from the conduit
</span><span class="cx"> yield remote_home.readAttachmentData(remote_id, attachment)
</span><ins>+
+
+ @inlineCallbacks
+ def linkAttachments(self):
+ """
+ Link attachments to the calendar objects they belong to.
+ """
+
+ # Get the map of links for the remote home
+ links = yield self.getAttachmentLinks()
+
+ # Get remote->local ID mappings
+ attachmentIDMap, objectIDMap = yield self.getAttachmentMappings()
+
+ # Batch setting links for the local home
+ len_links = len(links)
+ while len(links):
+ yield self.makeAttachmentLinks(links[:50], attachmentIDMap, objectIDMap)
+ links = links[50:]
+
+ returnValue(len_links)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def getAttachmentLinks(self, txn):
+ """
+ Get the remote link information.
+ """
+
+ # Get the map of links for the remote home
+ remote_home = yield self._remoteHome(txn)
+ links = yield remote_home.getAttachmentLinks()
+ returnValue(links)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def getAttachmentMappings(self, txn):
+ """
+ Get the remote link information.
+ """
+
+ # Get migration mappings
+ am = schema.ATTACHMENT_MIGRATION
+ rows = yield Select(
+ [am.REMOTE_RESOURCE_ID, am.LOCAL_RESOURCE_ID],
+ From=am,
+ Where=(am.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+ ).on(txn)
+ attachmentIDMap = dict(rows)
+
+ com = schema.CALENDAR_OBJECT_MIGRATION
+ rows = yield Select(
+ [com.REMOTE_RESOURCE_ID, com.LOCAL_RESOURCE_ID],
+ From=com,
+ Where=(com.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+ ).on(txn)
+ objectIDMap = dict(rows)
+
+ returnValue((attachmentIDMap, objectIDMap,))
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def makeAttachmentLinks(self, txn, links, attachmentIDMap, objectIDMap):
+ """
+ Map remote links to local links.
+ """
+
+ for link in links:
+ # Remote link has an invalid txn at this point so replace that first
+ link._txn = txn
+
+ # Now re-map the attachment ID and calendar_object_id to the local ones
+ link._attachmentID = attachmentIDMap[link._attachmentID]
+ link._calendarObjectID = objectIDMap[link._calendarObjectID]
+
+ yield link.insert()
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingmigrationtesttest_home_syncpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -18,6 +18,7 @@
</span><span class="cx"> from twext.enterprise.dal.syntax import Select
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><span class="cx"> from twistedcaldav.ical import Component, normalize_iCalStr
</span><ins>+from txdav.caldav.datastore.sql import ManagedAttachment
</ins><span class="cx"> from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
</span><span class="cx"> from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
</span><span class="cx"> from txdav.common.datastore.sql_tables import schema
</span><span class="lines">@@ -588,3 +589,100 @@
</span><span class="cx"> attachments = yield home1.getAllAttachments()
</span><span class="cx"> mapping1 = dict([(o.md5(), o.id()) for o in attachments])
</span><span class="cx"> yield _checkAttachmentObjectMigrationState(home1, mapping1)
</span><ins>+
+
+ @inlineCallbacks
+ def test_link_attachments(self):
+ """
+ Test that L{linkAttachments} links attachment data to the associated calendar object.
+ """
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ object0_1 = yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ object0_2 = yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
+ yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
+ remote_id = calendar0.id()
+
+ attachment, _ignore_location = yield object0_1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1."))
+ id0_1 = attachment.id()
+ md50_1 = attachment.md5()
+ managedid0_1 = attachment.managedID()
+ pathID0_1 = ManagedAttachment.lastSegmentOfUriPath(managedid0_1, attachment.name())
+
+ attachment, _ignore_location = yield object0_2.addAttachment(None, MimeType.fromString("text/plain"), "test2.txt", MemoryStream("Here is some text #2."))
+ id0_2 = attachment.id()
+ md50_2 = attachment.md5()
+ managedid0_2 = attachment.managedID()
+ pathID0_2 = ManagedAttachment.lastSegmentOfUriPath(managedid0_2, attachment.name())
+
+ yield self.commitTransaction(0)
+
+ # Add original to a different resource
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ component = yield object1.componentForUser()
+ attach = component.mainComponent().getProperty("ATTACH")
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="3.ics")
+ component = yield object1.componentForUser()
+ attach = component.mainComponent().addProperty(attach)
+ yield object1.setComponent(component)
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ syncer.homeId = yield syncer.prepareCalendarHome()
+
+ # Trigger sync of the one calendar
+ local_sync_state = {}
+ remote_sync_state = yield syncer.getCalendarSyncList()
+ yield syncer.syncCalendar(
+ remote_id,
+ local_sync_state,
+ remote_sync_state,
+ )
+ self.assertEqual(len(local_sync_state), 1)
+ self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set((id0_1, id0_2,)))
+ self.assertEqual(removed, set())
+
+ # Link attachments
+ len_links = yield syncer.linkAttachments()
+ self.assertEqual(len_links, 3)
+
+ # Local calendar exists
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is not None)
+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "2.ics", "3.ics",)))
+
+ # Make sure calendar object is associated with attachment
+ object1 = yield calendar1.objectResourceWithName("1.ics")
+ attachments = yield object1.managedAttachmentList()
+ self.assertEqual(attachments, [pathID0_1, ])
+
+ attachment = yield object1.attachmentWithManagedID(managedid0_1)
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), md50_1)
+
+ # Make sure calendar object is associated with attachment
+ object1 = yield calendar1.objectResourceWithName("2.ics")
+ attachments = yield object1.managedAttachmentList()
+ self.assertEqual(attachments, [pathID0_2, ])
+
+ attachment = yield object1.attachmentWithManagedID(managedid0_2)
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), md50_2)
+
+ # Make sure calendar object is associated with attachment
+ object1 = yield calendar1.objectResourceWithName("3.ics")
+ attachments = yield object1.managedAttachmentList()
+ self.assertEqual(attachments, [pathID0_1, ])
+
+ attachment = yield object1.attachmentWithManagedID(managedid0_1)
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), md50_1)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingrequestpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -79,7 +79,6 @@
</span><span class="cx"> data = json.loads(data)
</span><span class="cx"> else:
</span><span class="cx"> yield readStream(response.stream, self.writeStream.write)
</span><del>- yield self.writeStream.loseConnection()
</del><span class="cx"> content_type = response.headers.getHeader("content-type")
</span><span class="cx"> if content_type is None:
</span><span class="cx"> content_type = MimeType("application", "octet-stream")
</span><span class="lines">@@ -88,7 +87,9 @@
</span><span class="cx"> filename = ""
</span><span class="cx"> else:
</span><span class="cx"> filename = content_disposition.params["filename"]
</span><del>- response = {
</del><ins>+ self.writeStream.resetDetails(content_type, filename)
+ yield self.writeStream.loseConnection()
+ data = {
</ins><span class="cx"> "result": "ok",
</span><span class="cx"> "content-type": content_type,
</span><span class="cx"> "name": filename,
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -177,7 +177,7 @@
</span><span class="cx"> except ValueError:
</span><span class="cx"> code = responsecode.BAD_REQUEST
</span><span class="cx"> else:
</span><del>- headers = {"content-type": ct}
</del><ins>+ headers = {"content-type": MimeType.fromString(ct)}
</ins><span class="cx"> headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
</span><span class="cx"> returnValue(Response(responsecode.OK, headers, stream))
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingstore_apipy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -16,8 +16,8 @@
</span><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx">
</span><del>-from txdav.common.datastore.podding.base import FailedCrossPodRequestError
</del><span class="cx"> from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
</span><ins>+from txdav.common.datastore.podding.util import UtilityConduitMixin
</ins><span class="cx">
</span><span class="cx"> from twistedcaldav.caldavxml import TimeRange
</span><span class="cx">
</span><span class="lines">@@ -27,115 +27,7 @@
</span><span class="cx"> Defines common cross-pod API for generic access to remote resources.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- #
- # Utility methods to map from store objects to/from JSON
- #
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def _getRequestForStoreObject(self, action, storeObject, classMethod):
- """
- Create the JSON data needed to identify the remote resource by type and ids, along with any parent resources.
-
- @param action: the conduit action name
- @type action: L{str}
- @param storeObject: the store object that is being operated on
- @type storeObject: L{object}
- @param classMethod: indicates whether the method being called is a classmethod
- @type classMethod: L{bool}
-
- @return: the transaction in use, the JSON dict to send in the request,
- the server where the request should be sent
- @rtype: L{tuple} of (L{CommonStoreTransaction}, L{dict}, L{str})
- """
-
- from txdav.common.datastore.sql import CommonObjectResource, CommonHomeChild, CommonHome
- result = {
- "action": action,
- }
-
- # Extract the relevant store objects
- txn = storeObject._txn
- owner_home = None
- viewer_home = None
- home_child = None
- object_resource = None
- if isinstance(storeObject, CommonObjectResource):
- owner_home = storeObject.ownerHome()
- viewer_home = storeObject.viewerHome()
- home_child = storeObject.parentCollection()
- object_resource = storeObject
- elif isinstance(storeObject, CommonHomeChild):
- owner_home = storeObject.ownerHome()
- viewer_home = storeObject.viewerHome()
- home_child = storeObject
- result["classMethod"] = classMethod
- elif isinstance(storeObject, CommonHome):
- owner_home = storeObject
- viewer_home = storeObject
- txn = storeObject._txn
- result["classMethod"] = classMethod
-
- # Add store object identities to JSON request
- result["homeType"] = viewer_home._homeType
- result["homeUID"] = viewer_home.uid()
- if home_child:
- if home_child.owned():
- result["homeChildID"] = home_child.id()
- else:
- result["homeChildSharedID"] = home_child.name()
- if object_resource:
- result["objectResourceID"] = object_resource.id()
-
- # Note that the owner_home is always the ownerHome() because in the sharing case
- # a viewer is accessing the owner's data on another pod.
- recipient = yield self.store.directoryService().recordWithUID(owner_home.uid())
-
- returnValue((txn, result, recipient.server(),))
-
-
- @inlineCallbacks
- def _getStoreObjectForRequest(self, txn, request):
- """
- Resolve the supplied JSON data to get a store object to operate on.
- """
-
- returnObject = txn
- classObject = None
-
- if "homeUID" in request:
- home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
- if home is None:
- raise FailedCrossPodRequestError("Invalid owner UID specified")
- home._internalRequest = False
- returnObject = home
- if request.get("classMethod", False):
- classObject = home._childClass
-
- if "homeChildID" in request:
- homeChild = yield home.childWithID(request["homeChildID"])
- if homeChild is None:
- raise FailedCrossPodRequestError("Invalid home child specified")
- returnObject = homeChild
- if request.get("classMethod", False):
- classObject = homeChild._objectResourceClass
- elif "homeChildSharedID" in request:
- homeChild = yield home.childWithName(request["homeChildSharedID"])
- if homeChild is None:
- raise FailedCrossPodRequestError("Invalid home child specified")
- returnObject = homeChild
- if request.get("classMethod", False):
- classObject = homeChild._objectResourceClass
-
- if "objectResourceID" in request:
- objectResource = yield homeChild.objectResourceWithID(request["objectResourceID"])
- if objectResource is None:
- raise FailedCrossPodRequestError("Invalid object resource specified")
- returnObject = objectResource
-
- returnValue((returnObject, classObject,))
-
-
- @inlineCallbacks
</del><span class="cx"> def send_home_resource_id(self, txn, recipient):
</span><span class="cx"> """
</span><span class="cx"> Lookup the remote resourceID matching the specified directory uid.
</span><span class="lines">@@ -235,134 +127,29 @@
</span><span class="cx"> "matchtotal": matchtotal,
</span><span class="cx"> })
</span><span class="cx">
</span><del>-
- #
- # We can simplify code generation for simple calls by dynamically generating the appropriate class methods.
- #
-
- @inlineCallbacks
- def _simple_object_send(self, actionName, storeObject, classMethod=False, transform=None, args=None, kwargs=None):
- """
- A simple send operation that returns a value.
-
- @param actionName: name of the action.
- @type actionName: C{str}
- @param shareeView: sharee resource being operated on.
- @type shareeView: L{CommonHomeChildExternal}
- @param objectResource: the resource being operated on, or C{None} for classmethod.
- @type objectResource: L{CommonObjectResourceExternal}
- @param transform: a function used to convert the JSON response into return values.
- @type transform: C{callable}
- @param args: list of optional arguments.
- @type args: C{list}
- @param kwargs: optional keyword arguments.
- @type kwargs: C{dict}
- """
-
- txn, request, server = yield self._getRequestForStoreObject(actionName, storeObject, classMethod)
- if args is not None:
- request["arguments"] = args
- if kwargs is not None:
- request["keywords"] = kwargs
- response = yield self.sendRequestToServer(txn, server, request)
- returnValue(transform(response) if transform is not None else response)
-
-
- @inlineCallbacks
- def _simple_object_recv(self, txn, actionName, request, method, transform=None):
- """
- A simple recv operation that returns a value. We also look for an optional set of arguments/keywords
- and include those only if present.
-
- @param actionName: name of the action.
- @type actionName: C{str}
- @param request: request arguments
- @type request: C{dict}
- @param method: name of the method to execute on the shared resource to get the result.
- @type method: C{str}
- @param transform: method to call on returned JSON value to convert it to something useful.
- @type transform: C{callable}
- """
-
- storeObject, classObject = yield self._getStoreObjectForRequest(txn, request)
- if classObject is not None:
- value = yield getattr(classObject, method)(storeObject, *request.get("arguments", ()), **request.get("keywords", {}))
- else:
- value = yield getattr(storeObject, method)(*request.get("arguments", ()), **request.get("keywords", {}))
-
- returnValue(transform(value) if transform is not None else value)
-
-
- #
- # Factory methods for binding actions to the conduit class
- #
- @classmethod
- def _make_simple_action(cls, action, method, classMethod=False, transform_recv_result=None, transform_send_result=None):
- setattr(
- cls,
- "send_{}".format(action),
- lambda self, storeObject, *args, **kwargs:
- self._simple_object_send(action, storeObject, classMethod=classMethod, transform=transform_send_result, args=args, kwargs=kwargs)
- )
- setattr(
- cls,
- "recv_{}".format(action),
- lambda self, txn, message:
- self._simple_object_recv(txn, action, message, method, transform=transform_recv_result)
- )
-
-
- #
- # Transforms for returned data
- #
- @staticmethod
- def _to_externalize(value):
- """
- Convert the value to the external (JSON-based) representation.
- """
- return value.externalize() if value is not None else None
-
-
- @staticmethod
- def _to_externalize_list(value):
- """
- Convert the value to the external (JSON-based) representation.
- """
- return [v.externalize() for v in value]
-
-
- @staticmethod
- def _to_string(value):
- return str(value)
-
-
- @staticmethod
- def _to_tuple(value):
- return tuple(value)
-
</del><span class="cx"> # These are the actions on store objects we need to expose via the conduit api
</span><span class="cx">
</span><span class="cx"> # Calls on L{CommonHome} objects
</span><span class="cx">
</span><span class="cx"> # Calls on L{CommonHomeChild} objects
</span><del>-StoreAPIConduitMixin._make_simple_action("homechild_listobjects", "listObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("homechild_movehere", "moveObjectResourceHere")
-StoreAPIConduitMixin._make_simple_action("homechild_moveaway", "moveObjectResourceAway")
-StoreAPIConduitMixin._make_simple_action("homechild_synctokenrevision", "syncTokenRevision")
-StoreAPIConduitMixin._make_simple_action("homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=StoreAPIConduitMixin._to_tuple)
-StoreAPIConduitMixin._make_simple_action("homechild_search", "search")
</del><ins>+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_listobjects", "listObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_movehere", "moveObjectResourceHere")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_moveaway", "moveObjectResourceAway")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_synctokenrevision", "syncTokenRevision")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=UtilityConduitMixin._to_tuple)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_search", "search")
</ins><span class="cx">
</span><span class="cx"> # Calls on L{CommonObjectResource} objects
</span><del>-StoreAPIConduitMixin._make_simple_action("objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("objectresource_listobjects", "listObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_countobjects", "countObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_create", "create", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("objectresource_setcomponent", "setComponent")
-StoreAPIConduitMixin._make_simple_action("objectresource_component", "component", transform_recv_result=StoreAPIConduitMixin._to_string)
-StoreAPIConduitMixin._make_simple_action("objectresource_remove", "remove")
</del><ins>+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_listobjects", "listObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_countobjects", "countObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_create", "create", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_setcomponent", "setComponent")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_component", "component", transform_recv_result=UtilityConduitMixin._to_string)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_remove", "remove")
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtesttest_conduitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -32,7 +32,7 @@
</span><span class="cx"> from txdav.caldav.datastore.query.filter import Filter
</span><span class="cx"> from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
</span><span class="cx"> from txdav.caldav.datastore.scheduling.ischedule.localservers import ServersDB, Server
</span><del>-from txdav.caldav.datastore.sql import ManagedAttachment
</del><ins>+from txdav.caldav.datastore.sql import ManagedAttachment, AttachmentLink
</ins><span class="cx"> from txdav.caldav.datastore.test.common import CaptureProtocol
</span><span class="cx"> from txdav.common.datastore.podding.conduit import PoddingConduit, \
</span><span class="cx"> FailedCrossPodRequestError
</span><span class="lines">@@ -1107,3 +1107,32 @@
</span><span class="cx"> attachment._name = "test.txt"
</span><span class="cx"> yield shared_object.ownerHome().readAttachmentData(remote_id, attachment)
</span><span class="cx"> yield self.commitTransaction(1)
</span><ins>+
+
+ @inlineCallbacks
+ def test_get_attachment_links(self):
+ """
+ Test that action=get-attachment-links works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user01", name="calendar")
+ cobj1 = yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ calobjID = cobj1.id()
+ yield self.commitTransaction(0)
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ attID = attachment.id()
+ managedID = attachment.managedID()
+ yield self.commitTransaction(0)
+
+ shared_object = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ links = yield shared_object.ownerHome().getAttachmentLinks()
+ self.assertEqual(len(links), 1)
+ self.assertTrue(isinstance(links[0], AttachmentLink))
+ self.assertEqual(links[0]._attachmentID, attID)
+ self.assertEqual(links[0]._managedID, managedID)
+ self.assertEqual(links[0]._calendarObjectID, calobjID)
+ yield self.commitTransaction(1)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingtestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py (14411 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -21,20 +21,24 @@
</span><span class="cx"> Server, ServersDB
</span><span class="cx"> )
</span><span class="cx"> from txdav.common.datastore.podding.conduit import PoddingConduit
</span><ins>+from txdav.common.datastore.podding.request import ConduitRequest
</ins><span class="cx"> from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE
</span><span class="cx"> from txdav.common.datastore.test.util import (
</span><span class="cx"> CommonCommonTests, SQLStoreBuilder, buildTestDirectory
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> import txweb2.dav.test.util
</span><del>-from txweb2.stream import ProducerStream, readStream
</del><ins>+from txweb2 import responsecode
+from txweb2.http import Response, JSONResponse
+from txweb2.http_headers import MimeDisposition, MimeType
+from txweb2.stream import ProducerStream
</ins><span class="cx">
</span><span class="cx"> from twext.enterprise.ienterprise import AlreadyFinishedError
</span><span class="cx">
</span><span class="cx"> import json
</span><span class="cx">
</span><span class="cx">
</span><del>-class FakeConduitRequest(object):
</del><ins>+class FakeConduitRequest(ConduitRequest):
</ins><span class="cx"> """
</span><span class="cx"> A conduit request that sends messages internally rather than using HTTP
</span><span class="cx"> """
</span><span class="lines">@@ -66,32 +70,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def doRequest(self, txn):
-
- # Generate an HTTP client request
- try:
- response = (yield self._processRequest())
- if self.writeStream is None:
- response = json.loads(response)
- else:
- try:
- ct, name, stream = response
- response = {
- "result": "ok",
- "content-type": ct,
- "name": name,
- }
- yield readStream(stream, self.writeStream.write)
- yield self.writeStream.loseConnection()
- except ValueError:
- pass
- except Exception as e:
- raise ValueError("Failed cross-pod request: {}".format(e))
-
- returnValue(response)
-
-
- @inlineCallbacks
</del><span class="cx"> def _processRequest(self):
</span><span class="cx"> """
</span><span class="cx"> Process the request by sending it to the relevant server.
</span><span class="lines">@@ -121,11 +99,14 @@
</span><span class="cx"> try:
</span><span class="cx"> ct, name = result
</span><span class="cx"> except ValueError:
</span><del>- pass
</del><ins>+ code = responsecode.BAD_REQUEST
</ins><span class="cx"> else:
</span><del>- returnValue((ct, name, stream,))
</del><ins>+ headers = {"content-type": MimeType.fromString(ct)}
+ headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
+ returnValue(Response(responsecode.OK, headers, stream))
</ins><span class="cx"> else:
</span><span class="cx"> result = yield store.conduit.processRequest(j)
</span><ins>+ code = responsecode.OK
</ins><span class="cx"> except Exception as e:
</span><span class="cx"> # Send the exception over to the other side
</span><span class="cx"> result = {
</span><span class="lines">@@ -133,11 +114,13 @@
</span><span class="cx"> "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
</span><span class="cx"> "details": str(e),
</span><span class="cx"> }
</span><del>- result = json.dumps(result)
- returnValue(result)
</del><ins>+ code = responsecode.BAD_REQUEST
</ins><span class="cx">
</span><ins>+ response = JSONResponse(code, result)
+ returnValue(response)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class MultiStoreConduitTest(CommonCommonTests, txweb2.dav.test.util.TestCase):
</span><span class="cx">
</span><span class="cx"> numberOfStores = 2
</span></span></pre></div>
<a id="CalendarServerbranchesuserscdaboopod2podmigrationtxdavcommondatastorepoddingutilpy"></a>
<div class="addfile"><h4>Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py (0 => 14412)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py 2015-02-13 21:21:26 UTC (rev 14412)
</span><span class="lines">@@ -0,0 +1,237 @@
</span><ins>+##
+# Copyright (c) 2013-2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from txdav.common.datastore.podding.base import FailedCrossPodRequestError
+
+
+class UtilityConduitMixin(object):
+ """
+ Defines utility methods for cross-pod API and mix-ins.
+ """
+
+ #
+ # Utility methods to map from store objects to/from JSON
+ #
+
+ @inlineCallbacks
+ def _getRequestForStoreObject(self, action, storeObject, classMethod):
+ """
+ Create the JSON data needed to identify the remote resource by type and ids, along with any parent resources.
+
+ @param action: the conduit action name
+ @type action: L{str}
+ @param storeObject: the store object that is being operated on
+ @type storeObject: L{object}
+ @param classMethod: indicates whether the method being called is a classmethod
+ @type classMethod: L{bool}
+
+ @return: the transaction in use, the JSON dict to send in the request,
+ the server where the request should be sent
+ @rtype: L{tuple} of (L{CommonStoreTransaction}, L{dict}, L{str})
+ """
+
+ from txdav.common.datastore.sql import CommonObjectResource, CommonHomeChild, CommonHome
+ result = {
+ "action": action,
+ }
+
+ # Extract the relevant store objects
+ txn = storeObject._txn
+ owner_home = None
+ viewer_home = None
+ home_child = None
+ object_resource = None
+ if isinstance(storeObject, CommonObjectResource):
+ owner_home = storeObject.ownerHome()
+ viewer_home = storeObject.viewerHome()
+ home_child = storeObject.parentCollection()
+ object_resource = storeObject
+ elif isinstance(storeObject, CommonHomeChild):
+ owner_home = storeObject.ownerHome()
+ viewer_home = storeObject.viewerHome()
+ home_child = storeObject
+ result["classMethod"] = classMethod
+ elif isinstance(storeObject, CommonHome):
+ owner_home = storeObject
+ viewer_home = storeObject
+ txn = storeObject._txn
+ result["classMethod"] = classMethod
+
+ # Add store object identities to JSON request
+ result["homeType"] = viewer_home._homeType
+ result["homeUID"] = viewer_home.uid()
+ if home_child:
+ if home_child.owned():
+ result["homeChildID"] = home_child.id()
+ else:
+ result["homeChildSharedID"] = home_child.name()
+ if object_resource:
+ result["objectResourceID"] = object_resource.id()
+
+ # Note that the owner_home is always the ownerHome() because in the sharing case
+ # a viewer is accessing the owner's data on another pod.
+ recipient = yield self.store.directoryService().recordWithUID(owner_home.uid())
+
+ returnValue((txn, result, recipient.server(),))
+
+
+ @inlineCallbacks
+ def _getStoreObjectForRequest(self, txn, request):
+ """
+ Resolve the supplied JSON data to get a store object to operate on.
+ """
+
+ returnObject = txn
+ classObject = None
+
+ if "homeUID" in request:
+ home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
+ if home is None:
+ raise FailedCrossPodRequestError("Invalid owner UID specified")
+ home._internalRequest = False
+ returnObject = home
+ if request.get("classMethod", False):
+ classObject = home._childClass
+
+ if "homeChildID" in request:
+ homeChild = yield home.childWithID(request["homeChildID"])
+ if homeChild is None:
+ raise FailedCrossPodRequestError("Invalid home child specified")
+ returnObject = homeChild
+ if request.get("classMethod", False):
+ classObject = homeChild._objectResourceClass
+ elif "homeChildSharedID" in request:
+ homeChild = yield home.childWithName(request["homeChildSharedID"])
+ if homeChild is None:
+ raise FailedCrossPodRequestError("Invalid home child specified")
+ returnObject = homeChild
+ if request.get("classMethod", False):
+ classObject = homeChild._objectResourceClass
+
+ if "objectResourceID" in request:
+ objectResource = yield homeChild.objectResourceWithID(request["objectResourceID"])
+ if objectResource is None:
+ raise FailedCrossPodRequestError("Invalid object resource specified")
+ returnObject = objectResource
+
+ returnValue((returnObject, classObject,))
+
+
+ #
+ # We can simplify code generation for simple calls by dynamically generating the appropriate class methods.
+ #
+
+ @inlineCallbacks
+ def _simple_object_send(self, actionName, storeObject, classMethod=False, transform=None, args=None, kwargs=None):
+ """
+ A simple send operation that returns a value.
+
+ @param actionName: name of the action.
+ @type actionName: C{str}
+ @param shareeView: sharee resource being operated on.
+ @type shareeView: L{CommonHomeChildExternal}
+ @param objectResource: the resource being operated on, or C{None} for classmethod.
+ @type objectResource: L{CommonObjectResourceExternal}
+ @param transform: a function used to convert the JSON response into return values.
+ @type transform: C{callable}
+ @param args: list of optional arguments.
+ @type args: C{list}
+ @param kwargs: optional keyword arguments.
+ @type kwargs: C{dict}
+ """
+
+ txn, request, server = yield self._getRequestForStoreObject(actionName, storeObject, classMethod)
+ if args is not None:
+ request["arguments"] = args
+ if kwargs is not None:
+ request["keywords"] = kwargs
+ response = yield self.sendRequestToServer(txn, server, request)
+ returnValue(transform(response) if transform is not None else response)
+
+
+ @inlineCallbacks
+ def _simple_object_recv(self, txn, actionName, request, method, transform=None):
+ """
+ A simple recv operation that returns a value. We also look for an optional set of arguments/keywords
+ and include those only if present.
+
+ @param actionName: name of the action.
+ @type actionName: C{str}
+ @param request: request arguments
+ @type request: C{dict}
+ @param method: name of the method to execute on the shared resource to get the result.
+ @type method: C{str}
+ @param transform: method to call on returned JSON value to convert it to something useful.
+ @type transform: C{callable}
+ """
+
+ storeObject, classObject = yield self._getStoreObjectForRequest(txn, request)
+ if classObject is not None:
+ value = yield getattr(classObject, method)(storeObject, *request.get("arguments", ()), **request.get("keywords", {}))
+ else:
+ value = yield getattr(storeObject, method)(*request.get("arguments", ()), **request.get("keywords", {}))
+
+ returnValue(transform(value) if transform is not None else value)
+
+
+ #
+ # Factory methods for binding actions to the conduit class
+ #
+ @staticmethod
+ def _make_simple_action(bindcls, action, method, classMethod=False, transform_recv_result=None, transform_send_result=None):
+ setattr(
+ bindcls,
+ "send_{}".format(action),
+ lambda self, storeObject, *args, **kwargs:
+ self._simple_object_send(action, storeObject, classMethod=classMethod, transform=transform_send_result, args=args, kwargs=kwargs)
+ )
+ setattr(
+ bindcls,
+ "recv_{}".format(action),
+ lambda self, txn, message:
+ self._simple_object_recv(txn, action, message, method, transform=transform_recv_result)
+ )
+
+
+ #
+ # Transforms for returned data
+ #
+ @staticmethod
+ def _to_externalize(value):
+ """
+ Convert the value to the external (JSON-based) representation.
+ """
+ return value.externalize() if value is not None else None
+
+
+ @staticmethod
+ def _to_externalize_list(value):
+ """
+ Convert the value to the external (JSON-based) representation.
+ """
+ return [v.externalize() for v in value]
+
+
+ @staticmethod
+ def _to_string(value):
+ return str(value)
+
+
+ @staticmethod
+ def _to_tuple(value):
+ return tuple(value)
</ins></span></pre>
</div>
</div>
</body>
</html>