[CalendarServer-changes] [14412] CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav

source_changes at macosforge.org source_changes at macosforge.org
Fri Feb 13 13:21:26 PST 2015


Revision: 14412
          http://trac.calendarserver.org//changeset/14412
Author:   cdaboo at apple.com
Date:     2015-02-13 13:21:26 -0800 (Fri, 13 Feb 2015)
Log Message:
-----------
Checkpoint: migration final sync attachment links.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py

Added Paths:
-----------
    CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -143,7 +143,7 @@
         @type txn: L{txdav.common.datastore.sql.CommonStoreTransaction}
         """
 
-        at = schema.ATTACHMENT
+        at = Attachment._attachmentSchema
         rows = (yield Select(
             (at.DROPBOX_ID,),
             From=at,
@@ -175,8 +175,8 @@
         txn = self._store.newTransaction("CalendarStoreFeatures.upgradeToManagedAttachments - preliminary work")
         try:
             # Clear out unused CALENDAR_OBJECT.DROPBOX_IDs
-            co = schema.CALENDAR_OBJECT
-            at = schema.ATTACHMENT
+            co = CalendarObject._objectSchema
+            at = Attachment._attachmentSchema
             yield Update(
                 {co.DROPBOX_ID: None},
                 Where=co.RESOURCE_ID.In(Select(
@@ -249,7 +249,7 @@
         log.debug("  {0} affected calendar objects".format(len(cobjs),))
 
         # Get names of each matching attachment
-        at = schema.ATTACHMENT
+        at = Attachment._attachmentSchema
         names = (yield Select(
             (at.PATH,),
             From=at,
@@ -318,8 +318,8 @@
         @type dropbox_id: C{str}
         """
 
-        co = schema.CALENDAR_OBJECT
-        cb = schema.CALENDAR_BIND
+        co = CalendarObject._objectSchema
+        cb = Calendar._bindSchema
         rows = (yield Select(
             (cb.CALENDAR_HOME_RESOURCE_ID, co.CALENDAR_RESOURCE_ID, co.RESOURCE_ID,),
             From=co.join(cb, co.CALENDAR_RESOURCE_ID == cb.CALENDAR_RESOURCE_ID),
@@ -511,7 +511,7 @@
         assert self._txn._migrating
 
         # Simple attributes that can be copied over as-is
-        chm = schema.CALENDAR_HOME_METADATA
+        chm = self._homeMetaDataSchema
         values = {
             chm.ALARM_VEVENT_TIMED : other._alarm_vevent_timed,
             chm.ALARM_VEVENT_ALLDAY : other._alarm_vevent_allday,
@@ -617,8 +617,8 @@
         """
         Implement lookup via queries.
         """
-        co = schema.CALENDAR_OBJECT
-        cb = schema.CALENDAR_BIND
+        co = self._objectSchema
+        cb = self._bindSchema
         rows = (yield Select(
             [co.PARENT_RESOURCE_ID,
              co.RESOURCE_ID],
@@ -637,16 +637,22 @@
         returnValue(None)
 
 
-    @inlineCallbacks
     def getAllAttachments(self):
         """
         Return all the L{Attachment} objects associated with this calendar home.
         Needed during migration.
         """
-        attachments = yield Attachment.loadAllAttachments(self)
-        returnValue(attachments)
+        return Attachment.loadAllAttachments(self)
 
 
+    def getAttachmentLinks(self):
+        """
+        Read the attachment<->calendar object mapping data associated with this calendar home.
+        Needed during migration only.
+        """
+        return AttachmentLink.linksForHome(self)
+
+
     def getAttachmentByID(self, id):
         """
         Return a specific attachment associated with this calendar home.
@@ -657,8 +663,8 @@
 
     @inlineCallbacks
     def getAllDropboxIDs(self):
-        co = schema.CALENDAR_OBJECT
-        cb = schema.CALENDAR_BIND
+        co = self._objectSchema
+        cb = self._bindSchema
         rows = (yield Select(
             [co.DROPBOX_ID],
             From=co.join(cb, co.PARENT_RESOURCE_ID == cb.RESOURCE_ID),
@@ -671,7 +677,7 @@
 
     @inlineCallbacks
     def getAllAttachmentNames(self):
-        att = schema.ATTACHMENT
+        att = Attachment._attachmentSchema
         rows = (yield Select(
             [att.DROPBOX_ID],
             From=att,
@@ -683,8 +689,8 @@
 
     @inlineCallbacks
     def getAllManagedIDs(self):
-        at = schema.ATTACHMENT
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        at = Attachment._attachmentSchema
+        attco = Attachment._attachmentLinkSchema
         rows = (yield Select(
             [attco.MANAGED_ID, ],
             From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
@@ -1565,7 +1571,7 @@
         """
         Query to find resources that need to be re-expanded
         """
-        co = schema.CALENDAR_OBJECT
+        co = cls._objectSchema
         return Select(
             [co.RESOURCE_NAME],
             From=co,
@@ -3598,7 +3604,7 @@
                 recurrenceLowerLimit = None
                 recurrenceLimit = DateTime(1900, 1, 1, 0, 0, 0, tzid=Timezone(utc=True))
 
-        co = schema.CALENDAR_OBJECT
+        co = self._objectSchema
         tr = schema.TIME_RANGE
 
         # Do not update if reCreate (re-indexing - we don't want to re-write data
@@ -3809,7 +3815,7 @@
         """
         assert self._txn._migrating
 
-        co = schema.CALENDAR_OBJECT
+        co = self._objectSchema
         values = {
             co.ATTACHMENTS_MODE                : other._attachment,
             co.DROPBOX_ID                      : other._dropboxID,
@@ -4058,7 +4064,7 @@
         """
         DAL query to load RECURRANCE_MIN, RECURRANCE_MAX via an object's resource ID.
         """
-        co = schema.CALENDAR_OBJECT
+        co = cls._objectSchema
         return Select(
             [co.RECURRANCE_MIN, co.RECURRANCE_MAX, ],
             From=co,
@@ -4593,8 +4599,8 @@
         Get a list of managed attachments where the names returned are for the last path segment
         of the attachment URI.
         """
-        at = schema.ATTACHMENT
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        at = Attachment._attachmentSchema
+        attco = Attachment._attachmentLinkSchema
         rows = (yield Select(
             [attco.MANAGED_ID, at.PATH, ],
             From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
@@ -4610,8 +4616,8 @@
         """
 
         # Scan all the associated attachments for the one that matches
-        at = schema.ATTACHMENT
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        at = Attachment._attachmentSchema
+        attco = Attachment._attachmentLinkSchema
         rows = (yield Select(
             [attco.MANAGED_ID, at.PATH, ],
             From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
@@ -5050,32 +5056,136 @@
 
 
 
+class AttachmentLink(object):
+    """
+    A binding between an L{Attachment} and an L{CalendarObject}.
+    """
+
+    _attachmentSchema = schema.ATTACHMENT
+    _attachmentLinkSchema = schema.ATTACHMENT_CALENDAR_OBJECT
+
+    @classmethod
+    def makeClass(cls, txn, linkData):
+        """
+        Given the various database rows, build the actual class.
+
+        @param objectData: the standard set of object columns
+        @type objectData: C{list}
+
+        @return: the constructed child class
+        @rtype: L{CommonHomeChild}
+        """
+
+        child = cls(txn)
+        for attr, value in zip(child._rowAttributes(), linkData):
+            setattr(child, attr, value)
+        return child
+
+
+    @classmethod
+    def _allColumns(cls):
+        """
+        Full set of columns in the object table that need to be loaded to
+        initialize the object resource state.
+        """
+        aco = cls._attachmentLinkSchema
+        return [
+            aco.ATTACHMENT_ID,
+            aco.MANAGED_ID,
+            aco.CALENDAR_OBJECT_RESOURCE_ID,
+        ]
+
+
+    @classmethod
+    def _rowAttributes(cls):
+        """
+        Object attributes used to store the column values from L{_allColumns}. This used to create
+        a mapping when serializing the object for cross-pod requests.
+        """
+        return (
+            "_attachmentID",
+            "_managedID",
+            "_calendarObjectID",
+        )
+
+
+    @classmethod
+    @inlineCallbacks
+    def linksForHome(cls, home):
+        """
+        Load all attachment<->calendar object mappings for the specified home collection.
+        """
+
+        # Load from the main table first
+        att = cls._attachmentSchema
+        attco = cls._attachmentLinkSchema
+        dataRows = yield Select(
+            cls._allColumns(),
+            From=attco.join(att, on=(attco.ATTACHMENT_ID == att.ATTACHMENT_ID)),
+            Where=att.CALENDAR_HOME_RESOURCE_ID == home.id(),
+        ).on(home._txn)
+
+        # Create the actual objects
+        returnValue([cls.makeClass(home._txn, row) for row in dataRows])
+
+
+    def __init__(self, txn):
+        self._txn = txn
+        for attr in self._rowAttributes():
+            setattr(self, attr, None)
+
+
+    def externalize(self):
+        """
+        Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
+        and reconstituted at the other end. Note that the other end may have a different schema so
+        the attributes may not match exactly and will need to be processed accordingly.
+        """
+        return dict([(attr[1:], getattr(self, attr, None)) for attr in self._rowAttributes()])
+
+
+    @classmethod
+    def internalize(cls, txn, mapping):
+        """
+        Given a mapping generated by L{externalize}, convert the values into an array of database
+        like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
+        Note that there may be a schema mismatch with the external data, so treat missing items as
+        C{None} and ignore extra items.
+        """
+
+        return cls.makeClass(txn, [mapping.get(row[1:]) for row in cls._rowAttributes()])
+
+
+    def insert(self):
+        """
+        Insert the object.
+        """
+
+        row = dict([(column, getattr(self, attr)) for column, attr in itertools.izip(self._allColumns(), self._rowAttributes())])
+        return Insert(row).on(self._txn)
+
+
+
 class Attachment(object):
 
     implements(IAttachment)
 
     _attachmentSchema = schema.ATTACHMENT
+    _attachmentLinkSchema = schema.ATTACHMENT_CALENDAR_OBJECT
 
     @classmethod
     def makeClass(cls, txn, attachmentData):
         """
         Given the various database rows, build the actual class.
 
-        @param parent: the parent collection object
-        @type parent: L{CommonHomeChild}
+        @param attachmentData: the standard set of attachment columns
+        @type attachmentData: C{list}
 
-        @param objectData: the standard set of object columns
-        @type objectData: C{list}
-
-        @param propstore: a property store to use, or C{None} to load it
-            automatically
-        @type propstore: L{PropertyStore}
-
         @return: the constructed child class
-        @rtype: L{CommonHomeChild}
+        @rtype: L{Attachment}
         """
 
-        att = schema.ATTACHMENT
+        att = cls._attachmentSchema
         dropbox_id = attachmentData[cls._allColumns().index(att.DROPBOX_ID)]
         c = ManagedAttachment if dropbox_id == "." else DropBoxAttachment
         child = c(
@@ -5233,7 +5343,7 @@
 
         @return: C{True} if this attachment exists, C{False} otherwise.
         """
-        att = schema.ATTACHMENT
+        att = self._attachmentSchema
         if self._dropboxID and self._dropboxID != ".":
             where = (att.DROPBOX_ID == self._dropboxID).And(
                 att.PATH == self._name)
@@ -5255,7 +5365,6 @@
         returnValue(self)
 
 
-    @inlineCallbacks
     def copyRemote(self, remote):
         """
         Copy properties from a remote (external) attachment that is being migrated.
@@ -5263,7 +5372,7 @@
         @param remote: the external attachment
         @type remote: L{Attachment}
         """
-        yield self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
+        return self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
 
 
     def id(self):
@@ -5361,8 +5470,8 @@
 
         TODO: this needs to be transactional wrt the actual file deletes.
         """
-        att = schema.ATTACHMENT
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        att = cls._attachmentSchema
+        attco = cls._attachmentLinkSchema
 
         rows = (yield Select(
             [att.ATTACHMENT_ID, att.DROPBOX_ID, ],
@@ -5448,7 +5557,7 @@
             raise AttachmentDropboxNotAllowed
 
         # Now create the DB entry
-        att = schema.ATTACHMENT
+        att = cls._attachmentSchema
         rows = (yield Insert({
             att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
             att.DROPBOX_ID                : dropboxID,
@@ -5500,7 +5609,7 @@
         """
 
         # See if any other resources still reference this dropbox ID
-        co = schema.CALENDAR_OBJECT
+        co = CalendarObject._objectSchema
         rows = (yield Select(
             [co.RESOURCE_ID, ],
             From=co,
@@ -5510,7 +5619,7 @@
 
         if not rows:
             # Find each attachment with matching dropbox ID
-            att = schema.ATTACHMENT
+            att = cls._attachmentSchema
             rows = (yield Select(
                 [att.PATH],
                 From=att,
@@ -5532,7 +5641,7 @@
         self._md5 = md5
         self._size = size
 
-        att = schema.ATTACHMENT
+        att = self._attachmentSchema
         self._created, self._modified = map(
             sqltime,
             (yield Update(
@@ -5559,7 +5668,7 @@
         """
 
         # Change the DROPBOX_ID to a single "." to indicate a managed attachment.
-        att = schema.ATTACHMENT
+        att = self._attachmentSchema
         (yield Update(
             {att.DROPBOX_ID    : ".", },
             Where=(att.ATTACHMENT_ID == self._attachmentID),
@@ -5613,7 +5722,7 @@
         """
 
         # Now create the DB entry
-        att = schema.ATTACHMENT
+        att = cls._attachmentSchema
         rows = (yield Insert({
             att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
             att.DROPBOX_ID                : ".",
@@ -5663,7 +5772,7 @@
         attachment._objectResourceID = referencedBy
 
         # Create the attachment<->calendar object relationship for managed attachments
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = cls._attachmentLinkSchema
         yield Insert({
             attco.ATTACHMENT_ID               : attachment._attachmentID,
             attco.MANAGED_ID                  : attachment._managedID,
@@ -5698,7 +5807,7 @@
         attachment._objectResourceID = referencedBy
 
         # Update the attachment<->calendar object relationship for managed attachments
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = cls._attachmentLinkSchema
         yield Update(
             {
                 attco.ATTACHMENT_ID    : attachment._attachmentID,
@@ -5732,7 +5841,7 @@
         """
 
         if managedID:
-            attco = schema.ATTACHMENT_CALENDAR_OBJECT
+            attco = cls._attachmentLinkSchema
             where = (attco.MANAGED_ID == managedID)
             if referencedID is not None:
                 where = where.And(attco.CALENDAR_OBJECT_RESOURCE_ID == referencedID)
@@ -5760,7 +5869,7 @@
         """
         Find all the calendar object resourceIds referenced by this supplied managed-id.
         """
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = cls._attachmentLinkSchema
         rows = (yield Select(
             [attco.CALENDAR_OBJECT_RESOURCE_ID, ],
             From=attco,
@@ -5776,9 +5885,9 @@
         """
         Return the "owner" home and referencing resource is, and UID for a managed-id.
         """
-        att = schema.ATTACHMENT
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
-        co = schema.CALENDAR_OBJECT
+        att = cls._attachmentSchema
+        attco = cls._attachmentLinkSchema
+        co = CalendarObject._objectSchema
         rows = (yield Select(
             [
                 att.CALENDAR_HOME_RESOURCE_ID,
@@ -5801,7 +5910,7 @@
         """
 
         # Find all reference attachment-ids and dereference
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = cls._attachmentLinkSchema
         rows = (yield Select(
             [attco.MANAGED_ID, ],
             From=attco,
@@ -5821,7 +5930,7 @@
         """
 
         # Find the associated attachment-id and insert new reference
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = cls._attachmentLinkSchema
         aid = (yield Select(
             [attco.ATTACHMENT_ID, ],
             From=attco,
@@ -5897,7 +6006,7 @@
         self._name = dispositionName
         self._md5 = md5
         self._size = size
-        att = schema.ATTACHMENT
+        att = self._attachmentSchema
         self._created, self._modified = map(
             sqltime,
             (yield Update(
@@ -5926,7 +6035,7 @@
         @rtype: L{ManagedAttachment}
         """
 
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = self._attachmentLinkSchema
         yield Insert({
             attco.ATTACHMENT_ID               : self._attachmentID,
             attco.MANAGED_ID                  : self._managedID,
@@ -5941,7 +6050,7 @@
     def removeFromResource(self, resourceID):
 
         # Delete the reference
-        attco = schema.ATTACHMENT_CALENDAR_OBJECT
+        attco = self._attachmentLinkSchema
         yield Delete(
             From=attco,
             Where=(attco.ATTACHMENT_ID == self._attachmentID).And(

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -23,7 +23,7 @@
 from twext.python.log import Logger
 
 from txdav.caldav.datastore.sql import CalendarHome, Calendar, CalendarObject, \
-    Attachment
+    Attachment, AttachmentLink
 from txdav.caldav.icalendarstore import ComponentUpdateState, ComponentRemoveState
 from txdav.common.datastore.sql_external import CommonHomeExternal, CommonHomeChildExternal, \
     CommonObjectResourceExternal
@@ -68,14 +68,10 @@
         Return all the L{Attachment} objects associated with this calendar home.
         Needed during migration.
         """
-        raw_results = yield self._txn.store().conduit.send_get_all_attachments(self)
+        raw_results = yield self._txn.store().conduit.send_home_get_all_attachments(self)
+        returnValue([Attachment.internalize(self._txn, attachment) for attachment in raw_results])
 
-        results = []
-        for attachment in raw_results:
-            results.append(Attachment.internalize(self._txn, attachment))
-        returnValue(results)
 
-
     @inlineCallbacks
     def readAttachmentData(self, remote_id, attachment):
         """
@@ -86,6 +82,16 @@
         yield self._txn.store().conduit.send_get_attachment_data(self, remote_id, stream)
 
 
+    @inlineCallbacks
+    def getAttachmentLinks(self):
+        """
+        Read the attachment<->calendar object mapping data associated with this calendar home.
+        Needed during migration only.
+        """
+        raw_results = yield self._txn.store().conduit.send_home_get_attachment_links(self)
+        returnValue([AttachmentLink.internalize(self._txn, attachment) for attachment in raw_results])
+
+
     def getAllDropboxIDs(self):
         """
         No children.

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -522,6 +522,11 @@
             self._contentType = http_headers.MimeType.fromString(getType(self._attachment.name(), self.contentTypes))
 
 
+    def resetDetails(self, contentType, dispositionName):
+        self._contentType = contentType
+        self._dispositionName = dispositionName
+
+
     def write(self, data):
         """
         Children must override this to actually write the data, but should

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -16,6 +16,7 @@
 
 from twisted.internet.defer import inlineCallbacks, returnValue
 from txdav.caldav.icalendarstore import InvalidAttachmentOperation
+from txdav.common.datastore.podding.util import UtilityConduitMixin
 from txweb2.http_headers import generateContentType
 
 
@@ -155,36 +156,6 @@
 
 
     @inlineCallbacks
-    def send_get_all_attachments(self, home):
-        """
-        Managed attachment removeAttachment call.
-
-        @param home: the home whose attachments are being requested
-        @type home: L{CalendarHome}
-        """
-
-        actionName = "get-all-attachments"
-        txn, request, server = yield self._getRequestForStoreObject(actionName, home, False)
-
-        response = yield self.sendRequestToServer(txn, server, request)
-        returnValue(response)
-
-
-    @inlineCallbacks
-    def recv_get_all_attachments(self, txn, request):
-        """
-        Process an getAllAttachments cross-pod request. Request arguments as per L{send_get_all_attachments}.
-
-        @param request: request arguments
-        @type request: C{dict}
-        """
-
-        home, _ignore = yield self._getStoreObjectForRequest(txn, request)
-        attachments = yield home.getAllAttachments()
-        returnValue([attachment.externalize() for attachment in attachments])
-
-
-    @inlineCallbacks
     def send_get_attachment_data(self, home, attachment_id, stream):
         """
         Managed attachment readAttachmentData call. We are using streams on the sender and the receiver
@@ -222,3 +193,8 @@
 
         attachment.retrieve(stream)
         returnValue((generateContentType(attachment.contentType()), attachment.name(),))
+
+
+# Calls on L{CommonHome} objects
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_all_attachments", "getAllAttachments", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_attachment_links", "getAttachmentLinks", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -20,9 +20,10 @@
 from txdav.common.datastore.podding.attachments import AttachmentsConduitMixin
 from txdav.common.datastore.podding.base import FailedCrossPodRequestError
 from txdav.common.datastore.podding.directory import DirectoryPoddingConduitMixin
-from txdav.common.datastore.podding.store_api import StoreAPIConduitMixin
 from txdav.common.datastore.podding.request import ConduitRequest
 from txdav.common.datastore.podding.sharing_invites import SharingInvitesConduitMixin
+from txdav.common.datastore.podding.store_api import StoreAPIConduitMixin
+from txdav.common.datastore.podding.util import UtilityConduitMixin
 
 from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.python.reflect import namedClass
@@ -32,6 +33,7 @@
 
 
 class PoddingConduit(
+    UtilityConduitMixin,
     StoreAPIConduitMixin,
     AttachmentsConduitMixin,
     SharingInvitesConduitMixin,

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -162,10 +162,7 @@
         # TODO: sync attachments
         yield self.syncAttachments()
 
-        # TODO: group attendee/sharee reconcile
-        pass
 
-
     @inlineCallbacks
     def finalSync(self):
         """
@@ -174,14 +171,20 @@
         """
 
         # TODO: link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
-        pass
+        yield self.linkAttachments()
 
         # TODO: Re-write attachment URIs - not sure if we need this as reverse proxy may take care of it
         pass
 
+        # TODO: group attendee reconcile
+        pass
+
         # TODO: shared collections reconcile
         pass
 
+        # TODO: group sharee reconcile
+        pass
+
         # TODO: delegates reconcile
         pass
 
@@ -273,7 +276,7 @@
         """
 
         from txdav.caldav.datastore.sql_external import CalendarHomeExternal
-        resourceID = yield txn.store().conduit.send_home_resource_id(self, self.record)
+        resourceID = yield txn.store().conduit.send_home_resource_id(txn, self.record)
         home = CalendarHomeExternal(txn, self.record.uid, resourceID) if resourceID is not None else None
         if home:
             home._childClass = home._childClass._externalClass
@@ -754,3 +757,82 @@
 
         # Read the data from the conduit
         yield remote_home.readAttachmentData(remote_id, attachment)
+
+
+    @inlineCallbacks
+    def linkAttachments(self):
+        """
+        Link attachments to the calendar objects they belong to.
+        """
+
+        # Get the map of links for the remote home
+        links = yield self.getAttachmentLinks()
+
+        # Get remote->local ID mappings
+        attachmentIDMap, objectIDMap = yield self.getAttachmentMappings()
+
+        # Batch setting links for the local home
+        len_links = len(links)
+        while len(links):
+            yield self.makeAttachmentLinks(links[:50], attachmentIDMap, objectIDMap)
+            links = links[50:]
+
+        returnValue(len_links)
+
+
+    @inTransactionWrapper
+    @inlineCallbacks
+    def getAttachmentLinks(self, txn):
+        """
+        Get the remote link information.
+        """
+
+        # Get the map of links for the remote home
+        remote_home = yield self._remoteHome(txn)
+        links = yield remote_home.getAttachmentLinks()
+        returnValue(links)
+
+
+    @inTransactionWrapper
+    @inlineCallbacks
+    def getAttachmentMappings(self, txn):
+        """
+        Get the remote link information.
+        """
+
+        # Get migration mappings
+        am = schema.ATTACHMENT_MIGRATION
+        rows = yield Select(
+            [am.REMOTE_RESOURCE_ID, am.LOCAL_RESOURCE_ID],
+            From=am,
+            Where=(am.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+        ).on(txn)
+        attachmentIDMap = dict(rows)
+
+        com = schema.CALENDAR_OBJECT_MIGRATION
+        rows = yield Select(
+            [com.REMOTE_RESOURCE_ID, com.LOCAL_RESOURCE_ID],
+            From=com,
+            Where=(com.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+        ).on(txn)
+        objectIDMap = dict(rows)
+
+        returnValue((attachmentIDMap, objectIDMap,))
+
+
+    @inTransactionWrapper
+    @inlineCallbacks
+    def makeAttachmentLinks(self, txn, links, attachmentIDMap, objectIDMap):
+        """
+        Map remote links to local links.
+        """
+
+        for link in links:
+            # Remote link has an invalid txn at this point so replace that first
+            link._txn = txn
+
+            # Now re-map the attachment ID and calendar_object_id to the local ones
+            link._attachmentID = attachmentIDMap[link._attachmentID]
+            link._calendarObjectID = objectIDMap[link._calendarObjectID]
+
+            yield link.insert()

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -18,6 +18,7 @@
 from twext.enterprise.dal.syntax import Select
 from twisted.internet.defer import inlineCallbacks
 from twistedcaldav.ical import Component, normalize_iCalStr
+from txdav.caldav.datastore.sql import ManagedAttachment
 from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
 from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
 from txdav.common.datastore.sql_tables import schema
@@ -588,3 +589,100 @@
         attachments = yield home1.getAllAttachments()
         mapping1 = dict([(o.md5(), o.id()) for o in attachments])
         yield _checkAttachmentObjectMigrationState(home1, mapping1)
+
+
+    @inlineCallbacks
+    def test_link_attachments(self):
+        """
+        Test that L{linkAttachments} links attachment data to the associated calendar object.
+        """
+
+        home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+        calendar0 = yield home0.childWithName("calendar")
+        object0_1 = yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+        object0_2 = yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
+        yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
+        remote_id = calendar0.id()
+
+        attachment, _ignore_location = yield object0_1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1."))
+        id0_1 = attachment.id()
+        md50_1 = attachment.md5()
+        managedid0_1 = attachment.managedID()
+        pathID0_1 = ManagedAttachment.lastSegmentOfUriPath(managedid0_1, attachment.name())
+
+        attachment, _ignore_location = yield object0_2.addAttachment(None, MimeType.fromString("text/plain"), "test2.txt", MemoryStream("Here is some text #2."))
+        id0_2 = attachment.id()
+        md50_2 = attachment.md5()
+        managedid0_2 = attachment.managedID()
+        pathID0_2 = ManagedAttachment.lastSegmentOfUriPath(managedid0_2, attachment.name())
+
+        yield self.commitTransaction(0)
+
+        # Add original to a different resource
+        object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+        component = yield object1.componentForUser()
+        attach = component.mainComponent().getProperty("ATTACH")
+
+        object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="3.ics")
+        component = yield object1.componentForUser()
+        attach = component.mainComponent().addProperty(attach)
+        yield object1.setComponent(component)
+        yield self.commitTransaction(0)
+
+        syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+        yield syncer.loadRecord()
+        syncer.homeId = yield syncer.prepareCalendarHome()
+
+        # Trigger sync of the one calendar
+        local_sync_state = {}
+        remote_sync_state = yield syncer.getCalendarSyncList()
+        yield syncer.syncCalendar(
+            remote_id,
+            local_sync_state,
+            remote_sync_state,
+        )
+        self.assertEqual(len(local_sync_state), 1)
+        self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
+
+        # Sync attachments
+        changed, removed = yield syncer.syncAttachments()
+        self.assertEqual(changed, set((id0_1, id0_2,)))
+        self.assertEqual(removed, set())
+
+        # Link attachments
+        len_links = yield syncer.linkAttachments()
+        self.assertEqual(len_links, 3)
+
+        # Local calendar exists
+        home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+        calendar1 = yield home1.childWithName("calendar")
+        self.assertTrue(calendar1 is not None)
+        children = yield calendar1.objectResources()
+        self.assertEqual(set([child.name() for child in children]), set(("1.ics", "2.ics", "3.ics",)))
+
+        # Make sure calendar object is associated with attachment
+        object1 = yield calendar1.objectResourceWithName("1.ics")
+        attachments = yield object1.managedAttachmentList()
+        self.assertEqual(attachments, [pathID0_1, ])
+
+        attachment = yield object1.attachmentWithManagedID(managedid0_1)
+        self.assertTrue(attachment is not None)
+        self.assertEqual(attachment.md5(), md50_1)
+
+        # Make sure calendar object is associated with attachment
+        object1 = yield calendar1.objectResourceWithName("2.ics")
+        attachments = yield object1.managedAttachmentList()
+        self.assertEqual(attachments, [pathID0_2, ])
+
+        attachment = yield object1.attachmentWithManagedID(managedid0_2)
+        self.assertTrue(attachment is not None)
+        self.assertEqual(attachment.md5(), md50_2)
+
+        # Make sure calendar object is associated with attachment
+        object1 = yield calendar1.objectResourceWithName("3.ics")
+        attachments = yield object1.managedAttachmentList()
+        self.assertEqual(attachments, [pathID0_1, ])
+
+        attachment = yield object1.attachmentWithManagedID(managedid0_1)
+        self.assertTrue(attachment is not None)
+        self.assertEqual(attachment.md5(), md50_1)

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -79,7 +79,6 @@
                     data = json.loads(data)
                 else:
                     yield readStream(response.stream, self.writeStream.write)
-                    yield self.writeStream.loseConnection()
                     content_type = response.headers.getHeader("content-type")
                     if content_type is None:
                         content_type = MimeType("application", "octet-stream")
@@ -88,7 +87,9 @@
                         filename = ""
                     else:
                         filename = content_disposition.params["filename"]
-                    response = {
+                    self.writeStream.resetDetails(content_type, filename)
+                    yield self.writeStream.loseConnection()
+                    data = {
                         "result": "ok",
                         "content-type": content_type,
                         "name": filename,

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -177,7 +177,7 @@
                 except ValueError:
                     code = responsecode.BAD_REQUEST
                 else:
-                    headers = {"content-type": ct}
+                    headers = {"content-type": MimeType.fromString(ct)}
                     headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
                     returnValue(Response(responsecode.OK, headers, stream))
 

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -16,8 +16,8 @@
 
 from twisted.internet.defer import inlineCallbacks, returnValue
 
-from txdav.common.datastore.podding.base import FailedCrossPodRequestError
 from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
+from txdav.common.datastore.podding.util import UtilityConduitMixin
 
 from twistedcaldav.caldavxml import TimeRange
 
@@ -27,115 +27,7 @@
     Defines common cross-pod API for generic access to remote resources.
     """
 
-    #
-    # Utility methods to map from store objects to/from JSON
-    #
-
     @inlineCallbacks
-    def _getRequestForStoreObject(self, action, storeObject, classMethod):
-        """
-        Create the JSON data needed to identify the remote resource by type and ids, along with any parent resources.
-
-        @param action: the conduit action name
-        @type action: L{str}
-        @param storeObject: the store object that is being operated on
-        @type storeObject: L{object}
-        @param classMethod: indicates whether the method being called is a classmethod
-        @type classMethod: L{bool}
-
-        @return: the transaction in use, the JSON dict to send in the request,
-            the server where the request should be sent
-        @rtype: L{tuple} of (L{CommonStoreTransaction}, L{dict}, L{str})
-        """
-
-        from txdav.common.datastore.sql import CommonObjectResource, CommonHomeChild, CommonHome
-        result = {
-            "action": action,
-        }
-
-        # Extract the relevant store objects
-        txn = storeObject._txn
-        owner_home = None
-        viewer_home = None
-        home_child = None
-        object_resource = None
-        if isinstance(storeObject, CommonObjectResource):
-            owner_home = storeObject.ownerHome()
-            viewer_home = storeObject.viewerHome()
-            home_child = storeObject.parentCollection()
-            object_resource = storeObject
-        elif isinstance(storeObject, CommonHomeChild):
-            owner_home = storeObject.ownerHome()
-            viewer_home = storeObject.viewerHome()
-            home_child = storeObject
-            result["classMethod"] = classMethod
-        elif isinstance(storeObject, CommonHome):
-            owner_home = storeObject
-            viewer_home = storeObject
-            txn = storeObject._txn
-            result["classMethod"] = classMethod
-
-        # Add store object identities to JSON request
-        result["homeType"] = viewer_home._homeType
-        result["homeUID"] = viewer_home.uid()
-        if home_child:
-            if home_child.owned():
-                result["homeChildID"] = home_child.id()
-            else:
-                result["homeChildSharedID"] = home_child.name()
-        if object_resource:
-            result["objectResourceID"] = object_resource.id()
-
-        # Note that the owner_home is always the ownerHome() because in the sharing case
-        # a viewer is accessing the owner's data on another pod.
-        recipient = yield self.store.directoryService().recordWithUID(owner_home.uid())
-
-        returnValue((txn, result, recipient.server(),))
-
-
-    @inlineCallbacks
-    def _getStoreObjectForRequest(self, txn, request):
-        """
-        Resolve the supplied JSON data to get a store object to operate on.
-        """
-
-        returnObject = txn
-        classObject = None
-
-        if "homeUID" in request:
-            home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
-            if home is None:
-                raise FailedCrossPodRequestError("Invalid owner UID specified")
-            home._internalRequest = False
-            returnObject = home
-            if request.get("classMethod", False):
-                classObject = home._childClass
-
-        if "homeChildID" in request:
-            homeChild = yield home.childWithID(request["homeChildID"])
-            if homeChild is None:
-                raise FailedCrossPodRequestError("Invalid home child specified")
-            returnObject = homeChild
-            if request.get("classMethod", False):
-                classObject = homeChild._objectResourceClass
-        elif "homeChildSharedID" in request:
-            homeChild = yield home.childWithName(request["homeChildSharedID"])
-            if homeChild is None:
-                raise FailedCrossPodRequestError("Invalid home child specified")
-            returnObject = homeChild
-            if request.get("classMethod", False):
-                classObject = homeChild._objectResourceClass
-
-        if "objectResourceID" in request:
-            objectResource = yield homeChild.objectResourceWithID(request["objectResourceID"])
-            if objectResource is None:
-                raise FailedCrossPodRequestError("Invalid object resource specified")
-            returnObject = objectResource
-
-        returnValue((returnObject, classObject,))
-
-
-    @inlineCallbacks
     def send_home_resource_id(self, txn, recipient):
         """
         Lookup the remote resourceID matching the specified directory uid.
@@ -235,134 +127,29 @@
             "matchtotal": matchtotal,
         })
 
-
-    #
-    # We can simplify code generation for simple calls by dynamically generating the appropriate class methods.
-    #
-
-    @inlineCallbacks
-    def _simple_object_send(self, actionName, storeObject, classMethod=False, transform=None, args=None, kwargs=None):
-        """
-        A simple send operation that returns a value.
-
-        @param actionName: name of the action.
-        @type actionName: C{str}
-        @param shareeView: sharee resource being operated on.
-        @type shareeView: L{CommonHomeChildExternal}
-        @param objectResource: the resource being operated on, or C{None} for classmethod.
-        @type objectResource: L{CommonObjectResourceExternal}
-        @param transform: a function used to convert the JSON response into return values.
-        @type transform: C{callable}
-        @param args: list of optional arguments.
-        @type args: C{list}
-        @param kwargs: optional keyword arguments.
-        @type kwargs: C{dict}
-        """
-
-        txn, request, server = yield self._getRequestForStoreObject(actionName, storeObject, classMethod)
-        if args is not None:
-            request["arguments"] = args
-        if kwargs is not None:
-            request["keywords"] = kwargs
-        response = yield self.sendRequestToServer(txn, server, request)
-        returnValue(transform(response) if transform is not None else response)
-
-
-    @inlineCallbacks
-    def _simple_object_recv(self, txn, actionName, request, method, transform=None):
-        """
-        A simple recv operation that returns a value. We also look for an optional set of arguments/keywords
-        and include those only if present.
-
-        @param actionName: name of the action.
-        @type actionName: C{str}
-        @param request: request arguments
-        @type request: C{dict}
-        @param method: name of the method to execute on the shared resource to get the result.
-        @type method: C{str}
-        @param transform: method to call on returned JSON value to convert it to something useful.
-        @type transform: C{callable}
-        """
-
-        storeObject, classObject = yield self._getStoreObjectForRequest(txn, request)
-        if classObject is not None:
-            value = yield getattr(classObject, method)(storeObject, *request.get("arguments", ()), **request.get("keywords", {}))
-        else:
-            value = yield getattr(storeObject, method)(*request.get("arguments", ()), **request.get("keywords", {}))
-
-        returnValue(transform(value) if transform is not None else value)
-
-
-    #
-    # Factory methods for binding actions to the conduit class
-    #
-    @classmethod
-    def _make_simple_action(cls, action, method, classMethod=False, transform_recv_result=None, transform_send_result=None):
-        setattr(
-            cls,
-            "send_{}".format(action),
-            lambda self, storeObject, *args, **kwargs:
-                self._simple_object_send(action, storeObject, classMethod=classMethod, transform=transform_send_result, args=args, kwargs=kwargs)
-        )
-        setattr(
-            cls,
-            "recv_{}".format(action),
-            lambda self, txn, message:
-                self._simple_object_recv(txn, action, message, method, transform=transform_recv_result)
-        )
-
-
-    #
-    # Transforms for returned data
-    #
-    @staticmethod
-    def _to_externalize(value):
-        """
-        Convert the value to the external (JSON-based) representation.
-        """
-        return value.externalize() if value is not None else None
-
-
-    @staticmethod
-    def _to_externalize_list(value):
-        """
-        Convert the value to the external (JSON-based) representation.
-        """
-        return [v.externalize() for v in value]
-
-
-    @staticmethod
-    def _to_string(value):
-        return str(value)
-
-
-    @staticmethod
-    def _to_tuple(value):
-        return tuple(value)
-
 # These are the actions on store objects we need to expose via the conduit api
 
 # Calls on L{CommonHome} objects
 
 # Calls on L{CommonHomeChild} objects
-StoreAPIConduitMixin._make_simple_action("homechild_listobjects", "listObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("homechild_movehere", "moveObjectResourceHere")
-StoreAPIConduitMixin._make_simple_action("homechild_moveaway", "moveObjectResourceAway")
-StoreAPIConduitMixin._make_simple_action("homechild_synctokenrevision", "syncTokenRevision")
-StoreAPIConduitMixin._make_simple_action("homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=StoreAPIConduitMixin._to_tuple)
-StoreAPIConduitMixin._make_simple_action("homechild_search", "search")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_listobjects", "listObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_movehere", "moveObjectResourceHere")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_moveaway", "moveObjectResourceAway")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_synctokenrevision", "syncTokenRevision")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=UtilityConduitMixin._to_tuple)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_search", "search")
 
 # Calls on L{CommonObjectResource} objects
-StoreAPIConduitMixin._make_simple_action("objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("objectresource_listobjects", "listObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_countobjects", "countObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_create", "create", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("objectresource_setcomponent", "setComponent")
-StoreAPIConduitMixin._make_simple_action("objectresource_component", "component", transform_recv_result=StoreAPIConduitMixin._to_string)
-StoreAPIConduitMixin._make_simple_action("objectresource_remove", "remove")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_listobjects", "listObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_countobjects", "countObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_create", "create", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_setcomponent", "setComponent")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_component", "component", transform_recv_result=UtilityConduitMixin._to_string)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_remove", "remove")

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -32,7 +32,7 @@
 from txdav.caldav.datastore.query.filter import Filter
 from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
 from txdav.caldav.datastore.scheduling.ischedule.localservers import ServersDB, Server
-from txdav.caldav.datastore.sql import ManagedAttachment
+from txdav.caldav.datastore.sql import ManagedAttachment, AttachmentLink
 from txdav.caldav.datastore.test.common import CaptureProtocol
 from txdav.common.datastore.podding.conduit import PoddingConduit, \
     FailedCrossPodRequestError
@@ -1107,3 +1107,32 @@
         attachment._name = "test.txt"
         yield shared_object.ownerHome().readAttachmentData(remote_id, attachment)
         yield self.commitTransaction(1)
+
+
+    @inlineCallbacks
+    def test_get_attachment_links(self):
+        """
+        Test that action=get-attachment-links works.
+        """
+
+        yield self.createShare("user01", "puser01")
+
+        calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user01", name="calendar")
+        cobj1 = yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+        calobjID = cobj1.id()
+        yield self.commitTransaction(0)
+
+        object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+        attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+        attID = attachment.id()
+        managedID = attachment.managedID()
+        yield self.commitTransaction(0)
+
+        shared_object = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), home="puser01", calendar_name="shared-calendar", name="1.ics")
+        links = yield shared_object.ownerHome().getAttachmentLinks()
+        self.assertEqual(len(links), 1)
+        self.assertTrue(isinstance(links[0], AttachmentLink))
+        self.assertEqual(links[0]._attachmentID, attID)
+        self.assertEqual(links[0]._managedID, managedID)
+        self.assertEqual(links[0]._calendarObjectID, calobjID)
+        yield self.commitTransaction(1)

Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py	2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -21,20 +21,24 @@
     Server, ServersDB
 )
 from txdav.common.datastore.podding.conduit import PoddingConduit
+from txdav.common.datastore.podding.request import ConduitRequest
 from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE
 from txdav.common.datastore.test.util import (
     CommonCommonTests, SQLStoreBuilder, buildTestDirectory
 )
 
 import txweb2.dav.test.util
-from txweb2.stream import ProducerStream, readStream
+from txweb2 import responsecode
+from txweb2.http import Response, JSONResponse
+from txweb2.http_headers import MimeDisposition, MimeType
+from txweb2.stream import ProducerStream
 
 from twext.enterprise.ienterprise import AlreadyFinishedError
 
 import json
 
 
-class FakeConduitRequest(object):
+class FakeConduitRequest(ConduitRequest):
     """
     A conduit request that sends messages internally rather than using HTTP
     """
@@ -66,32 +70,6 @@
 
 
     @inlineCallbacks
-    def doRequest(self, txn):
-
-        # Generate an HTTP client request
-        try:
-            response = (yield self._processRequest())
-            if self.writeStream is None:
-                response = json.loads(response)
-            else:
-                try:
-                    ct, name, stream = response
-                    response = {
-                        "result": "ok",
-                        "content-type": ct,
-                        "name": name,
-                    }
-                    yield readStream(stream, self.writeStream.write)
-                    yield self.writeStream.loseConnection()
-                except ValueError:
-                    pass
-        except Exception as e:
-            raise ValueError("Failed cross-pod request: {}".format(e))
-
-        returnValue(response)
-
-
-    @inlineCallbacks
     def _processRequest(self):
         """
         Process the request by sending it to the relevant server.
@@ -121,11 +99,14 @@
                 try:
                     ct, name = result
                 except ValueError:
-                    pass
+                    code = responsecode.BAD_REQUEST
                 else:
-                    returnValue((ct, name, stream,))
+                    headers = {"content-type": MimeType.fromString(ct)}
+                    headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
+                    returnValue(Response(responsecode.OK, headers, stream))
             else:
                 result = yield store.conduit.processRequest(j)
+                code = responsecode.OK
         except Exception as e:
             # Send the exception over to the other side
             result = {
@@ -133,11 +114,13 @@
                 "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
                 "details": str(e),
             }
-        result = json.dumps(result)
-        returnValue(result)
+            code = responsecode.BAD_REQUEST
 
+        response = JSONResponse(code, result)
+        returnValue(response)
 
 
+
 class MultiStoreConduitTest(CommonCommonTests, txweb2.dav.test.util.TestCase):
 
     numberOfStores = 2

Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py	                        (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py	2015-02-13 21:21:26 UTC (rev 14412)
@@ -0,0 +1,237 @@
+##
+# Copyright (c) 2013-2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from txdav.common.datastore.podding.base import FailedCrossPodRequestError
+
+
+class UtilityConduitMixin(object):
+    """
+    Defines utility methods for cross-pod API and mix-ins.
+    """
+
+    #
+    # Utility methods to map from store objects to/from JSON
+    #
+
+    @inlineCallbacks
+    def _getRequestForStoreObject(self, action, storeObject, classMethod):
+        """
+        Create the JSON data needed to identify the remote resource by type and ids, along with any parent resources.
+
+        @param action: the conduit action name
+        @type action: L{str}
+        @param storeObject: the store object that is being operated on
+        @type storeObject: L{object}
+        @param classMethod: indicates whether the method being called is a classmethod
+        @type classMethod: L{bool}
+
+        @return: the transaction in use, the JSON dict to send in the request,
+            the server where the request should be sent
+        @rtype: L{tuple} of (L{CommonStoreTransaction}, L{dict}, L{str})
+        """
+
+        from txdav.common.datastore.sql import CommonObjectResource, CommonHomeChild, CommonHome
+        result = {
+            "action": action,
+        }
+
+        # Extract the relevant store objects
+        txn = storeObject._txn
+        owner_home = None
+        viewer_home = None
+        home_child = None
+        object_resource = None
+        if isinstance(storeObject, CommonObjectResource):
+            owner_home = storeObject.ownerHome()
+            viewer_home = storeObject.viewerHome()
+            home_child = storeObject.parentCollection()
+            object_resource = storeObject
+        elif isinstance(storeObject, CommonHomeChild):
+            owner_home = storeObject.ownerHome()
+            viewer_home = storeObject.viewerHome()
+            home_child = storeObject
+            result["classMethod"] = classMethod
+        elif isinstance(storeObject, CommonHome):
+            owner_home = storeObject
+            viewer_home = storeObject
+            txn = storeObject._txn
+            result["classMethod"] = classMethod
+
+        # Add store object identities to JSON request
+        result["homeType"] = viewer_home._homeType
+        result["homeUID"] = viewer_home.uid()
+        if home_child:
+            if home_child.owned():
+                result["homeChildID"] = home_child.id()
+            else:
+                result["homeChildSharedID"] = home_child.name()
+        if object_resource:
+            result["objectResourceID"] = object_resource.id()
+
+        # Note that the owner_home is always the ownerHome() because in the sharing case
+        # a viewer is accessing the owner's data on another pod.
+        recipient = yield self.store.directoryService().recordWithUID(owner_home.uid())
+
+        returnValue((txn, result, recipient.server(),))
+
+
+    @inlineCallbacks
+    def _getStoreObjectForRequest(self, txn, request):
+        """
+        Resolve the supplied JSON data to get a store object to operate on.
+        """
+
+        returnObject = txn
+        classObject = None
+
+        if "homeUID" in request:
+            home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
+            if home is None:
+                raise FailedCrossPodRequestError("Invalid owner UID specified")
+            home._internalRequest = False
+            returnObject = home
+            if request.get("classMethod", False):
+                classObject = home._childClass
+
+        if "homeChildID" in request:
+            homeChild = yield home.childWithID(request["homeChildID"])
+            if homeChild is None:
+                raise FailedCrossPodRequestError("Invalid home child specified")
+            returnObject = homeChild
+            if request.get("classMethod", False):
+                classObject = homeChild._objectResourceClass
+        elif "homeChildSharedID" in request:
+            homeChild = yield home.childWithName(request["homeChildSharedID"])
+            if homeChild is None:
+                raise FailedCrossPodRequestError("Invalid home child specified")
+            returnObject = homeChild
+            if request.get("classMethod", False):
+                classObject = homeChild._objectResourceClass
+
+        if "objectResourceID" in request:
+            objectResource = yield homeChild.objectResourceWithID(request["objectResourceID"])
+            if objectResource is None:
+                raise FailedCrossPodRequestError("Invalid object resource specified")
+            returnObject = objectResource
+
+        returnValue((returnObject, classObject,))
+
+
+    #
+    # We can simplify code generation for simple calls by dynamically generating the appropriate class methods.
+    #
+
+    @inlineCallbacks
+    def _simple_object_send(self, actionName, storeObject, classMethod=False, transform=None, args=None, kwargs=None):
+        """
+        A simple send operation that returns a value.
+
+        @param actionName: name of the action.
+        @type actionName: C{str}
+        @param shareeView: sharee resource being operated on.
+        @type shareeView: L{CommonHomeChildExternal}
+        @param objectResource: the resource being operated on, or C{None} for classmethod.
+        @type objectResource: L{CommonObjectResourceExternal}
+        @param transform: a function used to convert the JSON response into return values.
+        @type transform: C{callable}
+        @param args: list of optional arguments.
+        @type args: C{list}
+        @param kwargs: optional keyword arguments.
+        @type kwargs: C{dict}
+        """
+
+        txn, request, server = yield self._getRequestForStoreObject(actionName, storeObject, classMethod)
+        if args is not None:
+            request["arguments"] = args
+        if kwargs is not None:
+            request["keywords"] = kwargs
+        response = yield self.sendRequestToServer(txn, server, request)
+        returnValue(transform(response) if transform is not None else response)
+
+
+    @inlineCallbacks
+    def _simple_object_recv(self, txn, actionName, request, method, transform=None):
+        """
+        A simple recv operation that returns a value. We also look for an optional set of arguments/keywords
+        and include those only if present.
+
+        @param actionName: name of the action.
+        @type actionName: C{str}
+        @param request: request arguments
+        @type request: C{dict}
+        @param method: name of the method to execute on the shared resource to get the result.
+        @type method: C{str}
+        @param transform: method to call on returned JSON value to convert it to something useful.
+        @type transform: C{callable}
+        """
+
+        storeObject, classObject = yield self._getStoreObjectForRequest(txn, request)
+        if classObject is not None:
+            value = yield getattr(classObject, method)(storeObject, *request.get("arguments", ()), **request.get("keywords", {}))
+        else:
+            value = yield getattr(storeObject, method)(*request.get("arguments", ()), **request.get("keywords", {}))
+
+        returnValue(transform(value) if transform is not None else value)
+
+
+    #
+    # Factory methods for binding actions to the conduit class
+    #
+    @staticmethod
+    def _make_simple_action(bindcls, action, method, classMethod=False, transform_recv_result=None, transform_send_result=None):
+        setattr(
+            bindcls,
+            "send_{}".format(action),
+            lambda self, storeObject, *args, **kwargs:
+                self._simple_object_send(action, storeObject, classMethod=classMethod, transform=transform_send_result, args=args, kwargs=kwargs)
+        )
+        setattr(
+            bindcls,
+            "recv_{}".format(action),
+            lambda self, txn, message:
+                self._simple_object_recv(txn, action, message, method, transform=transform_recv_result)
+        )
+
+
+    #
+    # Transforms for returned data
+    #
+    @staticmethod
+    def _to_externalize(value):
+        """
+        Convert the value to the external (JSON-based) representation.
+        """
+        return value.externalize() if value is not None else None
+
+
+    @staticmethod
+    def _to_externalize_list(value):
+        """
+        Convert the value to the external (JSON-based) representation.
+        """
+        return [v.externalize() for v in value]
+
+
+    @staticmethod
+    def _to_string(value):
+        return str(value)
+
+
+    @staticmethod
+    def _to_tuple(value):
+        return tuple(value)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150213/1cbe9d48/attachment-0001.html>


More information about the calendarserver-changes mailing list