[CalendarServer-changes] [14412] CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Fri Feb 13 13:21:26 PST 2015
Revision: 14412
http://trac.calendarserver.org//changeset/14412
Author: cdaboo at apple.com
Date: 2015-02-13 13:21:26 -0800 (Fri, 13 Feb 2015)
Log Message:
-----------
Checkpoint: migration final sync attachment links.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -143,7 +143,7 @@
@type txn: L{txdav.common.datastore.sql.CommonStoreTransaction}
"""
- at = schema.ATTACHMENT
+ at = Attachment._attachmentSchema
rows = (yield Select(
(at.DROPBOX_ID,),
From=at,
@@ -175,8 +175,8 @@
txn = self._store.newTransaction("CalendarStoreFeatures.upgradeToManagedAttachments - preliminary work")
try:
# Clear out unused CALENDAR_OBJECT.DROPBOX_IDs
- co = schema.CALENDAR_OBJECT
- at = schema.ATTACHMENT
+ co = CalendarObject._objectSchema
+ at = Attachment._attachmentSchema
yield Update(
{co.DROPBOX_ID: None},
Where=co.RESOURCE_ID.In(Select(
@@ -249,7 +249,7 @@
log.debug(" {0} affected calendar objects".format(len(cobjs),))
# Get names of each matching attachment
- at = schema.ATTACHMENT
+ at = Attachment._attachmentSchema
names = (yield Select(
(at.PATH,),
From=at,
@@ -318,8 +318,8 @@
@type dropbox_id: C{str}
"""
- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
+ co = CalendarObject._objectSchema
+ cb = Calendar._bindSchema
rows = (yield Select(
(cb.CALENDAR_HOME_RESOURCE_ID, co.CALENDAR_RESOURCE_ID, co.RESOURCE_ID,),
From=co.join(cb, co.CALENDAR_RESOURCE_ID == cb.CALENDAR_RESOURCE_ID),
@@ -511,7 +511,7 @@
assert self._txn._migrating
# Simple attributes that can be copied over as-is
- chm = schema.CALENDAR_HOME_METADATA
+ chm = self._homeMetaDataSchema
values = {
chm.ALARM_VEVENT_TIMED : other._alarm_vevent_timed,
chm.ALARM_VEVENT_ALLDAY : other._alarm_vevent_allday,
@@ -617,8 +617,8 @@
"""
Implement lookup via queries.
"""
- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
+ co = self._objectSchema
+ cb = self._bindSchema
rows = (yield Select(
[co.PARENT_RESOURCE_ID,
co.RESOURCE_ID],
@@ -637,16 +637,22 @@
returnValue(None)
- @inlineCallbacks
def getAllAttachments(self):
"""
Return all the L{Attachment} objects associated with this calendar home.
Needed during migration.
"""
- attachments = yield Attachment.loadAllAttachments(self)
- returnValue(attachments)
+ return Attachment.loadAllAttachments(self)
+ def getAttachmentLinks(self):
+ """
+ Read the attachment<->calendar object mapping data associated with this calendar home.
+ Needed during migration only.
+ """
+ return AttachmentLink.linksForHome(self)
+
+
def getAttachmentByID(self, id):
"""
Return a specific attachment associated with this calendar home.
@@ -657,8 +663,8 @@
@inlineCallbacks
def getAllDropboxIDs(self):
- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
+ co = self._objectSchema
+ cb = self._bindSchema
rows = (yield Select(
[co.DROPBOX_ID],
From=co.join(cb, co.PARENT_RESOURCE_ID == cb.RESOURCE_ID),
@@ -671,7 +677,7 @@
@inlineCallbacks
def getAllAttachmentNames(self):
- att = schema.ATTACHMENT
+ att = Attachment._attachmentSchema
rows = (yield Select(
[att.DROPBOX_ID],
From=att,
@@ -683,8 +689,8 @@
@inlineCallbacks
def getAllManagedIDs(self):
- at = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ at = Attachment._attachmentSchema
+ attco = Attachment._attachmentLinkSchema
rows = (yield Select(
[attco.MANAGED_ID, ],
From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
@@ -1565,7 +1571,7 @@
"""
Query to find resources that need to be re-expanded
"""
- co = schema.CALENDAR_OBJECT
+ co = cls._objectSchema
return Select(
[co.RESOURCE_NAME],
From=co,
@@ -3598,7 +3604,7 @@
recurrenceLowerLimit = None
recurrenceLimit = DateTime(1900, 1, 1, 0, 0, 0, tzid=Timezone(utc=True))
- co = schema.CALENDAR_OBJECT
+ co = self._objectSchema
tr = schema.TIME_RANGE
# Do not update if reCreate (re-indexing - we don't want to re-write data
@@ -3809,7 +3815,7 @@
"""
assert self._txn._migrating
- co = schema.CALENDAR_OBJECT
+ co = self._objectSchema
values = {
co.ATTACHMENTS_MODE : other._attachment,
co.DROPBOX_ID : other._dropboxID,
@@ -4058,7 +4064,7 @@
"""
DAL query to load RECURRANCE_MIN, RECURRANCE_MAX via an object's resource ID.
"""
- co = schema.CALENDAR_OBJECT
+ co = cls._objectSchema
return Select(
[co.RECURRANCE_MIN, co.RECURRANCE_MAX, ],
From=co,
@@ -4593,8 +4599,8 @@
Get a list of managed attachments where the names returned are for the last path segment
of the attachment URI.
"""
- at = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ at = Attachment._attachmentSchema
+ attco = Attachment._attachmentLinkSchema
rows = (yield Select(
[attco.MANAGED_ID, at.PATH, ],
From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
@@ -4610,8 +4616,8 @@
"""
# Scan all the associated attachments for the one that matches
- at = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ at = Attachment._attachmentSchema
+ attco = Attachment._attachmentLinkSchema
rows = (yield Select(
[attco.MANAGED_ID, at.PATH, ],
From=attco.join(at, attco.ATTACHMENT_ID == at.ATTACHMENT_ID),
@@ -5050,32 +5056,136 @@
+class AttachmentLink(object):
+ """
+ A binding between an L{Attachment} and an L{CalendarObject}.
+ """
+
+ _attachmentSchema = schema.ATTACHMENT
+ _attachmentLinkSchema = schema.ATTACHMENT_CALENDAR_OBJECT
+
+ @classmethod
+ def makeClass(cls, txn, linkData):
+ """
+ Given the various database rows, build the actual class.
+
+ @param objectData: the standard set of object columns
+ @type objectData: C{list}
+
+ @return: the constructed child class
+ @rtype: L{CommonHomeChild}
+ """
+
+ child = cls(txn)
+ for attr, value in zip(child._rowAttributes(), linkData):
+ setattr(child, attr, value)
+ return child
+
+
+ @classmethod
+ def _allColumns(cls):
+ """
+ Full set of columns in the object table that need to be loaded to
+ initialize the object resource state.
+ """
+ aco = cls._attachmentLinkSchema
+ return [
+ aco.ATTACHMENT_ID,
+ aco.MANAGED_ID,
+ aco.CALENDAR_OBJECT_RESOURCE_ID,
+ ]
+
+
+ @classmethod
+ def _rowAttributes(cls):
+ """
+ Object attributes used to store the column values from L{_allColumns}. This used to create
+ a mapping when serializing the object for cross-pod requests.
+ """
+ return (
+ "_attachmentID",
+ "_managedID",
+ "_calendarObjectID",
+ )
+
+
+ @classmethod
+ @inlineCallbacks
+ def linksForHome(cls, home):
+ """
+ Load all attachment<->calendar object mappings for the specified home collection.
+ """
+
+ # Load from the main table first
+ att = cls._attachmentSchema
+ attco = cls._attachmentLinkSchema
+ dataRows = yield Select(
+ cls._allColumns(),
+ From=attco.join(att, on=(attco.ATTACHMENT_ID == att.ATTACHMENT_ID)),
+ Where=att.CALENDAR_HOME_RESOURCE_ID == home.id(),
+ ).on(home._txn)
+
+ # Create the actual objects
+ returnValue([cls.makeClass(home._txn, row) for row in dataRows])
+
+
+ def __init__(self, txn):
+ self._txn = txn
+ for attr in self._rowAttributes():
+ setattr(self, attr, None)
+
+
+ def externalize(self):
+ """
+ Create a dictionary mapping key attributes so this object can be sent over a cross-pod call
+ and reconstituted at the other end. Note that the other end may have a different schema so
+ the attributes may not match exactly and will need to be processed accordingly.
+ """
+ return dict([(attr[1:], getattr(self, attr, None)) for attr in self._rowAttributes()])
+
+
+ @classmethod
+ def internalize(cls, txn, mapping):
+ """
+ Given a mapping generated by L{externalize}, convert the values into an array of database
+ like items that conforms to the ordering of L{_allColumns} so it can be fed into L{makeClass}.
+ Note that there may be a schema mismatch with the external data, so treat missing items as
+ C{None} and ignore extra items.
+ """
+
+ return cls.makeClass(txn, [mapping.get(row[1:]) for row in cls._rowAttributes()])
+
+
+ def insert(self):
+ """
+ Insert the object.
+ """
+
+ row = dict([(column, getattr(self, attr)) for column, attr in itertools.izip(self._allColumns(), self._rowAttributes())])
+ return Insert(row).on(self._txn)
+
+
+
class Attachment(object):
implements(IAttachment)
_attachmentSchema = schema.ATTACHMENT
+ _attachmentLinkSchema = schema.ATTACHMENT_CALENDAR_OBJECT
@classmethod
def makeClass(cls, txn, attachmentData):
"""
Given the various database rows, build the actual class.
- @param parent: the parent collection object
- @type parent: L{CommonHomeChild}
+ @param attachmentData: the standard set of attachment columns
+ @type attachmentData: C{list}
- @param objectData: the standard set of object columns
- @type objectData: C{list}
-
- @param propstore: a property store to use, or C{None} to load it
- automatically
- @type propstore: L{PropertyStore}
-
@return: the constructed child class
- @rtype: L{CommonHomeChild}
+ @rtype: L{Attachment}
"""
- att = schema.ATTACHMENT
+ att = cls._attachmentSchema
dropbox_id = attachmentData[cls._allColumns().index(att.DROPBOX_ID)]
c = ManagedAttachment if dropbox_id == "." else DropBoxAttachment
child = c(
@@ -5233,7 +5343,7 @@
@return: C{True} if this attachment exists, C{False} otherwise.
"""
- att = schema.ATTACHMENT
+ att = self._attachmentSchema
if self._dropboxID and self._dropboxID != ".":
where = (att.DROPBOX_ID == self._dropboxID).And(
att.PATH == self._name)
@@ -5255,7 +5365,6 @@
returnValue(self)
- @inlineCallbacks
def copyRemote(self, remote):
"""
Copy properties from a remote (external) attachment that is being migrated.
@@ -5263,7 +5372,7 @@
@param remote: the external attachment
@type remote: L{Attachment}
"""
- yield self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
+ return self.changed(remote.contentType(), remote.name(), remote.md5(), remote.size())
def id(self):
@@ -5361,8 +5470,8 @@
TODO: this needs to be transactional wrt the actual file deletes.
"""
- att = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ att = cls._attachmentSchema
+ attco = cls._attachmentLinkSchema
rows = (yield Select(
[att.ATTACHMENT_ID, att.DROPBOX_ID, ],
@@ -5448,7 +5557,7 @@
raise AttachmentDropboxNotAllowed
# Now create the DB entry
- att = schema.ATTACHMENT
+ att = cls._attachmentSchema
rows = (yield Insert({
att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
att.DROPBOX_ID : dropboxID,
@@ -5500,7 +5609,7 @@
"""
# See if any other resources still reference this dropbox ID
- co = schema.CALENDAR_OBJECT
+ co = CalendarObject._objectSchema
rows = (yield Select(
[co.RESOURCE_ID, ],
From=co,
@@ -5510,7 +5619,7 @@
if not rows:
# Find each attachment with matching dropbox ID
- att = schema.ATTACHMENT
+ att = cls._attachmentSchema
rows = (yield Select(
[att.PATH],
From=att,
@@ -5532,7 +5641,7 @@
self._md5 = md5
self._size = size
- att = schema.ATTACHMENT
+ att = self._attachmentSchema
self._created, self._modified = map(
sqltime,
(yield Update(
@@ -5559,7 +5668,7 @@
"""
# Change the DROPBOX_ID to a single "." to indicate a managed attachment.
- att = schema.ATTACHMENT
+ att = self._attachmentSchema
(yield Update(
{att.DROPBOX_ID : ".", },
Where=(att.ATTACHMENT_ID == self._attachmentID),
@@ -5613,7 +5722,7 @@
"""
# Now create the DB entry
- att = schema.ATTACHMENT
+ att = cls._attachmentSchema
rows = (yield Insert({
att.CALENDAR_HOME_RESOURCE_ID : ownerHomeID,
att.DROPBOX_ID : ".",
@@ -5663,7 +5772,7 @@
attachment._objectResourceID = referencedBy
# Create the attachment<->calendar object relationship for managed attachments
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = cls._attachmentLinkSchema
yield Insert({
attco.ATTACHMENT_ID : attachment._attachmentID,
attco.MANAGED_ID : attachment._managedID,
@@ -5698,7 +5807,7 @@
attachment._objectResourceID = referencedBy
# Update the attachment<->calendar object relationship for managed attachments
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = cls._attachmentLinkSchema
yield Update(
{
attco.ATTACHMENT_ID : attachment._attachmentID,
@@ -5732,7 +5841,7 @@
"""
if managedID:
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = cls._attachmentLinkSchema
where = (attco.MANAGED_ID == managedID)
if referencedID is not None:
where = where.And(attco.CALENDAR_OBJECT_RESOURCE_ID == referencedID)
@@ -5760,7 +5869,7 @@
"""
Find all the calendar object resourceIds referenced by this supplied managed-id.
"""
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = cls._attachmentLinkSchema
rows = (yield Select(
[attco.CALENDAR_OBJECT_RESOURCE_ID, ],
From=attco,
@@ -5776,9 +5885,9 @@
"""
Return the "owner" home and referencing resource is, and UID for a managed-id.
"""
- att = schema.ATTACHMENT
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
- co = schema.CALENDAR_OBJECT
+ att = cls._attachmentSchema
+ attco = cls._attachmentLinkSchema
+ co = CalendarObject._objectSchema
rows = (yield Select(
[
att.CALENDAR_HOME_RESOURCE_ID,
@@ -5801,7 +5910,7 @@
"""
# Find all reference attachment-ids and dereference
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = cls._attachmentLinkSchema
rows = (yield Select(
[attco.MANAGED_ID, ],
From=attco,
@@ -5821,7 +5930,7 @@
"""
# Find the associated attachment-id and insert new reference
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = cls._attachmentLinkSchema
aid = (yield Select(
[attco.ATTACHMENT_ID, ],
From=attco,
@@ -5897,7 +6006,7 @@
self._name = dispositionName
self._md5 = md5
self._size = size
- att = schema.ATTACHMENT
+ att = self._attachmentSchema
self._created, self._modified = map(
sqltime,
(yield Update(
@@ -5926,7 +6035,7 @@
@rtype: L{ManagedAttachment}
"""
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = self._attachmentLinkSchema
yield Insert({
attco.ATTACHMENT_ID : self._attachmentID,
attco.MANAGED_ID : self._managedID,
@@ -5941,7 +6050,7 @@
def removeFromResource(self, resourceID):
# Delete the reference
- attco = schema.ATTACHMENT_CALENDAR_OBJECT
+ attco = self._attachmentLinkSchema
yield Delete(
From=attco,
Where=(attco.ATTACHMENT_ID == self._attachmentID).And(
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/sql_external.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -23,7 +23,7 @@
from twext.python.log import Logger
from txdav.caldav.datastore.sql import CalendarHome, Calendar, CalendarObject, \
- Attachment
+ Attachment, AttachmentLink
from txdav.caldav.icalendarstore import ComponentUpdateState, ComponentRemoveState
from txdav.common.datastore.sql_external import CommonHomeExternal, CommonHomeChildExternal, \
CommonObjectResourceExternal
@@ -68,14 +68,10 @@
Return all the L{Attachment} objects associated with this calendar home.
Needed during migration.
"""
- raw_results = yield self._txn.store().conduit.send_get_all_attachments(self)
+ raw_results = yield self._txn.store().conduit.send_home_get_all_attachments(self)
+ returnValue([Attachment.internalize(self._txn, attachment) for attachment in raw_results])
- results = []
- for attachment in raw_results:
- results.append(Attachment.internalize(self._txn, attachment))
- returnValue(results)
-
@inlineCallbacks
def readAttachmentData(self, remote_id, attachment):
"""
@@ -86,6 +82,16 @@
yield self._txn.store().conduit.send_get_attachment_data(self, remote_id, stream)
+ @inlineCallbacks
+ def getAttachmentLinks(self):
+ """
+ Read the attachment<->calendar object mapping data associated with this calendar home.
+ Needed during migration only.
+ """
+ raw_results = yield self._txn.store().conduit.send_home_get_attachment_links(self)
+ returnValue([AttachmentLink.internalize(self._txn, attachment) for attachment in raw_results])
+
+
def getAllDropboxIDs(self):
"""
No children.
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/caldav/datastore/util.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -522,6 +522,11 @@
self._contentType = http_headers.MimeType.fromString(getType(self._attachment.name(), self.contentTypes))
+ def resetDetails(self, contentType, dispositionName):
+ self._contentType = contentType
+ self._dispositionName = dispositionName
+
+
def write(self, data):
"""
Children must override this to actually write the data, but should
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/attachments.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -16,6 +16,7 @@
from twisted.internet.defer import inlineCallbacks, returnValue
from txdav.caldav.icalendarstore import InvalidAttachmentOperation
+from txdav.common.datastore.podding.util import UtilityConduitMixin
from txweb2.http_headers import generateContentType
@@ -155,36 +156,6 @@
@inlineCallbacks
- def send_get_all_attachments(self, home):
- """
- Managed attachment removeAttachment call.
-
- @param home: the home whose attachments are being requested
- @type home: L{CalendarHome}
- """
-
- actionName = "get-all-attachments"
- txn, request, server = yield self._getRequestForStoreObject(actionName, home, False)
-
- response = yield self.sendRequestToServer(txn, server, request)
- returnValue(response)
-
-
- @inlineCallbacks
- def recv_get_all_attachments(self, txn, request):
- """
- Process an getAllAttachments cross-pod request. Request arguments as per L{send_get_all_attachments}.
-
- @param request: request arguments
- @type request: C{dict}
- """
-
- home, _ignore = yield self._getStoreObjectForRequest(txn, request)
- attachments = yield home.getAllAttachments()
- returnValue([attachment.externalize() for attachment in attachments])
-
-
- @inlineCallbacks
def send_get_attachment_data(self, home, attachment_id, stream):
"""
Managed attachment readAttachmentData call. We are using streams on the sender and the receiver
@@ -222,3 +193,8 @@
attachment.retrieve(stream)
returnValue((generateContentType(attachment.contentType()), attachment.name(),))
+
+
+# Calls on L{CommonHome} objects
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_all_attachments", "getAllAttachments", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(AttachmentsConduitMixin, "home_get_attachment_links", "getAttachmentLinks", classMethod=False, transform_recv_result=UtilityConduitMixin._to_externalize_list)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/conduit.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -20,9 +20,10 @@
from txdav.common.datastore.podding.attachments import AttachmentsConduitMixin
from txdav.common.datastore.podding.base import FailedCrossPodRequestError
from txdav.common.datastore.podding.directory import DirectoryPoddingConduitMixin
-from txdav.common.datastore.podding.store_api import StoreAPIConduitMixin
from txdav.common.datastore.podding.request import ConduitRequest
from txdav.common.datastore.podding.sharing_invites import SharingInvitesConduitMixin
+from txdav.common.datastore.podding.store_api import StoreAPIConduitMixin
+from txdav.common.datastore.podding.util import UtilityConduitMixin
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.python.reflect import namedClass
@@ -32,6 +33,7 @@
class PoddingConduit(
+ UtilityConduitMixin,
StoreAPIConduitMixin,
AttachmentsConduitMixin,
SharingInvitesConduitMixin,
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/home_sync.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -162,10 +162,7 @@
# TODO: sync attachments
yield self.syncAttachments()
- # TODO: group attendee/sharee reconcile
- pass
-
@inlineCallbacks
def finalSync(self):
"""
@@ -174,14 +171,20 @@
"""
# TODO: link attachments to resources: ATTACHMENT_CALENDAR_OBJECT table
- pass
+ yield self.linkAttachments()
# TODO: Re-write attachment URIs - not sure if we need this as reverse proxy may take care of it
pass
+ # TODO: group attendee reconcile
+ pass
+
# TODO: shared collections reconcile
pass
+ # TODO: group sharee reconcile
+ pass
+
# TODO: delegates reconcile
pass
@@ -273,7 +276,7 @@
"""
from txdav.caldav.datastore.sql_external import CalendarHomeExternal
- resourceID = yield txn.store().conduit.send_home_resource_id(self, self.record)
+ resourceID = yield txn.store().conduit.send_home_resource_id(txn, self.record)
home = CalendarHomeExternal(txn, self.record.uid, resourceID) if resourceID is not None else None
if home:
home._childClass = home._childClass._externalClass
@@ -754,3 +757,82 @@
# Read the data from the conduit
yield remote_home.readAttachmentData(remote_id, attachment)
+
+
+ @inlineCallbacks
+ def linkAttachments(self):
+ """
+ Link attachments to the calendar objects they belong to.
+ """
+
+ # Get the map of links for the remote home
+ links = yield self.getAttachmentLinks()
+
+ # Get remote->local ID mappings
+ attachmentIDMap, objectIDMap = yield self.getAttachmentMappings()
+
+ # Batch setting links for the local home
+ len_links = len(links)
+ while len(links):
+ yield self.makeAttachmentLinks(links[:50], attachmentIDMap, objectIDMap)
+ links = links[50:]
+
+ returnValue(len_links)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def getAttachmentLinks(self, txn):
+ """
+ Get the remote link information.
+ """
+
+ # Get the map of links for the remote home
+ remote_home = yield self._remoteHome(txn)
+ links = yield remote_home.getAttachmentLinks()
+ returnValue(links)
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def getAttachmentMappings(self, txn):
+ """
+ Get the remote link information.
+ """
+
+ # Get migration mappings
+ am = schema.ATTACHMENT_MIGRATION
+ rows = yield Select(
+ [am.REMOTE_RESOURCE_ID, am.LOCAL_RESOURCE_ID],
+ From=am,
+ Where=(am.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+ ).on(txn)
+ attachmentIDMap = dict(rows)
+
+ com = schema.CALENDAR_OBJECT_MIGRATION
+ rows = yield Select(
+ [com.REMOTE_RESOURCE_ID, com.LOCAL_RESOURCE_ID],
+ From=com,
+ Where=(com.CALENDAR_HOME_RESOURCE_ID == self.homeId),
+ ).on(txn)
+ objectIDMap = dict(rows)
+
+ returnValue((attachmentIDMap, objectIDMap,))
+
+
+ @inTransactionWrapper
+ @inlineCallbacks
+ def makeAttachmentLinks(self, txn, links, attachmentIDMap, objectIDMap):
+ """
+ Map remote links to local links.
+ """
+
+ for link in links:
+ # Remote link has an invalid txn at this point so replace that first
+ link._txn = txn
+
+ # Now re-map the attachment ID and calendar_object_id to the local ones
+ link._attachmentID = attachmentIDMap[link._attachmentID]
+ link._calendarObjectID = objectIDMap[link._calendarObjectID]
+
+ yield link.insert()
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/migration/test/test_home_sync.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -18,6 +18,7 @@
from twext.enterprise.dal.syntax import Select
from twisted.internet.defer import inlineCallbacks
from twistedcaldav.ical import Component, normalize_iCalStr
+from txdav.caldav.datastore.sql import ManagedAttachment
from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
from txdav.common.datastore.sql_tables import schema
@@ -588,3 +589,100 @@
attachments = yield home1.getAllAttachments()
mapping1 = dict([(o.md5(), o.id()) for o in attachments])
yield _checkAttachmentObjectMigrationState(home1, mapping1)
+
+
+ @inlineCallbacks
+ def test_link_attachments(self):
+ """
+ Test that L{linkAttachments} links attachment data to the associated calendar object.
+ """
+
+ home0 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar0 = yield home0.childWithName("calendar")
+ object0_1 = yield calendar0.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ object0_2 = yield calendar0.createCalendarObjectWithName("2.ics", Component.fromString(self.caldata2))
+ yield calendar0.createCalendarObjectWithName("3.ics", Component.fromString(self.caldata3))
+ remote_id = calendar0.id()
+
+ attachment, _ignore_location = yield object0_1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text #1."))
+ id0_1 = attachment.id()
+ md50_1 = attachment.md5()
+ managedid0_1 = attachment.managedID()
+ pathID0_1 = ManagedAttachment.lastSegmentOfUriPath(managedid0_1, attachment.name())
+
+ attachment, _ignore_location = yield object0_2.addAttachment(None, MimeType.fromString("text/plain"), "test2.txt", MemoryStream("Here is some text #2."))
+ id0_2 = attachment.id()
+ md50_2 = attachment.md5()
+ managedid0_2 = attachment.managedID()
+ pathID0_2 = ManagedAttachment.lastSegmentOfUriPath(managedid0_2, attachment.name())
+
+ yield self.commitTransaction(0)
+
+ # Add original to a different resource
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ component = yield object1.componentForUser()
+ attach = component.mainComponent().getProperty("ATTACH")
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="3.ics")
+ component = yield object1.componentForUser()
+ attach = component.mainComponent().addProperty(attach)
+ yield object1.setComponent(component)
+ yield self.commitTransaction(0)
+
+ syncer = CrossPodHomeSync(self.theStoreUnderTest(1), "user01")
+ yield syncer.loadRecord()
+ syncer.homeId = yield syncer.prepareCalendarHome()
+
+ # Trigger sync of the one calendar
+ local_sync_state = {}
+ remote_sync_state = yield syncer.getCalendarSyncList()
+ yield syncer.syncCalendar(
+ remote_id,
+ local_sync_state,
+ remote_sync_state,
+ )
+ self.assertEqual(len(local_sync_state), 1)
+ self.assertEqual(local_sync_state[remote_id].lastSyncToken, remote_sync_state[remote_id].lastSyncToken)
+
+ # Sync attachments
+ changed, removed = yield syncer.syncAttachments()
+ self.assertEqual(changed, set((id0_1, id0_2,)))
+ self.assertEqual(removed, set())
+
+ # Link attachments
+ len_links = yield syncer.linkAttachments()
+ self.assertEqual(len_links, 3)
+
+ # Local calendar exists
+ home1 = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name=syncer.migratingUid())
+ calendar1 = yield home1.childWithName("calendar")
+ self.assertTrue(calendar1 is not None)
+ children = yield calendar1.objectResources()
+ self.assertEqual(set([child.name() for child in children]), set(("1.ics", "2.ics", "3.ics",)))
+
+ # Make sure calendar object is associated with attachment
+ object1 = yield calendar1.objectResourceWithName("1.ics")
+ attachments = yield object1.managedAttachmentList()
+ self.assertEqual(attachments, [pathID0_1, ])
+
+ attachment = yield object1.attachmentWithManagedID(managedid0_1)
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), md50_1)
+
+ # Make sure calendar object is associated with attachment
+ object1 = yield calendar1.objectResourceWithName("2.ics")
+ attachments = yield object1.managedAttachmentList()
+ self.assertEqual(attachments, [pathID0_2, ])
+
+ attachment = yield object1.attachmentWithManagedID(managedid0_2)
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), md50_2)
+
+ # Make sure calendar object is associated with attachment
+ object1 = yield calendar1.objectResourceWithName("3.ics")
+ attachments = yield object1.managedAttachmentList()
+ self.assertEqual(attachments, [pathID0_1, ])
+
+ attachment = yield object1.attachmentWithManagedID(managedid0_1)
+ self.assertTrue(attachment is not None)
+ self.assertEqual(attachment.md5(), md50_1)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/request.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -79,7 +79,6 @@
data = json.loads(data)
else:
yield readStream(response.stream, self.writeStream.write)
- yield self.writeStream.loseConnection()
content_type = response.headers.getHeader("content-type")
if content_type is None:
content_type = MimeType("application", "octet-stream")
@@ -88,7 +87,9 @@
filename = ""
else:
filename = content_disposition.params["filename"]
- response = {
+ self.writeStream.resetDetails(content_type, filename)
+ yield self.writeStream.loseConnection()
+ data = {
"result": "ok",
"content-type": content_type,
"name": filename,
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/resource.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -177,7 +177,7 @@
except ValueError:
code = responsecode.BAD_REQUEST
else:
- headers = {"content-type": ct}
+ headers = {"content-type": MimeType.fromString(ct)}
headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
returnValue(Response(responsecode.OK, headers, stream))
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/store_api.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -16,8 +16,8 @@
from twisted.internet.defer import inlineCallbacks, returnValue
-from txdav.common.datastore.podding.base import FailedCrossPodRequestError
from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
+from txdav.common.datastore.podding.util import UtilityConduitMixin
from twistedcaldav.caldavxml import TimeRange
@@ -27,115 +27,7 @@
Defines common cross-pod API for generic access to remote resources.
"""
- #
- # Utility methods to map from store objects to/from JSON
- #
-
@inlineCallbacks
- def _getRequestForStoreObject(self, action, storeObject, classMethod):
- """
- Create the JSON data needed to identify the remote resource by type and ids, along with any parent resources.
-
- @param action: the conduit action name
- @type action: L{str}
- @param storeObject: the store object that is being operated on
- @type storeObject: L{object}
- @param classMethod: indicates whether the method being called is a classmethod
- @type classMethod: L{bool}
-
- @return: the transaction in use, the JSON dict to send in the request,
- the server where the request should be sent
- @rtype: L{tuple} of (L{CommonStoreTransaction}, L{dict}, L{str})
- """
-
- from txdav.common.datastore.sql import CommonObjectResource, CommonHomeChild, CommonHome
- result = {
- "action": action,
- }
-
- # Extract the relevant store objects
- txn = storeObject._txn
- owner_home = None
- viewer_home = None
- home_child = None
- object_resource = None
- if isinstance(storeObject, CommonObjectResource):
- owner_home = storeObject.ownerHome()
- viewer_home = storeObject.viewerHome()
- home_child = storeObject.parentCollection()
- object_resource = storeObject
- elif isinstance(storeObject, CommonHomeChild):
- owner_home = storeObject.ownerHome()
- viewer_home = storeObject.viewerHome()
- home_child = storeObject
- result["classMethod"] = classMethod
- elif isinstance(storeObject, CommonHome):
- owner_home = storeObject
- viewer_home = storeObject
- txn = storeObject._txn
- result["classMethod"] = classMethod
-
- # Add store object identities to JSON request
- result["homeType"] = viewer_home._homeType
- result["homeUID"] = viewer_home.uid()
- if home_child:
- if home_child.owned():
- result["homeChildID"] = home_child.id()
- else:
- result["homeChildSharedID"] = home_child.name()
- if object_resource:
- result["objectResourceID"] = object_resource.id()
-
- # Note that the owner_home is always the ownerHome() because in the sharing case
- # a viewer is accessing the owner's data on another pod.
- recipient = yield self.store.directoryService().recordWithUID(owner_home.uid())
-
- returnValue((txn, result, recipient.server(),))
-
-
- @inlineCallbacks
- def _getStoreObjectForRequest(self, txn, request):
- """
- Resolve the supplied JSON data to get a store object to operate on.
- """
-
- returnObject = txn
- classObject = None
-
- if "homeUID" in request:
- home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
- if home is None:
- raise FailedCrossPodRequestError("Invalid owner UID specified")
- home._internalRequest = False
- returnObject = home
- if request.get("classMethod", False):
- classObject = home._childClass
-
- if "homeChildID" in request:
- homeChild = yield home.childWithID(request["homeChildID"])
- if homeChild is None:
- raise FailedCrossPodRequestError("Invalid home child specified")
- returnObject = homeChild
- if request.get("classMethod", False):
- classObject = homeChild._objectResourceClass
- elif "homeChildSharedID" in request:
- homeChild = yield home.childWithName(request["homeChildSharedID"])
- if homeChild is None:
- raise FailedCrossPodRequestError("Invalid home child specified")
- returnObject = homeChild
- if request.get("classMethod", False):
- classObject = homeChild._objectResourceClass
-
- if "objectResourceID" in request:
- objectResource = yield homeChild.objectResourceWithID(request["objectResourceID"])
- if objectResource is None:
- raise FailedCrossPodRequestError("Invalid object resource specified")
- returnObject = objectResource
-
- returnValue((returnObject, classObject,))
-
-
- @inlineCallbacks
def send_home_resource_id(self, txn, recipient):
"""
Lookup the remote resourceID matching the specified directory uid.
@@ -235,134 +127,29 @@
"matchtotal": matchtotal,
})
-
- #
- # We can simplify code generation for simple calls by dynamically generating the appropriate class methods.
- #
-
- @inlineCallbacks
- def _simple_object_send(self, actionName, storeObject, classMethod=False, transform=None, args=None, kwargs=None):
- """
- A simple send operation that returns a value.
-
- @param actionName: name of the action.
- @type actionName: C{str}
- @param shareeView: sharee resource being operated on.
- @type shareeView: L{CommonHomeChildExternal}
- @param objectResource: the resource being operated on, or C{None} for classmethod.
- @type objectResource: L{CommonObjectResourceExternal}
- @param transform: a function used to convert the JSON response into return values.
- @type transform: C{callable}
- @param args: list of optional arguments.
- @type args: C{list}
- @param kwargs: optional keyword arguments.
- @type kwargs: C{dict}
- """
-
- txn, request, server = yield self._getRequestForStoreObject(actionName, storeObject, classMethod)
- if args is not None:
- request["arguments"] = args
- if kwargs is not None:
- request["keywords"] = kwargs
- response = yield self.sendRequestToServer(txn, server, request)
- returnValue(transform(response) if transform is not None else response)
-
-
- @inlineCallbacks
- def _simple_object_recv(self, txn, actionName, request, method, transform=None):
- """
- A simple recv operation that returns a value. We also look for an optional set of arguments/keywords
- and include those only if present.
-
- @param actionName: name of the action.
- @type actionName: C{str}
- @param request: request arguments
- @type request: C{dict}
- @param method: name of the method to execute on the shared resource to get the result.
- @type method: C{str}
- @param transform: method to call on returned JSON value to convert it to something useful.
- @type transform: C{callable}
- """
-
- storeObject, classObject = yield self._getStoreObjectForRequest(txn, request)
- if classObject is not None:
- value = yield getattr(classObject, method)(storeObject, *request.get("arguments", ()), **request.get("keywords", {}))
- else:
- value = yield getattr(storeObject, method)(*request.get("arguments", ()), **request.get("keywords", {}))
-
- returnValue(transform(value) if transform is not None else value)
-
-
- #
- # Factory methods for binding actions to the conduit class
- #
- @classmethod
- def _make_simple_action(cls, action, method, classMethod=False, transform_recv_result=None, transform_send_result=None):
- setattr(
- cls,
- "send_{}".format(action),
- lambda self, storeObject, *args, **kwargs:
- self._simple_object_send(action, storeObject, classMethod=classMethod, transform=transform_send_result, args=args, kwargs=kwargs)
- )
- setattr(
- cls,
- "recv_{}".format(action),
- lambda self, txn, message:
- self._simple_object_recv(txn, action, message, method, transform=transform_recv_result)
- )
-
-
- #
- # Transforms for returned data
- #
- @staticmethod
- def _to_externalize(value):
- """
- Convert the value to the external (JSON-based) representation.
- """
- return value.externalize() if value is not None else None
-
-
- @staticmethod
- def _to_externalize_list(value):
- """
- Convert the value to the external (JSON-based) representation.
- """
- return [v.externalize() for v in value]
-
-
- @staticmethod
- def _to_string(value):
- return str(value)
-
-
- @staticmethod
- def _to_tuple(value):
- return tuple(value)
-
# These are the actions on store objects we need to expose via the conduit api
# Calls on L{CommonHome} objects
# Calls on L{CommonHomeChild} objects
-StoreAPIConduitMixin._make_simple_action("homechild_listobjects", "listObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("homechild_movehere", "moveObjectResourceHere")
-StoreAPIConduitMixin._make_simple_action("homechild_moveaway", "moveObjectResourceAway")
-StoreAPIConduitMixin._make_simple_action("homechild_synctokenrevision", "syncTokenRevision")
-StoreAPIConduitMixin._make_simple_action("homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=StoreAPIConduitMixin._to_tuple)
-StoreAPIConduitMixin._make_simple_action("homechild_search", "search")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_listobjects", "listObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_movehere", "moveObjectResourceHere")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_moveaway", "moveObjectResourceAway")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_synctokenrevision", "syncTokenRevision")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_resourcenamessincerevision", "resourceNamesSinceRevision", transform_send_result=UtilityConduitMixin._to_tuple)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "homechild_search", "search")
# Calls on L{CommonObjectResource} objects
-StoreAPIConduitMixin._make_simple_action("objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize_list)
-StoreAPIConduitMixin._make_simple_action("objectresource_listobjects", "listObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_countobjects", "countObjects", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
-StoreAPIConduitMixin._make_simple_action("objectresource_create", "create", classMethod=True, transform_recv_result=StoreAPIConduitMixin._to_externalize)
-StoreAPIConduitMixin._make_simple_action("objectresource_setcomponent", "setComponent")
-StoreAPIConduitMixin._make_simple_action("objectresource_component", "component", transform_recv_result=StoreAPIConduitMixin._to_string)
-StoreAPIConduitMixin._make_simple_action("objectresource_remove", "remove")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjects", "loadAllObjects", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_loadallobjectswithnames", "loadAllObjectsWithNames", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize_list)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_listobjects", "listObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_countobjects", "countObjects", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_objectwith", "objectWith", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourcenameforuid", "resourceNameForUID", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_resourceuidforname", "resourceUIDForName", classMethod=True)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_create", "create", classMethod=True, transform_recv_result=UtilityConduitMixin._to_externalize)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_setcomponent", "setComponent")
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_component", "component", transform_recv_result=UtilityConduitMixin._to_string)
+UtilityConduitMixin._make_simple_action(StoreAPIConduitMixin, "objectresource_remove", "remove")
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/test_conduit.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -32,7 +32,7 @@
from txdav.caldav.datastore.query.filter import Filter
from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
from txdav.caldav.datastore.scheduling.ischedule.localservers import ServersDB, Server
-from txdav.caldav.datastore.sql import ManagedAttachment
+from txdav.caldav.datastore.sql import ManagedAttachment, AttachmentLink
from txdav.caldav.datastore.test.common import CaptureProtocol
from txdav.common.datastore.podding.conduit import PoddingConduit, \
FailedCrossPodRequestError
@@ -1107,3 +1107,32 @@
attachment._name = "test.txt"
yield shared_object.ownerHome().readAttachmentData(remote_id, attachment)
yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_get_attachment_links(self):
+ """
+ Test that action=get-attachment-links works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(txn=self.theTransactionUnderTest(0), home="user01", name="calendar")
+ cobj1 = yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ calobjID = cobj1.id()
+ yield self.commitTransaction(0)
+
+ object1 = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ attID = attachment.id()
+ managedID = attachment.managedID()
+ yield self.commitTransaction(0)
+
+ shared_object = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(1), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ links = yield shared_object.ownerHome().getAttachmentLinks()
+ self.assertEqual(len(links), 1)
+ self.assertTrue(isinstance(links[0], AttachmentLink))
+ self.assertEqual(links[0]._attachmentID, attID)
+ self.assertEqual(links[0]._managedID, managedID)
+ self.assertEqual(links[0]._calendarObjectID, calobjID)
+ yield self.commitTransaction(1)
Modified: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-13 21:16:52 UTC (rev 14411)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/test/util.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -21,20 +21,24 @@
Server, ServersDB
)
from txdav.common.datastore.podding.conduit import PoddingConduit
+from txdav.common.datastore.podding.request import ConduitRequest
from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE
from txdav.common.datastore.test.util import (
CommonCommonTests, SQLStoreBuilder, buildTestDirectory
)
import txweb2.dav.test.util
-from txweb2.stream import ProducerStream, readStream
+from txweb2 import responsecode
+from txweb2.http import Response, JSONResponse
+from txweb2.http_headers import MimeDisposition, MimeType
+from txweb2.stream import ProducerStream
from twext.enterprise.ienterprise import AlreadyFinishedError
import json
-class FakeConduitRequest(object):
+class FakeConduitRequest(ConduitRequest):
"""
A conduit request that sends messages internally rather than using HTTP
"""
@@ -66,32 +70,6 @@
@inlineCallbacks
- def doRequest(self, txn):
-
- # Generate an HTTP client request
- try:
- response = (yield self._processRequest())
- if self.writeStream is None:
- response = json.loads(response)
- else:
- try:
- ct, name, stream = response
- response = {
- "result": "ok",
- "content-type": ct,
- "name": name,
- }
- yield readStream(stream, self.writeStream.write)
- yield self.writeStream.loseConnection()
- except ValueError:
- pass
- except Exception as e:
- raise ValueError("Failed cross-pod request: {}".format(e))
-
- returnValue(response)
-
-
- @inlineCallbacks
def _processRequest(self):
"""
Process the request by sending it to the relevant server.
@@ -121,11 +99,14 @@
try:
ct, name = result
except ValueError:
- pass
+ code = responsecode.BAD_REQUEST
else:
- returnValue((ct, name, stream,))
+ headers = {"content-type": MimeType.fromString(ct)}
+ headers["content-disposition"] = MimeDisposition("attachment", params={"filename": name})
+ returnValue(Response(responsecode.OK, headers, stream))
else:
result = yield store.conduit.processRequest(j)
+ code = responsecode.OK
except Exception as e:
# Send the exception over to the other side
result = {
@@ -133,11 +114,13 @@
"class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
"details": str(e),
}
- result = json.dumps(result)
- returnValue(result)
+ code = responsecode.BAD_REQUEST
+ response = JSONResponse(code, result)
+ returnValue(response)
+
class MultiStoreConduitTest(CommonCommonTests, txweb2.dav.test.util.TestCase):
numberOfStores = 2
Added: CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pod2pod-migration/txdav/common/datastore/podding/util.py 2015-02-13 21:21:26 UTC (rev 14412)
@@ -0,0 +1,237 @@
+##
+# Copyright (c) 2013-2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from txdav.common.datastore.podding.base import FailedCrossPodRequestError
+
+
+class UtilityConduitMixin(object):
+ """
+ Defines utility methods for cross-pod API and mix-ins.
+ """
+
+ #
+ # Utility methods to map from store objects to/from JSON
+ #
+
+ @inlineCallbacks
+ def _getRequestForStoreObject(self, action, storeObject, classMethod):
+ """
+ Create the JSON data needed to identify the remote resource by type and ids, along with any parent resources.
+
+ @param action: the conduit action name
+ @type action: L{str}
+ @param storeObject: the store object that is being operated on
+ @type storeObject: L{object}
+ @param classMethod: indicates whether the method being called is a classmethod
+ @type classMethod: L{bool}
+
+ @return: the transaction in use, the JSON dict to send in the request,
+ the server where the request should be sent
+ @rtype: L{tuple} of (L{CommonStoreTransaction}, L{dict}, L{str})
+ """
+
+ from txdav.common.datastore.sql import CommonObjectResource, CommonHomeChild, CommonHome
+ result = {
+ "action": action,
+ }
+
+ # Extract the relevant store objects
+ txn = storeObject._txn
+ owner_home = None
+ viewer_home = None
+ home_child = None
+ object_resource = None
+ if isinstance(storeObject, CommonObjectResource):
+ owner_home = storeObject.ownerHome()
+ viewer_home = storeObject.viewerHome()
+ home_child = storeObject.parentCollection()
+ object_resource = storeObject
+ elif isinstance(storeObject, CommonHomeChild):
+ owner_home = storeObject.ownerHome()
+ viewer_home = storeObject.viewerHome()
+ home_child = storeObject
+ result["classMethod"] = classMethod
+ elif isinstance(storeObject, CommonHome):
+ owner_home = storeObject
+ viewer_home = storeObject
+ txn = storeObject._txn
+ result["classMethod"] = classMethod
+
+ # Add store object identities to JSON request
+ result["homeType"] = viewer_home._homeType
+ result["homeUID"] = viewer_home.uid()
+ if home_child:
+ if home_child.owned():
+ result["homeChildID"] = home_child.id()
+ else:
+ result["homeChildSharedID"] = home_child.name()
+ if object_resource:
+ result["objectResourceID"] = object_resource.id()
+
+ # Note that the owner_home is always the ownerHome() because in the sharing case
+ # a viewer is accessing the owner's data on another pod.
+ recipient = yield self.store.directoryService().recordWithUID(owner_home.uid())
+
+ returnValue((txn, result, recipient.server(),))
+
+
+ @inlineCallbacks
+ def _getStoreObjectForRequest(self, txn, request):
+ """
+ Resolve the supplied JSON data to get a store object to operate on.
+ """
+
+ returnObject = txn
+ classObject = None
+
+ if "homeUID" in request:
+ home = yield txn.homeWithUID(request["homeType"], request["homeUID"])
+ if home is None:
+ raise FailedCrossPodRequestError("Invalid owner UID specified")
+ home._internalRequest = False
+ returnObject = home
+ if request.get("classMethod", False):
+ classObject = home._childClass
+
+ if "homeChildID" in request:
+ homeChild = yield home.childWithID(request["homeChildID"])
+ if homeChild is None:
+ raise FailedCrossPodRequestError("Invalid home child specified")
+ returnObject = homeChild
+ if request.get("classMethod", False):
+ classObject = homeChild._objectResourceClass
+ elif "homeChildSharedID" in request:
+ homeChild = yield home.childWithName(request["homeChildSharedID"])
+ if homeChild is None:
+ raise FailedCrossPodRequestError("Invalid home child specified")
+ returnObject = homeChild
+ if request.get("classMethod", False):
+ classObject = homeChild._objectResourceClass
+
+ if "objectResourceID" in request:
+ objectResource = yield homeChild.objectResourceWithID(request["objectResourceID"])
+ if objectResource is None:
+ raise FailedCrossPodRequestError("Invalid object resource specified")
+ returnObject = objectResource
+
+ returnValue((returnObject, classObject,))
+
+
+ #
+ # We can simplify code generation for simple calls by dynamically generating the appropriate class methods.
+ #
+
+ @inlineCallbacks
+ def _simple_object_send(self, actionName, storeObject, classMethod=False, transform=None, args=None, kwargs=None):
+ """
+ A simple send operation that returns a value.
+
+ @param actionName: name of the action.
+ @type actionName: C{str}
+ @param shareeView: sharee resource being operated on.
+ @type shareeView: L{CommonHomeChildExternal}
+ @param objectResource: the resource being operated on, or C{None} for classmethod.
+ @type objectResource: L{CommonObjectResourceExternal}
+ @param transform: a function used to convert the JSON response into return values.
+ @type transform: C{callable}
+ @param args: list of optional arguments.
+ @type args: C{list}
+ @param kwargs: optional keyword arguments.
+ @type kwargs: C{dict}
+ """
+
+ txn, request, server = yield self._getRequestForStoreObject(actionName, storeObject, classMethod)
+ if args is not None:
+ request["arguments"] = args
+ if kwargs is not None:
+ request["keywords"] = kwargs
+ response = yield self.sendRequestToServer(txn, server, request)
+ returnValue(transform(response) if transform is not None else response)
+
+
+ @inlineCallbacks
+ def _simple_object_recv(self, txn, actionName, request, method, transform=None):
+ """
+ A simple recv operation that returns a value. We also look for an optional set of arguments/keywords
+ and include those only if present.
+
+ @param actionName: name of the action.
+ @type actionName: C{str}
+ @param request: request arguments
+ @type request: C{dict}
+ @param method: name of the method to execute on the shared resource to get the result.
+ @type method: C{str}
+ @param transform: method to call on returned JSON value to convert it to something useful.
+ @type transform: C{callable}
+ """
+
+ storeObject, classObject = yield self._getStoreObjectForRequest(txn, request)
+ if classObject is not None:
+ value = yield getattr(classObject, method)(storeObject, *request.get("arguments", ()), **request.get("keywords", {}))
+ else:
+ value = yield getattr(storeObject, method)(*request.get("arguments", ()), **request.get("keywords", {}))
+
+ returnValue(transform(value) if transform is not None else value)
+
+
+ #
+ # Factory methods for binding actions to the conduit class
+ #
+ @staticmethod
+ def _make_simple_action(bindcls, action, method, classMethod=False, transform_recv_result=None, transform_send_result=None):
+ setattr(
+ bindcls,
+ "send_{}".format(action),
+ lambda self, storeObject, *args, **kwargs:
+ self._simple_object_send(action, storeObject, classMethod=classMethod, transform=transform_send_result, args=args, kwargs=kwargs)
+ )
+ setattr(
+ bindcls,
+ "recv_{}".format(action),
+ lambda self, txn, message:
+ self._simple_object_recv(txn, action, message, method, transform=transform_recv_result)
+ )
+
+
+ #
+ # Transforms for returned data
+ #
+ @staticmethod
+ def _to_externalize(value):
+ """
+ Convert the value to the external (JSON-based) representation.
+ """
+ return value.externalize() if value is not None else None
+
+
+ @staticmethod
+ def _to_externalize_list(value):
+ """
+ Convert the value to the external (JSON-based) representation.
+ """
+ return [v.externalize() for v in value]
+
+
+ @staticmethod
+ def _to_string(value):
+ return str(value)
+
+
+ @staticmethod
+ def _to_tuple(value):
+ return tuple(value)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150213/1cbe9d48/attachment-0001.html>
More information about the calendarserver-changes
mailing list