[CalendarServer-changes] [12190] CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:21:20 PDT 2014
Revision: 12190
http://trac.calendarserver.org//changeset/12190
Author: cdaboo at apple.com
Date: 2013-12-23 08:38:46 -0800 (Mon, 23 Dec 2013)
Log Message:
-----------
Checkpoint: cross-pod managed attachments.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/caldav/datastore/sql_external.py
CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/conduit.py
CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/request.py
CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/resource.py
CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/test_conduit.py
CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/util.py
Modified: CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/caldav/datastore/sql_external.py
===================================================================
--- CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/caldav/datastore/sql_external.py 2013-12-23 16:36:11 UTC (rev 12189)
+++ CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/caldav/datastore/sql_external.py 2013-12-23 16:38:46 UTC (rev 12190)
@@ -18,7 +18,7 @@
SQL backend for CalDAV storage when resources are external.
"""
-from twisted.internet.defer import succeed
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from twext.python.log import Logger
@@ -179,4 +179,39 @@
raise AssertionError("CalendarObjectExternal: not supported")
+ @inlineCallbacks
+ def addAttachment(self, rids, content_type, filename, stream):
+ result = yield self._txn.store().conduit.send_add_attachment(self, rids, content_type, filename, stream)
+ managedID, location = result
+ returnValue((ManagedAttachmentExternal(str(managedID)), str(location),))
+
+
+ @inlineCallbacks
+ def updateAttachment(self, managed_id, content_type, filename, stream):
+ result = yield self._txn.store().conduit.send_update_attachment(self, managed_id, content_type, filename, stream)
+ managedID, location = result
+ returnValue((ManagedAttachmentExternal(str(managedID)), str(location),))
+
+
+ @inlineCallbacks
+ def removeAttachment(self, rids, managed_id):
+ yield self._txn.store().conduit.send_remove_attachment(self, rids, managed_id)
+ returnValue(None)
+
+
+
+class ManagedAttachmentExternal(object):
+ """
+ Fake managed attachment object returned from L{CalendarObjectExternal.addAttachment} and
+ L{CalendarObjectExternal.updateAttachment}.
+ """
+
+ def __init__(self, managedID):
+ self._managedID = managedID
+
+
+ def managedID(self):
+ return self._managedID
+
+
CalendarExternal._objectResourceClass = CalendarObjectExternal
Modified: CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/conduit.py 2013-12-23 16:36:11 UTC (rev 12189)
+++ CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/conduit.py 2013-12-23 16:38:46 UTC (rev 12190)
@@ -108,9 +108,9 @@
@inlineCallbacks
- def sendRequest(self, txn, recipient, data):
+ def sendRequest(self, txn, recipient, data, stream=None, streamType=None):
- request = self.conduitRequestClass(recipient.server(), data)
+ request = self.conduitRequestClass(recipient.server(), data, stream, streamType)
try:
response = (yield request.doRequest(txn))
except Exception as e:
@@ -137,7 +137,7 @@
result = {"result": "ok"}
returnValue(result)
- method = "recv_{}".format(action)
+ method = "recv_{}".format(action.replace("-", "_"))
if not hasattr(self, method):
log.error("Unsupported action: {action}", action=action)
raise FailedCrossPodRequestError("Unsupported action: {}".format(action))
@@ -379,6 +379,184 @@
#
+ # Managed attachment related apis
+ #
+
+ @inlineCallbacks
+ def send_add_attachment(self, objectResource, rids, content_type, filename, stream):
+ """
+ Managed attachment addAttachment call.
+
+ @param objectResource: child resource having an attachment added
+ @type objectResource: L{CalendarObject}
+ @param rids: list of recurrence ids
+ @type rids: C{list}
+ @param content_type: content type of attachment data
+ @type content_type: L{MimeType}
+ @param filename: name of attachment
+ @type filename: C{str}
+ @param stream: attachment data stream
+ @type stream: L{IStream}
+ """
+
+ actionName = "add-attachment"
+ shareeView = objectResource._parentCollection
+ action, recipient = self._send(actionName, shareeView, objectResource)
+ action["rids"] = rids
+ action["filename"] = filename
+ result = yield self.sendRequest(shareeView._txn, recipient, action, stream, content_type)
+ if result["result"] == "ok":
+ returnValue(result["value"])
+ elif result["result"] == "exception":
+ raise namedClass(result["class"])(result["message"])
+
+
+ @inlineCallbacks
+ def recv_add_attachment(self, txn, message):
+ """
+ Process an addAttachment cross-pod message. Message arguments as per L{send_add_attachment}.
+
+ @param message: message arguments
+ @type message: C{dict}
+ """
+
+ actionName = "add-attachment"
+ _ignore_shareeView, objectResource = yield self._recv(txn, message, actionName)
+ try:
+ attachment, location = yield objectResource.addAttachment(
+ message["rids"],
+ message["streamType"],
+ message["filename"],
+ message["stream"],
+ )
+ except Exception as e:
+ returnValue({
+ "result": "exception",
+ "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
+ "message": str(e),
+ })
+
+ returnValue({
+ "result": "ok",
+ "value": (attachment.managedID(), location,),
+ })
+
+
+ @inlineCallbacks
+ def send_update_attachment(self, objectResource, managed_id, content_type, filename, stream):
+ """
+ Managed attachment updateAttachment call.
+
+ @param objectResource: child resource having an attachment added
+ @type objectResource: L{CalendarObject}
+ @param managed_id: managed-id to update
+ @type managed_id: C{str}
+ @param content_type: content type of attachment data
+ @type content_type: L{MimeType}
+ @param filename: name of attachment
+ @type filename: C{str}
+ @param stream: attachment data stream
+ @type stream: L{IStream}
+ """
+
+ actionName = "update-attachment"
+ shareeView = objectResource._parentCollection
+ action, recipient = self._send(actionName, shareeView, objectResource)
+ action["managedID"] = managed_id
+ action["filename"] = filename
+ result = yield self.sendRequest(shareeView._txn, recipient, action, stream, content_type)
+ if result["result"] == "ok":
+ returnValue(result["value"])
+ elif result["result"] == "exception":
+ raise namedClass(result["class"])(result["message"])
+
+
+ @inlineCallbacks
+ def recv_update_attachment(self, txn, message):
+ """
+ Process an updateAttachment cross-pod message. Message arguments as per L{send_update_attachment}.
+
+ @param message: message arguments
+ @type message: C{dict}
+ """
+
+ actionName = "update-attachment"
+ _ignore_shareeView, objectResource = yield self._recv(txn, message, actionName)
+ try:
+ attachment, location = yield objectResource.updateAttachment(
+ message["managedID"],
+ message["streamType"],
+ message["filename"],
+ message["stream"],
+ )
+ except Exception as e:
+ returnValue({
+ "result": "exception",
+ "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
+ "message": str(e),
+ })
+
+ returnValue({
+ "result": "ok",
+ "value": (attachment.managedID(), location,),
+ })
+
+
+ @inlineCallbacks
+ def send_remove_attachment(self, objectResource, rids, managed_id):
+ """
+ Managed attachment removeAttachment call.
+
+ @param objectResource: child resource having an attachment added
+ @type objectResource: L{CalendarObject}
+ @param rids: list of recurrence ids
+ @type rids: C{list}
+ @param managed_id: managed-id to update
+ @type managed_id: C{str}
+ """
+
+ actionName = "remove-attachment"
+ shareeView = objectResource._parentCollection
+ action, recipient = self._send(actionName, shareeView, objectResource)
+ action["rids"] = rids
+ action["managedID"] = managed_id
+ result = yield self.sendRequest(shareeView._txn, recipient, action)
+ if result["result"] == "ok":
+ returnValue(result["value"])
+ elif result["result"] == "exception":
+ raise namedClass(result["class"])(result["message"])
+
+
+ @inlineCallbacks
+ def recv_remove_attachment(self, txn, message):
+ """
+ Process an removeAttachment cross-pod message. Message arguments as per L{send_remove_attachment}.
+
+ @param message: message arguments
+ @type message: C{dict}
+ """
+
+ actionName = "remove-attachment"
+ _ignore_shareeView, objectResource = yield self._recv(txn, message, actionName)
+ try:
+ yield objectResource.removeAttachment(
+ message["rids"],
+ message["managedID"],
+ )
+ except Exception as e:
+ returnValue({
+ "result": "exception",
+ "class": ".".join((e.__class__.__module__, e.__class__.__name__,)),
+ "message": str(e),
+ })
+
+ returnValue({
+ "result": "ok",
+ "value": None,
+ })
+
+
+ #
# Sharer data access related apis
#
Modified: CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/request.py
===================================================================
--- CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/request.py 2013-12-23 16:36:11 UTC (rev 12189)
+++ CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/request.py 2013-12-23 16:38:46 UTC (rev 12190)
@@ -34,6 +34,7 @@
from twistedcaldav.util import utf8String
from cStringIO import StringIO
+import base64
import json
@@ -42,11 +43,17 @@
class ConduitRequest(object):
+ """
+ An HTTP request between pods. This is typically used to send and receive JSON data. However,
+ for attachments, we need to send the actual attachment data as the request body, so in that
+ case the JSON data is sent in an HTTP header.
+ """
- def __init__(self, server, data):
-
+ def __init__(self, server, data, stream=None, stream_type=None):
self.server = server
self.data = json.dumps(data)
+ self.stream = stream
+ self.streamType = stream_type
@inlineCallbacks
@@ -102,11 +109,15 @@
# We need to play a trick with the request stream as we can only read it once. So we
# read it, store the value in a MemoryStream, and replace the request's stream with that,
- # so the data can be read again.
- data = (yield allDataFromStream(request.stream))
- iostr.write(data)
- request.stream = MemoryStream(data if data is not None else "")
- request.stream.doStartReading = None
+ # so the data can be read again. Note if we are sending an attachment, we won't log
+ # the attachment data as we do not want to read it all into memory.
+ if self.stream is None:
+ data = (yield allDataFromStream(request.stream))
+ iostr.write(data)
+ request.stream = MemoryStream(data if data is not None else "")
+ request.stream.doStartReading = None
+ else:
+ iostr.write("<<Stream Type: {}>>\n".format(self.streamType))
iostr.write("\n\n>>>> Request end\n")
returnValue(iostr.getvalue())
@@ -155,7 +166,12 @@
headers = Headers()
headers.setHeader("Host", utf8String(host + ":{}".format(port)))
- headers.setHeader("Content-Type", MimeType("application", "json", params={"charset": "utf-8", }))
+ if self.streamType:
+ # For attachments we put the base64-encoded JSON data into a header
+ headers.setHeader("Content-Type", self.streamType)
+ headers.addRawHeader("XPOD", base64.b64encode(self.data))
+ else:
+ headers.setHeader("Content-Type", MimeType("application", "json", params={"charset": "utf-8", }))
headers.setHeader("User-Agent", "CalendarServer/{}".format(version))
headers.addRawHeader(*self.server.secretHeader())
@@ -165,7 +181,7 @@
ep = GAIEndpoint(reactor, host, port, _configuredClientContextFactory() if ssl else None)
proto = (yield ep.connect(f))
- request = ClientRequest("POST", path, headers, self.data)
+ request = ClientRequest("POST", path, headers, self.stream if self.stream is not None else self.data)
if accountingEnabledForCategory("xPod"):
self.loggedRequest = yield self.logRequest(request)
Modified: CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/resource.py 2013-12-23 16:36:11 UTC (rev 12189)
+++ CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/resource.py 2013-12-23 16:38:46 UTC (rev 12190)
@@ -14,8 +14,6 @@
# limitations under the License.
##
-import json
-
from twext.web2 import responsecode
from twext.web2.dav.noneprops import NonePropertyStore
from twext.web2.dav.util import allDataFromStream
@@ -34,6 +32,9 @@
from txdav.caldav.datastore.scheduling.ischedule.localservers import Servers
from txdav.common.datastore.podding.conduit import FailedCrossPodRequestError
+import base64
+import json
+
__all__ = [
"ConduitResource",
]
@@ -125,20 +126,33 @@
self.log.error("Invalid shared secret header in cross-pod request")
raise HTTPError(StatusResponse(responsecode.FORBIDDEN, "Not authorized to make this request"))
- # Check content first
+ # Look for XPOD header
+ xpod = request.headers.getRawHeaders("XPOD")
contentType = request.headers.getHeader("content-type")
+ if xpod is not None:
+ # Attachments are sent in the request body with the JSON data in a header. We
+ # decode the header and add the request.stream as an attribute of the JSON object.
+ xpod = xpod[0]
+ try:
+ j = json.loads(base64.b64decode(xpod))
+ except (TypeError, ValueError) as e:
+ self.log.error("Invalid JSON header in request: {ex}\n{xpod}", ex=e, xpod=xpod)
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Invalid JSON header in request: {}\n{}".format(e, xpod)))
+ j["stream"] = request.stream
+ j["streamType"] = contentType
+ else:
+ # Check content first
+ if "{}/{}".format(contentType.mediaType, contentType.mediaSubtype) != "application/json":
+ self.log.error("MIME type {mime} not allowed in request", mime=contentType)
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "MIME type {} not allowed in request".format(contentType)))
- if "{}/{}".format(contentType.mediaType, contentType.mediaSubtype) != "application/json":
- self.log.error("MIME type {mime} not allowed in request", mime=contentType)
- raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "MIME type {} not allowed in request".format(contentType)))
+ body = (yield allDataFromStream(request.stream))
+ try:
+ j = json.loads(body)
+ except ValueError as e:
+ self.log.error("Invalid JSON data in request: {ex}\n{body}", ex=e, body=body)
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Invalid JSON data in request: {}\n{}".format(e, body)))
- body = (yield allDataFromStream(request.stream))
- try:
- j = json.loads(body)
- except ValueError as e:
- self.log.error("Invalid JSON data in request: {ex}\n{body}", ex=e, body=body)
- raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Invalid JSON data in request: {}\n{}".format(e, body)))
-
# Log extended item
if not hasattr(request, "extendedLogItems"):
request.extendedLogItems = {}
Modified: CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/test_conduit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/test_conduit.py 2013-12-23 16:36:11 UTC (rev 12189)
+++ CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/test_conduit.py 2013-12-23 16:38:46 UTC (rev 12190)
@@ -18,6 +18,9 @@
from pycalendar.period import Period
from twext.python.clsprop import classproperty
+import twext.web2.dav.test.util
+from twext.web2.http_headers import MimeType
+from twext.web2.stream import MemoryStream
from twisted.internet.defer import inlineCallbacks, succeed, returnValue
@@ -28,6 +31,8 @@
from txdav.caldav.datastore.query.filter import Filter
from txdav.caldav.datastore.scheduling.freebusy import generateFreeBusyInfo
from txdav.caldav.datastore.scheduling.ischedule.localservers import Servers, Server
+from txdav.caldav.datastore.sql import ManagedAttachment
+from txdav.caldav.datastore.test.common import CaptureProtocol
from txdav.caldav.datastore.test.util import buildCalendarStore, \
TestCalendarStoreDirectoryRecord
from txdav.common.datastore.podding.conduit import PoddingConduit, \
@@ -41,7 +46,6 @@
ObjectResourceNameNotAllowedError
from txdav.common.idirectoryservice import DirectoryRecordNotFoundError
-import twext.web2.dav.test.util
class TestConduit (CommonCommonTests, twext.web2.dav.test.util.TestCase):
@@ -967,3 +971,112 @@
self.assertEqual(len(fbinfo[1]), 0)
self.assertEqual(len(fbinfo[2]), 0)
yield self.otherCommit()
+
+
+ def attachmentToString(self, attachment):
+ """
+ Convenience to convert an L{IAttachment} to a string.
+
+ @param attachment: an L{IAttachment} provider to convert into a string.
+
+ @return: a L{Deferred} that fires with the contents of the attachment.
+
+ @rtype: L{Deferred} firing C{bytes}
+ """
+ capture = CaptureProtocol()
+ attachment.retrieve(capture)
+ return capture.deferred
+
+
+ @inlineCallbacks
+ def test_add_attachment(self):
+ """
+ Test that action=add-attachment works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(home="user01", name="calendar")
+ object1 = yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ resourceID = object1.id()
+ yield self.commit()
+
+ shared_object = yield self.calendarObjectUnderTest(txn=self.newOtherTransaction(), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ attachment, location = yield shared_object.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ managedID = attachment.managedID()
+ from txdav.caldav.datastore.sql_external import ManagedAttachmentExternal
+ self.assertTrue(isinstance(attachment, ManagedAttachmentExternal))
+ self.assertTrue("user01/attachments/test" in location)
+ yield self.otherCommit()
+
+ cobjs = yield ManagedAttachment.referencesTo(self.transactionUnderTest(), managedID)
+ self.assertEqual(cobjs, set((resourceID,)))
+ attachment = yield ManagedAttachment.load(self.transactionUnderTest(), resourceID, managedID)
+ self.assertEqual(attachment.name(), "test.txt")
+ data = yield self.attachmentToString(attachment)
+ self.assertEqual(data, "Here is some text.")
+ yield self.commit()
+
+
+ @inlineCallbacks
+ def test_update_attachment(self):
+ """
+ Test that action=update-attachment works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(home="user01", name="calendar")
+ yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ yield self.commit()
+
+ object1 = yield self.calendarObjectUnderTest(home="user01", calendar_name="calendar", name="1.ics")
+ resourceID = object1.id()
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ managedID = attachment.managedID()
+ yield self.commit()
+
+ shared_object = yield self.calendarObjectUnderTest(txn=self.newOtherTransaction(), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ attachment, location = yield shared_object.updateAttachment(managedID, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some more text."))
+ managedID = attachment.managedID()
+ from txdav.caldav.datastore.sql_external import ManagedAttachmentExternal
+ self.assertTrue(isinstance(attachment, ManagedAttachmentExternal))
+ self.assertTrue("user01/attachments/test" in location)
+ yield self.otherCommit()
+
+ cobjs = yield ManagedAttachment.referencesTo(self.transactionUnderTest(), managedID)
+ self.assertEqual(cobjs, set((resourceID,)))
+ attachment = yield ManagedAttachment.load(self.transactionUnderTest(), resourceID, managedID)
+ self.assertEqual(attachment.name(), "test.txt")
+ data = yield self.attachmentToString(attachment)
+ self.assertEqual(data, "Here is some more text.")
+ yield self.commit()
+
+
+ @inlineCallbacks
+ def test_remove_attachment(self):
+ """
+ Test that action=remove-attachment works.
+ """
+
+ yield self.createShare("user01", "puser01")
+
+ calendar1 = yield self.calendarUnderTest(home="user01", name="calendar")
+ yield calendar1.createCalendarObjectWithName("1.ics", Component.fromString(self.caldata1))
+ yield self.commit()
+
+ object1 = yield self.calendarObjectUnderTest(home="user01", calendar_name="calendar", name="1.ics")
+ resourceID = object1.id()
+ attachment, _ignore_location = yield object1.addAttachment(None, MimeType.fromString("text/plain"), "test.txt", MemoryStream("Here is some text."))
+ managedID = attachment.managedID()
+ yield self.commit()
+
+ shared_object = yield self.calendarObjectUnderTest(txn=self.newOtherTransaction(), home="puser01", calendar_name="shared-calendar", name="1.ics")
+ yield shared_object.removeAttachment(None, managedID)
+ yield self.otherCommit()
+
+ cobjs = yield ManagedAttachment.referencesTo(self.transactionUnderTest(), managedID)
+ self.assertEqual(cobjs, set())
+ attachment = yield ManagedAttachment.load(self.transactionUnderTest(), resourceID, managedID)
+ self.assertTrue(attachment is None)
+ yield self.commit()
Modified: CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/util.py 2013-12-23 16:36:11 UTC (rev 12189)
+++ CalendarServer/branches/users/cdaboo/cross-pod-sharing/txdav/common/datastore/podding/test/util.py 2013-12-23 16:38:46 UTC (rev 12190)
@@ -51,10 +51,12 @@
cls.storeMap[server.details()] = store
- def __init__(self, server, data):
+ def __init__(self, server, data, stream=None, stream_type=None):
self.server = server
self.data = json.dumps(data)
+ self.stream = stream
+ self.streamType = stream_type
@inlineCallbacks
@@ -80,7 +82,11 @@
"""
store = self.storeMap[self.server.details()]
- result = yield store.conduit.processRequest(json.loads(self.data))
+ j = json.loads(self.data)
+ if self.stream is not None:
+ j["stream"] = self.stream
+ j["streamType"] = self.streamType
+ result = yield store.conduit.processRequest(j)
result = json.dumps(result)
returnValue(result)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/abecc928/attachment.html>
More information about the calendarserver-changes
mailing list