[CalendarServer-changes] [9900] CalendarServer/branches/release/CalendarServer-4.2-dev
source_changes at macosforge.org
source_changes at macosforge.org
Fri Oct 5 15:56:34 PDT 2012
Revision: 9900
http://trac.calendarserver.org//changeset/9900
Author: glyph at apple.com
Date: 2012-10-05 15:56:34 -0700 (Fri, 05 Oct 2012)
Log Message:
-----------
Pull up 9899 from trunk.
Modified Paths:
--------------
CalendarServer/branches/release/CalendarServer-4.2-dev/twistedcaldav/storebridge.py
CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/file.py
CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/sql.py
CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/test/common.py
CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/util.py
Property Changed:
----------------
CalendarServer/branches/release/CalendarServer-4.2-dev/
Property changes on: CalendarServer/branches/release/CalendarServer-4.2-dev
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:9867,9870,9876,9895
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:9867,9870,9876,9895,9899
Modified: CalendarServer/branches/release/CalendarServer-4.2-dev/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.2-dev/twistedcaldav/storebridge.py 2012-10-05 22:52:17 UTC (rev 9899)
+++ CalendarServer/branches/release/CalendarServer-4.2-dev/twistedcaldav/storebridge.py 2012-10-05 22:56:34 UTC (rev 9900)
@@ -1646,6 +1646,8 @@
stream = ProducerStream()
class StreamProtocol(Protocol):
+ def connectionMade(self):
+ stream.registerProducer(self.transport, False)
def dataReceived(self, data):
stream.write(data)
def connectionLost(self, reason):
@@ -1655,7 +1657,6 @@
except IOError, e:
log.error("Unable to read attachment: %s, due to: %s" % (self, e,))
raise HTTPError(responsecode.NOT_FOUND)
-
return Response(OK, {"content-type":self.contentType()}, stream)
Modified: CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/file.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/file.py 2012-10-05 22:52:17 UTC (rev 9899)
+++ CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/file.py 2012-10-05 22:56:34 UTC (rev 9900)
@@ -34,8 +34,6 @@
from twisted.internet.defer import inlineCallbacks, returnValue, succeed, fail
-from twisted.python.failure import Failure
-
from twext.python.vcomponent import VComponent
from txdav.xml import element as davxml
from txdav.xml.rfc2518 import ResourceType, GETContentType
@@ -56,7 +54,7 @@
IndexSchedule as OldInboxIndex
from txdav.caldav.datastore.util import (
validateCalendarComponent, dropboxIDFromCalendarObject, CalendarObjectBase,
- StorageTransportBase
+ StorageTransportBase, AttachmentRetrievalTransport
)
from txdav.common.datastore.file import (
@@ -737,6 +735,7 @@
def write(self, data):
# FIXME: multiple chunks
self._file.write(data)
+ return super(AttachmentStorageTransport, self).write(data)
def loseConnection(self):
@@ -801,12 +800,7 @@
def retrieve(self, protocol):
- # FIXME: makeConnection
- # FIXME: actually stream
- # FIMXE: connectionLost
- protocol.dataReceived(self._path.getContent())
- # FIXME: ConnectionDone, not NotImplementedError
- protocol.connectionLost(Failure(NotImplementedError()))
+ return AttachmentRetrievalTransport(self._path).start(protocol)
@property
Modified: CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/sql.py 2012-10-05 22:52:17 UTC (rev 9899)
+++ CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/sql.py 2012-10-05 22:56:34 UTC (rev 9900)
@@ -32,9 +32,7 @@
from twext.python.filepath import CachingFilePath
from twisted.internet.defer import inlineCallbacks, returnValue
-from twisted.internet.error import ConnectionLost
from twisted.python import hashlib
-from twisted.python.failure import Failure
from twistedcaldav import caldavxml, customxml
from twistedcaldav.caldavxml import ScheduleCalendarTransp, Opaque
@@ -81,6 +79,8 @@
from pycalendar.duration import PyCalendarDuration
from pycalendar.timezone import PyCalendarTimezone
+from txdav.caldav.datastore.util import AttachmentRetrievalTransport
+
from zope.interface.declarations import implements
import os
@@ -1533,8 +1533,7 @@
def retrieve(self, protocol):
- protocol.dataReceived(self._path.getContent())
- protocol.connectionLost(Failure(ConnectionLost()))
+ return AttachmentRetrievalTransport(self._path).start(protocol)
_removeStatement = Delete(
Modified: CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/test/common.py 2012-10-05 22:52:17 UTC (rev 9899)
+++ CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/test/common.py 2012-10-05 22:56:34 UTC (rev 9900)
@@ -2090,24 +2090,21 @@
t.write(sampleData)
yield t.loseConnection()
yield self.exceedQuotaTest(get)
+ @inlineCallbacks
def checkOriginal():
- catch = StringIO()
- catch.dataReceived = catch.write
- lost = []
- catch.connectionLost = lost.append
- attachment.retrieve(catch)
+ actual = yield self.attachmentToString(attachment)
expected = sampleData
# note: 60 is less than len(expected); trimming is just to make
# the error message look sane when the test fails.
- actual = catch.getvalue()[:60]
+ actual = actual[:60]
self.assertEquals(actual, expected)
- checkOriginal()
+ yield checkOriginal()
yield self.commit()
# Make sure that things go back to normal after a commit of that
# transaction.
obj = yield self.calendarObjectUnderTest()
attachment = yield get()
- checkOriginal()
+ yield checkOriginal()
def test_removeAttachmentWithName(self, refresh=lambda x:x):
Modified: CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/util.py 2012-10-05 22:52:17 UTC (rev 9899)
+++ CalendarServer/branches/release/CalendarServer-4.2-dev/txdav/caldav/datastore/util.py 2012-10-05 22:56:34 UTC (rev 9900)
@@ -45,6 +45,11 @@
InternalDataStoreError, HomeChildNameAlreadyExistsError
)
from txdav.base.datastore.util import normalizeUUIDOrNot
+from twisted.protocols.basic import FileSender
+from twisted.internet.interfaces import ITransport
+from twisted.internet.interfaces import IConsumer
+from twisted.internet.error import ConnectionLost
+from twisted.internet.task import LoopingCall
log = Logger()
@@ -299,6 +304,9 @@
self.storeTransport = storeTransport
self.done = Deferred()
+ def connectionMade(self):
+ self.storeTransport.registerProducer(self.transport, False)
+
def dataReceived(self, data):
self.storeTransport.write(data)
@@ -462,14 +470,34 @@
Create a storage transport with a reference to an L{IAttachment} and a
L{twext.web2.http_headers.MimeType}.
"""
+ from twisted.internet import reactor
+ self._clock = reactor
self._attachment = attachment
self._contentType = contentType
+ self._producer = None
# Make sure we have some kind of contrent-type
if self._contentType is None:
self._contentType = http_headers.MimeType.fromString(getType(self._attachment.name(), self.contentTypes))
+ def write(self, data):
+ """
+ Children must override this to actually write the data, but should
+ upcall this implementation to interact properly with producers.
+ """
+ if self._producer and self._streamingProducer:
+ # XXX this needs to be in a callLater because otherwise
+ # resumeProducing will call write which will call resumeProducing
+ # (etc) forever.
+ self._clock.callLater(0, self._producer.resumeProducing)
+
+
+ def registerProducer(self, producer, streaming):
+ self._producer = producer
+ self._streamingProducer = streaming
+
+
def getPeer(self):
return StorageTransportAddress(self._attachment, False)
@@ -482,7 +510,86 @@
return self.write(''.join(seq))
+ def stopProducing(self):
+ return self.loseConnection()
+
+
+class AttachmentRetrievalTransport(FileSender, object):
+ """
+ The transport for a protocol that does L{IAttachment.retrieve}.
+ """
+ implements(ITransport)
+
+ def __init__(self, filePath):
+ from twisted.internet import reactor
+ self.filePath = filePath
+ self.clock = reactor
+
+
+ def start(self, protocol):
+ this = self
+ class Consumer(object):
+ implements(IConsumer)
+ def registerProducer(self, producer, streaming):
+ protocol.makeConnection(producer)
+ this._maybeLoopDelivery()
+ def write(self, data):
+ protocol.dataReceived(data)
+ def unregisterProducer(self):
+ this._done(protocol)
+ self.beginFileTransfer(self.filePath.open(), Consumer())
+
+
+ def _done(self, protocol):
+ if self._deliveryLoop:
+ self._deliveryLoop.stop()
+ protocol.connectionLost(Failure(ConnectionLost()))
+
+
+ def write(self, data):
+ raise NotImplemented("This is a read-only transport.")
+
+
+ def writeSequence(self, datas):
+ self.write("".join(datas))
+
+
+ def loseConnection(self):
+ pass
+
+
+ def getPeer(self):
+ return self
+
+
+ def getHost(self):
+ return self
+
+
+ _everResumedProducing = False
+
+ def resumeProducing(self):
+ self._everResumedProducing = True
+ super(AttachmentRetrievalTransport, self).resumeProducing()
+
+
+ _deliveryLoop = None
+
+ def _maybeLoopDelivery(self):
+ """
+ If no consumer was registered (as inferred by the fact that
+ resumeProducing() wasn't called)
+ """
+ if not self._everResumedProducing:
+ # Not registered as a streaming producer.
+ def deliverNextChunk():
+ super(AttachmentRetrievalTransport, self).resumeProducing()
+ self._deliveryLoop = LoopingCall(deliverNextChunk)
+ self._deliveryLoop.start(0.01, True)
+
+
+
def fixOneCalendarObject(component):
"""
Correct the properties which may contain a user's directory UUID within a
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121005/927345d0/attachment-0001.html>
More information about the calendarserver-changes
mailing list