[CalendarServer-changes] [11654] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Sep 4 18:13:51 PDT 2013
Revision: 11654
http://trac.calendarserver.org//changeset/11654
Author: cdaboo at apple.com
Date: 2013-09-04 18:13:51 -0700 (Wed, 04 Sep 2013)
Log Message:
-----------
Fix issue with requests not timing out when client connection goes away. Fix bug in closing SSL
connections where an attempt to "half-close" prevents the connection from closing. Fix issue
where master socket was not being closed after being sent to a child.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/provision/root.py
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/twext/internet/sendfdport.py
CalendarServer/trunk/twext/web2/channel/http.py
CalendarServer/trunk/twext/web2/test/test_http.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
Modified: CalendarServer/trunk/calendarserver/provision/root.py
===================================================================
--- CalendarServer/trunk/calendarserver/provision/root.py 2013-09-05 00:58:35 UTC (rev 11653)
+++ CalendarServer/trunk/calendarserver/provision/root.py 2013-09-05 01:13:51 UTC (rev 11654)
@@ -94,15 +94,7 @@
from twext.web2.filter import gzip
self.contentFilters.append((gzip.gzipfilter, True))
- if not config.EnableKeepAlive:
- def addConnectionClose(request, response):
- response.headers.setHeader("connection", ("close",))
- if request.chanRequest is not None:
- request.chanRequest.channel.setReadPersistent(False)
- return response
- self.contentFilters.append((addConnectionClose, True))
-
def deadProperties(self):
if not hasattr(self, "_dead_properties"):
# Get the property store from super
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2013-09-05 00:58:35 UTC (rev 11653)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2013-09-05 01:13:51 UTC (rev 11654)
@@ -58,7 +58,8 @@
from twext.internet.ssl import ChainingOpenSSLContextFactory
from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
from twext.internet.fswatch import DirectoryChangeListener, IDirectoryChangeListenee
-from twext.web2.channel.http import LimitingHTTPFactory, SSLRedirectRequest
+from twext.web2.channel.http import LimitingHTTPFactory, SSLRedirectRequest, \
+ HTTPChannel
from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
from twext.enterprise.ienterprise import POSTGRES_DIALECT
from twext.enterprise.ienterprise import ORACLE_DIALECT
@@ -243,14 +244,15 @@
self.logRotateLength = logRotateLength
self.logMaxFiles = logMaxFiles
+
def setServiceParent(self, app):
MultiService.setServiceParent(self, app)
if self.logEnabled:
errorLogFile = LogFile.fromFullPath(
self.logPath,
- rotateLength = self.logRotateLength,
- maxRotatedFiles = self.logMaxFiles
+ rotateLength=self.logRotateLength,
+ maxRotatedFiles=self.logMaxFiles
)
errorLogObserver = FileLogObserver(errorLogFile).emit
@@ -978,6 +980,13 @@
def requestFactory(*args, **kw):
return SSLRedirectRequest(site=underlyingSite, *args, **kw)
+ # Setup HTTP connection behaviors
+ HTTPChannel.allowPersistentConnections = config.EnableKeepAlive
+ HTTPChannel.betweenRequestsTimeOut = config.PipelineIdleTimeOut
+ HTTPChannel.inputTimeOut = config.IncomingDataTimeOut
+ HTTPChannel.idleTimeOut = config.IdleConnectionTimeOut
+ HTTPChannel.closeTimeOut = config.CloseConnectionTimeOut
+
# Add the Strict-Transport-Security header to all secured requests
# if enabled.
if config.StrictTransportSecuritySeconds:
@@ -991,6 +1000,7 @@
"max-age={max_age:d}"
.format(max_age=config.StrictTransportSecuritySeconds))
return response
+ responseFilter.handleErrors = True
request.addResponseFilter(responseFilter)
return request
@@ -2423,6 +2433,7 @@
return uid, gid
+
class DataStoreMonitor(object):
implements(IDirectoryChangeListenee)
@@ -2434,18 +2445,21 @@
self._reactor = reactor
self._storageService = storageService
+
def disconnected(self):
self._storageService.hardStop()
self._reactor.stop()
+
def deleted(self):
self._storageService.hardStop()
self._reactor.stop()
+
def renamed(self):
self._storageService.hardStop()
self._reactor.stop()
+
def connectionLost(self, reason):
pass
-
Modified: CalendarServer/trunk/twext/internet/sendfdport.py
===================================================================
--- CalendarServer/trunk/twext/internet/sendfdport.py 2013-09-05 00:58:35 UTC (rev 11653)
+++ CalendarServer/trunk/twext/internet/sendfdport.py 2013-09-05 01:13:51 UTC (rev 11654)
@@ -153,6 +153,10 @@
self.outgoingSocketQueue.insert(0, (skt, desc))
return
raise
+
+ # Always close the socket on this end
+ skt.close()
+
if not self.outgoingSocketQueue:
self.stopWriting()
@@ -185,7 +189,7 @@
than the somewhat more abstract language that would be accurate.
"""
- def initialStatus():
+ def initialStatus(): #@NoSelf
"""
A new socket was created and added to the dispatcher. Compute an
initial value for its status.
@@ -193,8 +197,7 @@
@return: the new status.
"""
-
- def newConnectionStatus(previousStatus):
+ def newConnectionStatus(previousStatus): #@NoSelf
"""
A new connection was sent to a given socket. Compute its status based
on the previous status of that socket.
@@ -205,8 +208,7 @@
@return: the socket's status after incrementing its outstanding work.
"""
-
- def statusFromMessage(previousStatus, message):
+ def statusFromMessage(previousStatus, message): #@NoSelf
"""
A status message was received by a worker. Convert the previous status
value (returned from L{newConnectionStatus}, L{initialStatus}, or
@@ -412,4 +414,3 @@
"""
self.statusQueue.append(statusMessage)
self.startWriting()
-
Modified: CalendarServer/trunk/twext/web2/channel/http.py
===================================================================
--- CalendarServer/trunk/twext/web2/channel/http.py 2013-09-05 00:58:35 UTC (rev 11653)
+++ CalendarServer/trunk/twext/web2/channel/http.py 2013-09-05 01:13:51 UTC (rev 11654)
@@ -726,6 +726,10 @@
betweenRequestsTimeOut = 15
# Timeout between lines or bytes while reading a request
inputTimeOut = 60 * 4
+ # Timeout between end of request read and end of response write
+ idleTimeOut = 60 * 5
+ # Timeout when closing non-persistent connection
+ closeTimeOut = 20
# maximum length of headers (10KiB)
maxHeaderLength = 10240
@@ -744,7 +748,7 @@
_readLost = False
_writeLost = False
- _lingerTimer = None
+ _abortTimer = None
chanRequest = None
def _callLater(self, secs, fun):
@@ -823,10 +827,10 @@
self.chanRequest = None
self.setLineMode()
- # Disable the idle timeout, in case this request takes a long
+ # Set an idle timeout, in case this request takes a long
# time to finish generating output.
if len(self.requests) > 0:
- self.setTimeout(None)
+ self.setTimeout(self.idleTimeOut)
def _startNextRequest(self):
# notify next request, if present, it can start writing
@@ -881,57 +885,29 @@
# incoming requests.
self._callLater(0, self._startNextRequest)
else:
- self.lingeringClose()
+ # Set an abort timer in case an orderly close hangs
+ self.setTimeout(None)
+ self._abortTimer = reactor.callLater(self.closeTimeOut, self._abortTimeout)
+ #reactor.callLater(0.1, self.transport.loseConnection)
+ self.transport.loseConnection()
def timeoutConnection(self):
#log.info("Timing out client: %s" % str(self.transport.getPeer()))
+ # Set an abort timer in case an orderly close hangs
+ self._abortTimer = reactor.callLater(self.closeTimeOut, self._abortTimeout)
policies.TimeoutMixin.timeoutConnection(self)
- def lingeringClose(self):
- """
- This is a bit complicated. This process is necessary to ensure proper
- workingness when HTTP pipelining is in use.
+ def _abortTimeout(self):
+ log.error("Connection aborted - took too long to close: {c}", c=str(self.transport.getPeer()))
+ self._abortTimer = None
+ self.transport.abortConnection()
- Here is what it wants to do:
-
- 1. Finish writing any buffered data, then close our write side.
- While doing so, read and discard any incoming data.
-
- 2. When that happens (writeConnectionLost called), wait up to 20
- seconds for the remote end to close their write side (our read
- side).
-
- 3.
- - If they do (readConnectionLost called), close the socket,
- and cancel the timeout.
-
- - If that doesn't happen, the timer fires, and makes the
- socket close anyways.
- """
-
- # Close write half
- self.transport.loseWriteConnection()
-
- # Throw out any incoming data
- self.dataReceived = self.lineReceived = lambda *args: None
- self.transport.resumeProducing()
-
- def writeConnectionLost(self):
- # Okay, all data has been written
- # In 20 seconds, actually close the socket
- self._lingerTimer = reactor.callLater(20, self._lingerClose)
- self._writeLost = True
-
- def _lingerClose(self):
- self._lingerTimer = None
- self.transport.loseConnection()
-
def readConnectionLost(self):
"""Read connection lost"""
# If in the lingering-close state, lose the socket.
- if self._lingerTimer:
- self._lingerTimer.cancel()
- self._lingerTimer = None
+ if self._abortTimer:
+ self._abortTimer.cancel()
+ self._abortTimer = None
self.transport.loseConnection()
return
Modified: CalendarServer/trunk/twext/web2/test/test_http.py
===================================================================
--- CalendarServer/trunk/twext/web2/test/test_http.py 2013-09-05 00:58:35 UTC (rev 11653)
+++ CalendarServer/trunk/twext/web2/test/test_http.py 2013-09-05 01:13:51 UTC (rev 11654)
@@ -319,6 +319,10 @@
self.loseConnection()
+ def abortConnection(self):
+ self.aborted = True
+
+
def getHost(self):
"""
Synthesize a slightly more realistic 'host' thing.
@@ -850,6 +854,42 @@
self.compareResult(cxn, cmds, data)
return deferLater(reactor, 0.5, self.assertDone, cxn) # Wait for timeout
+ def testTimeout_idleRequest(self):
+ cxn = self.connect(idleTimeOut=0.3)
+ cmds = [[]]
+ data = ""
+
+ cxn.client.write("GET / HTTP/1.1\r\n\r\n")
+ cmds[0] += [('init', 'GET', '/', (1, 1), 0, ()),
+ ('contentComplete',)]
+ self.compareResult(cxn, cmds, data)
+
+ return deferLater(reactor, 0.5, self.assertDone, cxn) # Wait for timeout
+
+ def testTimeout_abortRequest(self):
+ cxn = self.connect(allowPersistentConnections=False, closeTimeOut=0.3)
+ cxn.client.transport.loseConnection = lambda : None
+ cmds = [[]]
+ data = ""
+
+ cxn.client.write("GET / HTTP/1.1\r\n\r\n")
+ cmds[0] += [('init', 'GET', '/', (1, 1), 0, ()),
+ ('contentComplete',)]
+ self.compareResult(cxn, cmds, data)
+
+ response = TestResponse()
+ response.headers.setRawHeaders("Content-Length", ("0",))
+ cxn.requests[0].writeResponse(response)
+ response.finish()
+
+ data += "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+
+ self.compareResult(cxn, cmds, data)
+ def _check(cxn):
+ self.assertDone(cxn)
+ self.assertTrue(cxn.serverToClient.aborted)
+ return deferLater(reactor, 0.5, self.assertDone, cxn) # Wait for timeout
+
def testConnectionCloseRequested(self):
cxn = self.connect()
cmds = [[]]
@@ -883,6 +923,26 @@
self.compareResult(cxn, cmds, data)
self.assertDone(cxn)
+ def testConnectionKeepAliveOff(self):
+ cxn = self.connect(allowPersistentConnections=False)
+ cmds = [[]]
+ data = ""
+
+ cxn.client.write("GET / HTTP/1.1\r\n\r\n")
+ cmds[0] += [('init', 'GET', '/', (1, 1), 0, ()),
+ ('contentComplete',)]
+ self.compareResult(cxn, cmds, data)
+
+ response = TestResponse()
+ response.headers.setRawHeaders("Content-Length", ("0",))
+ cxn.requests[0].writeResponse(response)
+ response.finish()
+
+ data += "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+
+ self.compareResult(cxn, cmds, data)
+ self.assertDone(cxn)
+
def testExtraCRLFs(self):
cxn = self.connect()
cmds = [[]]
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-09-05 00:58:35 UTC (rev 11653)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-09-05 01:13:51 UTC (rev 11654)
@@ -825,7 +825,12 @@
# connections used per worker process.
"ListenBacklog": 2024,
- "IdleConnectionTimeOut": 15,
+
+ "IncomingDataTimeOut": 60, # Max. time between request lines
+ "PipelineIdleTimeOut": 15, # Max. time between pipelined requests
+ "IdleConnectionTimeOut": 60 * 6, # Max. time for response processing
+ "CloseConnectionTimeOut": 15, # Max. time for client close
+
"UIDReservationTimeOut": 30 * 60,
"MaxMultigetWithDataHrefs": 5000,
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130904/3d7fb6d8/attachment-0001.html>
More information about the calendarserver-changes
mailing list