[CalendarServer-changes] [4976] CalendarServer/branches/users/cdaboo/deployment-partition-4722
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jan 29 07:15:30 PST 2010
Revision: 4976
http://trac.macosforge.org/projects/calendarserver/changeset/4976
Author: cdaboo at apple.com
Date: 2010-01-29 07:15:26 -0800 (Fri, 29 Jan 2010)
Log Message:
-----------
Merged changes from main deployment branch.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/deployment-partition-4722/lib-patches/Twisted/twisted.web2.dav.resource.patch
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/accesslog.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/cluster.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/config.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/directory/resource.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/httpfactory.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcache.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcachepool.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/method/copymove.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/tap.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcache.py
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcachepool.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/deployment-partition-4722/bin/caldav_watch
CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/inspection.py
Property Changed:
----------------
CalendarServer/branches/users/cdaboo/deployment-partition-4722/
Property changes on: CalendarServer/branches/users/cdaboo/deployment-partition-4722
___________________________________________________________________
Modified: svn:ignore
- *.tgz
data
logs
build
+ *.tgz
data
logs
build
_run
Modified: svn:mergeinfo
- /CalendarServer/branches/users/sagen/deployment-inherit-fds-4571:4573-4709
+ /CalendarServer/branches/users/cdaboo/deployment-partition-4524:4525-4594
/CalendarServer/branches/users/cdaboo/deployment-partition-4593:4594-4723
/CalendarServer/branches/users/cdaboo/deployment-partition-4722:4723-4856
/CalendarServer/branches/users/sagen/deployment-inherit-fds-4571:4573-4709
/CalendarServer/branches/users/sagen/deployment-inspection:4927-4937
/CalendarServer/branches/users/wsanchez/deployment:4723-4969
/CalendarServer/trunk:3189,3861,3941
Copied: CalendarServer/branches/users/cdaboo/deployment-partition-4722/bin/caldav_watch (from rev 4969, CalendarServer/branches/users/wsanchez/deployment/bin/caldav_watch)
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/bin/caldav_watch (rev 0)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/bin/caldav_watch 2010-01-29 15:15:26 UTC (rev 4976)
@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import os, sys
+
+from twisted.internet import reactor
+from twisted.protocols.basic import LineReceiver
+from twisted.internet.protocol import ClientFactory
+from subprocess import Popen, PIPE, STDOUT
+import time, datetime
+
+INACTIVITY_SECONDS = 60
+
+class WatchProtocol(LineReceiver):
+ def lineReceived(self, line):
+ self.history.append(line)
+ if len(self.history) > 40:
+ del self.history[0]
+
+ if self.later.active():
+ self.later.reset(INACTIVITY_SECONDS)
+
+ def inactivity(self):
+ print "Server process %d inactive for over %d seconds" % (self.id, INACTIVITY_SECONDS)
+ print "Inspection history:"
+ for line in self.history:
+ print line
+ pid = pidForID(self.id)
+ if pid is None:
+ print "Error determining PID from ID"
+ else:
+ print "Server process for ID %d is %d" % (self.id, pid)
+ dirname = capture(pid, self.history)
+ print "Data captured to:", dirname
+
+ self.later = reactor.callLater(INACTIVITY_SECONDS, self.inactivity)
+ print "--------------------------------------------------------------"
+
+class WatchFactory(ClientFactory):
+ protocol = WatchProtocol
+
+ def __init__(self, id):
+ self.id = id
+
+ def buildProtocol(self, addr):
+ p = ClientFactory.buildProtocol(self, addr)
+ p.id = self.id
+ p.history = []
+ p.later = reactor.callLater(INACTIVITY_SECONDS, p.inactivity)
+ return p
+
+
+def pidForID(id):
+ child = Popen(args=['/bin/ps', 'axo', 'pid,command'],
+ stdout=PIPE, stderr=STDOUT)
+ try:
+ output, error = child.communicate()
+ except IOError, e:
+ print "Couldn't determine PID of server process", e
+ return None
+ for line in output.split("\n"):
+ if line:
+ pid, command = line.split(" ", 1)
+ if "caldavd" in command and " LogID=%d " % (id,) in command:
+ return int(pid)
+ return None
+
+def sample(pid, filename):
+ child = Popen(args=['/usr/bin/sample', str(pid), "-file", filename],
+ stdout=PIPE, stderr=STDOUT)
+ output, error = child.communicate()
+
+def dtruss(pid, filename):
+ output = open(filename, "w")
+ try:
+ child = Popen(args=['/usr/bin/dtruss', '-p', str(pid)],
+ stdout=output, stderr=STDOUT)
+ time.sleep(5)
+ child.terminate()
+ except OSError, e:
+ print "Couldn't run dtruss:", e
+ finally:
+ output.close()
+
+def capture(pid, history):
+ now = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+ dirname = os.path.join("/tmp", "caldav_watch_%s_%d" % (now, pid))
+ os.mkdir(dirname)
+ historyname = os.path.join(dirname, "history.out")
+ historyfile = open(historyname, "w")
+ for line in history:
+ line = line + "\n"
+ historyfile.write(line)
+ historyfile.close()
+ samplename = os.path.join(dirname, "sample.out")
+ sample(pid, samplename)
+ dtrussname = os.path.join(dirname, "dtruss.out")
+ dtruss(pid, dtrussname)
+ return dirname
+
+def main():
+ for id in range(0, 75):
+ port = id + 10000
+ factory = WatchFactory(id)
+ reactor.connectTCP("localhost", port, factory)
+ print "Monitoring..."
+ reactor.run()
+
+main()
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/lib-patches/Twisted/twisted.web2.dav.resource.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/lib-patches/Twisted/twisted.web2.dav.resource.patch 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/lib-patches/Twisted/twisted.web2.dav.resource.patch 2010-01-29 15:15:26 UTC (rev 4976)
@@ -676,7 +676,7 @@
if isinstance(principal, davxml.HRef):
yield principal
-@@ -1517,6 +1611,270 @@
+@@ -1517,6 +1611,272 @@
return None
##
@@ -885,6 +885,8 @@
+ d = waitForDeferred(self.updateQuotaUse(request, adjust))
+ yield d
+ d.getResult()
++ yield None
++ return
+
+ # Check the next parent
+ url = request.urlForResource(self)
@@ -947,7 +949,7 @@
# HTTP
##
-@@ -1525,15 +1883,11 @@
+@@ -1525,15 +1885,11 @@
#litmus = request.headers.getRawHeaders("x-litmus")
#if litmus: log.msg("*** Litmus test: %s ***" % (litmus,))
@@ -965,7 +967,7 @@
def setHeaders(response):
response = IResponse(response)
-@@ -1567,7 +1921,7 @@
+@@ -1567,7 +1923,7 @@
def findChildren(self, depth, request, callback, privileges=None, inherited_aces=None):
return succeed(None)
@@ -974,7 +976,7 @@
"""
Resource representing a WebDAV principal. (RFC 3744, section 2)
"""
-@@ -1577,7 +1931,7 @@
+@@ -1577,7 +1933,7 @@
# WebDAV
##
@@ -983,7 +985,7 @@
(dav_namespace, "alternate-URI-set"),
(dav_namespace, "principal-URL" ),
(dav_namespace, "group-member-set" ),
-@@ -1585,14 +1939,11 @@
+@@ -1585,14 +1941,11 @@
)
def davComplianceClasses(self):
@@ -999,7 +1001,7 @@
def readProperty(self, property, request):
def defer():
if type(property) is tuple:
-@@ -1610,10 +1961,20 @@
+@@ -1610,10 +1963,20 @@
return davxml.PrincipalURL(davxml.HRef(self.principalURL()))
if name == "group-member-set":
@@ -1022,7 +1024,7 @@
if name == "resourcetype":
if self.isCollection():
-@@ -1655,7 +2016,7 @@
+@@ -1655,7 +2018,7 @@
principals. Subclasses should override this method to provide member
URLs for this resource if appropriate.
"""
@@ -1031,7 +1033,7 @@
def groupMemberships(self):
"""
-@@ -1666,6 +2027,7 @@
+@@ -1666,6 +2029,7 @@
"""
unimplemented(self)
@@ -1039,7 +1041,7 @@
def principalMatch(self, href):
"""
Check whether the supplied principal matches this principal or is a
-@@ -1675,10 +2037,33 @@
+@@ -1675,10 +2039,33 @@
"""
uri = str(href)
if self.principalURL() == uri:
@@ -1075,7 +1077,7 @@
class AccessDeniedError(Exception):
def __init__(self, errors):
"""
-@@ -1718,6 +2103,37 @@
+@@ -1718,6 +2105,37 @@
davxml.registerElement(TwistedACLInheritable)
davxml.ACE.allowed_children[(twisted_dav_namespace, "inheritable")] = (0, 1)
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/accesslog.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/accesslog.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/accesslog.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -64,6 +64,10 @@
request = eventDict['request']
response = eventDict['response']
loginfo = eventDict['loginfo']
+
+ channel = request.chanRequest.channel
+ if config.EnableInspection and channel._inspection:
+ channel._inspection.add("access_log")
# Try to determine authentication and authorization identifiers
uid = "-"
@@ -137,6 +141,15 @@
if " " in v:
v = '"%s"' % (v,)
formats.append("%s=%s" % (k, v))
+
+ fwdHeaders = request.headers.getRawHeaders("x-forwarded-for", "")
+ if fwdHeaders:
+ forwardedFor = ",".join(fwdHeaders)
+ forwardedFor = forwardedFor.replace(" ", "")
+ formats.append("fwd=%(fwd)s")
+ else:
+ forwardedFor = ""
+
format = " ".join(formats)
formatArgs = {
@@ -153,6 +166,8 @@
"serverInstance" : serverInstance,
"timeSpent" : (time.time() - request.initTime) * 1000,
"outstandingRequests" : request.chanRequest.channel.factory.outstandingRequests,
+ "outstandingRequests" : request.chanRequest.channel.factory.outstandingRequests,
+ "fwd" : forwardedFor,
}
self.logMessage(format % formatArgs)
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/cluster.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/cluster.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/cluster.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -117,6 +117,7 @@
'-o', 'PIDFile=None',
'-o', 'ErrorLogFile=None',
'-o', 'LogID=%s' % (self.id,),
+ '-o', 'InspectionPort=%s' % (config.BaseInspectionPort + self.id,),
'-o', 'MultiProcess/ProcessCount=%d' % (
config.MultiProcess['ProcessCount'],)])
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/config.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/config.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/config.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -217,6 +217,9 @@
"DefaultLogLevel" : "",
"LogLevels" : {},
"LogID" : "",
+ "EnableInspection" : False,
+ "BaseInspectionPort" : 10000,
+ "InspectionPort" : "",
"AccountingCategories": {
"iTIP": False,
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/directory/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/directory/resource.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/directory/resource.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -60,7 +60,10 @@
name = segments[0]
if name != "":
- child = self.provisionChild(name)
+ # If getChild() finds a child resource, return it
+ child = self.getChild(name)
+ if child is None:
+ child = self.provisionChild(name)
if child:
returnValue((child, segments[1:],))
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/httpfactory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/httpfactory.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/httpfactory.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -19,8 +19,11 @@
from twisted.internet import protocol
from twisted.python import log
from twisted.web2.channel.http import HTTPFactory, HTTPChannel
+from twisted.web2.server import Request, Site
+from twistedcaldav.inspection import Inspector
from twistedcaldav.config import config
+from twisted.web2 import iweb
__all__ = ['HTTP503LoggingFactory', ]
@@ -52,11 +55,54 @@
self.transport.loseConnection()
+class InspectableHTTPChannel(HTTPChannel):
+
+ _inspection = None
+
+ def connectionMade(self):
+ if self._inspection:
+ self._inspection.add("conn_made")
+
+ return super(InspectableHTTPChannel, self).connectionMade()
+
+ def connectionLost(self, reason):
+ if self._inspection:
+ self._inspection.add("conn_lost")
+ self._inspection.complete()
+
+ return super(InspectableHTTPChannel, self).connectionLost(reason)
+
+ _firstChunkReceived = False
+
+ def dataReceived(self, data):
+ if not self._firstChunkReceived:
+ self._firstChunkReceived = True
+ if self._inspection:
+ self._inspection.add("first_byte")
+
+ return super(InspectableHTTPChannel, self).dataReceived(data)
+
+ def requestReadFinished(self, request):
+ if self._inspection:
+ self._inspection.add("body_received")
+
+ return super(InspectableHTTPChannel, self).requestReadFinished(request)
+
+ def requestWriteFinished(self, request):
+ if self._inspection:
+ self._inspection.add("resp_finish")
+
+ return super(InspectableHTTPChannel, self).requestWriteFinished(request)
+
+
class HTTP503LoggingFactory(HTTPFactory):
"""Factory for HTTP server."""
+ protocol = InspectableHTTPChannel
+
def __init__(self, requestFactory, maxRequests=600, **kwargs):
HTTPFactory.__init__(self, requestFactory, maxRequests, **kwargs)
+ self.channelCounter = 0
def buildProtocol(self, addr):
if self.outstandingRequests >= self.maxRequests:
@@ -66,24 +112,33 @@
for arg,value in self.protocolArgs.iteritems():
setattr(p, arg, value)
+
+ self.channelCounter += 1
+ if config.EnableInspection:
+ p._inspection = Inspector.getInspection(self.channelCounter)
+
return p
-class LimitingHTTPChannel(HTTPChannel):
+class LimitingHTTPChannel(InspectableHTTPChannel):
def connectionMade(self):
- HTTPChannel.connectionMade(self)
+ retVal = super(LimitingHTTPChannel, self).connectionMade()
if self.factory.outstandingRequests >= self.factory.maxRequests:
# log.msg("Overloaded")
self.factory.myServer.myPort.stopReading()
+ return retVal
def connectionLost(self, reason):
- HTTPChannel.connectionLost(self, reason)
+ retVal = super(LimitingHTTPChannel, self).connectionLost(reason)
if self.factory.outstandingRequests < self.factory.resumeRequests:
# log.msg("Resuming")
self.factory.myServer.myPort.startReading()
+ return retVal
+
class LimitingHTTPFactory(HTTPFactory):
+
protocol = LimitingHTTPChannel
def __init__(self, requestFactory, maxRequests=600, maxAccepts=100, resumeRequests=550,
@@ -91,10 +146,40 @@
HTTPFactory.__init__(self, requestFactory, maxRequests, **kwargs)
self.maxAccepts = maxAccepts
self.resumeRequests = resumeRequests
+ self.channelCounter = 0
def buildProtocol(self, addr):
p = protocol.ServerFactory.buildProtocol(self, addr)
for arg, value in self.protocolArgs.iteritems():
setattr(p, arg, value)
+
+ self.channelCounter += 1
+ if config.EnableInspection:
+ p._inspection = Inspector.getInspection(self.channelCounter)
+
return p
+
+class LimitingRequest(Request):
+
+ def __init__(self, *args, **kwargs):
+ Request.__init__(self, *args, **kwargs)
+ self.extendedLogItems = {}
+ channel = self.chanRequest.channel
+ if config.EnableInspection and channel._inspection:
+ channel._inspection.add("headers_received")
+ self.extendedLogItems['insp'] = channel._inspection.id
+
+ def writeResponse(self, response):
+ channel = self.chanRequest.channel
+ if config.EnableInspection and channel._inspection:
+ channel._inspection.add("resp_start")
+
+ Request.writeResponse(self, response)
+
+
+class LimitingSite(Site):
+
+ def __call__(self, *args, **kwargs):
+ return LimitingRequest(site=self, *args, **kwargs)
+
Copied: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/inspection.py (from rev 4969, CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/inspection.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/inspection.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/inspection.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -0,0 +1,125 @@
+##
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Inspection framework for Calendar Server
+"""
+
+from twisted.internet.protocol import ServerFactory
+from twisted.protocols.basic import LineReceiver
+from twistedcaldav.log import LoggingMixIn
+import datetime
+import time
+
+__all__ = [
+ "Inspector",
+ "InspectionFactory",
+]
+
+
+class Inspector(object):
+
+ _inspector = None
+
+ @classmethod
+ def getInspector(klass):
+ if klass._inspector is None:
+ klass._inspector = klass()
+ return klass._inspector
+
+ @classmethod
+ def getInspection(klass, id, emit=True):
+ if klass.isInspecting():
+ return Inspection(id, emit=emit)
+ else:
+ return None
+
+ @classmethod
+ def isInspecting(klass):
+ return klass._inspector is not None and len(klass._inspector.observers) > 0
+
+ def __init__(self):
+ self.observers = set()
+
+ def addObserver(self, observer):
+ self.observers.add(observer)
+
+ def removeObserver(self, observer):
+ try:
+ self.observers.remove(observer)
+ except KeyError:
+ pass
+
+ def hasObservers(self):
+ return len(self.observers) > 0
+
+ def emit(self, msg):
+ if self.observers:
+ now = datetime.datetime.now()
+ msg = "%s | %s" % (now, msg)
+ for observer in self.observers:
+ observer.emit(msg)
+
+
+class Inspection(object):
+
+ def __init__(self, id, event="init", emit=True):
+ self.id = id
+ self.timeline = []
+ self.add(event, emit=emit)
+
+ def add(self, event, emit=True):
+ self.timeline.append((time.time(), event))
+ if emit:
+ if len(self.timeline) > 1:
+ Inspector.getInspector().emit("%d | %s=%.3f" %
+ (self.id, event, self.timeline[-1][0] - self.timeline[-2][0]))
+ else:
+ Inspector.getInspector().emit("%d | %s" % (self.id, event))
+
+ def complete(self):
+ timestrings = []
+ starttime, event = self.timeline[0]
+ basetime = starttime
+ for timestamp, event in self.timeline[1:]:
+ delta = timestamp - basetime
+ timestrings.append("%s=%.3f" % (event, delta))
+ basetime = timestamp
+ Inspector.getInspector().emit("%d | duration=%.3f | %s" %
+ (self.id, timestamp - starttime, " ".join(timestrings)))
+
+class InspectionProtocol(LineReceiver, LoggingMixIn):
+
+ def connectionMade(self):
+ Inspector.getInspector().addObserver(self)
+
+ def connectionLost(self, reason):
+ Inspector.getInspector().removeObserver(self)
+
+ def lineReceived(self, line):
+ line = line.strip()
+ if line == "ping":
+ self.sendLine("pong")
+ else:
+ self.sendLine("???")
+
+ def emit(self, msg):
+ self.sendLine(msg)
+
+
+class InspectionFactory(ServerFactory):
+
+ protocol = InspectionProtocol
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcache.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcache.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcache.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -34,7 +34,6 @@
from twisted.protocols.basic import LineReceiver
-from twisted.protocols.policies import TimeoutMixin
from twisted.internet.defer import Deferred, fail, TimeoutError
from twisted.python import log
@@ -109,13 +108,10 @@
-class MemCacheProtocol(LineReceiver, TimeoutMixin):
+class MemCacheProtocol(LineReceiver):
"""
MemCache protocol: connect to a memcached server to store/retrieve values.
- @ivar persistentTimeOut: the timeout period used to wait for a response.
- @type persistentTimeOut: C{int}
-
@ivar _current: current list of requests waiting for an answer from the
server.
@type _current: C{deque} of L{Command}
@@ -133,44 +129,20 @@
"""
MAX_KEY_LENGTH = 250
- def __init__(self, timeOut=60):
+ def __init__(self):
"""
Create the protocol.
-
- @param timeOut: the timeout to wait before detecting that the
- connection is dead and close it. It's expressed in seconds.
- @type timeOut: C{int}
"""
self._current = deque()
self._lenExpected = None
self._getBuffer = None
self._bufferLength = None
- self.persistentTimeOut = self.timeOut = timeOut
- def timeoutConnection(self):
- """
- Close the connection in case of timeout.
- """
- for cmd in self._current:
- cmd.fail(TimeoutError("Connection timeout"))
- self.transport.loseConnection()
-
-
- def sendLine(self, line):
- """
- Override sendLine to add a timeout to response.
- """
- if not self._current:
- self.setTimeout(self.persistentTimeOut)
- LineReceiver.sendLine(self, line)
-
-
def rawDataReceived(self, data):
"""
Collect data for a get.
"""
- self.resetTimeout()
self._getBuffer.append(data)
self._bufferLength += len(data)
if self._bufferLength >= self._lenExpected + 2:
@@ -310,7 +282,6 @@
"""
Receive line commands from the server.
"""
- self.resetTimeout()
token = line.split(" ", 1)[0]
# First manage standard commands without space
cmd = getattr(self, "cmd_%s" % (token,), None)
@@ -331,9 +302,6 @@
cmd = self._current.popleft()
val = int(line)
cmd.success(val)
- if not self._current:
- # No pending request, remove timeout
- self.setTimeout(None)
def increment(self, key, val=1):
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcachepool.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcachepool.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/memcachepool.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -17,7 +17,8 @@
from twisted.python.failure import Failure
from twisted.internet.address import IPv4Address
from twisted.internet.defer import Deferred, fail
-from twisted.internet.protocol import ReconnectingClientFactory
+from twisted.internet.error import ConnectionLost, ConnectionDone
+from twisted.internet.protocol import ClientFactory
from twistedcaldav.log import LoggingMixIn
from twistedcaldav.memcache import MemCacheProtocol, NoSuchCommand
@@ -44,7 +45,7 @@
-class MemCacheClientFactory(ReconnectingClientFactory, LoggingMixIn):
+class MemCacheClientFactory(ClientFactory, LoggingMixIn):
"""
A client factory for MemCache that reconnects and notifies a pool of it's
state.
@@ -71,12 +72,6 @@
if self._protocolInstance is not None:
self.connectionPool.clientBusy(self._protocolInstance)
- ReconnectingClientFactory.clientConnectionLost(
- self,
- connector,
- reason)
-
-
def clientConnectionFailed(self, connector, reason):
"""
Notify the connectionPool that we're unable to connect
@@ -85,12 +80,6 @@
if self._protocolInstance is not None:
self.connectionPool.clientBusy(self._protocolInstance)
- ReconnectingClientFactory.clientConnectionFailed(
- self,
- connector,
- reason)
-
-
def buildProtocol(self, addr):
"""
Attach the C{self.connectionPool} to the protocol so it can tell it,
@@ -197,6 +186,12 @@
self.clientFree(client)
return result
+ def _freeClientAfterError(error):
+ if not error.check(ConnectionLost, ConnectionDone):
+ self.clientFree(client)
+ return error
+
+
self.clientBusy(client)
method = getattr(client, command, None)
if method is not None:
@@ -204,7 +199,7 @@
else:
d = fail(Failure(NoSuchCommand()))
- d.addCallback(_freeClientAfterRequest)
+ d.addCallbacks(_freeClientAfterRequest, _freeClientAfterError)
return d
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/method/copymove.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/method/copymove.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/method/copymove.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -28,9 +28,10 @@
from twisted.web2.dav import davxml
from twisted.web2.dav.http import ErrorResponse
from twisted.web2.dav.util import parentForURL
-from twisted.web2.http import StatusResponse, HTTPError
+from twisted.web2.http import StatusResponse, HTTPError, splitHostPort
from twistedcaldav.caldavxml import caldav_namespace
+from twistedcaldav.config import config
from twistedcaldav.method.put_common import storeCalendarObjectResource
from twistedcaldav.resource import isCalendarCollectionResource
from twistedcaldav.log import Logger
@@ -252,6 +253,27 @@
log.err(msg)
raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, msg))
+ # Make sure it is a valid resource URI, but strip off the scheme:host details
+ (scheme, host, path, query, fragment) = urlsplit(destination_uri)
+ if query or fragment:
+ raise HTTPError(StatusResponse(
+ responsecode.BAD_REQUEST,
+ "URL may not contain a query or fragment: %s" % (destination_uri,)
+ ))
+ if scheme and scheme not in ("http", "https",):
+ raise HTTPError(StatusResponse(
+ responsecode.BAD_GATEWAY,
+ "URL is not on this site (%s://%s/): %s" % (request.scheme, request.headers.getHeader("host"), destination_uri)
+ ))
+ elif host:
+ host, port = splitHostPort(scheme, host)
+ if host != config.ServerHostName or port not in (config.HTTPPort, config.SSLPort,):
+ raise HTTPError(StatusResponse(
+ responsecode.BAD_GATEWAY,
+ "URL is not on this site (%s://%s/): %s" % (request.scheme, request.headers.getHeader("host"), destination_uri)
+ ))
+ destination_uri = path
+
destination = waitForDeferred(request.locateResource(destination_uri))
yield destination
destination = destination.getResult()
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/tap.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/tap.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/tap.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -39,7 +39,6 @@
from twisted.web2.dav import auth
from twisted.web2.auth.basic import BasicCredentialFactory
-from twisted.web2.server import Site
from twistedcaldav.log import Logger, logLevelForNamespace, setLogLevelForNamespace
from twistedcaldav.accesslog import DirectoryLogWrapperResource
@@ -55,7 +54,8 @@
from twistedcaldav.directory.principal import DirectoryPrincipalProvisioningResource
from twistedcaldav.directory.aggregate import AggregateDirectoryService
from twistedcaldav.directory.sudo import SudoDirectoryService
-from twistedcaldav.httpfactory import HTTP503LoggingFactory, LimitingHTTPFactory
+from twistedcaldav.httpfactory import HTTP503LoggingFactory, LimitingHTTPFactory, LimitingSite
+from twistedcaldav.inspection import InspectionFactory
from twistedcaldav.static import CalendarHomeProvisioningFile
from twistedcaldav.static import IScheduleInboxFile
from twistedcaldav.static import TimezoneServiceFile
@@ -763,7 +763,7 @@
service = CalDAVService(logObserver)
- site = Site(realRoot)
+ site = LimitingSite(realRoot)
# If inheriting file descriptors from the master, use those to handle
@@ -880,12 +880,21 @@
# Register USR1 handler
def sigusr1_handler(num, frame):
- log.debug("SIGUSR1 recieved, triggering directory refresh")
- baseDirectory.refresh()
+ from twisted.internet import reactor
+ log.debug("SIGUSR1 received, triggering directory refresh")
+ reactor.callLater(0, baseDirectory.refresh)
return
signal.signal(signal.SIGUSR1, sigusr1_handler)
+ if config.EnableInspection:
+ inspector = internet.TCPServer(
+ int(config.InspectionPort) if config.InspectionPort else config.BaseInspectionPort,
+ InspectionFactory(),
+ interface="127.0.0.1"
+ )
+ inspector.setServiceParent(service)
+
return service
makeService_Combined = makeService_Combined
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcache.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcache.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcache.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -11,7 +11,7 @@
from twisted.test.proto_helpers import StringTransportWithDisconnection
from twisted.internet.task import Clock
-from twisted.internet.defer import Deferred, gatherResults, TimeoutError
+from twisted.internet.defer import Deferred, gatherResults
class MemCacheTestCase(TestCase):
"""
@@ -221,119 +221,7 @@
"VALUE bar 0 %s\r\n%s\r\nEND\r\n" % (len(s), s))
- def test_timeOut(self):
- """
- Test the timeout on outgoing requests: when timeout is detected, all
- current commands should fail with a L{TimeoutError}, and the
- connection should be closed.
- """
- d1 = self.proto.get("foo")
- d2 = self.proto.get("bar")
- d3 = Deferred()
- self.proto.connectionLost = d3.callback
- self.clock.advance(self.proto.persistentTimeOut)
- self.assertFailure(d1, TimeoutError)
- self.assertFailure(d2, TimeoutError)
- def checkMessage(error):
- self.assertEquals(str(error), "Connection timeout")
- d1.addCallback(checkMessage)
- return gatherResults([d1, d2, d3])
-
-
- def test_timeoutRemoved(self):
- """
- When a request gets a response, no pending timeout call should remain
- around.
- """
- d = self.proto.get("foo")
-
- self.clock.advance(self.proto.persistentTimeOut - 1)
- self.proto.dataReceived("VALUE foo 0 3\r\nbar\r\nEND\r\n")
-
- def check(result):
- self.assertEquals(result, (0, "bar"))
- self.assertEquals(len(self.clock.calls), 0)
- d.addCallback(check)
- return d
-
-
- def test_timeOutRaw(self):
- """
- Test the timeout when raw mode was started: the timeout should not be
- reset until all the data has been received, so we can have a
- L{TimeoutError} when waiting for raw data.
- """
- d1 = self.proto.get("foo")
- d2 = Deferred()
- self.proto.connectionLost = d2.callback
-
- self.proto.dataReceived("VALUE foo 0 10\r\n12345")
- self.clock.advance(self.proto.persistentTimeOut)
- self.assertFailure(d1, TimeoutError)
- return gatherResults([d1, d2])
-
-
- def test_timeOutStat(self):
- """
- Test the timeout when stat command has started: the timeout should not
- be reset until the final B{END} is received.
- """
- d1 = self.proto.stats()
- d2 = Deferred()
- self.proto.connectionLost = d2.callback
-
- self.proto.dataReceived("STAT foo bar\r\n")
- self.clock.advance(self.proto.persistentTimeOut)
- self.assertFailure(d1, TimeoutError)
- return gatherResults([d1, d2])
-
-
- def test_timeoutPipelining(self):
- """
- When two requests are sent, a timeout call should remain around for the
- second request, and its timeout time should be correct.
- """
- d1 = self.proto.get("foo")
- d2 = self.proto.get("bar")
- d3 = Deferred()
- self.proto.connectionLost = d3.callback
-
- self.clock.advance(self.proto.persistentTimeOut - 1)
- self.proto.dataReceived("VALUE foo 0 3\r\nbar\r\nEND\r\n")
-
- def check(result):
- self.assertEquals(result, (0, "bar"))
- self.assertEquals(len(self.clock.calls), 1)
- for i in range(self.proto.persistentTimeOut):
- self.clock.advance(1)
- return self.assertFailure(d2, TimeoutError).addCallback(checkTime)
- def checkTime(ignored):
- # Check that the timeout happened C{self.proto.persistentTimeOut}
- # after the last response
- self.assertEquals(self.clock.seconds(),
- 2 * self.proto.persistentTimeOut - 1)
- d1.addCallback(check)
- return d1
-
-
- def test_timeoutNotReset(self):
- """
- Check that timeout is not resetted for every command, but keep the
- timeout from the first command without response.
- """
- d1 = self.proto.get("foo")
- d3 = Deferred()
- self.proto.connectionLost = d3.callback
-
- self.clock.advance(self.proto.persistentTimeOut - 1)
- d2 = self.proto.get("bar")
- self.clock.advance(1)
- self.assertFailure(d1, TimeoutError)
- self.assertFailure(d2, TimeoutError)
- return gatherResults([d1, d2, d3])
-
-
def test_tooLongKey(self):
"""
Test that an error is raised when trying to use a too long key: the
Modified: CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcachepool.py
===================================================================
--- CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcachepool.py 2010-01-29 01:20:59 UTC (rev 4975)
+++ CalendarServer/branches/users/cdaboo/deployment-partition-4722/twistedcaldav/test/test_memcachepool.py 2010-01-29 15:15:26 UTC (rev 4976)
@@ -175,12 +175,6 @@
[('gone', self.protocol)])
- def tearDown(self):
- """
- Make sure the L{MemCacheClientFactory} isn't trying to reconnect
- anymore.
- """
- self.factory.stopTrying()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100129/0bc7b774/attachment-0001.html>
More information about the calendarserver-changes
mailing list