[CalendarServer-changes] [5248] CalendarServer/trunk/calendarserver/tap

source_changes at macosforge.org source_changes at macosforge.org
Thu Mar 4 16:25:12 PST 2010


Revision: 5248
          http://trac.macosforge.org/projects/calendarserver/changeset/5248
Author:   glyph at apple.com
Date:     2010-03-04 16:25:12 -0800 (Thu, 04 Mar 2010)
Log Message:
-----------
Add a test case for long log lines being handled properly.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/caldav.py
    CalendarServer/trunk/calendarserver/tap/test/test_caldav.py

Added Paths:
-----------
    CalendarServer/trunk/calendarserver/tap/test/longlines.py

Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py	2010-03-04 20:54:20 UTC (rev 5247)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py	2010-03-05 00:25:12 UTC (rev 5248)
@@ -1076,7 +1076,7 @@
     def startProcess(self, name):
         if self.protocols.has_key(name):
             return
-        p = self.protocols[name] = procmon.LoggingProtocol()
+        p = self.protocols[name] = DelayedStartupLoggingProtocol()
         p.service = self
         p.name = name
         args, uid, gid, env = self.processes[name]
@@ -1096,6 +1096,74 @@
             childFDs=childFDs)
 
 
+class DelayedStartupLineLogger(object):
+    """
+    A line logger that can handle very long lines.
+    """
+
+    MAX_LENGTH = 80
+    tag = None
+    exceeded = False            # Am I in the middle of parsing a long line?
+    _buffer = ''
+
+    def makeConnection(self, transport):
+        """
+        Ignore this IProtocol method, since I don't need a transport.
+        """
+
+
+    def dataReceived(self, data):
+        lines = (self._buffer + data).split("\n")
+        while len(lines) > 1:
+            line = lines.pop(0)
+            if len(line) > self.MAX_LENGTH:
+                self.lineLengthExceeded(line)
+            elif self.exceeded:
+                self.lineLengthExceeded(line)
+                self.exceeded = False
+            else:
+                self.lineReceived(line)
+        lastLine = lines.pop(0)
+        if len(lastLine) > self.MAX_LENGTH:
+            self.lineLengthExceeded(lastLine)
+            self.exceeded = True
+            self._buffer = ''
+        else:
+            self._buffer = lastLine
+
+
+    def lineReceived(self, line):
+        from twisted.python.log import msg
+        msg('[%s] %s' % (self.tag, line))
+
+
+    def lineLengthExceeded(self, line):
+        """
+        A very long line is being received.  Log it immediately and forget
+        about buffering it.
+        """
+        for i in range(len(line)/self.MAX_LENGTH):
+            self.lineReceived(line[i*self.MAX_LENGTH:(i+1)*self.MAX_LENGTH]
+                              + " (truncated, continued)")
+
+
+
+class DelayedStartupLoggingProtocol(procmon.LoggingProtocol, object):
+    """
+    Logging protocol that handles lines which are too long.
+    """
+
+    def connectionMade(self):
+        """
+        Replace the superclass's output monitoring logic with one that can
+        handle lineLengthExceeded.
+        """
+        super(DelayedStartupLoggingProtocol, self).connectionMade()
+        self.output = DelayedStartupLineLogger()
+        self.output.tag = self.name
+
+
+
 def getSSLPassphrase(*ignored):
 
     if not config.SSLPrivateKey:

Added: CalendarServer/trunk/calendarserver/tap/test/longlines.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/longlines.py	                        (rev 0)
+++ CalendarServer/trunk/calendarserver/tap/test/longlines.py	2010-03-05 00:25:12 UTC (rev 5248)
@@ -0,0 +1,26 @@
+##
+# Copyright (c) 2007-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import sys
+
+length = int(sys.argv[1])
+
+data = (("x" * length) +
+        ("y" * length) + "\n" +
+        ("z" + "\n"))
+
+sys.stdout.write(data)
+sys.stdout.flush()

Modified: CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_caldav.py	2010-03-04 20:54:20 UTC (rev 5247)
+++ CalendarServer/trunk/calendarserver/tap/test/test_caldav.py	2010-03-05 00:25:12 UTC (rev 5248)
@@ -14,6 +14,7 @@
 # limitations under the License.
 ##
 
+import sys
 import os
 import stat
 import grp
@@ -22,16 +23,21 @@
 
 from twisted.trial.unittest import TestCase as BaseTestCase
 
+from twisted.python.threadable import isInIOThread
+from twisted.internet.reactor import callFromThread
 from twisted.python.usage import Options, UsageError
 from twisted.python.reflect import namedAny
+from twisted.python import log
 
 from twisted.internet.protocol import ServerFactory
+from twisted.internet.defer import Deferred
 
 from twisted.application.service import IService
 from twisted.application import internet
 
 from twext.web2.dav import auth
 from twext.web2.log import LogWrapperResource
+from twext.python.filepath import CachingFilePath as FilePath
 
 from twext.python.plistlib import writePlist
 from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
@@ -45,7 +51,9 @@
 from twistedcaldav.test.util import TestCase
 
 from calendarserver.tap.caldav import (CalDAVOptions, CalDAVServiceMaker,
-                                       CalDAVService, GroupOwnedUNIXServer)
+                                       CalDAVService, GroupOwnedUNIXServer,
+                                       DelayedStartupProcessMonitor,
+                                       DelayedStartupLineLogger)
 
 
 # Points to top of source tree.
@@ -775,3 +783,84 @@
         configuredDirectory = namedAny(self.config.DirectoryService.type)
 
         self.failUnless(isinstance(realDirectory, configuredDirectory))
+
+
+
+class DummyProcessObject(object):
+    """
+    Simple stub for the Process Object API that will run a test script.
+    """
+
+    def __init__(self, scriptname, *args):
+        self.scriptname = scriptname
+        self.args = list(args)
+
+
+    def getCommandLine(self):
+        """
+        Get the command line to invoke this script.
+        """
+        return [sys.executable, FilePath(__file__).sibling(self.scriptname).path] + self.args
+
+
+    def getName(self):
+        """
+        Get a dummy name.
+        """
+        return 'Dummy'
+
+
+
+class DelayedStartupProcessMonitorTests(TestCase):
+    """
+    Test cases for L{DelayedStartupProcessMonitor}.
+    """
+
+    def test_lineAfterLongLine(self):
+        """
+        A "long" line of output from a monitored process (longer than
+        L{LineReceiver.MAX_LENGTH}) should be logged in chunks rather than all
+        at once, to avoid resource exhaustion.
+        """
+        dspm = DelayedStartupProcessMonitor()
+        dspm.addProcessObject(DummyProcessObject(
+                'longlines.py', str(DelayedStartupLineLogger.MAX_LENGTH)),
+                          os.environ)
+        dspm.startService()
+        self.addCleanup(dspm.stopService)
+
+        logged = []
+
+        def tempObserver(event):
+            # Probably won't be a problem, but let's not have any intermittent
+            # test issues that stem from multi-threaded log messages randomly
+            # going off...
+            if not isInIOThread():
+                callFromThread(tempObserver, event)
+                return
+            if event.get('isError'):
+                d.errback()
+            m = event.get('message')[0]
+            if m.startswith('[Dummy] '):
+                logged.append(event)
+                if m == '[Dummy] z':
+                    d.callback("done")
+            
+        log.addObserver(tempObserver)
+        self.addCleanup(log.removeObserver, tempObserver)
+        d = Deferred()
+        def assertions(result):
+            self.assertEquals(["[Dummy] x",
+                               "[Dummy] y",
+                               "[Dummy] z"],
+                              [''.join(evt['message'])[:len('[Dummy]') + 2]
+                               for evt in logged])
+            self.assertEquals([" (truncated, continued)",
+                               " (truncated, continued)",
+                               "[Dummy] z"],
+                              [''.join(evt['message'])[-len(" (truncated, continued)"):]
+                               for evt in logged])
+        d.addCallback(assertions)
+        return d
+
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100304/3d6e5837/attachment-0001.html>


More information about the calendarserver-changes mailing list