[CalendarServer-changes] [5243] CalendarServer/trunk/twext/python

source_changes at macosforge.org source_changes at macosforge.org
Thu Mar 4 10:44:23 PST 2010


Revision: 5243
          http://trac.macosforge.org/projects/calendarserver/changeset/5243
Author:   glyph at apple.com
Date:     2010-03-04 10:44:23 -0800 (Thu, 04 Mar 2010)
Log Message:
-----------
Add test coverage, and a workaround for EINVAL exceptions, to `CachingFilePath`.

Modified Paths:
--------------
    CalendarServer/trunk/twext/python/filepath.py

Added Paths:
-----------
    CalendarServer/trunk/twext/python/test/test_filepath.py

Modified: CalendarServer/trunk/twext/python/filepath.py
===================================================================
--- CalendarServer/trunk/twext/python/filepath.py	2010-03-04 17:41:20 UTC (rev 5242)
+++ CalendarServer/trunk/twext/python/filepath.py	2010-03-04 18:44:23 UTC (rev 5243)
@@ -1,3 +1,4 @@
+# -*- test-case-name: twext.python.test.test_filepath -*-
 ##
 # Copyright (c) 2010 Apple Inc. All rights reserved.
 #
@@ -19,6 +20,17 @@
 calendar server.
 """
 
+from os import listdir as _listdir
+
+from os.path import (join as _joinpath,
+                     basename as _basename,
+                     exists as _exists,
+                     dirname as _dirname)
+
+from time import sleep as _sleep
+from types import FunctionType, MethodType
+from errno import EINVAL
+
 from twisted.python.filepath import FilePath
 
 from stat import S_ISDIR
@@ -29,12 +41,33 @@
     aggressive caching policy.
     """
 
+    _listdir = _listdir         # integration points for tests
+    _sleep = _sleep
+
+    BACKOFF_MAX = 5.0           # Maximum time to wait between calls to
+                                # listdir()
+
     def __init__(self, path, alwaysCreate=False):
         super(CachingFilePath, self).__init__(path, alwaysCreate)
         self.existsCached = None
         self.isDirCached = None
 
 
+    @property
+    def siblingExtensionSearch(self):
+        """
+        Dynamically create a version of L{FilePath.siblingExtensionSearch} that
+        uses a pluggable 'listdir' implementation.
+        """
+        return MethodType(FunctionType(
+                FilePath.siblingExtensionSearch.im_func.func_code,
+                {'listdir': self._retryListdir,
+                 'basename': _basename,
+                 'dirname': _dirname,
+                 'joinpath': _joinpath,
+                 'exists': _exists}), self, self.__class__)
+
+
     def changed(self):
         """
         This path may have changed in the filesystem, so forget all cached
@@ -45,6 +78,32 @@
         self.isDirCached = None
 
 
+    def _retryListdir(self, pathname):
+        """
+        Implementation of retry logic for C{listdir} and
+        C{siblingExtensionSearch}.
+        """
+        delay = 0.1
+        while True:
+            try:
+                return self._listdir(pathname)
+            except OSError, e:
+                if e.errno == EINVAL:
+                    self._sleep(delay)
+                    delay = min(self.BACKOFF_MAX, delay * 2.0)
+                else:
+                    raise
+        raise RuntimeError("unreachable code.")
+
+
+    def listdir(self):
+        """
+        List the directory which C{self.path} points to, compensating for
+        EINVAL from C{os.listdir}.
+        """
+        return self._retryListdir(self.path)
+
+
     def restat(self, reraise=True):
         """
         Re-cache stat information.

Added: CalendarServer/trunk/twext/python/test/test_filepath.py
===================================================================
--- CalendarServer/trunk/twext/python/test/test_filepath.py	                        (rev 0)
+++ CalendarServer/trunk/twext/python/test/test_filepath.py	2010-03-04 18:44:23 UTC (rev 5243)
@@ -0,0 +1,160 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Tests for specialized behavior of L{CachingFilePath}
+"""
+from errno import EINVAL
+from os.path import join as pathjoin
+
+from twisted.internet.task import Clock
+
+from twisted.trial.unittest import TestCase
+
+from twext.python.filepath import CachingFilePath
+
+# Cheat and pull in the Twisted test cases for FilePath.  XXX: Twisteds should
+# provide a supported way of doing this for exported interfaces.  Also, it
+# should export IFilePath. --glyph
+
+from twisted.test.test_paths import FilePathTestCase
+
+class BaseVerification(FilePathTestCase):
+    """
+    Make sure that L{CachingFilePath} doesn't break the contracts that
+    L{FilePath} tries to provide.
+    """
+
+    def setUp(self):
+        """
+        Set up the test case to set the base attributes to point at
+        L{AbstractFilePathTestCase}.
+        """
+        FilePathTestCase.setUp(self)
+        self.root = CachingFilePath(self.root.path)
+        self.path = CachingFilePath(self.path.path)
+
+
+
+class EINVALTestCase(TestCase):
+    """
+    Sometimes, L{os.listdir} will raise C{EINVAL}.  This is a transient error,
+    and L{CachingFilePath.listdir} should work around it by retrying the
+    C{listdir} operation until it succeeds.
+    """
+
+    def setUp(self):
+        """
+        Create a L{CachingFilePath} for the test to use.
+        """
+        self.cfp = CachingFilePath(self.mktemp())
+        self.clock = Clock()
+        self.cfp._sleep = self.clock.advance
+
+
+    def test_testValidity(self):
+        """
+        If C{listdir} is replaced on a L{CachingFilePath}, we should be able to
+        observe exceptions raised by the replacement.  This verifies that the
+        test patching done here is actually testing something.
+        """
+        class CustomException(Exception): "Just for testing."
+        def blowUp(dirname):
+            raise CustomException()
+        self.cfp._listdir = blowUp
+        self.assertRaises(CustomException, self.cfp.listdir)
+        self.assertRaises(CustomException, self.cfp.children)
+
+
+    def test_retryLoop(self):
+        """
+        L{CachingFilePath} should catch C{EINVAL} and respond by retrying the
+        C{listdir} operation until it succeeds.
+        """
+        calls = []
+        def raiseEINVAL(dirname):
+            calls.append(dirname)
+            if len(calls) < 5:
+                raise OSError(EINVAL, "This should be caught by the test.")
+            return ['a', 'b', 'c']
+        self.cfp._listdir = raiseEINVAL
+        self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
+        self.assertEquals(self.cfp.children(), [
+                CachingFilePath(pathjoin(self.cfp.path, 'a')),
+                CachingFilePath(pathjoin(self.cfp.path, 'b')),
+                CachingFilePath(pathjoin(self.cfp.path, 'c')),])
+
+
+    def requireTimePassed(self, filenames):
+        """
+        Create a replacement for listdir() which only fires after a certain
+        amount of time.
+        """
+        self.calls = []
+        def thunk(dirname):
+            now = self.clock.seconds()
+            if now < 20.0:
+                self.calls.append(now)
+                raise OSError(EINVAL, "Not enough time has passed yet.")
+            else:
+                return filenames
+        self.cfp._listdir = thunk
+
+
+    def assertRequiredTimePassed(self):
+        """
+        Assert that calls to the simulated time.sleep() installed by
+        C{requireTimePassed} have been invoked the required number of times.
+        """
+        # Waiting should be growing by *2 each time until the additional wait
+        # exceeds BACKOFF_MAX (5), at which point we should wait for 5s each
+        # time.
+        def cumulative(values):
+            current = 0.0
+            for value in values:
+                current += value
+                yield current
+
+        self.assertEquals(self.calls,
+                          list(cumulative(
+                    [0.0, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0])))
+
+
+    def test_backoff(self):
+        """
+        L{CachingFilePath} will wait for an increasing interval up to
+        C{BACKOFF_MAX} between calls to listdir().
+        """
+        self.requireTimePassed(['a', 'b', 'c'])
+        self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
+
+
+    def test_siblingExtensionSearch(self):
+        """
+        L{FilePath.siblingExtensionSearch} is unfortunately not implemented in
+        terms of L{FilePath.listdir}, so we need to verify that it will also
+        retry.
+        """
+        filenames = [self.cfp.basename()+'.a',
+                     self.cfp.basename() + '.b',
+                     self.cfp.basename() + '.c']
+        siblings = map(self.cfp.sibling, filenames)
+        for sibling in siblings:
+            sibling.touch()
+        self.requireTimePassed(filenames)
+        self.assertEquals(self.cfp.siblingExtensionSearch("*"),
+                          siblings[0])
+        self.assertRequiredTimePassed()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100304/b9d38c24/attachment-0001.html>


More information about the calendarserver-changes mailing list