[CalendarServer-changes] [5243] CalendarServer/trunk/twext/python
source_changes at macosforge.org
source_changes at macosforge.org
Thu Mar 4 10:44:23 PST 2010
Revision: 5243
http://trac.macosforge.org/projects/calendarserver/changeset/5243
Author: glyph at apple.com
Date: 2010-03-04 10:44:23 -0800 (Thu, 04 Mar 2010)
Log Message:
-----------
Add test coverage, and a workaround for EINVAL exceptions, to `CachingFilePath`.
Modified Paths:
--------------
CalendarServer/trunk/twext/python/filepath.py
Added Paths:
-----------
CalendarServer/trunk/twext/python/test/test_filepath.py
Modified: CalendarServer/trunk/twext/python/filepath.py
===================================================================
--- CalendarServer/trunk/twext/python/filepath.py 2010-03-04 17:41:20 UTC (rev 5242)
+++ CalendarServer/trunk/twext/python/filepath.py 2010-03-04 18:44:23 UTC (rev 5243)
@@ -1,3 +1,4 @@
+# -*- test-case-name: twext.python.test.test_filepath -*-
##
# Copyright (c) 2010 Apple Inc. All rights reserved.
#
@@ -19,6 +20,17 @@
calendar server.
"""
+from os import listdir as _listdir
+
+from os.path import (join as _joinpath,
+ basename as _basename,
+ exists as _exists,
+ dirname as _dirname)
+
+from time import sleep as _sleep
+from types import FunctionType, MethodType
+from errno import EINVAL
+
from twisted.python.filepath import FilePath
from stat import S_ISDIR
@@ -29,12 +41,33 @@
aggressive caching policy.
"""
+ _listdir = _listdir # integration points for tests
+ _sleep = _sleep
+
+ BACKOFF_MAX = 5.0 # Maximum time to wait between calls to
+ # listdir()
+
def __init__(self, path, alwaysCreate=False):
super(CachingFilePath, self).__init__(path, alwaysCreate)
self.existsCached = None
self.isDirCached = None
+ @property
+ def siblingExtensionSearch(self):
+ """
+ Dynamically create a version of L{FilePath.siblingExtensionSearch} that
+ uses a pluggable 'listdir' implementation.
+ """
+ return MethodType(FunctionType(
+ FilePath.siblingExtensionSearch.im_func.func_code,
+ {'listdir': self._retryListdir,
+ 'basename': _basename,
+ 'dirname': _dirname,
+ 'joinpath': _joinpath,
+ 'exists': _exists}), self, self.__class__)
+
+
def changed(self):
"""
This path may have changed in the filesystem, so forget all cached
@@ -45,6 +78,32 @@
self.isDirCached = None
+ def _retryListdir(self, pathname):
+ """
+ Implementation of retry logic for C{listdir} and
+ C{siblingExtensionSearch}.
+ """
+ delay = 0.1
+ while True:
+ try:
+ return self._listdir(pathname)
+ except OSError, e:
+ if e.errno == EINVAL:
+ self._sleep(delay)
+ delay = min(self.BACKOFF_MAX, delay * 2.0)
+ else:
+ raise
+ raise RuntimeError("unreachable code.")
+
+
+ def listdir(self):
+ """
+ List the directory which C{self.path} points to, compensating for
+ EINVAL from C{os.listdir}.
+ """
+ return self._retryListdir(self.path)
+
+
def restat(self, reraise=True):
"""
Re-cache stat information.
Added: CalendarServer/trunk/twext/python/test/test_filepath.py
===================================================================
--- CalendarServer/trunk/twext/python/test/test_filepath.py (rev 0)
+++ CalendarServer/trunk/twext/python/test/test_filepath.py 2010-03-04 18:44:23 UTC (rev 5243)
@@ -0,0 +1,160 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Tests for specialized behavior of L{CachingFilePath}
+"""
+from errno import EINVAL
+from os.path import join as pathjoin
+
+from twisted.internet.task import Clock
+
+from twisted.trial.unittest import TestCase
+
+from twext.python.filepath import CachingFilePath
+
+# Cheat and pull in the Twisted test cases for FilePath. XXX: Twisteds should
+# provide a supported way of doing this for exported interfaces. Also, it
+# should export IFilePath. --glyph
+
+from twisted.test.test_paths import FilePathTestCase
+
+class BaseVerification(FilePathTestCase):
+ """
+ Make sure that L{CachingFilePath} doesn't break the contracts that
+ L{FilePath} tries to provide.
+ """
+
+ def setUp(self):
+ """
+ Set up the test case to set the base attributes to point at
+ L{AbstractFilePathTestCase}.
+ """
+ FilePathTestCase.setUp(self)
+ self.root = CachingFilePath(self.root.path)
+ self.path = CachingFilePath(self.path.path)
+
+
+
+class EINVALTestCase(TestCase):
+ """
+ Sometimes, L{os.listdir} will raise C{EINVAL}. This is a transient error,
+ and L{CachingFilePath.listdir} should work around it by retrying the
+ C{listdir} operation until it succeeds.
+ """
+
+ def setUp(self):
+ """
+ Create a L{CachingFilePath} for the test to use.
+ """
+ self.cfp = CachingFilePath(self.mktemp())
+ self.clock = Clock()
+ self.cfp._sleep = self.clock.advance
+
+
+ def test_testValidity(self):
+ """
+ If C{listdir} is replaced on a L{CachingFilePath}, we should be able to
+ observe exceptions raised by the replacement. This verifies that the
+ test patching done here is actually testing something.
+ """
+ class CustomException(Exception): "Just for testing."
+ def blowUp(dirname):
+ raise CustomException()
+ self.cfp._listdir = blowUp
+ self.assertRaises(CustomException, self.cfp.listdir)
+ self.assertRaises(CustomException, self.cfp.children)
+
+
+ def test_retryLoop(self):
+ """
+ L{CachingFilePath} should catch C{EINVAL} and respond by retrying the
+ C{listdir} operation until it succeeds.
+ """
+ calls = []
+ def raiseEINVAL(dirname):
+ calls.append(dirname)
+ if len(calls) < 5:
+ raise OSError(EINVAL, "This should be caught by the test.")
+ return ['a', 'b', 'c']
+ self.cfp._listdir = raiseEINVAL
+ self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
+ self.assertEquals(self.cfp.children(), [
+ CachingFilePath(pathjoin(self.cfp.path, 'a')),
+ CachingFilePath(pathjoin(self.cfp.path, 'b')),
+ CachingFilePath(pathjoin(self.cfp.path, 'c')),])
+
+
+ def requireTimePassed(self, filenames):
+ """
+ Create a replacement for listdir() which only fires after a certain
+ amount of time.
+ """
+ self.calls = []
+ def thunk(dirname):
+ now = self.clock.seconds()
+ if now < 20.0:
+ self.calls.append(now)
+ raise OSError(EINVAL, "Not enough time has passed yet.")
+ else:
+ return filenames
+ self.cfp._listdir = thunk
+
+
+ def assertRequiredTimePassed(self):
+ """
+ Assert that calls to the simulated time.sleep() installed by
+ C{requireTimePassed} have been invoked the required number of times.
+ """
+ # Waiting should be growing by *2 each time until the additional wait
+ # exceeds BACKOFF_MAX (5), at which point we should wait for 5s each
+ # time.
+ def cumulative(values):
+ current = 0.0
+ for value in values:
+ current += value
+ yield current
+
+ self.assertEquals(self.calls,
+ list(cumulative(
+ [0.0, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0])))
+
+
+ def test_backoff(self):
+ """
+ L{CachingFilePath} will wait for an increasing interval up to
+ C{BACKOFF_MAX} between calls to listdir().
+ """
+ self.requireTimePassed(['a', 'b', 'c'])
+ self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
+
+
+ def test_siblingExtensionSearch(self):
+ """
+ L{FilePath.siblingExtensionSearch} is unfortunately not implemented in
+ terms of L{FilePath.listdir}, so we need to verify that it will also
+ retry.
+ """
+ filenames = [self.cfp.basename()+'.a',
+ self.cfp.basename() + '.b',
+ self.cfp.basename() + '.c']
+ siblings = map(self.cfp.sibling, filenames)
+ for sibling in siblings:
+ sibling.touch()
+ self.requireTimePassed(filenames)
+ self.assertEquals(self.cfp.siblingExtensionSearch("*"),
+ siblings[0])
+ self.assertRequiredTimePassed()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100304/b9d38c24/attachment-0001.html>
More information about the calendarserver-changes
mailing list