[CalendarServer-changes] [4968] CalendarServer/branches/users/wsanchez/deployment-fileprops/ twistedcaldav

source_changes at macosforge.org source_changes at macosforge.org
Tue Jan 26 21:12:15 PST 2010


Revision: 4968
          http://trac.macosforge.org/projects/calendarserver/changeset/4968
Author:   wsanchez at apple.com
Date:     2010-01-26 21:12:12 -0800 (Tue, 26 Jan 2010)
Log Message:
-----------
Add write tests, more fixes

Modified Paths:
--------------
    CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py
    CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/test/test_fileprops.py

Modified: CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py	2010-01-27 05:02:55 UTC (rev 4967)
+++ CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/fileprops.py	2010-01-27 05:12:12 UTC (rev 4968)
@@ -48,7 +48,7 @@
     def __init__(self, collection, cacheTimeout=0):
         self.collection = collection
         self.cacheTimeout = cacheTimeout
-        self._dirty = False
+        self._dirty = set()
 
     @classmethod
     def memcacheClient(cls, refresh=False):
@@ -66,7 +66,7 @@
     def propertyCache(self):
         # The property cache has this format:
         #  {
-        #    "/path/to/resource/file":
+        #    self._keyForPath("/path/to/resource/file"):
         #      (
         #        {
         #          (namespace, name): property,
@@ -186,10 +186,10 @@
         self.log_info("Building property file from xattrs for %s" % (self.collection,))
 
         cache = {}
-
         propertyStores = []
 
-        for childName in self.collection.listChildren():
+        childNames = self.collection.listChildren()
+        for childName in childNames:
             child = self.collection.getChild(childName)
             if child is None:
                 continue
@@ -201,14 +201,14 @@
             for qname in propertyStore.list():
                 props[qname] = propertyStore.get(qname)
 
-            cache[self._keyForPath(child.fp.path)] = props
+            cache[self._keyForPath(child.fp.path)] = (props, None)
 
             propertyStores.append(propertyStore)
 
         self.log_info("Done building property file from xattrs for %s" % (self.collection,))
 
         self._propertyCache = cache
-        self._dirty = True
+        self._dirty.update(childNames)
         self.flush()
 
         # Erase old property store
@@ -219,7 +219,7 @@
     def setProperty(self, child, property, delete=False):
         propertyCache, key, childCache, token = self.childCache(child)
 
-        self._dirty = True
+        self._dirty.add(child)
 
         if delete:
             qname = property
@@ -229,44 +229,6 @@
             qname = property.qname()
             childCache[qname] = property
 
-        client = self.memcacheClient()
-
-        if client is not None:
-            retries = 10
-            while retries:
-                try:
-                    if client.set(key, childCache, time=self.cacheTimeout,
-                        token=token):
-                        # Success
-                        break
-
-                except TokenMismatchError:
-                    # The value in memcache has changed since we last
-                    # fetched it
-                    log.debug("memcacheprops setProperty TokenMismatchError; retrying...")
-
-                finally:
-                    # Re-fetch the properties for this child
-                    loaded = self._loadCache(childNames=(child.fp.basename(),))
-                    propertyCache.update(loaded.iteritems())
-
-                retries -= 1
-
-                propertyCache, key, childCache, token = self.childCache(child)
-
-                if delete:
-                    if childCache.has_key(qname):
-                        del childCache[qname]
-                else:
-                    childCache[qname] = property
-
-            else:
-                log.error("memcacheprops setProperty had too many failures")
-                delattr(self, "_propertyCache")
-                raise MemcacheError("Unable to %s property {%s}%s on %s"
-                    % ("delete" if delete else "set",
-                    qname[0], qname[1], child))
-
     def flush(self):
         if self._dirty:
             def argh(what):
@@ -287,8 +249,49 @@
             finally:
                 cacheFile.close()
 
-            self._dirty = False
+            return ############################################
 
+            client = self.memcacheClient()
+
+            if client is not None:
+                retries = 10
+                while retries:
+                    try:
+                        if client.set(key, childCache, time=self.cacheTimeout,
+                            token=token):
+                            # Success
+                            break
+
+                    except TokenMismatchError:
+                        # The value in memcache has changed since we last
+                        # fetched it
+                        log.debug("memcacheprops setProperty TokenMismatchError; retrying...")
+
+                    finally:
+                        # Re-fetch the properties for this child
+                        loaded = self._loadCache(childNames=(child.fp.basename(),))
+                        print "-"*10, loaded
+                        propertyCache.update(loaded.iteritems())
+
+                    retries -= 1
+
+                    propertyCache, key, childCache, token = self.childCache(child)
+
+                    if delete:
+                        if childCache.has_key(qname):
+                            del childCache[qname]
+                    else:
+                        childCache[qname] = property
+
+                else:
+                    log.error("memcacheprops setProperty had too many failures")
+                    delattr(self, "_propertyCache")
+                    raise MemcacheError("Unable to %s property {%s}%s on %s"
+                        % ("delete" if delete else "set",
+                        qname[0], qname[1], child))
+
+            self._dirty = set()
+
     def deleteProperty(self, child, qname):
         return self.setProperty(child, qname, delete=True)
 
@@ -319,7 +322,7 @@
             path = self.child.fp.path
             key = self.parentPropertyCollection._keyForPath(path)
             parentPropertyCache = self.parentPropertyCollection.propertyCache()
-            return parentPropertyCache.get(key, {})
+            return parentPropertyCache.get(key, ({}, None))[0]
 
         def get(self, qname):
             propertyCache = self.propertyCache()

Modified: CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/test/test_fileprops.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/test/test_fileprops.py	2010-01-27 05:02:55 UTC (rev 4967)
+++ CalendarServer/branches/users/wsanchez/deployment-fileprops/twistedcaldav/test/test_fileprops.py	2010-01-27 05:12:12 UTC (rev 4968)
@@ -35,13 +35,9 @@
 
 from twistedcaldav.config import config
 from twistedcaldav.static import CalDAVFile, CalendarHomeProvisioningFile
-#from twistedcaldav.fileprops import PropertyCollection
 
-#from twistedcaldav.test.test_memcacheprops import StubCollection, StubResource, StubFP
-from twistedcaldav.test import test_memcacheprops
 from twistedcaldav.test.util import TestCase
 
-#class PropertyCollectionTestCase(test_memcacheprops.MemcachePropertyCollectionTestCase):
 class PropertyCollectionTestCase(TestCase):
     """
     Test PropertyCollection
@@ -100,6 +96,8 @@
             )
         )
 
+        self.md5s = {}
+
         for objectName in self.calendarObjectNames:
             data = (
                 """BEGIN:VCALENDAR"""
@@ -128,6 +126,10 @@
             finally:
                 fh.close()
 
+            # Get and remember md5 hash
+            hash = md5(data).hexdigest()
+            self.md5s[objectName] = hash
+
             #
             # Instantiate CalDAVFile directory, not through the
             # calendar home, because we want to use the old-style
@@ -136,7 +138,7 @@
             # store works.
             #
             child = CalDAVFile(childFP)
-            child.writeDeadProperty(TwistedGETContentMD5.fromString(md5(data).hexdigest()))
+            child.writeDeadProperty(TwistedGETContentMD5.fromString(hash))
             child.writeDeadProperty(GETContentType.fromString("text/calendar"))
 
         #
@@ -144,10 +146,7 @@
         #
         self.calendar.fp.child(".db.sqlite").remove()
 
-    def tearDown(self):
-        raise NotImplementedError()
-
-    def test_upgrade(self):
+    def test_upgrade_read(self):
         #print "*"*80
         #print self.calendar
         #print self.calendar.listChildren()
@@ -162,14 +161,48 @@
             # and correct via the API
             self.failUnless(child.hasDeadProperty(TwistedGETContentMD5))
             self.failUnless(child.hasDeadProperty(GETContentType))
-            self.assertEquals(len(str(child.readDeadProperty(TwistedGETContentMD5))), 32)
-            self.assertEquals(str(child.readDeadProperty(GETContentType)), "text/calendar")
+            self.assertEquals(child.readDeadProperty(TwistedGETContentMD5), self.md5s[childName])
+            self.assertEquals(child.readDeadProperty(GETContentType), "text/calendar")
 
             # Old dead properties should be gone now
             oldSchool = CalDAVFile(child.fp.path)
             oldProperties = [x for x in oldSchool.deadProperties().list()]
             self.assertEquals(len(oldProperties), 0, oldProperties)
 
+        self.failUnless(self.calendar.fp.child(".davprops.pickle").exists(), self.calendar.fp.listdir())
+
+    def test_upgrade_write(self):
+        child = self.calendar.getChild("Earth.ics")
+
+        # Ensure that write and read works
+        child.writeDeadProperty(TwistedGETContentMD5.fromString("d9baeb281f407f3381089df85e4dc2e5"))
+        self.assertEquals(child.readDeadProperty(TwistedGETContentMD5), "d9baeb281f407f3381089df85e4dc2e5")
+
+        # Old dead properties should be gone now
+        oldSchool = CalDAVFile(child.fp.path)
+        oldProperties = [x for x in oldSchool.deadProperties().list()]
+        self.assertEquals(len(oldProperties), 0, oldProperties)
+
+    def test_readwrite(self):
+        childName = "Earth.ics"
+
+        child = self.calendar.getChild(childName)
+
+        # Ensure that write and read works
+        child.writeDeadProperty(TwistedGETContentMD5.fromString("d9baeb281f407f3381089df85e4dc2e5"))
+        self.assertEquals(child.readDeadProperty(TwistedGETContentMD5), "d9baeb281f407f3381089df85e4dc2e5")
+
+        # Ensure that write and read works (post-upgrade)
+        child.writeDeadProperty(TwistedGETContentMD5.fromString("e5786e7b9a94ad428219c82c9428d1e4"))
+        self.assertEquals(child.readDeadProperty(TwistedGETContentMD5), "e5786e7b9a94ad428219c82c9428d1e4")
+
+        ### Doesn't work; need new calendar home (and cache):
+        # No flush above, so writes should not show up in a new instance
+        child2 = self.calendar.getChild(childName)
+        self.assertEquals(child2.readDeadProperty(TwistedGETContentMD5), self.md5s[childName])
+
+
+
 #
 # Utilities
 #
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100126/861ead04/attachment-0001.html>


More information about the calendarserver-changes mailing list