[CalendarServer-changes] [1209] CalendarServer/branches/users/cdaboo/sqlprops-1202

source_changes at macosforge.org source_changes at macosforge.org
Mon Feb 19 08:38:26 PST 2007


Revision: 1209
          http://trac.macosforge.org/projects/calendarserver/changeset/1209
Author:   cdaboo at apple.com
Date:     2007-02-19 08:38:25 -0800 (Mon, 19 Feb 2007)

Log Message:
-----------
Copy/move properties when the resource is copied/moved. Note this does not take into account partial failures. It also
breaks use of xattrs as that does not support the getAll/setAll props methods yet.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch
    CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch
    CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/method/put_common.py
    CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/sqlprops.py
    CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/test/test_sqlprops.py

Modified: CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch	2007-02-18 17:26:59 UTC (rev 1208)
+++ CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.copymove.patch	2007-02-19 16:38:25 UTC (rev 1209)
@@ -53,16 +53,27 @@
          yield destparent
          destparent = destparent.getResult()
  
-@@ -144,7 +155,19 @@
+@@ -144,10 +155,31 @@
          log.err(msg)
          raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, msg))
  
 -    x = waitForDeferred(move(self.fp, request.uri, destination.fp, destination_uri, depth))
+-    yield x
+-    yield x.getResult()
 +    # Lets optimise a move within the same directory to a new resource as a simple move
 +    # rather than using the full transaction based storeResource api. This allows simple
 +    # "rename" operations to work quickly.
 +    if (not destination.exists()) and destparent == parent:
 +        x = waitForDeferred(move(self.fp, request.uri, destination.fp, destination_uri, depth))
++        yield x
++        result =  x.getResult()
+ 
++        # Explicitly move properties - no one else does it
++        properties = self.deadProperties().getAll()
++        destination.deadProperties().setSeveral([p for p in properties.itervalues()])
++        self.deadProperties().deleteAll()
++        
++        yield result
 +    else:
 +        x = waitForDeferred(put_common.storeResource(request,
 +                                                     source=self,
@@ -71,6 +82,9 @@
 +                                                     destination_uri=destination_uri,
 +                                                     deletesource=True,
 +                                                     depth=depth))
-     yield x
-     yield x.getResult()
++        yield x
++        yield x.getResult()
++
+ http_MOVE = deferredGenerator(http_MOVE)
  
+ def prepareForCopy(self, request):

Modified: CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch	2007-02-18 17:26:59 UTC (rev 1208)
+++ CalendarServer/branches/users/cdaboo/sqlprops-1202/lib-patches/Twisted/twisted.web2.dav.method.put_common.patch	2007-02-19 16:38:25 UTC (rev 1209)
@@ -2,7 +2,7 @@
 ===================================================================
 --- twisted/web2/dav/method/put_common.py	(revision 0)
 +++ twisted/web2/dav/method/put_common.py	(revision 0)
-@@ -0,0 +1,265 @@
+@@ -0,0 +1,273 @@
 +##
 +# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
 +#
@@ -210,6 +210,11 @@
 +        yield response
 +        response = response.getResult()
 +
++        # Copy dead properties first, before adding overridden values
++        if source is not None:
++            properties = source.deadProperties().getAll()
++            destination.deadProperties().setSeveral([p for p in properties.itervalues()])
++
 +        # Update the MD5 value on the resource
 +        if source is not None:
 +            # Copy MD5 value from source to destination
@@ -255,6 +260,9 @@
 +            delete(source_uri, source.fp, depth)
 +            rollback.source_deleted = True
 +
++            # Explicitly delete properties - no one else does it
++            source.deadProperties().deleteAll()
++        
 +        # Can now commit changes and forget the rollback details
 +        rollback.Commit()
 +

Modified: CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/method/put_common.py	2007-02-18 17:26:59 UTC (rev 1208)
+++ CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/method/put_common.py	2007-02-19 16:38:25 UTC (rev 1209)
@@ -434,6 +434,11 @@
         yield response
         response = response.getResult()
 
+        # Copy dead properties first, before adding overridden values
+        if source is not None:
+            properties = source.deadProperties().getAll()
+            destination.deadProperties().setSeveral([p for p in properties.itervalues()])
+
         # Update the MD5 value on the resource
         if source is not None:
             # Copy MD5 value from source to destination
@@ -494,6 +499,10 @@
             # Delete the source resource
             delete(source_uri, source.fp, "0")
             rollback.source_deleted = True
+
+            # Explicitly delete properties - no one else does it
+            source.deadProperties().deleteAll()
+
             logging.debug("Source removed %s" % (source.fp.path,), system="Store Resource")
 
         def doSourceIndexRecover():

Modified: CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/sqlprops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/sqlprops.py	2007-02-18 17:26:59 UTC (rev 1208)
+++ CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/sqlprops.py	2007-02-19 16:38:25 UTC (rev 1209)
@@ -73,7 +73,7 @@
                 "No such property: {%s}%s" % qname
             ))
             
-        value = self.index.getPropertyValue(self.rname, qname)
+        value = self.index.getOnePropertyValue(self.rname, qname)
         if not value:
             raise HTTPError(StatusResponse(
                 responsecode.NOT_FOUND,
@@ -82,12 +82,12 @@
             
         return value
 
-    def getAll(self, qnames):
+    def getSeveral(self, qnames):
         """
-        Read properties from index.
+        Read specific properties from index.
         
         @param qnames: C{list} of C{tuple} of property namespace and name.
-        @return: a C{list} of property classes
+        @return: a C{dict} containing property name/value.
         """
         if not qnames:
             return None
@@ -98,13 +98,28 @@
                 "No such property: {%s}%s" % qnames[0]
             ))
             
-        return self.index.getAllPropertyValues(self.rname, qnames)
+        return self.index.getSeveralPropertyValues(self.rname, qnames)
 
-    def getAllResources(self, qnames):
+    def getAll(self):
         """
-        Read properties for all child resources from index.
+        Read all properties from index.
         
         @param qnames: C{list} of C{tuple} of property namespace and name.
+        @return: a C{dict} containing property name/value.
+        """
+        if not self.index:
+            raise HTTPError(StatusResponse(
+                responsecode.NOT_FOUND,
+                "No properties"
+            ))
+            
+        return self.index.getAllPropertyValues(self.rname)
+
+    def getSeveralResources(self, qnames):
+        """
+        Read specific properties for all child resources from index.
+        
+        @param qnames: C{list} of C{tuple} of property namespace and name.
         @return: a C{dict} with resource name as keys and C{dict} of property name/classes as values
         """
         if not qnames:
@@ -116,7 +131,7 @@
                 "No such property: {%s}%s" % qnames[0]
             ))
             
-        return self.index.getAllResourcePropertyValues(qnames)
+        return self.index.getSeveralResourcePropertyValues(qnames)
 
     def set(self, property):
         """
@@ -126,17 +141,17 @@
         """
 
         if self.index:
-            self.index.setPropertyValue(self.rname, property.qname(), property)
+            self.index.setOnePropertyValue(self.rname, property.qname(), property)
 
-    def setAll(self, properties):
+    def setSeveral(self, properties):
         """
-        Write all properties into index.
+        Write specific properties into index.
         
         @param properties: C{list} of properties to write
         """
 
         if self.index:
-            self.index.setAllPropertyValues(self.rname, [(p.qname(), p) for p in properties])
+            self.index.setSeveralPropertyValues(self.rname, [(p.qname(), p) for p in properties])
 
     def delete(self, qname):
         """
@@ -152,7 +167,7 @@
 
     def deleteAll(self):
         """
-        Delete proeprty from index.
+        Delete property from index.
 
         DELETE from PROPERTIES where NAME=<<rname>> and PROPNAME=<<pname>>
 
@@ -164,7 +179,7 @@
 
     def contains(self, qname):
         if self.index:
-            value = self.index.getPropertyValue(self.rname, qname)
+            value = self.index.getOnePropertyValue(self.rname, qname)
             return value is not None
         else:
             return False
@@ -216,7 +231,7 @@
         path = os.path.join(path, SQLPropertiesDatabase.dbFilename)
         super(SQLPropertiesDatabase, self).__init__(path, SQLPropertiesDatabase.dbFormatVersion, utf8=True)
 
-    def setPropertyValue(self, rname, pname, pvalue):
+    def setOnePropertyValue(self, rname, pname, pvalue):
         """
         Add a property.
     
@@ -230,9 +245,9 @@
         self._add_to_db(rname, self._encode(pname), cPickle.dumps(pvalue))
         self._db_commit()
 
-    def setAllPropertyValues(self, rname, properties):
+    def setSeveralPropertyValues(self, rname, properties):
         """
-        Add a property.
+        Add a set of properties.
     
         @param rname: a C{str} containing the resource name.
         @param pname: a C{str} containing the name of the property to set.
@@ -245,7 +260,7 @@
             self._add_to_db(rname, self._encode(p[0]), cPickle.dumps(p[1]))
         self._db_commit()
 
-    def getPropertyValue(self, rname, pname):
+    def getOnePropertyValue(self, rname, pname):
         """
         Get a property.
     
@@ -268,7 +283,7 @@
         else:
             raise ValueError("Multiple properties of the same name \"%s\" stored for resource \"%s\"" % (pname, rname,))
 
-    def getAllPropertyValues(self, rname, pnames):
+    def getSeveralPropertyValues(self, rname, pnames):
         """
         Get specified property values from specific resource.
     
@@ -279,7 +294,7 @@
         
         # Remove what is there, then add it back.
         if DEBUG_LOG:
-            log.msg("getAllPropertyValues: %s \"%s\"" % (self.dbpath, pnames))
+            log.msg("getSeveralPropertyValues: %s \"%s\"" % (self.dbpath, pnames))
         properties = {}
         statement = "select PROPERTYNAME, PROPERTYVALUE from PROPERTIES where RESOURCENAME = :1 and ("
         args = [rname]
@@ -295,8 +310,25 @@
 
         return properties
 
-    def getAllResourcePropertyValues(self, pnames):
+    def getAllPropertyValues(self, rname):
         """
+        Get specified property values from specific resource.
+    
+        @param rname: a C{str} containing the resource name.
+        @return: a C{dict} containing property name/value.
+        """
+        
+        # Remove what is there, then add it back.
+        if DEBUG_LOG:
+            log.msg("getAllPropertyValues: %s" % (self.dbpath,))
+        properties = {}
+        for row in self._db_execute("select PROPERTYNAME, PROPERTYVALUE from PROPERTIES where RESOURCENAME = :1", rname):
+            properties[self._decode(row[0])] = cPickle.loads(row[1])
+
+        return properties
+
+    def getSeveralResourcePropertyValues(self, pnames):
+        """
         Get specified property values from all resources.
     
         @param pnames: a C{list} of C{str} containing the name of the properties to get.

Modified: CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/test/test_sqlprops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/test/test_sqlprops.py	2007-02-18 17:26:59 UTC (rev 1208)
+++ CalendarServer/branches/users/cdaboo/sqlprops-1202/twistedcaldav/test/test_sqlprops.py	2007-02-19 16:38:25 UTC (rev 1209)
@@ -15,6 +15,8 @@
 #
 # DRI: Wilfredo Sanchez, wsanchez at apple.com
 ##
+from twistedcaldav.ical import Component
+from twisted.web2.dav.element.rfc2518 import DisplayName
 
 import os
 import time
@@ -39,6 +41,7 @@
     """
     SQL properties tests
     """
+    data_dir = os.path.join(os.path.dirname(__file__), "data")
 
     props = (
         davxml.DisplayName.fromString("My Name"),
@@ -69,11 +72,11 @@
     def _setProperty(self, index, prop):
         index.set(prop)
         
-    def _testProperty(self, index, prop):
+    def _testProperty(self, index, prop, description = ""):
         self.assertTrue(index.contains(prop.qname()),
-                        msg="Could not find property %s." % prop)
+                        msg="Could not find property %s %s." % (description, prop,))
         self.assertTrue(index.get(prop.qname()) == prop,
-                        msg="Could not get property %s." % prop)
+                        msg="Could not get property %s %s." % (description, prop,))
     
     def _testPropertyList(self, proplist):
         self.assertTrue(len(proplist) == len(SQLProps.props),
@@ -100,7 +103,7 @@
         for i in xrange(number):
             rsrc = CalDAVFile(os.path.join(self.collection_name, "file%04s.ics" % (i,)))
             index = sqlPropertyStore(rsrc)
-            index.setAll(SQLProps.props)
+            index.setSeveral(SQLProps.props)
         return index
 
     def test_db_init_directory(self):
@@ -146,7 +149,7 @@
 
     def test_setallproperties(self):
         index = self._setUpIndex()
-        index.setAll(SQLProps.props)
+        index.setSeveral(SQLProps.props)
         for prop in SQLProps.props:
             self._testProperty(index, prop)
         proplist = set(index.list())
@@ -183,28 +186,31 @@
         for prop in SQLProps.props:
             self._setProperty(index, prop)
         
-        result = index.getAll([p.qname() for p in SQLProps.props])
+        result = index.getSeveral([p.qname() for p in SQLProps.props])
         self._testPropertyList(result)
 
     def test_getallresourceproperties(self):
         num_resources = 10
         index = self._setupMultipleResources(num_resources)
-        result = index.getAllResources([p.qname() for p in SQLProps.props])
+        result = index.getSeveralResources([p.qname() for p in SQLProps.props])
         self._testResourcePropertyList(num_resources, result)
 
 #    def test_timegetallresourceproperties(self):
 #        num_resources = 1000
 #        index = self._setupMultipleResources(num_resources)
 #        t1 = time.time()
-#        result = index.getAllResources([p.qname() for p in SQLProps.props])
+#        result = index.getSeveralResources([p.qname() for p in SQLProps.props])
 #        t2 = time.time()
 #        self.assertTrue(t1 == t2,
 #                        msg="Time for 1000 prop query = %s" % (t2 - t1,))
 #
 #        self._testResourcePropertyList(num_resources, result)
 
-    def test_deleteresource(self):
-        fpath = os.path.join(self.docroot, "file.ics")
+    def _do_delete(self, parent):
+        fpath = self.docroot
+        if parent:
+            fpath = os.path.join(fpath, parent)
+        fpath = os.path.join(fpath, "file.ics")
         rsrc = CalDAVFile(fpath)
         ms = MemoryStream("Some Data")
 
@@ -231,7 +237,7 @@
 
             def work():
                 # Delete resource and test
-                request = SimpleRequest(self.site, "DELETE", "/file.ics")
+                request = SimpleRequest(self.site, "DELETE", "/%sfile.ics" % (parent,))
                 yield (request, doneDelete)
 
             return serialize(self.send, work())
@@ -239,9 +245,171 @@
         d = put(ms, rsrc.fp)
         d.addCallback(donePut)
         return d
+
+    def test_deleteresource(self):
+        return self._do_delete("")
+
+    def test_deletecalendarresource(self):
+
+        def doneMake(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            return self._do_delete("calendar/")
+
+        # Make a calendar
+        request = SimpleRequest(self.site, "MKCALENDAR", "/calendar/")
+        return self.send(request, doneMake)
+
+    event = """BEGIN:VCALENDAR
+PRODID:Test Case
+CALSCALE:GREGORIAN
+BEGIN:VEVENT
+DTSTAMP:20070219T120000Z
+DTSTART:20070219T120000Z
+DTEND:20070219T130000Z
+UID:12345-67890-54321
+END:VEVENT
+END:VCALENDAR
+"""
+
+    def _do_copy(self, src, dst):
+        fpath = self.docroot
+        if src:
+            fpath = os.path.join(fpath, src)
+        fpath = os.path.join(fpath, "file.ics")
+        fpath_new = self.docroot
+        if dst:
+            fpath_new = os.path.join(fpath_new, dst)
+        fpath_new = os.path.join(fpath_new, "copy.ics")
+        rsrc = CalDAVFile(fpath)
+
+        def donePut(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            displayname = DisplayName.fromString("adisplayname")
+            rsrc.writeDeadProperty(displayname)
+            
+            # Check index
+            index = sqlPropertyStore(rsrc)
+            self._testProperty(index, displayname)
+            
+            def doneCopy(response):
+                response = IResponse(response)
     
+                if response.code != responsecode.CREATED:
+                    self.fail("COPY response %s != %s" % (response.code, responsecode.NO_CONTENT))
+            
+                if not os.path.exists(fpath):
+                    self.fail("COPY removed original path %s" % (fpath,))
+
+                if not os.path.exists(fpath_new):
+                    self.fail("COPY did not create new path %s" % (fpath_new,))
+
+                self._testProperty(index, displayname, "on original resource")
+
+                rsrc_new = CalDAVFile(fpath_new)
+                index_new = sqlPropertyStore(rsrc_new)
+                self._testProperty(index_new, displayname, "on new resource")
+
+            # Copy resource and test
+            request = SimpleRequest(self.site, "COPY", "/%sfile.ics" % (src,))
+            request.headers.setHeader("destination", "/%scopy.ics" % (dst,))
+            return self.send(request, doneCopy)
+
+        stream = file(os.path.join(self.data_dir, "Holidays", "C318AA54-1ED0-11D9-A5E0-000A958A3252.ics"))
+        try: calendar = str(Component.fromStream(stream))
+        finally: stream.close()
+
+        request = SimpleRequest(self.site, "PUT", "/%sfile.ics" % (src,))
+        request.stream = MemoryStream(calendar)
+        return self.send(request, donePut)
+    
     def test_copyresource(self):
-        raise SkipTest("test unimplemented")
+        return self._do_copy("", "")
+
+    def test_copycalendarresource(self):
+
+        def doneMake2(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            return self._do_copy("calendar1/", "calendar2/")
+
+        def doneMake1(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            request = SimpleRequest(self.site, "MKCALENDAR", "/calendar2/")
+            return self.send(request, doneMake2)
+
+        # Make a calendar
+        request = SimpleRequest(self.site, "MKCALENDAR", "/calendar1/")
+        return self.send(request, doneMake1)
+
+    def _do_move(self, src, dst):
+        fpath = self.docroot
+        if src:
+            fpath = os.path.join(fpath, src)
+        fpath = os.path.join(fpath, "file.ics")
+        fpath_new = self.docroot
+        if dst:
+            fpath_new = os.path.join(fpath_new, dst)
+        fpath_new = os.path.join(fpath_new, "move.ics")
+        rsrc = CalDAVFile(fpath)
+
+        def donePut(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            displayname = DisplayName.fromString("adisplayname")
+            rsrc.writeDeadProperty(displayname)
+            
+            # Check index
+            index = sqlPropertyStore(rsrc)
+            self._testProperty(index, displayname)
+            
+            def doneMove(response):
+                response = IResponse(response)
     
+                if response.code != responsecode.CREATED:
+                    self.fail("MOVE response %s != %s" % (response.code, responsecode.NO_CONTENT))
+            
+                if os.path.exists(fpath):
+                    self.fail("MOVE did not remove original path %s" % (fpath,))
+
+                if not os.path.exists(fpath_new):
+                    self.fail("MOVE did not create new path %s" % (fpath_new,))
+
+                self.assertFalse(index.contains(displayname.qname()),
+                                 msg="Property %s exists after resource was moved." % displayname)
+
+                rsrc_new = CalDAVFile(fpath_new)
+                index_new = sqlPropertyStore(rsrc_new)
+                self._testProperty(index_new, displayname, "on new resource")
+
+            def work():
+                # Move resource and test
+                request = SimpleRequest(self.site, "MOVE", "/%sfile.ics" % (src,))
+                request.headers.setHeader("destination", "/%smove.ics" % (dst,))
+                yield (request, doneMove)
+
+            return serialize(self.send, work())
+
+        stream = file(os.path.join(self.data_dir, "Holidays", "C318AA54-1ED0-11D9-A5E0-000A958A3252.ics"))
+        try: calendar = str(Component.fromStream(stream))
+        finally: stream.close()
+
+        request = SimpleRequest(self.site, "PUT", "/%sfile.ics" % (src,))
+        request.stream = MemoryStream(calendar)
+        return self.send(request, donePut)
+    
     def test_moveresource(self):
-        raise SkipTest("test unimplemented")
+        return self._do_move("", "")
+
+    def test_movecalendarresource(self):
+
+        def doneMake2(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            return self._do_move("calendar1/", "calendar2/")
+
+        def doneMake1(response):
+            self.assertTrue(response.code == responsecode.CREATED)
+            request = SimpleRequest(self.site, "MKCALENDAR", "/calendar2/")
+            return self.send(request, doneMake2)
+
+        # Make a calendar
+        request = SimpleRequest(self.site, "MKCALENDAR", "/calendar1/")
+        return self.send(request, doneMake1)
+

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20070219/fbf076dc/attachment.html


More information about the calendarserver-changes mailing list