[CalendarServer-changes] [4907] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Wed Jan 6 08:54:58 PST 2010


Revision: 4907
          http://trac.macosforge.org/projects/calendarserver/changeset/4907
Author:   cdaboo at apple.com
Date:     2010-01-06 08:54:57 -0800 (Wed, 06 Jan 2010)
Log Message:
-----------
WebDAV sync REPORT support.

Modified Paths:
--------------
    CalendarServer/trunk/support/build.sh
    CalendarServer/trunk/twext/web2/dav/__init__.py
    CalendarServer/trunk/twext/web2/dav/davxml.py
    CalendarServer/trunk/twistedcaldav/index.py
    CalendarServer/trunk/twistedcaldav/method/__init__.py
    CalendarServer/trunk/twistedcaldav/method/delete_common.py
    CalendarServer/trunk/twistedcaldav/method/put_common.py
    CalendarServer/trunk/twistedcaldav/method/report_common.py
    CalendarServer/trunk/twistedcaldav/resource.py
    CalendarServer/trunk/twistedcaldav/static.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/twistedcaldav/test/test_index.py

Added Paths:
-----------
    CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py

Modified: CalendarServer/trunk/support/build.sh
===================================================================
--- CalendarServer/trunk/support/build.sh	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/support/build.sh	2010-01-06 16:54:57 UTC (rev 4907)
@@ -2,7 +2,7 @@
 # -*- sh-basic-offset: 2 -*-
 
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -496,7 +496,7 @@
     false true true true 212;
 
   # Tool dependencies.  The code itself doesn't depend on these, but you probably want them.
-  svn_get "CalDAVTester" "${top}/CalDAVTester" "${svn_uri_base}/CalDAVTester/trunk" 4874;
+  svn_get "CalDAVTester" "${top}/CalDAVTester" "${svn_uri_base}/CalDAVTester/trunk" 4906;
   svn_get "Pyflakes" "${top}/Pyflakes" http://divmod.org/svn/Divmod/trunk/Pyflakes 17198;
 }
 

Modified: CalendarServer/trunk/twext/web2/dav/__init__.py
===================================================================
--- CalendarServer/trunk/twext/web2/dav/__init__.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twext/web2/dav/__init__.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2009 Apple Inc. All rights reserved.
+# Copyright (c) 2009-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,5 +15,14 @@
 ##
 
 """
-Extentions to twisted.web2.dav
+Extensions to twisted.web2.dav
 """
+
+#
+# Register additional WebDAV XML elements
+#
+
+from twext.web2.dav import davxml
+import twisted.web2.dav.davxml
+
+twisted.web2.dav.davxml.registerElements(davxml)

Modified: CalendarServer/trunk/twext/web2/dav/davxml.py
===================================================================
--- CalendarServer/trunk/twext/web2/dav/davxml.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twext/web2/dav/davxml.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,12 +21,15 @@
 __all__ = [
     "sname2qname",
     "qname2sname",
+    "ErrorDescription",
     "ErrorResponse",
+    "SyncCollection",
+    "SyncToken",
 ]
 
 from twisted.web2.http import Response
 from twisted.web2.dav.http import ErrorResponse as SuperErrorResponse
-from twisted.web2.dav.davxml import twisted_dav_namespace, WebDAVTextElement
+from twisted.web2.dav.davxml import dav_namespace, twisted_dav_namespace, WebDAVElement, WebDAVTextElement
 from twisted.web2.dav.davxml import WebDAVUnknownElement, Error
 from twisted.web2.http_headers import MimeType
 
@@ -88,6 +91,7 @@
     Renders itself as a DAV:error XML document.
     """
     error = None
+    unregistered = True     # base class is already registered
 
     def __init__(self, code, error, description=None):
         """
@@ -120,3 +124,47 @@
     def __repr__(self):
         return "<%s %s %s>" % (self.__class__.__name__, self.code, self.error.sname())
 
+class SyncCollection (WebDAVElement):
+    """
+    DAV report used to retrieve specific calendar component items via their
+    URIs.
+    (CalDAV-access-09, section 9.9)
+    """
+    name = "sync-collection"
+
+    # To allow for an empty element in a supported-report-set property we need
+    # to relax the child restrictions
+    allowed_children = {
+        (dav_namespace, "sync-token"): (0, 1), # When used in the REPORT this is required
+        (dav_namespace, "prop"    ):   (0, 1),
+    }
+
+    def __init__(self, *children, **attributes):
+        super(SyncCollection, self).__init__(*children, **attributes)
+
+        self.property = None
+        self.sync_token = None
+
+        for child in self.children:
+            qname = child.qname()
+
+            if qname == (dav_namespace, "sync-token"):
+                
+                self.sync_token = str(child)
+
+            elif qname in (
+                (dav_namespace, "prop"    ),
+            ):
+                if property is not None:
+                    raise ValueError("Only one of DAV:prop allowed")
+                self.property = child
+
+class SyncToken (WebDAVTextElement):
+    """
+    Synchronization token used in report and as a property.
+    """
+    name = "sync-token"
+    hidden = True
+    protected = True
+
+

Modified: CalendarServer/trunk/twistedcaldav/index.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/index.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/index.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,7 +57,7 @@
 log = Logger()
 
 db_basename = db_prefix + "sqlite"
-schema_version = "8"
+schema_version = "9"
 collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
 
 icalfbtype_to_indexfbtype = {
@@ -101,6 +101,9 @@
 class IndexedSearchException(ValueError):
     pass
 
+class SyncTokenValidException(ValueError):
+    pass
+
 class AbstractCalendarIndex(AbstractSQLDatabase, LoggingMixIn):
     """
     Calendar collection index abstract base class that defines the apis for the index.
@@ -180,7 +183,7 @@
             if name is not None and self.resource.getChild(name_utf8) is None:
                 # Clean up
                 log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
-                self._delete_from_db(name, uid)
+                self._delete_from_db(name, uid, None)
                 self._db_commit()
             else:
                 resources.append(name)
@@ -212,7 +215,7 @@
 
         return uid
 
-    def addResource(self, name, calendar, fast=False, reCreate=False):
+    def addResource(self, name, calendar, revision, fast=False, reCreate=False):
         """
         Adding or updating an existing resource.
         To check for an update we attempt to get an existing UID
@@ -225,12 +228,12 @@
         """
         oldUID = self.resourceUIDForName(name)
         if oldUID is not None:
-            self._delete_from_db(name, oldUID)
-        self._add_to_db(name, calendar, reCreate=reCreate)
+            self._delete_from_db(name, oldUID, None)
+        self._add_to_db(name, calendar, revision, reCreate=reCreate)
         if not fast:
             self._db_commit()
 
-    def deleteResource(self, name):
+    def deleteResource(self, name, revision):
         """
         Remove this resource from the index.
         @param name: the name of the resource to add.
@@ -238,7 +241,7 @@
         """
         uid = self.resourceUIDForName(name)
         if uid is not None:
-            self._delete_from_db(name, uid)
+            self._delete_from_db(name, uid, revision)
             self._db_commit()
 
     def resourceExists(self, name):
@@ -274,6 +277,26 @@
             self.log_info("Search falls outside range of index for %s %s" % (name, minDate))
             self.reExpandResource(name, minDate)
 
+    def whatchanged(self, revision):
+
+        results = [(name.encode("utf-8"), created, wasdeleted) for name, created, wasdeleted in self._db_execute("select NAME, CREATEDREVISION, WASDELETED from REVISIONS where REVISION > :1", revision)]
+        results.sort(key=lambda x:x[1])
+        
+        changed = []
+        deleted = []
+        for name, created, wasdeleted in results:
+            if name:
+                if wasdeleted == 'Y':
+                    # Don't report items that were created/deleted since the requested revision
+                    if created <= revision:
+                        deleted.append(name)
+                else:
+                    changed.append(name)
+            else:
+                raise SyncTokenValidException
+        
+        return changed, deleted,
+
     def indexedSearch(self, filter, fbtype=False):
         """
         Finds resources matching the given qualifiers.
@@ -357,7 +380,7 @@
         """
         return schema_version
 
-    def _add_to_db(self, name, calendar, cursor = None, expand_until=None, reCreate=False):
+    def _add_to_db(self, name, calendar, revision, cursor=None, expand_until=None, reCreate=False):
         """
         Records the given calendar resource in the index with the given name.
         Resource names and UIDs must both be unique; only one resource name may
@@ -370,7 +393,7 @@
         """
         raise NotImplementedError
 
-    def _delete_from_db(self, name, uid):
+    def _delete_from_db(self, name, uid, revision):
         """
         Deletes the specified entry from all dbs.
         @param name: the name of the resource to delete.
@@ -445,6 +468,28 @@
             """
         )
 
+        #
+        # REVISIONS table tracks changes
+        #   NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
+        #   REVISION: revision number
+        #   WASDELETED: Y if revision deleted, N if added or changed
+        #
+        q.execute(
+            """
+            create table REVISIONS (
+                NAME            text unique,
+                REVISION        integer,
+                CREATEDREVISION integer,
+                WASDELETED      text(1)
+            )
+            """
+        )
+        q.execute(
+            """
+            create index REVISION on REVISIONS (REVISION)
+            """
+        )
+
         if uidunique:
             #
             # RESERVED table tracks reserved UIDs
@@ -473,15 +518,29 @@
         Upgrade the data from an older version of the DB.
         """
 
-        # When going to version 8 all we need to do is add an index
-        if old_version < "8":
-            q.execute("create index STARTENDFLOAT on TIMESPAN (START, END, FLOAT)")
-
-        # When going to version 7,8 all we need to do is add a column to the resource and timespan
+        # When going to version 7+ all we need to do is add a column to the resource and timespan
         if old_version < "7":
             q.execute("alter table RESOURCE add column ORGANIZER text default '?'")
             q.execute("alter table TIMESPAN add column FBTYPE text(1) default '?'")
 
+        # When going to version 8+ all we need to do is add an index
+        if old_version < "8":
+            q.execute("create index STARTENDFLOAT on TIMESPAN (START, END, FLOAT)")
+
+        # When going to version 9+ all we need to do is add revision table and index
+        if old_version < "9":
+            q.execute(
+                """
+                create table REVISIONS (
+                    NAME            text unique,
+                    REVISION        integer,
+                    CREATEDREVISION integer,
+                    WASDELETED      text(1)
+                )
+                """
+            )
+            q.execute("create index REVISION on REVISIONS (REVISION)")
+
     def notExpandedBeyond(self, minDate):
         """
         Gives all resources which have not been expanded beyond a given date
@@ -495,10 +554,10 @@
         with a longer expansion.
         """
         calendar = self.resource.getChild(name).iCalendar()
-        self._add_to_db(name, calendar, expand_until=expand_until, reCreate=True)
+        self._add_to_db(name, calendar, None, expand_until=expand_until, reCreate=True)
         self._db_commit()
 
-    def _add_to_db(self, name, calendar, cursor = None, expand_until=None, reCreate=False):
+    def _add_to_db(self, name, calendar, revision, cursor = None, expand_until=None, reCreate=False):
         """
         Records the given calendar resource in the index with the given name.
         Resource names and UIDs must both be unique; only one resource name may
@@ -536,7 +595,7 @@
             log.err("Invalid instance %s when indexing %s in %s" % (e.rid, name, self.resource,))
             raise
 
-        self._delete_from_db(name, uid)
+        self._delete_from_db(name, uid, None)
 
         for key in instances:
             instance = instances[key]
@@ -562,7 +621,7 @@
                 values (:1, :2, :3, :4, :5)
                 """, name, float, start, end, '?'
             )
-             
+            
         self._db_execute(
             """
             insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX, ORGANIZER)
@@ -570,7 +629,18 @@
             """, name, uid, calendar.resourceType(), instances.limit, organizer
         )
 
-    def _delete_from_db(self, name, uid):
+        if revision is not None:
+            created = self._db_value_for_sql("select CREATEDREVISION from REVISIONS where NAME = :1", name)
+            if created is None:
+                created = revision
+            self._db_execute(
+                """
+                insert or replace into REVISIONS (NAME, REVISION, CREATEDREVISION, WASDELETED)
+                values (:1, :2, :3, :4)
+                """, name, revision, revision, 'N',
+            )
+
+    def _delete_from_db(self, name, uid, revision):
         """
         Deletes the specified entry from all dbs.
         @param name: the name of the resource to delete.
@@ -578,6 +648,13 @@
         """
         self._db_execute("delete from TIMESPAN where NAME = :1", name)
         self._db_execute("delete from RESOURCE where NAME = :1", name)
+        if revision is not None:
+            self._db_execute(
+                """
+                update REVISIONS SET REVISION = :1, WASDELETED = :2
+                where NAME = :3
+                """, revision, 'Y', name
+            )
 
 
 def wrapInDeferred(f):
@@ -832,13 +909,16 @@
                 log.err("Non-calendar resource: %s" % (name,))
             else:
                 #log.msg("Indexing resource: %s" % (name,))
-                self.addResource(name, calendar, True, reCreate=True)
+                self.addResource(name, calendar, 0, True, reCreate=True)
             finally:
                 stream.close()
 
         # Do commit outside of the loop for better performance
         if do_commit:
             self._db_commit()
+            
+        # This is a deferred but we can't defer at this point...
+        self.resource.bumpSyncToken(True)
 
 class IndexSchedule (CalendarIndex):
     """
@@ -944,10 +1024,13 @@
                 log.err("Non-calendar resource: %s" % (name,))
             else:
                 #log.msg("Indexing resource: %s" % (name,))
-                self.addResource(name, calendar, True, reCreate=True)
+                self.addResource(name, calendar, 0, True, reCreate=True)
             finally:
                 stream.close()
 
         # Do commit outside of the loop for better performance
         if do_commit:
             self._db_commit()
+            
+        # This is a deferred but we can't defer at this point...
+        self.resource.bumpSyncToken(True)

Modified: CalendarServer/trunk/twistedcaldav/method/__init__.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/__init__.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/method/__init__.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2007 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,4 +33,5 @@
     "report_calquery",
     "report_freebusy",
     "report_multiget",
+    "report_sync_collection",
 ]

Modified: CalendarServer/trunk/twistedcaldav/method/delete_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/delete_common.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/method/delete_common.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -105,11 +105,9 @@
 
         if response == responsecode.NO_CONTENT:
             if isPseudoCalendarCollectionResource(parent):
+                newrevision = (yield parent.bumpSyncToken())
                 index = parent.index()
-                index.deleteResource(delresource.fp.basename())
-
-                # Change CTag on the parent calendar collection
-                yield parent.updateCTag()
+                index.deleteResource(delresource.fp.basename(), newrevision)
                 
         returnValue(response)
 
@@ -161,12 +159,10 @@
                 yield delresource.quotaSizeAdjust(self.request, -old_size)
     
             if response == responsecode.NO_CONTENT:
+                newrevision = (yield parent.bumpSyncToken())
                 index = parent.index()
-                index.deleteResource(delresource.fp.basename())
+                index.deleteResource(delresource.fp.basename(), newrevision)
     
-                # Change CTag on the parent calendar collection
-                yield parent.updateCTag()
-    
                 # Do scheduling
                 if scheduler and not self.internal_request:
                     yield scheduler.doImplicitScheduling()
@@ -217,7 +213,7 @@
 
         # Now do normal delete
         # Change CTag
-        yield delresource.updateCTag()
+        yield delresource.bumpSyncToken()
         more_responses = (yield self.deleteResource(delresource, deluri, parent))
         
         if isinstance(more_responses, MultiStatusResponse):

Modified: CalendarServer/trunk/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/put_common.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/method/put_common.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,6 +1,6 @@
 # -*- test-case-name: twistedcaldav.test.test_validation -*-
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -273,6 +273,7 @@
         
         self.rollback = None
         self.access = None
+        self.newrevision = None
 
     @inlineCallbacks
     def fullValidation(self):
@@ -830,7 +831,8 @@
     def doSourceDelete(self):
         # Delete index for original item
         if self.sourcecal:
-            self.source_index.deleteResource(self.source.fp.basename())
+            self.newrevision = (yield self.sourceparent.bumpSyncToken())
+            self.source_index.deleteResource(self.source.fp.basename(), self.newrevision)
             self.rollback.source_index_deleted = True
             log.debug("Source index removed %s" % (self.source.fp.path,))
 
@@ -838,10 +840,6 @@
         delete(self.source_uri, self.source.fp, "0")
         self.rollback.source_deleted = True
         log.debug("Source removed %s" % (self.source.fp.path,))
-
-        # Change CTag on the parent calendar collection
-        if self.sourcecal:
-            yield self.sourceparent.updateCTag()
   
         returnValue(None)
 
@@ -878,7 +876,7 @@
         
         # Add or update the index for this resource.
         try:
-            self.source_index.addResource(self.source.fp.basename(), self.calendar)
+            self.source_index.addResource(self.source.fp.basename(), self.calendar, self.newrevision)
         except TooManyInstancesError, ex:
             raise HTTPError(ErrorResponse(
                 responsecode.FORBIDDEN,
@@ -899,7 +897,7 @@
         
         # Add or update the index for this resource.
         try:
-            self.destination_index.addResource(self.destination.fp.basename(), caltoindex)
+            self.destination_index.addResource(self.destination.fp.basename(), caltoindex, self.newrevision)
             log.debug("Destination indexed %s" % (self.destination.fp.path,))
         except TooManyInstancesError, ex:
             log.err("Cannot index calendar resource as there are too many recurrence instances %s" % self.destination)
@@ -926,7 +924,7 @@
         
         # Delete index for original item
         if self.destinationcal:
-            self.destination_index.deleteResource(self.destination.fp.basename())
+            self.destination_index.deleteResource(self.destination.fp.basename(), None)
             self.rollback.destination_index_deleted = True
             log.debug("Destination index removed %s" % (self.destination.fp.path,))
 
@@ -1088,6 +1086,7 @@
     
             # Index the new resource if storing to a calendar.
             if self.destinationcal:
+                self.newrevision = (yield self.destinationparent.bumpSyncToken())
                 result = self.doDestinationIndex(self.calendar)
                 if result is not None:
                     self.rollback.Rollback()
@@ -1101,10 +1100,6 @@
             if self.destquota is not None:
                 yield self.doDestinationQuotaCheck()
     
-            if self.destinationcal:
-                # Change CTag on the parent calendar collection
-                yield self.destinationparent.updateCTag()
-    
             # Can now commit changes and forget the rollback details
             self.rollback.Commit()
     

Modified: CalendarServer/trunk/twistedcaldav/method/report_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_common.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/method/report_common.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2006-2008 Apple Inc. All rights reserved.
+# Copyright (c) 2006-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -452,7 +452,7 @@
                     processEventFreeBusy(calendar, fbinfo, timerange, tzinfo)
                     
                     # Lets also force an index rebuild for this resource so that next time we have the fbtype set
-                    calresource.index().addResource(name, calendar, reCreate=True)
+                    calresource.index().addResource(name, calendar, None, reCreate=True)
 
                 elif calendar.mainType() == "VFREEBUSY":
                     processFreeBusyFreeBusy(calendar, fbinfo, timerange)

Copied: CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py (from rev 4900, CalendarServer/branches/users/cdaboo/webdav-extensions-4888/twistedcaldav/method/report_sync_collection.py)
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py	                        (rev 0)
+++ CalendarServer/trunk/twistedcaldav/method/report_sync_collection.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -0,0 +1,146 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+DAV sync-collection report
+"""
+
+__all__ = ["report_DAV__sync_collection"]
+
+from twext.web2.dav.davxml import ErrorResponse, SyncToken
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python.failure import Failure
+from twisted.web2 import responsecode
+from twisted.web2.dav import davxml
+from twisted.web2.dav.element.base import WebDAVElement
+from twisted.web2.dav.http import MultiStatusResponse, statusForFailure
+from twisted.web2.dav.method.prop_common import responseForHref
+from twisted.web2.dav.method.propfind import propertyName
+from twisted.web2.dav.util import joinURL
+from twisted.web2.http import HTTPError
+
+from twistedcaldav.config import config
+from twistedcaldav.log import Logger
+
+import functools
+
+log = Logger()
+
+ at inlineCallbacks
+def report_DAV__sync_collection(self, request, sync_collection):
+    """
+    Generate a sync-collection REPORT.
+    """
+    if not self.isPseudoCalendarCollection() or not config.EnableSyncReport:
+        log.err("sync-collection report is only allowed on calendar/inbox collection resources %s" % (self,))
+        raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, davxml.SupportedReport()))
+   
+    responses = []
+
+    propertyreq = sync_collection.property
+    
+    @inlineCallbacks
+    def _namedPropertiesForResource(request, props, resource, forbidden=False):
+        """
+        Return the specified properties on the specified resource.
+        @param request: the L{IRequest} for the current request.
+        @param props: a list of property elements or qname tuples for the properties of interest.
+        @param resource: the L{DAVFile} for the targeted resource.
+        @return: a map of OK and NOT FOUND property values.
+        """
+        properties_by_status = {
+            responsecode.OK        : [],
+            responsecode.FORBIDDEN : [],
+            responsecode.NOT_FOUND : [],
+        }
+        
+        for property in props:
+            if isinstance(property, WebDAVElement):
+                qname = property.qname()
+            else:
+                qname = property
+
+            if forbidden:
+                properties_by_status[responsecode.FORBIDDEN].append(propertyName(qname))
+            else:
+                props = (yield resource.listProperties(request))
+                if qname in props:
+                    try:
+                        prop = (yield resource.readProperty(qname, request))
+                        properties_by_status[responsecode.OK].append(prop)
+                    except:
+                        f = Failure()
+                        log.err("Error reading property %r for resource %s: %s" % (qname, request.uri, f.value))
+                        status = statusForFailure(f, "getting property: %s" % (qname,))
+                        if status not in properties_by_status: properties_by_status[status] = []
+                        properties_by_status[status].append(propertyName(qname))
+                else:
+                    properties_by_status[responsecode.NOT_FOUND].append(propertyName(qname))
+        
+        yield properties_by_status
+    
+    # Do some optimization of access control calculation by determining any inherited ACLs outside of
+    # the child resource loop and supply those to the checkPrivileges on each child.
+    filteredaces = (yield self.inheritedACEsforChildren(request))
+
+    changed, removed, newtoken = self.whatchanged(sync_collection.sync_token)
+
+    # Now determine which valid resources are readable and which are not
+    ok_resources = []
+    forbidden_resources = []
+    if changed:
+        yield self.findChildrenFaster(
+            "1",
+            request,
+            lambda x, y: ok_resources.append((x, y)),
+            lambda x, y: forbidden_resources.append((x, y)),
+            changed,
+            (davxml.Read(),),
+            inherited_aces=filteredaces
+        )
+
+    for child, child_uri in ok_resources:
+        href = davxml.HRef.fromString(child_uri)
+        yield responseForHref(
+            request,
+            responses,
+            href,
+            child,
+            functools.partial(_namedPropertiesForResource, forbidden=False) if propertyreq else None,
+            propertyreq)
+
+    for child, child_uri in forbidden_resources:
+        href = davxml.HRef.fromString(child_uri)
+        yield responseForHref(
+            request,
+            responses,
+            href,
+            child,
+            functools.partial(_namedPropertiesForResource, forbidden=True) if propertyreq else None,
+            propertyreq)
+
+    for name in removed:
+        href = davxml.HRef.fromString(joinURL(request.uri, name))
+        responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+    
+    if not hasattr(request, "extendedLogItems"):
+        request.extendedLogItems = {}
+    request.extendedLogItems["responses"] = len(responses)
+
+    responses.append(SyncToken.fromString(newtoken))
+
+    returnValue(MultiStatusResponse(responses))

Modified: CalendarServer/trunk/twistedcaldav/resource.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/resource.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/resource.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@
 
 from zope.interface import implements
 
-from twext.web2.dav.davxml import ErrorResponse
+from twext.web2.dav.davxml import ErrorResponse, SyncCollection
 
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, maybeDeferred, succeed
@@ -690,9 +690,12 @@
         result = super(CalDAVResource, self).supportedReports()
         result.append(davxml.Report(caldavxml.CalendarQuery(),))
         result.append(davxml.Report(caldavxml.CalendarMultiGet(),))
-        if (self.isCollection()):
+        if self.isCollection():
             # Only allowed on collections
             result.append(davxml.Report(caldavxml.FreeBusyQuery(),))
+        if self.isPseudoCalendarCollection() and config.EnableSyncReport:
+            # Only allowed on calendar/inbox collections
+            result.append(davxml.Report(SyncCollection(),))
         return result
 
     def writeNewACEs(self, newaces):

Modified: CalendarServer/trunk/twistedcaldav/static.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/static.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/static.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,6 +38,7 @@
 import os
 import errno
 from urlparse import urlsplit
+from uuid import uuid4
 
 from twext.web2.dav.davxml import ErrorResponse
 
@@ -47,6 +48,7 @@
 from twisted.web2 import responsecode, http, http_headers
 from twisted.web2.http import HTTPError, StatusResponse
 from twisted.web2.dav import davxml
+from twisted.web2.dav.element.base import dav_namespace
 from twisted.web2.dav.fileop import mkcollection, rmdir
 from twisted.web2.dav.idav import IDAVResource
 from twisted.web2.dav.noneprops import NonePropertyStore
@@ -60,11 +62,12 @@
 from twistedcaldav.config import config
 from twistedcaldav.customxml import TwistedCalendarAccessProperty, TwistedScheduleMatchETags
 from twistedcaldav.extensions import DAVFile, CachingPropertyStore
+from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
 from twistedcaldav.memcacheprops import MemcachePropertyCollection
 from twistedcaldav.freebusyurl import FreeBusyURLResource
 from twistedcaldav.ical import Component as iComponent
 from twistedcaldav.ical import Property as iProperty
-from twistedcaldav.index import Index, IndexSchedule
+from twistedcaldav.index import Index, IndexSchedule, SyncTokenValidException
 from twistedcaldav.resource import CalDAVResource, isCalendarCollectionResource, isPseudoCalendarCollectionResource
 from twistedcaldav.schedule import ScheduleInboxResource, ScheduleOutboxResource, IScheduleInboxResource
 from twistedcaldav.dropbox import DropBoxHomeResource, DropBoxCollectionResource
@@ -208,7 +211,7 @@
                 raise HTTPError(status)
 
             # Initialize CTag on the calendar collection
-            d1 = self.updateCTag()
+            d1 = self.bumpSyncToken()
 
             # Calendar is initially transparent to freebusy
             self.writeDeadProperty(caldavxml.ScheduleCalendarTransp(caldavxml.Transparent()))
@@ -373,6 +376,94 @@
         """
         return Index(self)
 
+    def whatchanged(self, client_token):
+        
+        current_token = str(self.readDeadProperty(customxml.GETCTag))
+        current_uuid, current_revision = current_token.split("#", 1)
+        current_revision = int(current_revision)
+
+        if client_token:
+            try:
+                caluuid, revision = client_token.split("#", 1)
+                revision = int(revision)
+                
+                # Check client token validity
+                if caluuid != current_uuid:
+                    raise ValueError
+                if revision > current_revision:
+                    raise ValueError
+            except ValueError:
+                raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (dav_namespace, "valid-sync-token")))
+        else:
+            revision = 0
+
+        try:
+            changed, removed = self.index().whatchanged(revision)
+        except SyncTokenValidException:
+            raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (dav_namespace, "valid-sync-token")))
+
+        return changed, removed, current_token
+
+    @inlineCallbacks
+    def bumpSyncToken(self, reset=False):
+        """
+        Increment the sync-token which is also the ctag.
+        
+        return: a deferred that returns the new revision number
+        """
+        assert self.isCollection()
+        
+        # Need to lock
+        lock = MemcacheLock("ResourceLock", self.fp.path, timeout=60.0)
+        try:
+            try:
+                yield lock.acquire()
+            except MemcacheLockTimeoutError:
+                raise HTTPError(StatusResponse(responsecode.CONFLICT, "Resource: %s currently in use on the server." % (self.uri,)))
+
+            try:
+                if reset:
+                    raise ValueError
+                token = str(self.readDeadProperty(customxml.GETCTag))
+                caluuid, revision = token.split("#", 1)
+                revision = int(revision) + 1
+                token = "%s#%d" % (caluuid, revision,)
+    
+            except (HTTPError, ValueError):
+                # Initialise it
+                caluuid = uuid4()
+                revision = 1
+                token = "%s#%d" % (caluuid, revision,)
+    
+            yield self.updateCTag(token)
+            returnValue(revision)
+        finally:
+            yield lock.clean()
+
+    def updateCTag(self, token=None):
+        assert self.isCollection()
+        
+        if not token:
+            token = str(datetime.datetime.now())
+        try:
+            self.writeDeadProperty(customxml.GETCTag(token))
+        except:
+            return fail(Failure())
+
+        if hasattr(self, 'clientNotifier'):
+            self.clientNotifier.notify(op="update")
+        else:
+            log.debug("%r does not have a clientNotifier but the CTag changed"
+                      % (self,))
+
+        if hasattr(self, 'cacheNotifier'):
+            return self.cacheNotifier.changed()
+        else:
+            log.debug("%r does not have a cacheNotifier but the CTag changed"
+                      % (self,))
+
+        return succeed(True)
+
     ##
     # File
     ##
@@ -439,28 +530,6 @@
 
         return similar
 
-    def updateCTag(self):
-        assert self.isCollection()
-        try:
-            self.writeDeadProperty(customxml.GETCTag(
-                    str(datetime.datetime.now())))
-        except:
-            return fail(Failure())
-
-        if hasattr(self, 'clientNotifier'):
-            self.clientNotifier.notify(op="update")
-        else:
-            log.debug("%r does not have a clientNotifier but the CTag changed"
-                      % (self,))
-
-        if hasattr(self, 'cacheNotifier'):
-            return self.cacheNotifier.changed()
-        else:
-            log.debug("%r does not have a cacheNotifier but the CTag changed"
-                      % (self,))
-
-        return succeed(True)
-
     ##
     # Quota
     ##
@@ -883,7 +952,7 @@
         if self.provisionFile():
 
             # Initialize CTag on the calendar collection
-            self.updateCTag()
+            self.bumpSyncToken()
 
             # Initialize the index
             self.index().create()
@@ -910,10 +979,7 @@
         ScheduleOutboxResource.__init__(self, parent)
 
     def provision(self):
-        if self.provisionFile():
-            # Initialize CTag on the calendar collection
-            self.updateCTag()
-
+        self.provisionFile()
         return super(ScheduleOutboxFile, self).provision()
 
     def __repr__(self):

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,6 +1,6 @@
 # -*- test-case-name: twistedcaldav.test.test_stdconfig -*-
 ##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -201,6 +201,11 @@
     "EnableSACLs": False,
 
     #
+    # Standard (or draft) WebDAV extensions
+    #
+    "EnableSyncReport"        : True,  # REPORT collection-sync
+
+    #
     # Non-standard CalDAV extensions
     #
     "EnableDropBox"           : False, # Calendar Drop Box

Modified: CalendarServer/trunk/twistedcaldav/test/test_index.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_index.py	2010-01-06 16:51:44 UTC (rev 4906)
+++ CalendarServer/trunk/twistedcaldav/test/test_index.py	2010-01-06 16:54:57 UTC (rev 4907)
@@ -1,5 +1,5 @@
 ##
-# Copyright (c) 2007 Apple Inc. All rights reserved.
+# Copyright (c) 2010 Apple Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
 from twistedcaldav.ical import Component
 from twistedcaldav.index import Index, default_future_expansion_duration,\
     maximum_future_expansion_duration, IndexedSearchException,\
-    AbstractCalendarIndex
+    AbstractCalendarIndex, icalfbtype_to_indexfbtype
 from twistedcaldav.index import ReservationError, MemcachedUIDReserver
 from twistedcaldav.instance import InvalidOverriddenInstanceError
 from twistedcaldav.test.util import InMemoryMemcacheProtocol
@@ -230,17 +230,19 @@
             ),
         )
 
+        revision = 0
         for description, name, calendar_txt, reCreate, ok in data:
+            revision += 1
             calendar = Component.fromString(calendar_txt)
             if ok:
                 f = open(os.path.join(self.site.resource.fp.path, name), "w")
                 f.write(calendar_txt)
                 del f
 
-                self.db.addResource(name, calendar, reCreate=reCreate)
+                self.db.addResource(name, calendar, revision, reCreate=reCreate)
                 self.assertTrue(self.db.resourceExists(name), msg=description)
             else:
-                self.assertRaises(InvalidOverriddenInstanceError, self.db.addResource, name, calendar)
+                self.assertRaises(InvalidOverriddenInstanceError, self.db.addResource, name, calendar, revision)
                 self.assertFalse(self.db.resourceExists(name), msg=description)
 
         self.db._db_recreate()
@@ -402,14 +404,16 @@
             ),
         )
 
+        revision = 0
         for description, name, calendar_txt, trstart, trend, organizer, instances in data:
+            revision += 1
             calendar = Component.fromString(calendar_txt)
 
             f = open(os.path.join(self.site.resource.fp.path, name), "w")
             f.write(calendar_txt)
             del f
 
-            self.db.addResource(name, calendar)
+            self.db.addResource(name, calendar, revision)
             self.assertTrue(self.db.resourceExists(name), msg=description)
 
             # Create fake filter element to match time-range
@@ -434,6 +438,69 @@
 
             self.assertEqual(set(instances), index_results, msg=description)
 
+    def test_index_revisions(self):
+        data1 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+"""
+        data2 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+"""
+        data3 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.3
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+"""
+
+        calendar = Component.fromString(data1)
+        self.db.addResource("data1.ics", calendar, 1)
+        calendar = Component.fromString(data2)
+        self.db.addResource("data2.ics", calendar, 2)
+        calendar = Component.fromString(data3)
+        self.db.addResource("data3.ics", calendar, 3)
+        self.db.deleteResource("data3.ics", 4)
+
+        tests = (
+            (0, (["data1.ics", "data2.ics",], [],)),
+            (1, (["data2.ics",], [],)),
+            (2, ([], [],)),
+            (3, ([], ["data3.ics",],)),
+            (4, ([], [],)),
+            (5, ([], [],)),
+        )
+        
+        for revision, results in tests:
+            self.assertEquals(self.db.whatchanged(revision), results, "Mismatched results for whatchanged with revision %d" % (revision,))
+
 class SQLIndexUpgradeTests (twistedcaldav.test.util.TestCase):
     """
     Test abstract SQL DB class
@@ -556,7 +623,7 @@
             except InvalidOverriddenInstanceError:
                 raise
     
-            self._delete_from_db(name, uid)
+            self._delete_from_db(name, uid, None)
     
             for key in instances:
                 instance = instances[key]
@@ -675,6 +742,77 @@
                     """
                 )
 
+        def _add_to_db(self, name, calendar, cursor = None, expand_until=None, reCreate=False):
+            """
+            Records the given calendar resource in the index with the given name.
+            Resource names and UIDs must both be unique; only one resource name may
+            be associated with any given UID and vice versa.
+            NB This method does not commit the changes to the db - the caller
+            MUST take care of that
+            @param name: the name of the resource to add.
+            @param calendar: a L{Calendar} object representing the resource
+                contents.
+            """
+            uid = calendar.resourceUID()
+            organizer = calendar.getOrganizer()
+            if not organizer:
+                organizer = ""
+    
+            # Decide how far to expand based on the component
+            master = calendar.masterComponent()
+            if master is None or not calendar.isRecurring() and not calendar.isRecurringUnbounded():
+                # When there is no master we have a set of overridden components - index them all.
+                # When there is one instance - index it.
+                # When bounded - index all.
+                expand = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
+            else:
+                if expand_until:
+                    expand = expand_until
+                else:
+                    expand = datetime.date.today() + default_future_expansion_duration
+        
+                if expand > (datetime.date.today() + maximum_future_expansion_duration):
+                    raise IndexedSearchException
+    
+            try:
+                instances = calendar.expandTimeRanges(expand, ignoreInvalidInstances=reCreate)
+            except InvalidOverriddenInstanceError, e:
+                raise
+    
+            self._delete_from_db(name, uid, None)
+    
+            for key in instances:
+                instance = instances[key]
+                start = instance.start.replace(tzinfo=utc)
+                end = instance.end.replace(tzinfo=utc)
+                float = 'Y' if instance.start.tzinfo is None else 'N'
+                self._db_execute(
+                    """
+                    insert into TIMESPAN (NAME, FLOAT, START, END, FBTYPE)
+                    values (:1, :2, :3, :4, :5)
+                    """, name, float, start, end, icalfbtype_to_indexfbtype.get(instance.component.getFBType(), 'F')
+                )
+    
+            # Special - for unbounded recurrence we insert a value for "infinity"
+            # that will allow an open-ended time-range to always match it.
+            if calendar.isRecurringUnbounded():
+                start = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
+                end = datetime.datetime(2100, 1, 1, 1, 0, 0, tzinfo=utc)
+                float = 'N'
+                self._db_execute(
+                    """
+                    insert into TIMESPAN (NAME, FLOAT, START, END, FBTYPE)
+                    values (:1, :2, :3, :4, :5)
+                    """, name, float, start, end, '?'
+                )
+                 
+            self._db_execute(
+                """
+                insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX, ORGANIZER)
+                values (:1, :2, :3, :4, :5)
+                """, name, uid, calendar.resourceType(), instances.limit, organizer
+            )
+
     def setUp(self):
         super(SQLIndexUpgradeTests, self).setUp()
         self.site.resource.isCalendarCollection = lambda: True
@@ -749,7 +887,7 @@
 END:VCALENDAR
 """
     
-            olddb.addResource(calendar_name, Component.fromString(calendar_data))
+            olddb.addResource(calendar_name, Component.fromString(calendar_data), 1)
             self.assertTrue(olddb.resourceExists(calendar_name))
     
             if olddb._db_version() == "6":
@@ -772,7 +910,7 @@
             else:
                 self.assertEqual(value, "B")
     
-            self.db.addResource(calendar_name, Component.fromString(calendar_data))
+            self.db.addResource(calendar_name, Component.fromString(calendar_data), 2)
             self.assertTrue(olddb.resourceExists(calendar_name))
     
             value = self.db._db_value_for_sql("select ORGANIZER from RESOURCE where NAME = :1", calendar_name)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100106/21119cdf/attachment-0001.html>


More information about the calendarserver-changes mailing list