[CalendarServer-changes] [11096] CalendarServer/branches/users/cdaboo/store-scheduling

source_changes at macosforge.org source_changes at macosforge.org
Wed Apr 24 11:58:18 PDT 2013


Revision: 11096
          http://trac.calendarserver.org//changeset/11096
Author:   cdaboo at apple.com
Date:     2013-04-24 11:58:18 -0700 (Wed, 24 Apr 2013)
Log Message:
-----------
Checkpoint: CardDAV implemented. Various test fixes.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/test_principal.py
    CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/util.py
    CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/storebridge.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/scheduling/processing.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/sql.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/calendar_store/ho/me/home1/calendar_1/1.ics
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/common.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_attachments.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_implicit.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/icalendarstore.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/sql.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/common.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/test_file.py
    CalendarServer/branches/users/cdaboo/store-scheduling/txdav/common/icommondatastore.py

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/test_principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/test_principal.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/test_principal.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -69,9 +69,9 @@
         for directory in self.directoryServices:
             name = directory.__class__.__name__
             url = "/" + name + "/"
-            directory.setPrincipalPath(name)
 
             provisioningResource = DirectoryPrincipalProvisioningResource(url, directory)
+            directory.setPrincipalCollection(provisioningResource)
 
             self.site.resource.putChild(name, provisioningResource)
 
@@ -437,12 +437,7 @@
         """
         for _ignore_provisioningResource, _ignore_recordType, recordResource, record in self._allRecords():
             if record.enabledForCalendaring:
-                self.failUnless(
-                    (
-                        set((recordResource.principalURL(),)) |
-                        set(record.calendarUserAddresses)
-                    ).issubset(set(recordResource.calendarUserAddresses()))
-                )
+                self.assertEqual(set(record.calendarUserAddresses), set(recordResource.calendarUserAddresses()))
 
                 # Verify that if not enabled for calendaring, no CUAs:
                 record.enabledForCalendaring = False

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/util.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/directory/test/util.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -200,6 +200,8 @@
         addresses = set(value("addresses"))
         if record.enabledForCalendaring:
             addresses.add("urn:uuid:%s" % (record.guid,))
+            addresses.add("/principals/__uids__/%s/" % (record.guid,))
+            addresses.add("/principals/%s/%s/" % (record.recordType, record.shortNames[0],))
 
         if hasattr(record.service, "recordTypePrefix"):
             prefix = record.service.recordTypePrefix

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/storebridge.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/twistedcaldav/storebridge.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -41,8 +41,8 @@
 from twistedcaldav.cache import CacheStoreNotifier, ResponseCacheMixin, \
     DisabledCacheNotifier
 from twistedcaldav.caldavxml import caldav_namespace, MaxAttendeesPerInstance, \
-    NoUIDConflict, MaxInstances
-from twistedcaldav.carddavxml import carddav_namespace
+    MaxInstances, NoUIDConflict
+from twistedcaldav.carddavxml import carddav_namespace, NoUIDConflict as NovCardUIDConflict
 from twistedcaldav.config import config
 from twistedcaldav.directory.wiki import WikiDirectoryService, getWikiAccess
 from twistedcaldav.ical import Component as VCalendar, Property as VProperty, \
@@ -59,15 +59,16 @@
     AttachmentStoreValidManagedID, AttachmentRemoveFailed, \
     AttachmentDropboxNotAllowed, InvalidComponentTypeError, \
     TooManyAttendeesError, InvalidCalendarAccessError, ValidOrganizerError, \
-    UIDExistsError, InvalidUIDError, InvalidPerUserDataMerge, \
-    AttendeeAllowedError, ResourceDeletedError, InvalidComponentForStoreError, \
-    InvalidResourceMove, UIDExistsElsewhereError, InvalidAttachmentOperation
+    InvalidPerUserDataMerge, \
+    AttendeeAllowedError, ResourceDeletedError, InvalidAttachmentOperation
 from txdav.common.datastore.sql_tables import _BIND_MODE_READ, _BIND_MODE_WRITE, \
     _BIND_MODE_DIRECT
 from txdav.common.icommondatastore import NoSuchObjectResourceError, \
     TooManyObjectResourcesError, ObjectResourceTooBigError, \
     InvalidObjectResourceError, ObjectResourceNameNotAllowedError, \
-    ObjectResourceNameAlreadyExistsError
+    ObjectResourceNameAlreadyExistsError, UIDExistsError, \
+    UIDExistsElsewhereError, InvalidUIDError, InvalidResourceMove, \
+    InvalidComponentForStoreError
 from txdav.idav import PropertyChangeNotAllowedError
 from txdav.xml import element as davxml
 from txdav.xml.base import dav_namespace, WebDAVUnknownElement, encodeXMLName
@@ -651,7 +652,7 @@
                 # Get a resource for the new item
                 newchildURL = joinURL(request.path, name)
                 newchild = (yield request.locateResource(newchildURL))
-                dataChanged = (yield self.storeResourceData(newchild, component, returnData=return_changed))
+                dataChanged = (yield self.storeResourceData(newchild, component, returnChangedData=return_changed))
 
             except HTTPError, e:
                 # Extract the pre-condition
@@ -1210,10 +1211,10 @@
 
 
     @inlineCallbacks
-    def storeResourceData(self, newchild, component, returnData=False):
+    def storeResourceData(self, newchild, component, returnChangedData=False):
 
         yield newchild.storeComponent(component)
-        if returnData:
+        if returnChangedData and newchild._newStoreObject._componentChanged:
             result = (yield newchild.componentForUser())
             returnValue(str(result))
         else:
@@ -2934,10 +2935,10 @@
 
 
     @inlineCallbacks
-    def storeResourceData(self, newchild, component, returnData=False):
+    def storeResourceData(self, newchild, component, returnChangedData=False):
 
         yield newchild.storeComponent(component)
-        if returnData:
+        if returnChangedData and newchild._newStoreObject._componentChanged:
             result = (yield newchild.component())
             returnValue(str(result))
         else:
@@ -3027,8 +3028,129 @@
 
     vCard = _CommonObjectResource.component
 
+    StoreExceptionsStatusErrors = set((
+        ObjectResourceNameNotAllowedError,
+        ObjectResourceNameAlreadyExistsError,
+    ))
 
+    StoreExceptionsErrors = {
+        TooManyObjectResourcesError: customxml.MaxResources(),
+        ObjectResourceTooBigError: (carddav_namespace, "max-resource-size"),
+        InvalidObjectResourceError: (carddav_namespace, "valid-address-data"),
+        InvalidComponentForStoreError: (carddav_namespace, "valid-addressbook-object-resource"),
+        UIDExistsError: NovCardUIDConflict(),
+        InvalidUIDError: NovCardUIDConflict(),
+        InvalidPerUserDataMerge: (carddav_namespace, "valid-address-data"),
+    }
 
+    StoreMoveExceptionsStatusErrors = set((
+        ObjectResourceNameNotAllowedError,
+        ObjectResourceNameAlreadyExistsError,
+    ))
+
+    StoreMoveExceptionsErrors = {
+        TooManyObjectResourcesError: customxml.MaxResources(),
+        InvalidResourceMove: (calendarserver_namespace, "valid-move"),
+    }
+
+    @inlineCallbacks
+    def http_PUT(self, request):
+
+        # Content-type check
+        content_type = request.headers.getHeader("content-type")
+        if content_type is not None and (content_type.mediaType, content_type.mediaSubtype) != ("text", "vcard"):
+            log.err("MIME type %s not allowed in vcard collection" % (content_type,))
+            raise HTTPError(ErrorResponse(
+                responsecode.FORBIDDEN,
+                (carddav_namespace, "supported-address-data"),
+                "Invalid MIME type for vcard collection",
+            ))
+
+        # Read the vcard from the stream
+        try:
+            vcarddata = (yield allDataFromStream(request.stream))
+            if not hasattr(request, "extendedLogItems"):
+                request.extendedLogItems = {}
+            request.extendedLogItems["cl"] = str(len(vcarddata)) if vcarddata else "0"
+
+            # We must have some data at this point
+            if vcarddata is None:
+                # Use correct DAV:error response
+                raise HTTPError(ErrorResponse(
+                    responsecode.FORBIDDEN,
+                    (carddav_namespace, "valid-address-data"),
+                    description="No vcard data"
+                ))
+
+            try:
+                component = VCard.fromString(vcarddata)
+            except ValueError, e:
+                log.err(str(e))
+                raise HTTPError(ErrorResponse(
+                    responsecode.FORBIDDEN,
+                    (carddav_namespace, "valid-address-data"),
+                    "Could not parse vCard",
+                ))
+
+            # storeComponent needs to know who the auth'd user is for access control
+            # TODO: this needs to be done in a better way - ideally when the txn is created for the request,
+            # we should set a txn.authzid attribute.
+            authz = None
+            authz_principal = self._parentResource.currentPrincipal(request).children[0]
+            if isinstance(authz_principal, davxml.HRef):
+                principalURL = str(authz_principal)
+                if principalURL:
+                    authz = (yield request.locateResource(principalURL))
+                    self._parentResource._newStoreObject._txn._authz_uid = authz.record.guid
+
+            try:
+                response = (yield self.storeComponent(component))
+            except ResourceDeletedError:
+                # This is OK - it just means the server deleted the resource during the PUT. We make it look
+                # like the PUT succeeded.
+                response = responsecode.CREATED if self.exists() else responsecode.NO_CONTENT
+
+                # Re-initialize to get stuff setup again now we have no object
+                self._initializeWithObject(None, self._newStoreParent)
+
+                returnValue(response)
+
+            response = IResponse(response)
+
+            # Must not set ETag in response if data changed
+            if self._newStoreObject._componentChanged:
+                def _removeEtag(request, response):
+                    response.headers.removeHeader('etag')
+                    return response
+                _removeEtag.handleErrors = True
+
+                request.addResponseFilter(_removeEtag, atEnd=True)
+
+            # Look for Prefer header
+            prefer = request.headers.getHeader("prefer", {})
+            returnRepresentation = any([key == "return" and value == "representation" for key, value, _ignore_args in prefer])
+
+            if returnRepresentation and response.code / 100 == 2:
+                oldcode = response.code
+                response = (yield self.http_GET(request))
+                if oldcode == responsecode.CREATED:
+                    response.code = responsecode.CREATED
+                response.headers.removeHeader("content-location")
+                response.headers.setHeader("content-location", self.url())
+
+            returnValue(response)
+
+        # Handle the various store errors
+        except Exception as err:
+
+            if isinstance(err, ValueError):
+                log.err("Error while handling (vCard) PUT: %s" % (err,))
+                raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, str(err)))
+            else:
+                raise
+
+
+
 class _NotificationChildHelper(object):
     """
     Methods for things which are like notification objects.

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/scheduling/processing.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/scheduling/processing.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/scheduling/processing.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -253,17 +253,20 @@
         """
         Queue up an update to attendees and use a memcache lock to ensure we don't update too frequently.
 
-        @param exclude_attendees: list of attendees who should not be refreshed (e.g., the one that triggeed the refresh)
+        @param exclude_attendees: list of attendees who should not be refreshed (e.g., the one that triggered the refresh)
         @type exclude_attendees: C{list}
         """
 
         # When doing auto-processing of replies, only refresh attendees when the last auto-accept is done.
         # Note that when we do this we also need to refresh the attendee that is generating the reply because they
-        # are no longer up to date with changes of other auto-accept attendees.
-        if hasattr(self.txn, "auto_reply_processing_count") and self.txn.auto_reply_processing_count > 1:
+        # are no longer up to date with changes of other auto-accept attendees. See docstr for sendAttendeeAutoReply
+        # below for more details of what is going on here.
+        if getattr(self.txn, "auto_reply_processing_count", 0) > 1:
+            log.debug("ImplicitProcessing - refreshing UID: '%s', Suppressed: %s" % (self.uid, self.txn.auto_reply_processing_count,))
             self.txn.auto_reply_suppressed = True
             returnValue(None)
-        if hasattr(self.txn, "auto_reply_suppressed"):
+        if getattr(self.txn, "auto_reply_suppressed", False):
+            log.debug("ImplicitProcessing - refreshing UID: '%s', Suppression lifted" % (self.uid,))
             exclude_attendees = ()
 
         self.uid = self.recipient_calendar.resourceUID()
@@ -517,10 +520,8 @@
 
             if send_reply:
                 # Track outstanding auto-reply processing
-                if not hasattr(self.txn, "auto_reply_processing_count"):
-                    self.txn.auto_reply_processing_count = 1
-                else:
-                    self.txn.auto_reply_processing_count += 1
+                self.txn.auto_reply_processing_count = getattr(self.txn, "auto_reply_processing_count", 0) + 1
+                log.debug("ImplicitProcessing - recipient '%s' processing UID: '%s' - auto-reply queued: %s" % (self.recipient.cuaddr, self.uid, self.txn.auto_reply_processing_count,))
                 reactor.callLater(2.0, self.sendAttendeeAutoReply, *(new_calendar, new_resource, partstat))
 
             # Build the schedule-changes XML element
@@ -560,10 +561,8 @@
 
                 if send_reply:
                     # Track outstanding auto-reply processing
-                    if not hasattr(self.txn, "auto_reply_processing_count"):
-                        self.txn.auto_reply_processing_count = 1
-                    else:
-                        self.txn.auto_reply_processing_count += 1
+                    self.txn.auto_reply_processing_count = getattr(self.txn, "auto_reply_processing_count", 0) + 1
+                    log.debug("ImplicitProcessing - recipient '%s' processing UID: '%s' - auto-reply queued: %s" % (self.recipient.cuaddr, self.uid, self.txn.auto_reply_processing_count,))
                     reactor.callLater(2.0, self.sendAttendeeAutoReply, *(new_calendar, new_resource, partstat))
 
                 # Build the schedule-changes XML element
@@ -667,6 +666,17 @@
         Auto-process the calendar option to generate automatic accept/decline status and
         send a reply if needed.
 
+        There is some tricky behavior here: when multiple auto-accept attendees are present in a
+        calendar object, we want to suppress the processing of other attendee refreshes until all
+        auto-accepts have replied, to avoid a flood of refreshes. We do that by tracking the pending
+        auto-replies via a "auto_reply_processing_count" attribute on the original txn objection (even
+        though that has been committed). We also use a "auto_reply_suppressed" attribute on that txn
+        to indicate when suppression has occurred, to ensure that when the refresh is finally sent, we
+        send it to everyone to make sure all are in sync. In order for the actual refreshes to be
+        suppressed we have to "transfer" those two attributes from the original txn to the new one
+        used to send the reply. Then we transfer "auto_reply_suppressed" back when done, and decrement
+        "auto_reply_processing_count" (all done under a UID lock to prevent race conditions).
+
         @param calendar: calendar data to examine
         @type calendar: L{Component}
 
@@ -677,12 +687,17 @@
         # transaction to do this work.
         txn = yield self.txn.store().newTransaction("Attendee (%s) auto-reply for UID: %s" % (self.recipient.cuaddr, self.uid,))
 
+        aborted = False
         try:
             # We need to get the UID lock for implicit processing whilst we send the auto-reply
             # as the Organizer processing will attempt to write out data to other attendees to
             # refresh them. To prevent a race we need a lock.
             yield NamedLock.acquire(txn, "ImplicitUIDLock:%s" % (hashlib.md5(calendar.resourceUID()).hexdigest(),))
 
+            # Must be done after acquiring the lock to avoid a race-condition
+            txn.auto_reply_processing_count = getattr(self.txn, "auto_reply_processing_count", 0)
+            txn.auto_reply_suppressed = getattr(self.txn, "auto_reply_suppressed", False)
+
             # Send out a reply
             log.debug("ImplicitProcessing - recipient '%s' processing UID: '%s' - auto-reply: %s" % (self.recipient.cuaddr, self.uid, partstat))
             from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
@@ -690,16 +705,19 @@
             yield scheduler.sendAttendeeReply(txn, resource, calendar, self.recipient)
         except Exception, e:
             log.debug("ImplicitProcessing - auto-reply exception UID: '%s', %s" % (self.uid, str(e)))
-            yield txn.abort()
+            aborted = True
         except:
             log.debug("ImplicitProcessing - auto-reply bare exception UID: '%s'" % (self.uid,))
+            aborted = True
+
+        # Track outstanding auto-reply processing - must be done before commit/abort which releases the lock
+        self.txn.auto_reply_processing_count = getattr(self.txn, "auto_reply_processing_count", 0) - 1
+        self.txn.auto_reply_suppressed = txn.auto_reply_suppressed
+
+        if aborted:
             yield txn.abort()
         else:
             yield txn.commit()
-        finally:
-            # Track outstanding auto-reply processing
-            if hasattr(self.txn, "auto_reply_processing_count"):
-                self.txn.auto_reply_processing_count -= 1
 
 
     @inlineCallbacks

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/sql.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/sql.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -69,11 +69,11 @@
     IAttachment, AttachmentStoreFailed, AttachmentStoreValidManagedID, \
     AttachmentMigrationFailed, AttachmentDropboxNotAllowed, \
     TooManyAttendeesError, InvalidComponentTypeError, InvalidCalendarAccessError, \
-    InvalidUIDError, UIDExistsError, ResourceDeletedError, \
+    ResourceDeletedError, \
     AttendeeAllowedError, InvalidPerUserDataMerge, ComponentUpdateState, \
     ValidOrganizerError, ShareeAllowedError, ComponentRemoveState, \
-    InvalidComponentForStoreError, InvalidResourceMove, InvalidDefaultCalendar, \
-    UIDExistsElsewhereError, InvalidAttachmentOperation
+    InvalidDefaultCalendar, \
+    InvalidAttachmentOperation
 from txdav.caldav.icalendarstore import QuotaExceeded
 from txdav.common.datastore.sql import CommonHome, CommonHomeChild, \
     CommonObjectResource, ECALENDARTYPE
@@ -90,7 +90,9 @@
     InternalDataStoreError, HomeChildNameAlreadyExistsError, \
     HomeChildNameNotAllowedError, ObjectResourceTooBigError, \
     InvalidObjectResourceError, ObjectResourceNameAlreadyExistsError, \
-    ObjectResourceNameNotAllowedError, TooManyObjectResourcesError
+    ObjectResourceNameNotAllowedError, TooManyObjectResourcesError, \
+    InvalidUIDError, UIDExistsError, UIDExistsElsewhereError, \
+    InvalidResourceMove, InvalidComponentForStoreError
 from txdav.xml.rfc2518 import ResourceType
 
 from pycalendar.datetime import PyCalendarDateTime
@@ -315,67 +317,6 @@
 
 
 
-#class CalendarPrincipal(object):
-#
-#    def __init__(self, uid, cuaddrs):
-#        self.principal_uid = uid
-#        self.cuaddrs = cuaddrs
-#
-#
-#    def uid(self):
-#        return self.principal_uid
-#
-#
-#    def shortNames(self):
-#        return [self.principal_uid, ]
-#
-#
-#    def fullName(self):
-#        return "%s %s" % (self.principal_uid[:4].capitalize(), self.principal_uid[4:])
-#
-#
-#    def displayName(self):
-#        fullName = self.fullName()
-#        return fullName if fullName else self.shortNames()[0]
-#
-#
-#    def calendarUserAddresses(self):
-#        return self.cuaddrs
-#
-#
-#    def canonicalCalendarUserAddress(self):
-#        return [cuaddr for cuaddr in self.cuaddrs if cuaddr.startswith("urn:uuid")][0]
-#
-#
-#    def locallyHosted(self):
-#        return True
-#
-#
-#    def thisServer(self):
-#        return True
-#
-#
-#    def calendarsEnabled(self):
-#        return True
-#
-#
-#    def getCUType(self):
-#        return "INDIVIDUAL"
-#
-#
-#    def enabledAsOrganizer(self):
-#        return True
-#
-#
-#    def canAutoSchedule(self, organizer):
-#        return False
-#
-#
-#    def getAutoScheduleMode(self, organizer):
-#        return "auto"
-
-
-
 class CalendarHome(CommonHome):
 
     implements(ICalendarHome)
@@ -869,27 +810,6 @@
         self.properties()[PropertyName.fromElement(prop)] = prop.fromString(alarm)
 
 
-#    def principal(self):
-#        return self.principalForUID(self.uid())
-#
-#
-#    def principalForUID(self, uid):
-#        return CalendarPrincipal(uid, ("urn:uuid:%s" % (uid,), "mailto:%s at example.com" % (uid,),))
-#
-#
-#    def principalForCalendarUserAddress(self, cuaddr):
-#        if cuaddr.startswith("mailto:"):
-#            uid, domain = cuaddr[7:].split('@')
-#            if domain != "example.com":
-#                return None
-#            return CalendarPrincipal(uid, (cuaddr, "urn:uuid:%s" % (uid,)))
-#        elif cuaddr.startswith("urn:uuid:"):
-#            uid = cuaddr[9:]
-#            return CalendarPrincipal(uid, (cuaddr, "mailto:%s at example.com" % (uid,)))
-#        else:
-#            return None
-
-
 CalendarHome._register(ECALENDARTYPE)
 
 

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/calendar_store/ho/me/home1/calendar_1/1.ics
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/calendar_store/ho/me/home1/calendar_1/1.ics	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/calendar_store/ho/me/home1/calendar_1/1.ics	2013-04-24 18:58:18 UTC (rev 11096)
@@ -20,13 +20,8 @@
 END:STANDARD
 END:VTIMEZONE
 BEGIN:VEVENT
-ATTENDEE;CN="Wilfredo Sanchez";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailt
- o:wsanchez at example.com
-ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cda
- boo at example.com
 DTEND;TZID=US/Pacific:%(now)s0324T124500
 TRANSP:OPAQUE
-ORGANIZER;CN="Wilfredo Sanchez":mailto:wsanchez at example.com
 UID:uid1
 DTSTAMP:20090326T145447Z
 LOCATION:Wilfredo's Office

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/common.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/common.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -35,7 +35,7 @@
 from txdav.idav import IPropertyStore, IDataStore
 from txdav.base.propertystore.base import PropertyName
 from txdav.common.icommondatastore import HomeChildNameAlreadyExistsError, \
-    ICommonTransaction
+    ICommonTransaction, InvalidComponentForStoreError, InvalidUIDError
 from txdav.common.icommondatastore import InvalidObjectResourceError
 from txdav.common.icommondatastore import NoSuchHomeChildError
 from txdav.common.icommondatastore import ObjectResourceNameAlreadyExistsError
@@ -45,8 +45,8 @@
 
 from txdav.caldav.icalendarstore import (
     ICalendarObject, ICalendarHome,
-    ICalendar, ICalendarTransaction, InvalidUIDError,
-    InvalidComponentForStoreError, ComponentUpdateState)
+    ICalendar, ICalendarTransaction,
+    ComponentUpdateState)
 
 from twistedcaldav.customxml import InviteNotification, InviteSummary
 from txdav.common.datastore.test.util import transactionClean

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_attachments.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_attachments.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_attachments.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -1206,11 +1206,11 @@
 
         # Create attachment
         obj = yield self.calendarObjectUnderTest()
-        cdata = yield obj.componentForUser()
 
-        attachment, _ignore_location = yield obj.addAttachment(None, MimeType("text", "x-fixture"), "new.attachment", MemoryStream("new attachment text"), cdata)
+        attachment, _ignore_location = yield obj.addAttachment(None, MimeType("text", "x-fixture"), "new.attachment", MemoryStream("new attachment text"))
         apath = attachment._path.path
 
+        cdata = yield obj.componentForUser()
         newcdata = Component.fromString(str(cdata).replace("uid1", "uid1-attached"))
         calendar = yield self.calendarUnderTest()
         yield calendar.createCalendarObjectWithName("test.ics", newcdata)

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_implicit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_implicit.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/datastore/test/test_implicit.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -24,11 +24,10 @@
 from twext.python.clsprop import classproperty
 from twistedcaldav.config import config
 from txdav.common.icommondatastore import ObjectResourceTooBigError, \
-    InvalidObjectResourceError
+    InvalidObjectResourceError, InvalidComponentForStoreError, InvalidUIDError, \
+    UIDExistsError, UIDExistsElsewhereError
 from txdav.caldav.icalendarstore import InvalidComponentTypeError, \
-    TooManyAttendeesError, InvalidCalendarAccessError, InvalidUIDError, \
-    UIDExistsError, ComponentUpdateState, InvalidComponentForStoreError, \
-    UIDExistsElsewhereError
+    TooManyAttendeesError, InvalidCalendarAccessError, ComponentUpdateState
 from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE
 from txdav.caldav.datastore.test.util import buildCalendarStore
 

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/icalendarstore.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/icalendarstore.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/caldav/icalendarstore.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -120,13 +120,6 @@
 
 
 
-class InvalidComponentForStoreError(CommonStoreError):
-    """
-    Invalid component for an object resource.
-    """
-
-
-
 class InvalidComponentTypeError(CommonStoreError):
     """
     Invalid object resource component type for collection.
@@ -148,27 +141,6 @@
 
 
 
-class InvalidUIDError(CommonStoreError):
-    """
-    The UID of the component in a store operation does not match the existing value.
-    """
-
-
-
-class UIDExistsError(CommonStoreError):
-    """
-    The UID of the component in a store operation exists in the same calendar belonging to the owner.
-    """
-
-
-
-class UIDExistsElsewhereError(CommonStoreError):
-    """
-    The UID of the component in a store operation exists in different calendar belonging to the owner.
-    """
-
-
-
 class ResourceDeletedError(CommonStoreError):
     """
     The resource was determined to be redundant and was deleted by the server.
@@ -204,13 +176,6 @@
 
 
 
-class InvalidResourceMove(CommonStoreError):
-    """
-    Moving a resource failed.
-    """
-
-
-
 class InvalidDefaultCalendar(CommonStoreError):
     """
     Setting a default calendar failed.

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/sql.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/sql.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##
-from txdav.common.icommondatastore import InternalDataStoreError
 
 """
 SQL backend for CardDAV storage.
@@ -26,41 +25,42 @@
     "AddressBookObject",
 ]
 
-from zope.interface.declarations import implements
+from twext.enterprise.dal.syntax import Delete
+from twext.enterprise.dal.syntax import Insert
+from twext.enterprise.dal.syntax import Update
+from twext.enterprise.dal.syntax import utcNowSQL
+from twext.enterprise.locking import NamedLock
+from twext.web2.http_headers import MimeType
 
 from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.python import hashlib
 
-from txdav.xml.rfc2518 import ResourceType
-from twext.web2.http_headers import MimeType
-
 from twistedcaldav import carddavxml, customxml
+from twistedcaldav.config import config
 from twistedcaldav.memcacher import Memcacher
 from twistedcaldav.vcard import Component as VCard, InvalidVCardDataError
 
-from txdav.common.datastore.sql_legacy import PostgresLegacyABIndexEmulator
-
-from txdav.carddav.datastore.util import validateAddressBookComponent
+from txdav.base.propertystore.base import PropertyName
 from txdav.carddav.iaddressbookstore import IAddressBookHome, IAddressBook, \
     IAddressBookObject
-
 from txdav.common.datastore.sql import CommonHome, CommonHomeChild, \
     CommonObjectResource, EADDRESSBOOKTYPE
-from twext.enterprise.dal.syntax import Delete
-from twext.enterprise.dal.syntax import Insert
-
-from twext.enterprise.dal.syntax import Update
-from twext.enterprise.dal.syntax import utcNowSQL
+from txdav.common.datastore.sql_legacy import PostgresLegacyABIndexEmulator
 from txdav.common.datastore.sql_tables import ADDRESSBOOK_TABLE, \
     ADDRESSBOOK_BIND_TABLE, ADDRESSBOOK_OBJECT_REVISIONS_TABLE, \
     ADDRESSBOOK_OBJECT_TABLE, ADDRESSBOOK_HOME_TABLE, \
     ADDRESSBOOK_HOME_METADATA_TABLE, ADDRESSBOOK_AND_ADDRESSBOOK_BIND, \
     ADDRESSBOOK_OBJECT_AND_BIND_TABLE, \
     ADDRESSBOOK_OBJECT_REVISIONS_AND_BIND_TABLE, schema
-from txdav.base.propertystore.base import PropertyName
+from txdav.common.icommondatastore import InternalDataStoreError, \
+    InvalidUIDError, UIDExistsError, ObjectResourceTooBigError, \
+    InvalidObjectResourceError, InvalidComponentForStoreError
+from txdav.xml.rfc2518 import ResourceType
 
+from zope.interface.declarations import implements
 
 
+
 class AddressBookHome(CommonHome):
 
     implements(IAddressBookHome)
@@ -219,7 +219,10 @@
 
         super(AddressBookObject, self).__init__(addressbook, name, uid, resourceID)
 
+        # Component caching
+        self._cachedComponent = None
 
+
     @property
     def _addressbook(self):
         return self._parentCollection
@@ -229,11 +232,81 @@
         return self._addressbook
 
 
+    # Stuff from put_addressbook_common
+    def fullValidation(self, component, inserting):
+        """
+        Do full validation of source and destination calendar data.
+        """
+
+        # Basic validation
+
+        # Valid data sizes
+        if config.MaxResourceSize:
+            vcardsize = len(str(component))
+            if vcardsize > config.MaxResourceSize:
+                raise ObjectResourceTooBigError()
+
+        # Valid calendar data checks
+        self.validAddressDataCheck(component, inserting)
+
+
+    def validAddressDataCheck(self, component, inserting):
+        """
+        Check that the calendar data is valid iCalendar.
+        @return:         tuple: (True/False if the calendar data is valid,
+                                 log message string).
+        """
+
+        # Valid calendar data checks
+        if not isinstance(component, VCard):
+            raise InvalidObjectResourceError("Wrong type of object: %s" % (type(component),))
+
+        try:
+            component.validVCardData()
+        except InvalidVCardDataError, e:
+            raise InvalidObjectResourceError(str(e))
+        try:
+            component.validForCardDAV()
+        except InvalidVCardDataError, e:
+            raise InvalidComponentForStoreError(str(e))
+
+
     @inlineCallbacks
-    def setComponent(self, component, inserting=False, options=None):
+    def _lockUID(self, component, inserting):
+        """
+        Create a lock on the component's UID and verify, after getting the lock, that the incoming UID
+        meets the requirements of the store.
+        """
 
-        validateAddressBookComponent(self, self._addressbook, component, inserting)
+        new_uid = component.resourceUID()
+        yield NamedLock.acquire(self._txn, "vCardUIDLock:%s" % (hashlib.md5(new_uid).hexdigest(),))
 
+        # UID conflict check - note we do this after reserving the UID to avoid a race condition where two requests
+        # try to write the same calendar data to two different resource URIs.
+
+        # Cannot overwrite a resource with different UID
+        if not inserting:
+            if self._uid != new_uid:
+                raise InvalidUIDError("Cannot change the UID in an existing resource.")
+        else:
+            # New UID must be unique for the owner - no need to do this on an overwrite as we can assume
+            # the store is already consistent in this regard
+            elsewhere = (yield self.addressbook().addressbookObjectWithUID(new_uid))
+            if elsewhere is not None:
+                raise UIDExistsError("UID already exists in same addressbook.")
+
+
+    @inlineCallbacks
+    def setComponent(self, component, inserting=False):
+
+        self._componentChanged = False
+
+        # Handle all validation operations here.
+        self.fullValidation(component, inserting)
+
+        # UID lock - this will remain active until the end of the current txn
+        yield self._lockUID(component, inserting)
+
         yield self.updateDatabase(component, inserting=inserting)
         if inserting:
             yield self._addressbook._insertRevision(self._name)
@@ -242,7 +315,9 @@
 
         yield self._addressbook.notifyChanged()
 
+        returnValue(self._componentChanged)
 
+
     @inlineCallbacks
     def updateDatabase(self, component, expand_until=None, reCreate=False, inserting=False):
         """
@@ -256,6 +331,7 @@
 
         componentText = str(component)
         self._objectText = componentText
+        self._cachedComponent = component
 
         # ADDRESSBOOK_OBJECT table update
         self._md5 = hashlib.md5(componentText).hexdigest()
@@ -295,29 +371,45 @@
         the caller - that is not ideal but in theory we should have checked everything on the way in and
         only allowed in good data.
         """
-        text = yield self._text()
+        if self._cachedComponent is None:
 
-        try:
-            component = VCard.fromString(text)
-        except InvalidVCardDataError, e:
-            # This is a really bad situation, so do raise
-            raise InternalDataStoreError(
-                "Data corruption detected (%s) in id: %s"
-                % (e, self._resourceID)
-            )
+            text = yield self._text()
 
-        # Fix any bogus data we can
-        fixed, unfixed = component.validVCardData(doFix=True, doRaise=False)
+            try:
+                component = VCard.fromString(text)
+            except InvalidVCardDataError, e:
+                # This is a really bad situation, so do raise
+                raise InternalDataStoreError(
+                    "Data corruption detected (%s) in id: %s"
+                    % (e, self._resourceID)
+                )
 
-        if unfixed:
-            self.log_error("Address data id=%s had unfixable problems:\n  %s" % (self._resourceID, "\n  ".join(unfixed),))
+            # Fix any bogus data we can
+            fixed, unfixed = component.validVCardData(doFix=True, doRaise=False)
 
-        if fixed:
-            self.log_error("Address data id=%s had fixable problems:\n  %s" % (self._resourceID, "\n  ".join(fixed),))
+            if unfixed:
+                self.log_error("Address data id=%s had unfixable problems:\n  %s" % (self._resourceID, "\n  ".join(unfixed),))
 
-        returnValue(component)
+            if fixed:
+                self.log_error("Address data id=%s had fixable problems:\n  %s" % (self._resourceID, "\n  ".join(fixed),))
 
+            self._cachedComponent = component
 
+        returnValue(self._cachedComponent)
+
+
+    def moveValidation(self, destination, name):
+        """
+        Validate whether a move to the specified collection is allowed.
+
+        @param destination: destination calendar collection
+        @type destination: L{CalendarCollection}
+        @param name: name of new resource
+        @type name: C{str}
+        """
+        pass
+
+
     # IDataStoreObject
     def contentType(self):
         """

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/common.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/common.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -25,7 +25,7 @@
 
 from txdav.common.icommondatastore import (
     HomeChildNameAlreadyExistsError, ICommonTransaction
-)
+, InvalidUIDError)
 from txdav.common.icommondatastore import InvalidObjectResourceError
 from txdav.common.icommondatastore import NoSuchHomeChildError
 from txdav.common.icommondatastore import ObjectResourceNameAlreadyExistsError
@@ -755,7 +755,7 @@
         addressbookObject = yield addressbook1.addressbookObjectWithName("1.vcf")
         yield self.failUnlessFailure(
             maybeDeferred(addressbookObject.setComponent, component),
-            InvalidObjectResourceError
+            InvalidObjectResourceError, InvalidUIDError,
         )
 
 

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/test_file.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/test_file.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/carddav/datastore/test/test_file.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -379,14 +379,6 @@
 
 
     @testUnimplemented
-    def test_addressbookObjectsInTimeRange(self):
-        """
-        Find addressbook objects occuring in a given time range.
-        """
-        raise NotImplementedError()
-
-
-    @testUnimplemented
     def test_addressbookObjectsSinceToken(self):
         """
         Find addressbook objects that have been modified since a given

Modified: CalendarServer/branches/users/cdaboo/store-scheduling/txdav/common/icommondatastore.py
===================================================================
--- CalendarServer/branches/users/cdaboo/store-scheduling/txdav/common/icommondatastore.py	2013-04-24 16:35:53 UTC (rev 11095)
+++ CalendarServer/branches/users/cdaboo/store-scheduling/txdav/common/icommondatastore.py	2013-04-24 18:58:18 UTC (rev 11096)
@@ -149,6 +149,13 @@
 
 
 
+class InvalidComponentForStoreError(CommonStoreError):
+    """
+    Invalid component for an object resource.
+    """
+
+
+
 class ObjectResourceTooBigError(CommonStoreError):
     """
     Object resource data is larger than allowed limit.
@@ -156,6 +163,34 @@
 
 
 
+class InvalidUIDError(CommonStoreError):
+    """
+    The UID of the component in a store operation does not match the existing value.
+    """
+
+
+
+class UIDExistsError(CommonStoreError):
+    """
+    The UID of the component in a store operation exists in the same calendar belonging to the owner.
+    """
+
+
+
+class UIDExistsElsewhereError(CommonStoreError):
+    """
+    The UID of the component in a store operation exists in different calendar belonging to the owner.
+    """
+
+
+
+class InvalidResourceMove(CommonStoreError):
+    """
+    Moving a resource failed.
+    """
+
+
+
 class InternalDataStoreError(CommonStoreError):
     """
     Uh, oh.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130424/58cf0a67/attachment-0001.html>


More information about the calendarserver-changes mailing list