[CalendarServer-changes] [12549] CalendarServer/branches/users/cdaboo/scheduling-queue-refresh

source_changes at macosforge.org source_changes at macosforge.org
Wed Mar 12 11:16:08 PDT 2014


Revision: 12549
          http://trac.calendarserver.org//changeset/12549
Author:   cdaboo at apple.com
Date:     2014-02-03 17:26:39 -0800 (Mon, 03 Feb 2014)
Log Message:
-----------
Fully enable scheduling queues. This currently does a "monolithic" organizer work item (all attendees in one txn),
but the organizer PUT is not blocked on that. A later revision will split the organizer work into smaller chunks.
There is also a schedule work queue visualization tool showing the active work items.

Modified Paths:
--------------
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/conf/caldavd-test.plist
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/implicit.py
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current.sql
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_33_to_34.sql
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_33_to_34.sql

Added Paths:
-----------
    CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/calendarserver/tools/schedule_workitems.py

Added: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/calendarserver/tools/schedule_workitems.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/calendarserver/tools/schedule_workitems.py	                        (rev 0)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/calendarserver/tools/schedule_workitems.py	2014-02-04 01:26:39 UTC (rev 12549)
@@ -0,0 +1,228 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2006-2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from __future__ import print_function
+
+from getopt import getopt, GetoptError
+import os
+import sys
+import curses
+import datetime
+
+from twisted.internet.defer import inlineCallbacks, succeed
+from calendarserver.tools.cmdline import utilityMain, WorkerService
+from calendarserver.push.notifier import PushNotificationWork
+from txdav.caldav.datastore.scheduling.work import ScheduleOrganizerWork, \
+    ScheduleReplyWork, ScheduleRefreshWork
+
+useCurses = True
+
+def usage(e=None):
+
+    name = os.path.basename(sys.argv[0])
+    print("usage: %s [options]" % (name,))
+    print("")
+    print("  TODO: describe usage")
+    print("")
+    print("options:")
+    print("  -h --help: print this help and exit")
+    print("  -e --error: send stderr to stdout")
+    print("  -f --config <path>: Specify caldavd.plist configuration path")
+    print("")
+
+    if e:
+        sys.exit(64)
+    else:
+        sys.exit(0)
+
+
+
+def main():
+
+    try:
+        (optargs, _ignore_args) = getopt(
+            sys.argv[1:], "hef:", [
+                "help",
+                "error",
+                "config=",
+            ],
+        )
+    except GetoptError, e:
+        usage(e)
+
+    #
+    # Get configuration
+    #
+    configFileName = None
+    debug = False
+
+    for opt, arg in optargs:
+        if opt in ("-h", "--help"):
+            usage()
+
+        if opt in ("-e", "--error"):
+            debug = True
+
+        elif opt in ("-f", "--config"):
+            configFileName = arg
+
+        else:
+            raise NotImplementedError(opt)
+
+    utilityMain(configFileName, WorkItemMonitorService, verbose=debug)
+
+
+
+class WorkItemMonitorService(WorkerService, object):
+
+    def __init__(self, store):
+        super(WorkItemMonitorService, self).__init__(store)
+        from twisted.internet import reactor
+        self.reactor = reactor
+
+
+    def doWork(self):
+        self.screen = curses.initscr() if useCurses else None
+        self.windows = []
+        self.updateScreenGeometry()
+        self.reactor.callLater(0, self.updateDisplay)
+        return succeed(None)
+
+
+    def postStartService(self):
+        """
+        Don't quit right away
+        """
+        pass
+
+
+    def updateScreenGeometry(self):
+        for win in self.windows:
+            del win
+        winY, winX = self.screen.getmaxyx() if useCurses else (100, 100)
+        seencolumns = [1]
+        seenrows = [1]
+        heightSoFar = 0
+        begin_x = 0
+        begin_y = 0
+        # Specify height and width of each window as one of:
+        #    absolute value (int), e.g.: 42
+        #    percentage of window height / width (string), e.g.: "42%"
+        # Specify row and column for each window as though it is a cell in an invisible html table
+        # Itemize windows in ascending order by row, col
+        for title, height, width, row, col, workItemClass, fmt, attrs in (
+            ("Organizer Requests", "100%", "25%", 1, 1, ScheduleOrganizerWork, "%s: %d", ("icalendarUid", "attendeeCount")),
+            ("Attendee Replies", "100%", "25%", 1, 2, ScheduleReplyWork, "%s", ("icalendarUid",)),
+            ("Attendee Refresh", "100%", "25%", 1, 3, ScheduleRefreshWork, "%s: %d", ("icalendarUid", "attendeeCount")),
+#            ("Auto Reply", "100%", "25%", 1, 4, ScheduleAutoReplyWork, "%s", ("icalendarUid")),
+            ("Push Notifications", "100%", "25%", 1, 4, PushNotificationWork, "%s: %d", ("pushID", "priority")),
+        ):
+            if (isinstance(height, basestring)):
+                height = max(int(winY * (float(height.strip("%")) / 100.0)), 3)
+            if (isinstance(width, basestring)):
+                width = max(int(winX * (float(width.strip("%")) / 100.0)), 10)
+            if col not in seencolumns:
+                heightSoFar = max(height, heightSoFar)
+                seencolumns.append(col)
+            if row not in seenrows:
+                begin_y = heightSoFar
+                heightSoFar += height
+                begin_x = 0
+                seenrows.append(row)
+                seencolumns = [col]
+            window = WorkWindow(height, width, begin_y, begin_x,
+                self.store, title, workItemClass, fmt, attrs)
+            self.windows.append(window)
+            begin_x += width
+
+
+    @inlineCallbacks
+    def updateDisplay(self):
+        for window in self.windows:
+            try:
+                yield window.update()
+            except Exception as e:
+                print(str(e))
+        if not useCurses:
+            print("-------------")
+
+        self.reactor.callLater(0.1, self.updateDisplay)
+
+
+
+class WorkWindow(object):
+    def __init__(self, nlines, ncols, begin_y, begin_x,
+        store, title, workItemClass, fmt, attrs):
+        self.window = curses.newwin(nlines, ncols, begin_y, begin_x) if useCurses else None
+        self.ncols = ncols
+        self.store = store
+        self.title = title
+        self.workItemClass = workItemClass
+        self.fmt = fmt
+        self.attrs = attrs
+        self.iter = 0
+
+
+    @inlineCallbacks
+    def update(self):
+        txn = self.store.newTransaction()
+        records = (yield self.workItemClass.all(txn))
+        self.iter += 1
+
+        if useCurses:
+            self.window.erase()
+            self.window.border()
+            self.window.addstr(0, 2, self.title + " %d (%d)" % (len(records), self.iter,))
+
+        x = 1
+        y = 1
+        for record in records:
+            txt = ""
+            seconds = record.notBefore - datetime.datetime.utcnow()
+            try:
+                if useCurses:
+                    self.window.addstr(y, x, "%d seconds" % int(seconds.total_seconds()))
+                else:
+                    txt = "%s:" % (self.title,)
+            except curses.error:
+                continue
+            y += 1
+            if self.attrs:
+                try:
+                    s = self.fmt % tuple([getattr(record, str(a)) for a in self.attrs])
+                except Exception, e:
+                    s = "Error: %s" % (str(e),)
+                try:
+                    if useCurses:
+                        self.window.addnstr(y, x, s, self.ncols - 2)
+                    else:
+                        txt += " " + s
+                except curses.error:
+                    pass
+                y += 1
+
+            if not useCurses:
+                print(txt)
+
+        if useCurses:
+            self.window.refresh()
+
+        yield txn.commit()
+
+
+if __name__ == "__main__":
+    main()


Property changes on: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/calendarserver/tools/schedule_workitems.py
___________________________________________________________________
Added: svn:executable
   + *

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/conf/caldavd-test.plist	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/conf/caldavd-test.plist	2014-02-04 01:26:39 UTC (rev 12549)
@@ -796,7 +796,7 @@
 		<key>WorkQueues</key>
 		<dict>
 			<key>Enabled</key>
-			<false/>
+			<true/>
             <key>RequestDelaySeconds</key>
             <integer>1</integer>
             <key>ReplyDelaySeconds</key>

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/implicit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/implicit.py	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/implicit.py	2014-02-04 01:26:39 UTC (rev 12549)
@@ -395,7 +395,7 @@
 
 
     @inlineCallbacks
-    def queuedOrganizerProcessing(self, txn, action, home, resource, uid, calendar, smart_merge):
+    def queuedOrganizerProcessing(self, txn, action, home, resource, uid, calendar_old, calendar_new, smart_merge):
         """
         Process an organizer scheduling work queue item. The basic goal here is to setup the ImplicitScheduler as if
         this operation were the equivalent of the PUT that enqueued the work, and then do the actual work.
@@ -407,46 +407,44 @@
         self.calendar_home = home
         self.resource = resource
         self.do_smart_merge = smart_merge
+        self.queuedResponses = []
 
+        cal_uid = calendar_old.resourceUID() if calendar_old is not None else (calendar_new.resourceUID() if calendar_new is not None else "unknown")
+
         # Handle different action scenarios
         if action == "create":
-            # resource is None, calendar is None
+            # resource is None, calendar_old is None
             # Find the newly created resource
             resources = (yield self.calendar_home.objectResourcesWithUID(uid, ignore_children=["inbox"], allowShared=False))
             if len(resources) != 1:
                 # Ughh - what has happened? It is possible the resource was created then deleted before we could start work processing,
                 # so simply ignore this
-                log.debug("ImplicitScheduler - queuedOrganizerProcessing 'create' cannot find organizer resource for UID: {uid}", uid=calendar.resourceUID())
+                log.debug("ImplicitScheduler - queuedOrganizerProcessing 'create' cannot find organizer resource for UID: {uid}", uid=cal_uid)
                 returnValue(None)
             self.resource = resources[0]
+            self.calendar = calendar_new
 
-            # The calendar data to use is the current calendar data, not what was stored in the work item, since it might have been
-            # updated a few times after the create, but those modifications are effectively coalesced into the create
-            self.calendar = (yield self.resource.componentForUser())
-
-        elif action == "modify":
+        elif action in ("modify", "modify-cancelled"):
             # Check that the resource still exists - it may have been deleted after this work item was queued, in which
             # case we have to ignore this (on the assumption that the "remove" action will have queued some work that will
             # execute soon).
             if self.resource is None:
-                log.debug("ImplicitScheduler - queuedOrganizerProcessing 'modify' cannot find organizer resource for UID: {uid}", uid=calendar.resourceUID())
+                log.debug("ImplicitScheduler - queuedOrganizerProcessing 'modify' cannot find organizer resource for UID: {uid}", uid=cal_uid)
                 returnValue(None)
 
-            # The new calendar data is what is currently stored - other modifications may have causes coalescing.
-            # Old calendar data is what was stored int he work item
-            self.calendar = (yield self.resource.componentForUser())
-            self.oldcalendar = calendar
+            # The new calendar_old data is what is currently stored - other modifications may have causes coalescing.
+            # Old calendar_old data is what was stored int he work item
+            self.calendar = calendar_new
+            self.oldcalendar = calendar_old
 
         elif action == "remove":
-            # Check whether the resource still exists - it cannot be in existence as once it is deleted, its resource-id
-            # should never be used again.
-            if self.resource is not None:
-                log.debug("ImplicitScheduler - queuedOrganizerProcessing 'remove' found an organizer resource for UID: {uid}", uid=calendar.resourceUID())
-                raise ImplicitSchedulingWorkError("Resource exists for queued 'remove' scheduling work")
+            # A remove can happen when the underlying resource is deleted, or when all scheduling properties
+            # (organizer and attendees) are removed from its content. So sometimes the resource will not exist, other
+            # times it might. Thus we cannot make any assumptions about resource existence.
 
-            # The "new" calendar data is in fact the calendar data at the time of the remove - which is the data stored
+            # The "new" calendar_old data is in fact the calendar_old data at the time of the remove - which is the data stored
             # in the work item.
-            self.calendar = calendar
+            self.calendar = calendar_old
 
         yield self.extractCalendarData()
         self.organizerPrincipal = self.calendar_home.directoryService().recordWithCalendarUserAddress(self.organizer)
@@ -615,11 +613,10 @@
             self.cancelledAttendees = [(attendee, None) for attendee in self.attendees]
 
             # CANCEL always bumps sequence
-            if not queued or not config.Scheduling.Options.WorkQueues.Enabled:
-                self.needs_sequence_change = True
+            self.needs_sequence_change = True
 
         # Check for a new resource or an update
-        elif self.action == "modify":
+        elif self.action in ("modify", "modify-cancelled"):
 
             # Read in existing data
             if not queued or not config.Scheduling.Options.WorkQueues.Enabled:
@@ -678,8 +675,7 @@
 
                 # For now we always bump the sequence number on modifications because we cannot track DTSTAMP on
                 # the Attendee side. But we check the old and the new and only bump if the client did not already do it.
-                if not queued or not config.Scheduling.Options.WorkQueues.Enabled:
-                    self.needs_sequence_change = self.calendar.needsiTIPSequenceChange(self.oldcalendar)
+                self.needs_sequence_change = self.calendar.needsiTIPSequenceChange(self.oldcalendar)
 
         elif self.action == "create":
             if self.split_details is None:
@@ -694,12 +690,12 @@
             if attendee.parameterValue("PARTSTAT", "NEEDS-ACTION").upper() == "NEEDS-ACTION":
                 attendee.setParameter("RSVP", "TRUE")
 
-        if self.needs_sequence_change:
-            self.calendar.bumpiTIPInfo(oldcalendar=self.oldcalendar, doSequence=True)
-
         # If processing a queue item, actually execute the scheduling operations, else queue it.
         # Note a split is always queued, so we do not need to re-queue
         if queued or not config.Scheduling.Options.WorkQueues.Enabled or self.split_details is not None:
+            if self.needs_sequence_change:
+                self.calendar.bumpiTIPInfo(oldcalendar=self.oldcalendar, doSequence=True)
+
             yield self.scheduleWithAttendees()
         else:
             yield self.queuedScheduleWithAttendees()
@@ -1040,10 +1036,18 @@
             self.calendar_home,
             self.resource,
             self.oldcalendar,
+            self.calendar,
             self.organizerPrincipal.canonicalCalendarUserAddress(),
+            len(self.calendar.getAllUniqueAttendees()) - 1,
             self.do_smart_merge,
         )
 
+        # We bump the sequence AFTER storing the work item data to make sure that the sequence
+        # change does not cause unchanged components to be treated as changed when the work
+        # item executes.
+        if self.needs_sequence_change:
+            self.calendar.bumpiTIPInfo(oldcalendar=self.oldcalendar, doSequence=True)
+
         # First process cancelled attendees
         total = (yield self.processQueuedCancels())
 
@@ -1287,20 +1291,24 @@
 
     def handleSchedulingResponse(self, response, is_organizer):
 
-        # Map each recipient in the response to a status code
-        responses = {}
-        propname = self.calendar.mainComponent().recipientPropertyName() if is_organizer else "ORGANIZER"
-        for item in response.responses:
-            recipient = str(item.recipient.children[0])
-            status = str(item.reqstatus)
-            responses[recipient] = status
+        # For a queued operation we stash the response away for the work item to deal with
+        if hasattr(self, "queuedResponses"):
+            self.queuedResponses.append(response)
+        else:
+            # Map each recipient in the response to a status code
+            responses = {}
+            propname = self.calendar.mainComponent().recipientPropertyName() if is_organizer else "ORGANIZER"
+            for item in response.responses:
+                recipient = str(item.recipient.children[0])
+                status = str(item.reqstatus)
+                responses[recipient] = status
 
-            # Now apply to each ATTENDEE/ORGANIZER in the original data
-            self.calendar.setParameterToValueForPropertyWithValue(
-                "SCHEDULE-STATUS",
-                status.split(";")[0],
-                propname,
-                recipient)
+                # Now apply to each ATTENDEE/ORGANIZER in the original data
+                self.calendar.setParameterToValueForPropertyWithValue(
+                    "SCHEDULE-STATUS",
+                    status.split(";")[0],
+                    propname,
+                    recipient)
 
 
     @inlineCallbacks

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py	2014-02-04 01:26:39 UTC (rev 12549)
@@ -33,6 +33,7 @@
 import datetime
 import hashlib
 from pycalendar.datetime import DateTime
+import traceback
 
 __all__ = [
     "ScheduleOrganizerWork",
@@ -87,8 +88,7 @@
                 self.transaction.postCommit(_post)
 
 
-    @inlineCallbacks
-    def handleSchedulingResponse(self, response, calendar, resource, is_organizer):
+    def handleSchedulingResponse(self, response, calendar, is_organizer):
         """
         Update a user's calendar object resource based on the results of a queued scheduling
         message response. Note we only need to update in the case where there is an error response
@@ -99,8 +99,6 @@
         @type response: L{caldavxml.ScheduleResponse}
         @param calendar: original calendar component
         @type calendar: L{Component}
-        @param resource: calendar object resource to update
-        @type resource: L{CalendarObject}
         @param is_organizer: whether or not iTIP message was sent by the organizer
         @type is_organizer: C{bool}
         """
@@ -123,8 +121,7 @@
                 )
                 changed = True
 
-        if changed:
-            yield resource._setComponentInternal(calendar, internal_state=ComponentUpdateState.ATTENDEE_ITIP_UPDATE)
+        return changed
 
 
 
@@ -138,16 +135,23 @@
 
     @classmethod
     @inlineCallbacks
-    def schedule(cls, txn, uid, action, home, resource, calendar, organizer, smart_merge):
+    def schedule(cls, txn, uid, action, home, resource, calendar_old, calendar_new, organizer, attendee_count, smart_merge):
         """
         The actual arguments depend on the action:
 
-        1) If action is "create", resource is None, calendar is None
-        2) If action is "modify", resource is existing resource, calendar is the old calendar data
-        3) If action is "remove", resource is the existing resource, calendar is the old calendar data
+        1) If action is "create", resource is None, calendar_old is None, calendar_new is the new data
+        2) If action is "modify", resource is existing resource, calendar_old is the old calendar_old data, and
+            calendar_new is the new data
+        3) If action is "remove", resource is the existing resource, calendar_old is the old calendar_old data,
+            and calendar_new is None
 
-        Note that for (1), when the work executes the resource will be in existence so we need to load it.
-        Note that for (3), when work executes the resource will have been removed.
+        Right now we will also create the iTIP message based on the diff of calendar_old and calendar_new rather than
+        looking at the current state of the orgnaizer's resource (which may have changed since this work item was
+        filed). That means that we are basically NOT doing any coalescing of changes - instead every change results
+        in its own iTIP message (pretty much as it would without the queue). Ultimately we need to support coalescing
+        for performance benefit, but the logic involved in doing that is tricky (e.g., certain properties like
+        SCHEDULE-FORCE-SEND are not preserved in the saved data, yet need to be accounted for because they change the
+        nature of the iTIP processing).
         """
         # Always queue up new work - coalescing happens when work is executed
         notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=config.Scheduling.Options.WorkQueues.RequestDelaySeconds)
@@ -158,7 +162,9 @@
             scheduleAction=scheduleActionToSQL[action],
             homeResourceID=home.id(),
             resourceID=resource.id() if resource else None,
-            icalendarText=calendar.getTextWithTimezones(includeTimezones=not config.EnableTimezonesByReference) if calendar else None,
+            icalendarTextOld=calendar_old.getTextWithTimezones(includeTimezones=not config.EnableTimezonesByReference) if calendar_old else None,
+            icalendarTextNew=calendar_new.getTextWithTimezones(includeTimezones=not config.EnableTimezonesByReference) if calendar_new else None,
+            attendeeCount=attendee_count,
             smartMerge=smart_merge
         ))
         cls._enqueued()
@@ -185,7 +191,8 @@
             resource = (yield home.objectResourceWithID(self.resourceID))
             organizerPrincipal = home.directoryService().recordWithUID(home.uid())
             organizer = organizerPrincipal.canonicalCalendarUserAddress()
-            calendar = Component.fromString(self.icalendarText) if self.icalendarText else None
+            calendar_old = Component.fromString(self.icalendarTextOld) if self.icalendarTextOld else None
+            calendar_new = Component.fromString(self.icalendarTextNew) if self.icalendarTextNew else None
 
             log.debug("ScheduleOrganizerWork - running for ID: {id}, UID: {uid}, organizer: {org}", id=self.workID, uid=self.icalendarUid, org=organizer)
 
@@ -194,15 +201,39 @@
 
             from txdav.caldav.datastore.scheduling.implicit import ImplicitScheduler
             scheduler = ImplicitScheduler()
-            yield scheduler.queuedOrganizerProcessing(self.transaction, scheduleActionFromSQL[self.scheduleAction], home, resource, self.icalendarUid, calendar, self.smartMerge)
+            yield scheduler.queuedOrganizerProcessing(
+                self.transaction,
+                scheduleActionFromSQL[self.scheduleAction],
+                home,
+                resource,
+                self.icalendarUid,
+                calendar_old,
+                calendar_new,
+                self.smartMerge
+            )
 
+            # Handle responses - update the actual resource in the store. Note that for a create the resource did not previously
+            # exist and is stored as None for the work item, but the scheduler will attempt to find the new resources and use
+            # that. We need to grab the scheduler's resource for further processing.
+            resource = scheduler.resource
+            if resource is not None:
+                changed = False
+                calendar = (yield resource.componentForUser())
+                for response in scheduler.queuedResponses:
+                    changed |= yield self.handleSchedulingResponse(response, calendar, True)
+
+                if changed:
+                    yield resource._setComponentInternal(calendar, internal_state=ComponentUpdateState.ORGANIZER_ITIP_UPDATE)
+
             self._dequeued()
 
         except Exception, e:
             log.debug("ScheduleOrganizerWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=self.icalendarUid, err=str(e))
+            log.debug(traceback.format_exc())
             raise
         except:
             log.debug("ScheduleOrganizerWork - bare exception ID: {id}, UID: '{uid}'", id=self.workID, uid=self.icalendarUid)
+            log.debug(traceback.format_exc())
             raise
 
         log.debug("ScheduleOrganizerWork - done for ID: {id}, UID: {uid}, organizer: {org}", id=self.workID, uid=self.icalendarUid, org=organizer)
@@ -297,8 +328,11 @@
 
             # Send scheduling message and process response
             response = (yield self.sendToOrganizer(home, "REPLY", itipmsg, attendee, organizer))
-            yield self.handleSchedulingResponse(response, calendar, resource, False)
+            changed = yield self.handleSchedulingResponse(response, calendar, False)
 
+            if changed:
+                yield resource._setComponentInternal(calendar, internal_state=ComponentUpdateState.ATTENDEE_ITIP_UPDATE)
+
             self._dequeued()
 
         except Exception, e:
@@ -441,7 +475,8 @@
             icalendarUid=organizer_resource.uid(),
             homeResourceID=organizer_resource._home.id(),
             resourceID=organizer_resource.id(),
-            notBefore=notBefore
+            attendeeCount=len(attendees),
+            notBefore=notBefore,
         ))
         cls._enqueued()
         yield proposal.whenProposed()
@@ -494,8 +529,10 @@
             notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=config.Scheduling.Options.WorkQueues.AttendeeRefreshBatchIntervalSeconds)
             yield self.transaction.enqueue(
                 self.__class__,
+                icalendarUid=self.icalendarUid,
                 homeResourceID=self.homeResourceID,
                 resourceID=self.resourceID,
+                attendeeCount=len(pendingAttendees),
                 notBefore=notBefore
             )
 

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current-oracle-dialect.sql	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current-oracle-dialect.sql	2014-02-04 01:26:39 UTC (rev 12549)
@@ -400,7 +400,8 @@
     "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
     "ICALENDAR_UID" nvarchar2(255),
     "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade,
-    "RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade
+    "RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade,
+    "ATTENDEE_COUNT" integer
 );
 
 create table SCHEDULE_REFRESH_ATTENDEES (
@@ -424,7 +425,9 @@
     "SCHEDULE_ACTION" integer not null,
     "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade,
     "RESOURCE_ID" integer,
-    "ICALENDAR_TEXT" nclob,
+    "ICALENDAR_TEXT_OLD" nclob,
+    "ICALENDAR_TEXT_NEW" nclob,
+    "ATTENDEE_COUNT" integer,
     "SMART_MERGE" integer
 );
 
@@ -435,7 +438,8 @@
 
 insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('create', 0);
 insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('modify', 1);
-insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('remove', 2);
+insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('modify-cancelled', 2);
+insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('remove', 3);
 create table SCHEDULE_REPLY_WORK (
     "WORK_ID" integer primary key not null,
     "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current.sql	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/current.sql	2014-02-04 01:26:39 UTC (rev 12549)
@@ -751,7 +751,8 @@
   NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
   ICALENDAR_UID        			varchar(255) not null,
   HOME_RESOURCE_ID              integer      not null references CALENDAR_HOME on delete cascade,
-  RESOURCE_ID                   integer      not null references CALENDAR_OBJECT on delete cascade
+  RESOURCE_ID                   integer      not null references CALENDAR_OBJECT on delete cascade,
+  ATTENDEE_COUNT				integer
 );
 
 create index SCHEDULE_REFRESH_WORK_HOME_RESOURCE_ID on
@@ -796,7 +797,9 @@
   SCHEDULE_ACTION				integer		 not null, -- Enum SCHEDULE_ACTION
   HOME_RESOURCE_ID              integer      not null references CALENDAR_HOME on delete cascade,
   RESOURCE_ID                   integer,	 -- this references a possibly non-existent CALENDR_OBJECT
-  ICALENDAR_TEXT				text,
+  ICALENDAR_TEXT_OLD			text,
+  ICALENDAR_TEXT_NEW			text,
+  ATTENDEE_COUNT				integer,
   SMART_MERGE					boolean
 );
 
@@ -814,7 +817,8 @@
 
 insert into SCHEDULE_ACTION values (0, 'create');
 insert into SCHEDULE_ACTION values (1, 'modify');
-insert into SCHEDULE_ACTION values (2, 'remove');
+insert into SCHEDULE_ACTION values (2, 'modify-cancelled');
+insert into SCHEDULE_ACTION values (3, 'remove');
 
 -------------------------
 -- Schedule Reply Work --

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_33_to_34.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_33_to_34.sql	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_33_to_34.sql	2014-02-04 01:26:39 UTC (rev 12549)
@@ -25,7 +25,8 @@
     "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
     "ICALENDAR_UID" nvarchar2(255),
     "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade,
-    "RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade
+    "RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade,
+    "ATTENDEE_COUNT" integer
 );
 
 create index SCHEDULE_REFRESH_WORK_26084c7b on SCHEDULE_REFRESH_WORK (
@@ -72,7 +73,9 @@
     "SCHEDULE_ACTION" integer not null,
     "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade,
     "RESOURCE_ID" integer,
-    "ICALENDAR_TEXT" nclob,
+    "ICALENDAR_TEXT_OLD" nclob,
+    "ICALENDAR_TEXT_NEW" nclob,
+    "ATTENDEE_COUNT" integer,
     "SMART_MERGE" integer
 );
 
@@ -91,7 +94,8 @@
 
 insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('create', 0);
 insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('modify', 1);
-insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('remove', 2);
+insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('modify-cancelled', 2);
+insert into SCHEDULE_ACTION (DESCRIPTION, ID) values ('remove', 3);
 
 
 create table SCHEDULE_REPLY_WORK (

Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_33_to_34.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_33_to_34.sql	2014-02-04 01:25:14 UTC (rev 12548)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_33_to_34.sql	2014-02-04 01:26:39 UTC (rev 12549)
@@ -29,7 +29,8 @@
   NOT_BEFORE                    timestamp    default timezone('UTC', CURRENT_TIMESTAMP),
   ICALENDAR_UID        			varchar(255) not null,
   HOME_RESOURCE_ID              integer      not null references CALENDAR_HOME on delete cascade,
-  RESOURCE_ID                   integer      not null references CALENDAR_OBJECT on delete cascade
+  RESOURCE_ID                   integer      not null references CALENDAR_OBJECT on delete cascade,
+  ATTENDEE_COUNT				integer
 );
 
 create index SCHEDULE_REFRESH_WORK_HOME_RESOURCE_ID on
@@ -74,7 +75,9 @@
   SCHEDULE_ACTION				integer		 not null, -- Enum SCHEDULE_ACTION
   HOME_RESOURCE_ID              integer      not null references CALENDAR_HOME on delete cascade,
   RESOURCE_ID                   integer,	 -- this references a possibly non-existent CALENDR_OBJECT
-  ICALENDAR_TEXT				text,
+  ICALENDAR_TEXT_OLD			text,
+  ICALENDAR_TEXT_NEW			text,
+  ATTENDEE_COUNT				integer,
   SMART_MERGE					boolean
 );
 
@@ -92,7 +95,8 @@
 
 insert into SCHEDULE_ACTION values (0, 'create');
 insert into SCHEDULE_ACTION values (1, 'modify');
-insert into SCHEDULE_ACTION values (2, 'remove');
+insert into SCHEDULE_ACTION values (2, 'modify-cancelled');
+insert into SCHEDULE_ACTION values (3, 'remove');
 
 -------------------------
 -- Schedule Reply Work --
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/2b7b0bbc/attachment.html>


More information about the calendarserver-changes mailing list