[CalendarServer-changes] [12468] CalendarServer/branches/users/cdaboo/scheduling-queue-refresh
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:24:25 PDT 2014
Revision: 12468
http://trac.calendarserver.org//changeset/12468
Author: cdaboo at apple.com
Date: 2014-01-28 17:45:15 -0800 (Tue, 28 Jan 2014)
Log Message:
-----------
Checkpoint: tests for queued invite/reply (and fix reply).
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/twistedcaldav/ical.py
CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py
CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/test/util.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/test/test_queue_scheduling.py
Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/twistedcaldav/ical.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/twistedcaldav/ical.py 2014-01-28 19:56:12 UTC (rev 12467)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/twistedcaldav/ical.py 2014-01-29 01:45:15 UTC (rev 12468)
@@ -3652,7 +3652,12 @@
icalstr = str(icalstr).replace("\r\n ", "")
icalstr = icalstr.replace("\n ", "")
- icalstr = "\r\n".join([line for line in icalstr.splitlines() if not line.startswith("DTSTAMP")])
+ lines = [line for line in icalstr.splitlines() if not line.startswith("DTSTAMP")]
+ for ctr, line in enumerate(lines[:]):
+ pos = line.find(";X-CALENDARSERVER-DTSTAMP=")
+ if pos != -1:
+ lines[ctr] = line[:pos] + line[pos + len(";X-CALENDARSERVER-DTSTAMP=") + 16:]
+ icalstr = "\r\n".join(lines)
return icalstr + "\r\n"
Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py 2014-01-28 19:56:12 UTC (rev 12467)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/scheduling/work.py 2014-01-29 01:45:15 UTC (rev 12468)
@@ -22,7 +22,6 @@
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
-from twistedcaldav import caldavxml
from twistedcaldav.config import config
from twistedcaldav.ical import Component
@@ -33,6 +32,7 @@
import datetime
import hashlib
+from pycalendar.datetime import DateTime
__all__ = [
"ScheduleOrganizerWork",
@@ -49,10 +49,44 @@
Base class for common schedule work item behavior.
"""
+ # Track when all work is complete (needed for unit tests)
+ _allDoneCallback = None
+ _queued = 0
+
# Schedule work is grouped based on calendar object UID
group = property(lambda self: "ScheduleWork:%s" % (self.icalendarUid,))
+ @classmethod
+ def allDone(cls):
+ d = Deferred()
+ cls._allDoneCallback = d.callback
+ cls._queued = 0
+ return d
+
+
+ @classmethod
+ def _enqueued(cls):
+ """
+ Called when a new item is enqueued - using for tracking purposes.
+ """
+ ScheduleWorkMixin._queued += 1
+
+
+ def _dequeued(self):
+ """
+ Called when an item is dequeued - using for tracking purposes. We call
+ the callback when the last item is dequeued.
+ """
+ ScheduleWorkMixin._queued -= 1
+ if ScheduleWorkMixin._queued == 0:
+ if ScheduleWorkMixin._allDoneCallback:
+ def _post():
+ ScheduleWorkMixin._allDoneCallback(None)
+ ScheduleWorkMixin._allDoneCallback = None
+ self.transaction.postCommit(_post)
+
+
@inlineCallbacks
def handleSchedulingResponse(self, response, calendar, resource, is_organizer):
"""
@@ -73,10 +107,10 @@
# Map each recipient in the response to a status code
changed = False
+ propname = calendar.mainComponent().recipientPropertyName() if is_organizer else "ORGANIZER"
for item in response.responses:
- assert isinstance(item, caldavxml.Response), "Wrong element in response"
- recipient = str(item.children[0].children[0])
- status = str(item.children[1])
+ recipient = str(item.recipient.children[0])
+ status = str(item.reqstatus)
statusCode = status.split(";")[0]
# Now apply to each ATTENDEE/ORGANIZER in the original data only if not 1.2
@@ -84,7 +118,7 @@
calendar.setParameterToValueForPropertyWithValue(
"SCHEDULE-STATUS",
statusCode,
- "ATTENDEE" if is_organizer else "ORGANIZER",
+ propname,
recipient,
)
changed = True
@@ -102,8 +136,6 @@
their calendar object resource.
"""
- _allDoneCallback = None
-
@classmethod
@inlineCallbacks
def schedule(cls, txn, uid, action, home, resource, calendar, organizer, smart_merge):
@@ -129,6 +161,7 @@
icalendarText=calendar.getTextWithTimezones(includeTimezones=not config.EnableTimezonesByReference) if calendar else None,
smartMerge=smart_merge
))
+ cls._enqueued()
yield proposal.whenProposed()
log.debug("ScheduleOrganizerWork - enqueued for ID: {id}, UID: {uid}, organizer: {org}", id=proposal.workItem.workID, uid=uid, org=organizer)
@@ -144,13 +177,6 @@
returnValue(len(rows) > 0)
- @classmethod
- def allDone(cls):
- d = Deferred()
- cls._allDoneCallback = d.callback
- return d
-
-
@inlineCallbacks
def doWork(self):
@@ -170,8 +196,7 @@
scheduler = ImplicitScheduler()
yield scheduler.queuedOrganizerProcessing(self.transaction, scheduleActionFromSQL[self.scheduleAction], home, resource, self.icalendarUid, calendar, self.smartMerge)
- if self._allDoneCallback:
- self._allDoneCallback(None)
+ self._dequeued()
except Exception, e:
log.debug("ScheduleOrganizerWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=self.icalendarUid, err=str(e))
@@ -229,8 +254,11 @@
icalendarUid=resource.uid(),
homeResourceID=home.id(),
resourceID=resource.id(),
- changedRids=",".join(changedRids) if changedRids else None,
+
+ # Serialize None as ""
+ changedRids=",".join(map(lambda x: "" if x is None else str(x), changedRids)) if changedRids else None,
))
+ cls._enqueued()
yield proposal.whenProposed()
log.debug("ScheduleReplyWork - enqueued for ID: {id}, UID: {uid}, attendee: {att}", id=proposal.workItem.workID, uid=resource.uid(), att=attendee)
@@ -257,7 +285,8 @@
calendar = (yield resource.componentForUser())
organizer = calendar.validOrganizerForScheduling()
- changedRids = self.changedRids.split(",") if self.changedRids else None
+ # Deserialize "" as None
+ changedRids = map(lambda x: DateTime.parseText(x) if x else None, self.changedRids.split(",")) if self.changedRids else None
log.debug("ScheduleReplyWork - running for ID: {id}, UID: {uid}, attendee: {att}", id=self.workID, uid=calendar.resourceUID(), att=attendee)
@@ -270,6 +299,8 @@
response = (yield self.sendToOrganizer(home, "REPLY", itipmsg, attendee, organizer))
yield self.handleSchedulingResponse(response, calendar, resource, False)
+ self._dequeued()
+
except Exception, e:
log.debug("ScheduleReplyWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=calendar.resourceUID(), err=str(e))
raise
@@ -306,6 +337,7 @@
homeResourceID=home.id(),
icalendarText=calendar.getTextWithTimezones(includeTimezones=not config.EnableTimezonesByReference),
))
+ cls._enqueued()
yield proposal.whenProposed()
log.debug("ScheduleReplyCancelWork - enqueued for ID: {id}, UID: {uid}, attendee: {att}", id=proposal.workItem.workID, uid=calendar.resourceUID(), att=attendee)
@@ -330,6 +362,8 @@
# Send scheduling message - no need to process response as original resource is gone
yield self.sendToOrganizer(home, "CANCEL", itipmsg, attendee, organizer)
+ self._dequeued()
+
except Exception, e:
log.debug("ScheduleReplyCancelWork - exception ID: {id}, UID: '{uid}', {err}", id=self.workID, uid=calendar.resourceUID(), err=str(e))
raise
@@ -409,6 +443,7 @@
resourceID=organizer_resource.id(),
notBefore=notBefore
))
+ cls._enqueued()
yield proposal.whenProposed()
log.debug("ScheduleRefreshWork - enqueued for ID: {id}, UID: {uid}, attendees: {att}", id=proposal.workItem.workID, uid=organizer_resource.uid(), att=",".join(attendeesToRefresh))
@@ -464,9 +499,13 @@
notBefore=notBefore
)
+ self._enqueued()
+
# Do refresh
yield self._doDelayedRefresh(attendeesToProcess)
+ self._dequeued()
+
log.debug("ScheduleRefreshWork - done for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUid)
@@ -542,6 +581,7 @@
partstat=partstat,
notBefore=notBefore,
))
+ cls._enqueued()
yield proposal.whenProposed()
log.debug("ScheduleAutoReplyWork - enqueued for ID: {id}, UID: {uid}", id=proposal.workItem.workID, uid=resource.uid())
@@ -559,6 +599,8 @@
# Do reply
yield self._sendAttendeeAutoReply()
+ self._dequeued()
+
log.debug("ScheduleAutoReplyWork - done for ID: {id}, UID: {uid}", id=self.workID, uid=self.icalendarUid)
Added: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/test/test_queue_scheduling.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/test/test_queue_scheduling.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/caldav/datastore/test/test_queue_scheduling.py 2014-01-29 01:45:15 UTC (rev 12468)
@@ -0,0 +1,254 @@
+##
+# Copyright (c) 2014 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from twext.python.clsprop import classproperty
+from twistedcaldav.config import config
+from txdav.caldav.datastore.scheduling.work import ScheduleWorkMixin
+from txdav.caldav.datastore.test.util import CommonStoreTests
+from txdav.common.datastore.test.util import componentUpdate
+from twistedcaldav.ical import normalize_iCalStr
+
+class BaseQueueSchedulingTests(CommonStoreTests):
+
+ """
+ Test store-based calendar sharing.
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+ yield super(BaseQueueSchedulingTests, self).setUp()
+
+ # Enable the queue and make it fast
+ self.patch(config.Scheduling.Options.WorkQueues, "Enabled", True)
+ self.patch(config.Scheduling.Options.WorkQueues, "RequestDelaySeconds", 1)
+ self.patch(config.Scheduling.Options.WorkQueues, "ReplyDelaySeconds", 1)
+ self.patch(config.Scheduling.Options.WorkQueues, "AutoReplyDelaySeconds", 1)
+ self.patch(config.Scheduling.Options.WorkQueues, "AttendeeRefreshBatchDelaySeconds", 1)
+ self.patch(config.Scheduling.Options.WorkQueues, "AttendeeRefreshBatchIntervalSeconds", 1)
+
+
+ @classproperty(cache=False)
+ def requirements(cls): #@NoSelf
+ return {
+ "user01": {
+ "calendar": {
+ },
+ "inbox": {
+ },
+ },
+ "user02": {
+ "calendar": {
+ },
+ "inbox": {
+ },
+ },
+ "user03": {
+ "calendar": {
+ },
+ "inbox": {
+ },
+ },
+ }
+
+
+ @inlineCallbacks
+ def _getOneResource(self, home, calendar_name):
+ """
+ Get the one resources expected in a collection
+ """
+ inbox = yield self.calendarUnderTest(home=home, name=calendar_name)
+ objs = yield inbox.objectResources()
+ self.assertEqual(len(objs), 1)
+ returnValue(objs[0])
+
+
+ @inlineCallbacks
+ def _testOneResource(self, home, calendar_name, data):
+ """
+ Get the one resources expected in a collection
+ """
+ inbox = yield self.calendarUnderTest(home=home, name=calendar_name)
+ objs = yield inbox.objectResources()
+ self.assertEqual(len(objs), 1)
+
+ caldata = yield objs[0].componentForUser()
+ self.assertEqual(normalize_iCalStr(caldata), normalize_iCalStr(componentUpdate(data)))
+
+
+
+class SimpleSchedulingTests(BaseQueueSchedulingTests):
+
+ @inlineCallbacks
+ def test_invite_reply(self):
+ """
+ Test simple invite/reply roundtrip.
+ """
+
+ data1 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;PARTSTAT=ACCEPTED:mailto:user01 at example.com
+ATTENDEE:mailto:user02 at example.com
+DTSTAMP:20051222T210507Z
+ORGANIZER:mailto:user01 at example.com
+SUMMARY:1
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data2 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 01;EMAIL=user01 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user01
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;RSVP=TRUE;SCHEDULE-STATUS=1.2:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com:urn:uuid:user01
+SUMMARY:1
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data3 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+METHOD:REQUEST
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 01;EMAIL=user01 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user01
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;RSVP=TRUE:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com:urn:uuid:user01
+SUMMARY:1
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data4 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 01;EMAIL=user01 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user01
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;RSVP=TRUE:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com:urn:uuid:user01
+SUMMARY:1
+TRANSP:TRANSPARENT
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data5 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 01;EMAIL=user01 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user01
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com:urn:uuid:user01
+SUMMARY:1
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data6 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 01;EMAIL=user01 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user01
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;PARTSTAT=ACCEPTED;SCHEDULE-STATUS=2.0:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com:urn:uuid:user01
+SUMMARY:1
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data7 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+METHOD:REPLY
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com:urn:uuid:user01
+SUMMARY:1
+REQUEST-STATUS:2.0;Success
+END:VEVENT
+END:VCALENDAR
+"""
+
+ data8 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890
+DTSTART:{now}T000000Z
+DURATION:PT1H
+ATTENDEE;CN=User 01;EMAIL=user01 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user01
+ATTENDEE;CN=User 02;EMAIL=user02 at example.com;PARTSTAT=ACCEPTED:urn:uuid:user02
+DTSTAMP:20051222T210507Z
+ORGANIZER;CN=User 01;EMAIL=user01 at example.com;SCHEDULE-STATUS=1.2:urn:uuid:user01
+SUMMARY:1
+END:VEVENT
+END:VCALENDAR
+"""
+
+ waitForWork = ScheduleWorkMixin.allDone()
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ yield calendar.createCalendarObjectWithName("data1.ics", componentUpdate(data1))
+ yield self.commit()
+
+ yield waitForWork
+
+ yield self._testOneResource("user01", "calendar", data2)
+ yield self._testOneResource("user02", "inbox", data3)
+ yield self._testOneResource("user02", "calendar", data4)
+ yield self.commit()
+
+ waitForWork = ScheduleWorkMixin.allDone()
+ cobj = yield self._getOneResource("user02", "calendar")
+ yield cobj.setComponent(componentUpdate(data5))
+ yield self.commit()
+
+ yield waitForWork
+
+ yield self._testOneResource("user01", "calendar", data6)
+ yield self._testOneResource("user01", "inbox", data7)
+ yield self._testOneResource("user02", "calendar", data8)
Modified: CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/test/util.py 2014-01-28 19:56:12 UTC (rev 12467)
+++ CalendarServer/branches/users/cdaboo/scheduling-queue-refresh/txdav/common/datastore/test/util.py 2014-01-29 01:45:15 UTC (rev 12468)
@@ -529,7 +529,7 @@
return data % {"now": nowYear}
-dateSubstitutions = {}
+relativeDateSubstitutions = {}
def componentUpdate(data):
@@ -537,24 +537,24 @@
Update the supplied iCalendar data so that all dates are updated to the current year.
"""
- if len(dateSubstitutions) == 0:
+ if len(relativeDateSubstitutions) == 0:
now = DateTime.getToday()
- dateSubstitutions["now"] = now
+ relativeDateSubstitutions["now"] = now
for i in range(30):
attrname = "now_back%s" % (i + 1,)
- setattr(self, attrname, now.duplicate())
- getattr(self, attrname).offsetDay(-(i + 1))
- dateSubstitutions[attrname] = getattr(self, attrname)
+ dt = now.duplicate()
+ dt.offsetDay(-(i + 1))
+ relativeDateSubstitutions[attrname] = dt
for i in range(30):
attrname = "now_fwd%s" % (i + 1,)
- setattr(self, attrname, now.duplicate())
- getattr(self, attrname).offsetDay(i + 1)
- dateSubstitutions[attrname] = getattr(self, attrname)
+ dt = now.duplicate()
+ dt.offsetDay(i + 1)
+ relativeDateSubstitutions[attrname] = dt
- return Component.fromString(data.format(dateSubstitutions))
+ return Component.fromString(data.format(**relativeDateSubstitutions))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/991aff63/attachment.html>
More information about the calendarserver-changes
mailing list