[CalendarServer-changes] [14664] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Apr 8 06:50:32 PDT 2015
Revision: 14664
http://trac.calendarserver.org//changeset/14664
Author: cdaboo at apple.com
Date: 2015-04-08 06:50:32 -0700 (Wed, 08 Apr 2015)
Log Message:
-----------
Schedule work items fail temporarily if remote servers are down.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/delivery.py
CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py
Added Paths:
-----------
CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/test/test_cross_pod_scheduling.py
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-04-06 21:13:15 UTC (rev 14663)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-04-08 13:50:32 UTC (rev 14664)
@@ -761,6 +761,8 @@
"AutoReplyDelaySeconds" : 5, # Time delay for sending an auto reply iTIP message
"AttendeeRefreshBatchDelaySeconds" : 5, # Time after an iTIP REPLY for first batched attendee refresh
"AttendeeRefreshBatchIntervalSeconds" : 5, # Time between attendee batch refreshes
+ "TemporaryFailureDelay" : 60, # Delay in seconds before a work item is executed again after a temp failure
+ "MaxTemporaryFailures" : 10, # Max number of temp failure retries before treating as a permanent failure
},
"Splitting": {
Modified: CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/delivery.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/delivery.py 2015-04-06 21:13:15 UTC (rev 14663)
+++ CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/delivery.py 2015-04-08 13:50:32 UTC (rev 14664)
@@ -18,7 +18,7 @@
from calendarserver.version import version
-from twext.internet.gaiendpoint import GAIEndpoint
+from twext.internet.gaiendpoint import GAIEndpoint, MultiFailure
from twext.python.log import Logger
from txweb2 import responsecode
from txweb2.client.http import ClientRequest
@@ -31,7 +31,7 @@
from txweb2.stream import MemoryStream
from twisted.internet.defer import inlineCallbacks, DeferredList, returnValue
-from twisted.internet.error import ConnectionDone
+from twisted.internet.error import ConnectionDone, ConnectionRefusedError
from twisted.internet.protocol import Factory
from twisted.python.failure import Failure
@@ -264,6 +264,15 @@
raise ValueError("Incorrect server response status code: {code}".format(code=response.code))
except Exception, e:
+ # Check for connection failure
+ if isinstance(e, MultiFailure) and not self.scheduler.isfreebusy:
+ all_connections_failed = all([isinstance(err.value, ConnectionRefusedError) for err in e.failures])
+ else:
+ all_connections_failed = False
+
+ # We will return MESSAGE_PENDING if we failed to connect to the remote server, otherwise SERVICE_UNAVAILABLE
+ failed_status = iTIPRequestStatus.MESSAGE_PENDING if all_connections_failed else iTIPRequestStatus.SERVICE_UNAVAILABLE
+
# Generated failed responses for each recipient
log.error(
"Could not do server-to-server request : {req} {exc}",
@@ -276,7 +285,7 @@
(ischedule_namespace, "recipient-failed"),
"Server-to-server request failed",
))
- self.responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.SERVICE_UNAVAILABLE)
+ self.responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=failed_status)
@inlineCallbacks
Added: CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/test/test_cross_pod_scheduling.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/test/test_cross_pod_scheduling.py (rev 0)
+++ CalendarServer/trunk/txdav/caldav/datastore/scheduling/ischedule/test/test_cross_pod_scheduling.py 2015-04-08 13:50:32 UTC (rev 14664)
@@ -0,0 +1,281 @@
+##
+# Copyright (c) 2012-2015 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from pycalendar.datetime import DateTime
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twistedcaldav.ical import Component
+from txdav.caldav.datastore.scheduling.ischedule.delivery import IScheduleRequest
+from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
+from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
+from txweb2.dav.test.util import SimpleRequest
+from twext.internet.gaiendpoint import MultiFailure
+from twisted.python.failure import Failure
+from twisted.internet.error import ConnectionRefusedError
+from twext.enterprise.jobqueue import JobItem
+
+class TestCrossPodScheduling (MultiStoreConduitTest):
+
+
+ now = {
+ "now": DateTime.getToday().getYear(),
+ "now1": DateTime.getToday().getYear() + 1,
+ }
+
+ @inlineCallbacks
+ def setUp(self):
+ """
+ Setup fake hook-up between pods
+ """
+ @inlineCallbacks
+ def _fakeSubmitRequest(iself, ssl, host, port, request):
+
+ if self.refuseConnection:
+ raise MultiFailure((Failure(ConnectionRefusedError()),))
+ else:
+ pod = (port - 8008) / 100
+ inbox = IScheduleInboxResource(self.site.resource, self.theStoreUnderTest(pod), podding=True)
+ response = yield inbox.http_POST(SimpleRequest(
+ self.site,
+ "POST",
+ "http://{host}:{port}/podding".format(host=host, port=port),
+ request.headers,
+ request.stream.mem,
+ ))
+ returnValue(response)
+
+ self.refuseConnection = False
+ self.patch(IScheduleRequest, "_submitRequest", _fakeSubmitRequest)
+ yield super(TestCrossPodScheduling, self).setUp()
+
+
+ def configure(self):
+ super(TestCrossPodScheduling, self).configure()
+
+ # Enable the queue and make it slow
+ self.patch(self.config.Scheduling.Options.WorkQueues, "Enabled", True)
+ self.patch(self.config.Scheduling.Options.WorkQueues, "RequestDelaySeconds", 0.1)
+ self.patch(self.config.Scheduling.Options.WorkQueues, "ReplyDelaySeconds", 0.1)
+ self.patch(self.config.Scheduling.Options.WorkQueues, "AttendeeRefreshBatchDelaySeconds", 0.1)
+ self.patch(self.config.Scheduling.Options.WorkQueues, "TemporaryFailureDelay", 5)
+
+
+ @inlineCallbacks
+ def test_simpleInvite(self):
+ data_organizer = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data
+DTSTART:{now1:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_2
+ORGANIZER:mailto:user01 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE:mailto:puser02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**self.now)
+
+ # Organizer schedules
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("1.ics", Component.fromString(data_organizer))
+ yield self.commitTransaction(0)
+
+ yield self.waitAllEmpty()
+
+ # Data for user02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 1)
+ self.assertEqual(cobjs[0].uid(), "uid_data")
+ yield self.commitTransaction(0)
+
+ # Data for puser02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 1)
+ self.assertEqual(cobjs[0].uid(), "uid_data")
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_connectionRefusedForOrganizer(self):
+ data_organizer = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data
+DTSTART:{now1:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_2
+ORGANIZER:mailto:user01 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE:mailto:puser02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**self.now)
+
+ # Stop cross-pod connection from working
+ self.refuseConnection = True
+
+ # Organizer schedules
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("1.ics", Component.fromString(data_organizer))
+ yield self.commitTransaction(0)
+
+ while True:
+ jobs = yield JobItem.all(self.theTransactionUnderTest(0))
+ yield self.commitTransaction(0)
+ if len(jobs) == 1 and jobs[0].failed > 0:
+ break
+
+ # Data for user02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 1)
+ self.assertEqual(cobjs[0].uid(), "uid_data")
+ yield self.commitTransaction(0)
+
+ # Data for puser02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 0)
+ yield self.commitTransaction(1)
+
+ # Now allow cross-pod to work
+ self.refuseConnection = False
+
+ yield self.waitAllEmpty()
+
+ # Data for puser02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 1)
+ self.assertEqual(cobjs[0].uid(), "uid_data")
+ yield self.commitTransaction(1)
+
+
+ @inlineCallbacks
+ def test_connectionRefusedForAttendee(self):
+ data_organizer = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data
+DTSTART:{now1:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_2
+ORGANIZER:mailto:user01 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE:mailto:puser02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**self.now)
+
+ data_attendee = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid_data
+DTSTART:{now1:04d}0102T160000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data01_2
+ORGANIZER:mailto:user01 at example.com
+ATTENDEE:mailto:user01 at example.com
+ATTENDEE:mailto:user02 at example.com
+ATTENDEE;PARTSTAT=DECLINED:mailto:puser02 at example.com
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n").format(**self.now)
+
+ # Organizer schedules
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user01", create=True)
+ calendar = yield home.childWithName("calendar")
+ yield calendar.createCalendarObjectWithName("1.ics", Component.fromString(data_organizer))
+ yield self.commitTransaction(0)
+
+ yield self.waitAllEmpty()
+
+ # Data for user02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(0), name="user02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 1)
+ self.assertEqual(cobjs[0].uid(), "uid_data")
+ yield self.commitTransaction(0)
+
+ # Data for puser02
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ self.assertEqual(len(cobjs), 1)
+ self.assertEqual(cobjs[0].uid(), "uid_data")
+ yield self.commitTransaction(1)
+
+ # Stop cross-pod connection from working
+ self.refuseConnection = True
+
+ # Attendee changes
+ home = yield self.homeUnderTest(txn=self.theTransactionUnderTest(1), name="puser02", create=True)
+ calendar = yield home.childWithName("calendar")
+ cobjs = yield calendar.calendarObjects()
+ yield cobjs[0].setComponent(Component.fromString(data_attendee))
+ yield self.commitTransaction(1)
+
+ while True:
+ jobs = yield JobItem.all(self.theTransactionUnderTest(1))
+ yield self.commitTransaction(1)
+ if len(jobs) == 1 and jobs[0].failed > 0:
+ break
+
+ # Organizer data unchanged
+ cobj = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ comp = yield cobj.componentForUser()
+ self.assertTrue("DECLINED" not in str(comp))
+ yield self.commitTransaction(0)
+
+ # Now allow cross-pod to work
+ self.refuseConnection = False
+
+ yield self.waitAllEmpty()
+
+ # Organizer data changed
+ cobj = yield self.calendarObjectUnderTest(txn=self.theTransactionUnderTest(0), home="user01", calendar_name="calendar", name="1.ics")
+ comp = yield cobj.componentForUser()
+ self.assertTrue("DECLINED" in str(comp))
+ yield self.commitTransaction(0)
Modified: CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py 2015-04-06 21:13:15 UTC (rev 14663)
+++ CalendarServer/trunk/txdav/caldav/datastore/scheduling/work.py 2015-04-08 13:50:32 UTC (rev 14664)
@@ -18,7 +18,7 @@
from twext.enterprise.dal.syntax import Select, Insert, Delete, Parameter
from twext.enterprise.locking import NamedLock
from twext.enterprise.jobqueue import WorkItem, WORK_PRIORITY_MEDIUM, JobItem, \
- WORK_WEIGHT_5
+ WORK_WEIGHT_5, JobTemporaryError
from twext.python.log import Logger
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
@@ -279,7 +279,26 @@
return changed
+ @inlineCallbacks
+ def checkTemporaryFailure(self, results):
+ """
+ Check to see whether whether a temporary failure should be raised as opposed to continuing on with a permanent failure.
+ @param results: set of results gathered in L{extractSchedulingResponse}
+ @type results: L{list}
+ """
+ if all([result[1] == iTIPRequestStatus.MESSAGE_PENDING_CODE for result in results]):
+ job = yield JobItem.load(self.transaction, self.jobID)
+ if job.failed >= config.Scheduling.Options.WorkQueues.MaxTemporaryFailures:
+ # Set results to SERVICE_UNAVAILABLE
+ for ctr, result in enumerate(results):
+ results[ctr] = (result[0], iTIPRequestStatus.SERVICE_UNAVAILABLE_CODE,)
+ returnValue(None)
+ else:
+ raise JobTemporaryError(config.Scheduling.Options.WorkQueues.TemporaryFailureDelay)
+
+
+
class ScheduleWork(Record, fromTable(schema.SCHEDULE_WORK)):
"""
A L{Record} based table whose rows are used for locking scheduling work by iCalendar UID value.
@@ -478,6 +497,11 @@
if resource is not None:
responses, all_delivered = self.extractSchedulingResponse(scheduler.queuedResponses)
if not all_delivered:
+
+ # Check for all connection failed
+ yield self.checkTemporaryFailure(responses)
+
+ # Update calendar data to reflect error status
calendar = (yield resource.componentForUser())
changed = self.handleSchedulingResponse(responses, calendar, True)
if changed:
@@ -567,6 +591,11 @@
if resource is not None:
responses, all_delivered = self.extractSchedulingResponse((response,))
if not all_delivered:
+
+ # Check for all connection failed
+ yield self.checkTemporaryFailure(responses)
+
+ # Update calendar data to reflect error status
calendar = (yield resource.componentForUser())
changed = yield self.handleSchedulingResponse(responses, calendar, False)
if changed:
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150408/9b23f138/attachment-0001.html>
More information about the calendarserver-changes
mailing list