[CalendarServer-changes] [1680]
CalendarServer/branches/users/cdaboo/reserveduid-db-1672/
twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Mon Jul 16 13:50:39 PDT 2007
Revision: 1680
http://trac.macosforge.org/projects/calendarserver/changeset/1680
Author: cdaboo at apple.com
Date: 2007-07-16 13:50:39 -0700 (Mon, 16 Jul 2007)
Log Message:
-----------
Allow multiple process to co-operate on UID reservation by using an sqlite DB. Also attempt to resolve lock
contention by deferring for a short time when a reservation cannot be made. Unit test added for new DB
reservation support.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/index.py
CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/method/put_common.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/test/test_index.py
Modified: CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/index.py
===================================================================
--- CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/index.py 2007-07-16 00:43:52 UTC (rev 1679)
+++ CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/index.py 2007-07-16 20:50:39 UTC (rev 1680)
@@ -30,6 +30,7 @@
import datetime
import os
+import time
try:
import sqlite3 as sqlite
@@ -49,6 +50,8 @@
schema_version = "5"
collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
+reservation_timeout_secs = 5 * 60
+
#
# Duration into the future through which recurrances are expanded in the index
# by default. This is a caching parameter which affects the size of the index;
@@ -374,6 +377,21 @@
"""
)
+ if uidunique:
+ #
+ # RESERVED table tracks reserved UIDs
+ # UID: The UID being reserved
+ # TIME: When the reservation was made
+ #
+ q.execute(
+ """
+ create table RESERVED (
+ UID text unique,
+ TIME date
+ )
+ """
+ )
+
def _add_to_db(self, name, calendar, cursor = None):
"""
Records the given calendar resource in the index with the given name.
@@ -436,7 +454,6 @@
# A dict of sets. The dict keys are calendar collection paths,
# and the sets contains reserved UIDs for each path.
#
- _reservations = {}
def reserveUID(self, uid):
"""
@@ -444,18 +461,20 @@
@param uid: the UID to reserve
@raise ReservationError: if C{uid} is already reserved
"""
- fpath = self.resource.fp.path
- if fpath in self._reservations and uid in self._reservations[fpath]:
+ try:
+ self._db_execute("insert into RESERVED (UID, TIME) values (:1, :2)", uid, datetime.datetime.now())
+ self._db_commit()
+ except sqlite.IntegrityError:
+ self._db_rollback()
raise ReservationError(
"UID %s already reserved for calendar collection %s."
% (uid, self.resource)
)
-
- if fpath not in self._reservations:
- self._reservations[fpath] = set()
-
- self._reservations[fpath].add(uid)
+ except sqlite.Error, e:
+ log.err("Unable to reserve UID: %s", (e,))
+ self._db_rollback()
+ raise
def unreserveUID(self, uid):
"""
@@ -463,15 +482,20 @@
@param uid: the UID to reserve
@raise ReservationError: if C{uid} is not reserved
"""
- fpath = self.resource.fp.path
-
- if fpath not in self._reservations or uid not in self._reservations[fpath]:
+
+ if not self.isReservedUID(uid):
raise ReservationError(
"UID %s is not reserved for calendar collection %s."
% (uid, self.resource)
)
- self._reservations[fpath].remove(uid)
+ try:
+ self._db_execute("delete from RESERVED where UID = :1", uid)
+ self._db_commit()
+ except sqlite.Error, e:
+ log.err("Unable to unreserve UID: %s", (e,))
+ self._db_rollback()
+ raise
def isReservedUID(self, uid):
"""
@@ -479,16 +503,32 @@
@param uid: the UID to check
@return: True if C{uid} is reserved, False otherwise.
"""
- fpath = self.resource.fp.path
+
+ rowiter = self._db_execute("select UID, TIME from RESERVED where UID = :1", uid)
+ for uid, attime in rowiter:
+ # Double check that the time is within a reasonable period of now
+ # otherwise we probably have a stale reservation
+ tm = time.strptime(attime[:19], "%Y-%m-%d %H:%M:%S")
+ dt = datetime.datetime(year=tm.tm_year, month=tm.tm_mon, day=tm.tm_mday, hour=tm.tm_hour, minute=tm.tm_min, second = tm.tm_sec)
+ if datetime.datetime.now() - dt > datetime.timedelta(seconds=reservation_timeout_secs):
+ try:
+ self._db_execute("delete from RESERVED where UID = :1", uid)
+ self._db_commit()
+ except sqlite.Error, e:
+ log.err("Unable to unreserve UID: %s", (e,))
+ self._db_rollback()
+ raise
+ return False
+ else:
+ return True
- return fpath in self._reservations and uid in self._reservations[fpath]
+ return False
def isAllowedUID(self, uid, *names):
"""
- Checks to see whether to allow an operation with adds the the specified
- UID is allowed to the index. Specifically, the operation may not
- violate the constraint that UIDs must be unique, and the UID must not
- be reserved.
+ Checks to see whether to allow an operation which would add the
+ specified UID to the index. Specifically, the operation may not
+ violate the constraint that UIDs must be unique.
@param uid: the UID to check
@param names: the names of resources being replaced or deleted by the
operation; UIDs associated with these resources are not checked.
@@ -496,7 +536,7 @@
False otherwise.
"""
rname = self.resourceNameForUID(uid)
- return not self.isReservedUID(uid) and (rname is None or rname in names)
+ return (rname is None or rname in names)
def _db_type(self):
"""
Modified: CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/method/put_common.py 2007-07-16 00:43:52 UTC (rev 1679)
+++ CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/method/put_common.py 2007-07-16 20:50:39 UTC (rev 1680)
@@ -22,8 +22,11 @@
__all__ = ["storeCalendarObjectResource"]
+from twisted.internet import reactor
+from twisted.internet.defer import Deferred
from twisted.internet.defer import deferredGenerator
from twisted.internet.defer import maybeDeferred
+from twisted.internet.defer import succeed
from twisted.internet.defer import waitForDeferred
from twisted.python import failure, log
from twisted.python.filepath import FilePath
@@ -37,6 +40,7 @@
from twisted.web2.dav.stream import MD5StreamWrapper
from twisted.web2.dav.util import joinURL, parentForURL
from twisted.web2.http import HTTPError
+from twisted.web2.http import StatusResponse
from twisted.web2.iweb import IResponse
from twisted.web2.stream import MemoryStream
@@ -46,6 +50,7 @@
from twistedcaldav.caldavxml import NumberOfRecurrencesWithinLimits
from twistedcaldav.caldavxml import caldav_namespace
from twistedcaldav.ical import Component
+from twistedcaldav.index import ReservationError
from twistedcaldav.instance import TooManyInstancesError
from twistedcaldav.resource import CalDAVResource
@@ -342,7 +347,34 @@
log.err(message)
raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (caldav_namespace, "max-resource-size")))
- # uid conflict check
+ # Reserve UID
+ destination_index = destinationparent.index()
+
+ # Lets use a deferred for this and loop a few times if we cannot reserve so that we give
+ # time to whoever has the reservation to finish and release it.
+ failure_count = 0
+ while(failure_count < 5):
+ try:
+ destination_index.reserveUID(uid)
+ reserved = True
+ break
+ except ReservationError:
+ reserved = False
+ failure_count += 1
+
+ d = Deferred()
+ def _timedDeferred():
+ d.callback(True)
+ reactor.callLater(0.1, _timedDeferred)
+ pause = waitForDeferred(d)
+ yield pause
+ _ignore = pause.getResult()
+
+ if not reserved:
+ raise HTTPError(StatusResponse(responsecode.CONFLICT, "Resource: %s currently in use." % (destination_uri,)))
+
+ # uid conflict check - note we do this after reserving the UID to avoid a race condition where two requests
+ # try to write the same calendar data to two different resource URIs.
if not isiTIP:
result, message, rname = noUIDConflict(uid)
if not result:
@@ -351,15 +383,6 @@
NoUIDConflict(davxml.HRef.fromString(joinURL(parentForURL(destination_uri), rname.encode("utf-8"))))
))
- # Reserve UID
- # FIXME: A race-condition could exist here if a deferred action were to be inserted between this statement
- # and the isAllowedUID statement above. It would probably be best to merge isAllowedUID and reserveUID
- # into a single 'atomic' 'test-and-set' operation to avoid this. Right now just make sure there are no
- # deferreds.
- destination_index = destinationparent.index()
- destination_index.reserveUID(uid)
- reserved = True
-
"""
Handle rollback setup here.
"""
Added: CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/test/test_index.py
===================================================================
--- CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/test/test_index.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/reserveduid-db-1672/twistedcaldav/test/test_index.py 2007-07-16 20:50:39 UTC (rev 1680)
@@ -0,0 +1,65 @@
+##
+# Copyright (c) 2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DRI: Cyrus Daboo, cdaboo at apple.com
+##
+
+from twistedcaldav.index import Index
+
+import twistedcaldav.test.util
+from twistedcaldav.resource import CalDAVResource
+from twistedcaldav.index import ReservationError
+import os
+import time
+
+class TestIndex (twistedcaldav.test.util.TestCase):
+ """
+ Test abstract SQL DB class
+ """
+
+ def setUp(self):
+ super(TestIndex, self).setUp()
+ self.site.resource.isCalendarCollection = lambda: True
+
+ def test_reserve_uid_ok(self):
+ uid = "test-test-test"
+ db = Index(self.site.resource)
+ self.assertFalse(db.isReservedUID(uid))
+ db.reserveUID(uid)
+ self.assertTrue(db.isReservedUID(uid))
+ db.unreserveUID(uid)
+ self.assertFalse(db.isReservedUID(uid))
+
+ def test_reserve_uid_twice(self):
+ uid = "test-test-test"
+ db = Index(self.site.resource)
+ db.reserveUID(uid)
+ self.assertTrue(db.isReservedUID(uid))
+ self.assertRaises(ReservationError, db.reserveUID, uid)
+
+ def test_unreserve_unreserved(self):
+ uid = "test-test-test"
+ db = Index(self.site.resource)
+ self.assertRaises(ReservationError, db.unreserveUID, uid)
+
+ def test_reserve_uid_timeout(self):
+ uid = "test-test-test"
+ twistedcaldav.index.reservation_timeout_secs = 2
+ db = Index(self.site.resource)
+ self.assertFalse(db.isReservedUID(uid))
+ db.reserveUID(uid)
+ self.assertTrue(db.isReservedUID(uid))
+ time.sleep(3)
+ self.assertFalse(db.isReservedUID(uid))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20070716/ac412095/attachment.html
More information about the calendarserver-changes
mailing list