[CalendarServer-changes] [8978] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Apr 4 13:56:56 PDT 2012
Revision: 8978
http://trac.macosforge.org/projects/calendarserver/changeset/8978
Author: cdaboo at apple.com
Date: 2012-04-04 13:56:55 -0700 (Wed, 04 Apr 2012)
Log Message:
-----------
Clean up recent on-demand time range expansion work.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/dateops.py
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/sql_legacy.py
Added Paths:
-----------
CalendarServer/trunk/twistedcaldav/test/test_dateops.py
Modified: CalendarServer/trunk/twistedcaldav/dateops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/dateops.py 2012-04-04 17:19:48 UTC (rev 8977)
+++ CalendarServer/trunk/twistedcaldav/dateops.py 2012-04-04 20:56:55 UTC (rev 8978)
@@ -259,8 +259,15 @@
@return: L{PyCalendarDateTime} result
"""
- dt = datetime.datetime.strptime(ts[:19], "%Y-%m-%d %H:%M:%S")
- return PyCalendarDateTime(year=dt.year, month=dt.month, day=dt.day, hours=dt.hour, minutes=dt.minute, seconds=dt.second)
+ # Format is "%Y-%m-%d %H:%M:%S"
+ return PyCalendarDateTime(
+ year=int(ts[0:4]),
+ month=int(ts[5:7]),
+ day=int(ts[8:10]),
+ hours=int(ts[11:13]),
+ minutes=int(ts[14:16]),
+ seconds=int(ts[17:19])
+ )
def parseSQLDateToPyCalendar(ts):
"""
@@ -271,8 +278,12 @@
@return: L{PyCalendarDateTime} result
"""
- dt = datetime.datetime.strptime(ts[:10], "%Y-%m-%d")
- return PyCalendarDateTime(year=dt.year, month=dt.month, day=dt.day)
+ # Format is "%Y-%m-%d", though Oracle may add zero time which we ignore
+ return PyCalendarDateTime(
+ year=int(ts[0:4]),
+ month=int(ts[5:7]),
+ day=int(ts[8:10])
+ )
def datetimeMktime(dt):
Added: CalendarServer/trunk/twistedcaldav/test/test_dateops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_dateops.py (rev 0)
+++ CalendarServer/trunk/twistedcaldav/test/test_dateops.py 2012-04-04 20:56:55 UTC (rev 8978)
@@ -0,0 +1,111 @@
+##
+# Copyright (c) 2005-2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import twistedcaldav.test.util
+from twisted.trial.unittest import SkipTest
+from pycalendar.datetime import PyCalendarDateTime
+from twistedcaldav.dateops import parseSQLTimestampToPyCalendar,\
+ parseSQLDateToPyCalendar, parseSQLTimestamp, pyCalendarTodatetime
+import datetime
+import dateutil
+
+class Dateops(twistedcaldav.test.util.TestCase):
+ """
+ dateops.py tests
+ """
+
+ def test_normalizeForIndex(self):
+ raise SkipTest("test unimplemented")
+
+ def test_normalizeToUTC(self):
+ raise SkipTest("test unimplemented")
+
+ def test_floatoffset(self):
+ raise SkipTest("test unimplemented")
+
+ def test_adjustFloatingToTimezone(self):
+ raise SkipTest("test unimplemented")
+
+ def test_compareDateTime(self):
+ raise SkipTest("test unimplemented")
+
+ def test_differenceDateTime(self):
+ raise SkipTest("test unimplemented")
+
+ def test_timeRangesOverlap(self):
+ raise SkipTest("test unimplemented")
+
+ def test_normalizePeriodList(self):
+ raise SkipTest("test unimplemented")
+
+ def test_clipPeriod(self):
+ raise SkipTest("test unimplemented")
+
+ def test_pyCalendarTodatetime(self):
+ """
+ dateops.pyCalendarTodatetime
+ """
+
+ tests = (
+ (PyCalendarDateTime(2012, 4, 4, 12, 34, 56), datetime.datetime(2012, 4, 4, 12, 34, 56, tzinfo=dateutil.tz.tzutc())),
+ (PyCalendarDateTime(2012, 12, 31), datetime.date(2012, 12, 31)),
+ )
+
+ for pycal, result in tests:
+ self.assertEqual(pyCalendarTodatetime(pycal), result)
+
+ def test_parseSQLTimestamp(self):
+ """
+ dateops.parseSQLTimestamp
+ """
+
+ tests = (
+ ("2012-04-04 12:34:56", datetime.datetime(2012, 4, 4, 12, 34, 56)),
+ ("2012-12-31 01:01:01", datetime.datetime(2012, 12, 31, 1, 1, 1)),
+ )
+
+ for sqlStr, result in tests:
+ self.assertEqual(parseSQLTimestamp(sqlStr), result)
+
+ def test_parseSQLTimestampToPyCalendar(self):
+ """
+ dateops.parseSQLTimestampToPyCalendar
+ """
+
+ tests = (
+ ("2012-04-04 12:34:56", PyCalendarDateTime(2012, 4, 4, 12, 34, 56)),
+ ("2012-12-31 01:01:01", PyCalendarDateTime(2012, 12, 31, 1, 1, 1)),
+ )
+
+ for sqlStr, result in tests:
+ self.assertEqual(parseSQLTimestampToPyCalendar(sqlStr), result)
+
+ def test_parseSQLDateToPyCalendar(self):
+ """
+ dateops.parseSQLDateToPyCalendar
+ """
+
+ tests = (
+ ("2012-04-04", PyCalendarDateTime(2012, 4, 4)),
+ ("2012-12-31 00:00:00", PyCalendarDateTime(2012, 12, 31)),
+ )
+
+ for sqlStr, result in tests:
+ self.assertEqual(parseSQLDateToPyCalendar(sqlStr), result)
+
+ def test_datetimeMktime(self):
+ raise SkipTest("test unimplemented")
+
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-04-04 17:19:48 UTC (rev 8977)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2012-04-04 20:56:55 UTC (rev 8978)
@@ -753,15 +753,21 @@
@inlineCallbacks
- def updateDatabase(self, component,
- expand_until=None, reCreate=False, inserting=False):
+ def updateDatabase(self, component, expand_until=None, reCreate=False,
+ inserting=False, useTxn=None):
"""
- Update the database tables for the new data being written.
+ Update the database tables for the new data being written. Occasionally we might need to do an update to
+ time-range data via a separate transaction, so we allow that to be passed in. Note that in that case
+ access to the parent resources will not occur in this method, so the queries on the new txn won't depend
+ on any parent objects having the same txn set.
@param component: calendar data to store
@type component: L{Component}
"""
+ # Setup appropriate txn
+ txn = useTxn if useTxn is not None else self._txn
+
# Decide how far to expand based on the component
doInstanceIndexing = False
master = component.masterComponent()
@@ -774,7 +780,7 @@
else:
# If migrating or re-creating or config option for delayed indexing is off, always index
- if reCreate or self._txn._migrating or not config.FreeBusyIndexDelayedExpand:
+ if reCreate or txn._migrating or not config.FreeBusyIndexDelayedExpand:
doInstanceIndexing = True
# Duration into the future through which recurrences are expanded in the index
@@ -812,7 +818,7 @@
self.log_error("Invalid instance %s when indexing %s in %s" %
(e.rid, self._name, self._calendar,))
- if self._txn._migrating:
+ if txn._migrating:
# TODO: fix the data here by re-writing component then re-index
instances = component.expandTimeRanges(expand, ignoreInvalidInstances=True)
recurrenceLimit = instances.limit
@@ -843,7 +849,7 @@
self._size = len(componentText)
# Special - if migrating we need to preserve the original md5
- if self._txn._migrating and hasattr(component, "md5"):
+ if txn._migrating and hasattr(component, "md5"):
self._md5 = component.md5
# Determine attachment mode (ignore inbox's) - NB we have to do this
@@ -885,7 +891,7 @@
yield Insert(
values,
Return=(co.RESOURCE_ID, co.CREATED, co.MODIFIED)
- ).on(self._txn)
+ ).on(txn)
)[0]
else:
values[co.MODIFIED] = utcNowSQL
@@ -893,13 +899,13 @@
yield Update(
values, Return=co.MODIFIED,
Where=co.RESOURCE_ID == self._resourceID
- ).on(self._txn)
+ ).on(txn)
)[0][0]
# Need to wipe the existing time-range for this and rebuild
yield Delete(
From=tr,
Where=tr.CALENDAR_OBJECT_RESOURCE_ID == self._resourceID
- ).on(self._txn)
+ ).on(txn)
else:
values = {
co.RECURRANCE_MAX :
@@ -909,13 +915,13 @@
yield Update(
values,
Where=co.RESOURCE_ID == self._resourceID
- ).on(self._txn)
+ ).on(txn)
# Need to wipe the existing time-range for this and rebuild
yield Delete(
From=tr,
Where=tr.CALENDAR_OBJECT_RESOURCE_ID == self._resourceID
- ).on(self._txn)
+ ).on(txn)
if doInstanceIndexing:
# TIME_RANGE table update
@@ -938,14 +944,14 @@
instance.component.getFBType(),
icalfbtype_to_indexfbtype["FREE"]),
tr.TRANSPARENT : transp,
- }, Return=tr.INSTANCE_ID).on(self._txn))[0][0]
+ }, Return=tr.INSTANCE_ID).on(txn))[0][0]
peruserdata = component.perUserTransparency(instance.rid)
for useruid, transp in peruserdata:
(yield Insert({
tpy.TIME_RANGE_INSTANCE_ID : instanceid,
tpy.USER_ID : useruid,
tpy.TRANSPARENT : transp,
- }).on(self._txn))
+ }).on(txn))
# Special - for unbounded recurrence we insert a value for "infinity"
# that will allow an open-ended time-range to always match it.
@@ -963,14 +969,14 @@
tr.FBTYPE :
icalfbtype_to_indexfbtype["UNKNOWN"],
tr.TRANSPARENT : transp,
- }, Return=tr.INSTANCE_ID).on(self._txn))[0][0]
+ }, Return=tr.INSTANCE_ID).on(txn))[0][0]
peruserdata = component.perUserTransparency(None)
for useruid, transp in peruserdata:
(yield Insert({
tpy.TIME_RANGE_INSTANCE_ID : instanceid,
tpy.USER_ID : useruid,
tpy.TRANSPARENT : transp,
- }).on(self._txn))
+ }).on(txn))
@inlineCallbacks
@@ -1018,14 +1024,18 @@
@inlineCallbacks
- def recurrenceMax(self):
+ def recurrenceMax(self, useTxn=None):
"""
- Get the RECURRANCE_MAX value.
+ Get the RECURRANCE_MAX value. Occasionally we might need to do an update to
+ time-range data via a separate transaction, so we allow that to be passed in.
@return: L{PyCalendarDateTime} result
"""
+ # Setup appropriate txn
+ txn = useTxn if useTxn is not None else self._txn
+
rMax = (
- yield self._recurrenceMaxByIDQuery.on(self._txn,
+ yield self._recurrenceMaxByIDQuery.on(txn,
resourceID=self._resourceID)
)[0][0]
returnValue(parseSQLDateToPyCalendar(rMax))
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-04-04 17:19:48 UTC (rev 8977)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-04-04 20:56:55 UTC (rev 8978)
@@ -38,6 +38,7 @@
from txdav.common.datastore.sql_tables import schema, _BIND_MODE_DIRECT,\
_BIND_STATUS_ACCEPTED
from txdav.common.datastore.test.util import buildStore, populateCalendarsFrom
+from txdav.common.icommondatastore import NoSuchObjectResourceError
from twistedcaldav import caldavxml
from twistedcaldav.caldavxml import CalendarDescription
@@ -1104,3 +1105,67 @@
supported_components.add(result)
self.assertEqual(supported_components, set(("VEVENT", "VTODO",)))
+
+ @inlineCallbacks
+ def test_resourceLock(self):
+ """
+ Test CommonObjectResource.lock to make sure it locks, raises on missing resource,
+ and raises when locked and NOWAIT used.
+ """
+
+ # Valid object
+ resource = yield self.calendarObjectUnderTest()
+
+ # Valid lock
+ yield resource.lock()
+ self.assertTrue(resource._locked)
+
+ # Setup a new transaction to verify the lock and also verify NOWAIT behavior
+ newTxn = self._sqlCalendarStore.newTransaction()
+ newResource = yield self.calendarObjectUnderTest(txn=newTxn)
+ try:
+ yield newResource.lock(nowait=True)
+ except:
+ pass # OK
+ else:
+ self.fail("Expected an exception")
+ self.assertFalse(newResource._locked)
+ yield newTxn.abort()
+
+ # Commit existing transaction and verify we can get the lock using
+ yield self.commit()
+
+ resource = yield self.calendarObjectUnderTest()
+ yield resource.lock()
+ self.assertTrue(resource._locked)
+
+ # Setup a new transaction to verify the lock but pass in an alternative txn directly
+ newTxn = self._sqlCalendarStore.newTransaction()
+
+ # FIXME: not sure why, but without this statement here, this portion of the test fails in a funny way.
+ # Basically the query in the try block seems to execute twice, failing each time, one of which is caught,
+ # and the other not - causing the test to fail. Seems like some state on newTxn is not being initialized?
+ _ignore = yield self.calendarObjectUnderTest("2.ics", txn=newTxn)
+
+ try:
+ yield resource.lock(nowait=True, useTxn=newTxn)
+ except:
+ pass # OK
+ else:
+ self.fail("Expected an exception")
+ self.assertTrue(resource._locked)
+
+ # Test missing resource
+ resource2 = yield self.calendarObjectUnderTest("2.ics")
+ resource2._resourceID = 123456789
+ try:
+ yield resource2.lock()
+ except NoSuchObjectResourceError:
+ pass # OK
+ except:
+ self.fail("Expected a NoSuchObjectResourceError exception")
+ else:
+ self.fail("Expected an exception")
+ self.assertFalse(resource2._locked)
+
+
\ No newline at end of file
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2012-04-04 17:19:48 UTC (rev 8977)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2012-04-04 20:56:55 UTC (rev 8978)
@@ -2695,7 +2695,7 @@
self._modified = None
self._objectText = None
- self._overrideTxn = None
+ self._locked = False
@classproperty
@@ -2903,13 +2903,12 @@
@property
def _txn(self):
- return self._overrideTxn if self._overrideTxn else self._parentCollection._txn
+ return self._parentCollection._txn
+
def transaction(self):
- return self._overrideTxn if self._overrideTxn else self._parentCollection._txn
+ return self._parentCollection._txn
- def useTxn(self, newTxn):
- self._overrideTxn = newTxn
@classmethod
def _selectForUpdateQuery(cls, nowait): #@NoSelf
@@ -2918,10 +2917,29 @@
"""
return Select(From=cls._objectSchema, ForUpdate=True, NoWait=nowait, Where=cls._objectSchema.RESOURCE_ID == Parameter("resourceID"))
+
@inlineCallbacks
- def lock(self, nowait=False):
- yield self._selectForUpdateQuery(nowait).on(self._txn, NoSuchObjectResourceError, resourceID=self._resourceID)
+ def lock(self, nowait=False, useTxn=None):
+ """
+ Attempt to obtain a row lock on the object resource. Lock will remain until
+ transaction is complete, or fail if resource is missing, or it is already locked
+ and NOWAIT is used. Occasionally we need to lock via a separate transaction so we
+ pass that in too.
+ @param nowait: whether or not to use NOWAIT option
+ @type nowait: C{bool}
+ @param useTxn: alternatiuve transaction to use
+ @type useTxn: L{CommonStoreTransaction}
+
+ @raise: L{NoSuchObjectResourceError} if resource does not exist, other L{Exception}
+ if already locked and NOWAIT is used.
+ """
+
+ txn = useTxn if useTxn is not None else self._txn
+ yield self._selectForUpdateQuery(nowait).on(txn, NoSuchObjectResourceError, resourceID=self._resourceID)
+ self._locked = True
+
+
def setComponent(self, component, inserting=False):
raise NotImplementedError
Modified: CalendarServer/trunk/txdav/common/datastore/sql_legacy.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_legacy.py 2012-04-04 17:19:48 UTC (rev 8977)
+++ CalendarServer/trunk/txdav/common/datastore/sql_legacy.py 2012-04-04 20:56:55 UTC (rev 8978)
@@ -1161,16 +1161,13 @@
# transaction may have the row locked, so use NOWAIT and if that fails, fakll back to using the original txn.
newTxn = obj.transaction().store().newTransaction()
- obj.useTxn(newTxn)
try:
- yield obj.lock(nowait=True)
+ yield obj.lock(nowait=True, useTxn=newTxn)
except NoSuchObjectResourceError:
yield newTxn.commit()
- obj.useTxn(None)
returnValue(None)
except:
yield newTxn.abort()
- obj.useTxn(None)
newTxn = None
# Now do the re-expand using the appropriate transaction
@@ -1178,16 +1175,18 @@
if newTxn is None:
rmax = None
else:
- rmax = (yield obj.recurrenceMax())
+ rmax = (yield obj.recurrenceMax(useTxn=newTxn))
if rmax is None or rmax < expand_until:
yield obj.updateDatabase(
- (yield obj.component()), expand_until=expand_until, reCreate=True
+ (yield obj.component()),
+ expand_until=expand_until,
+ reCreate=True,
+ useTxn=newTxn,
)
finally:
if newTxn is not None:
yield newTxn.commit()
- obj.useTxn(None)
@inlineCallbacks
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120404/46a4e19c/attachment-0001.html>
More information about the calendarserver-changes
mailing list