[CalendarServer-changes] [14925] CalendarServer/trunk/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Fri Jun 26 14:21:39 PDT 2015
Revision: 14925
http://trac.calendarserver.org//changeset/14925
Author: cdaboo at apple.com
Date: 2015-06-26 14:21:39 -0700 (Fri, 26 Jun 2015)
Log Message:
-----------
Fix revision clean-up and revision MODIFIED updating.
Modified Paths:
--------------
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/carddav/datastore/sql.py
CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/sql_sharing.py
CalendarServer/trunk/txdav/common/datastore/sql_util.py
CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py
CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -2568,11 +2568,8 @@
}
accesstype_to_accessMode = dict([(v, k) for k, v in accessMode_to_type.items()])
-def _pathToName(path):
- return path.rsplit(".", 1)[0]
-
class CalendarObject(CommonObjectResource, CalendarObjectBase):
implements(ICalendarObject)
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -58,10 +58,10 @@
from txdav.common.datastore.sql_tables import schema, _BIND_MODE_DIRECT, \
_BIND_STATUS_ACCEPTED, _TRANSP_OPAQUE
from txdav.caldav.datastore.test.common import CommonTests as CalendarCommonTests, \
- test_event_text
+ test_event_text, cal1Root
from txdav.caldav.datastore.test.test_file import setUpCalendarStore
from txdav.common.datastore.test.util import populateCalendarsFrom, \
- CommonCommonTests
+ CommonCommonTests, updateToCurrentYear
from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
from txdav.caldav.icalendarstore import ComponentUpdateState, InvalidDefaultCalendar, \
InvalidSplit, UnknownTimezone
@@ -1913,117 +1913,6 @@
@inlineCallbacks
- def test_calendarRevisionChangeConcurrency(self):
- """
- Test that two concurrent attempts to add resources in two separate
- calendar homes does not deadlock on the revision table update.
- """
-
- calendarStore = self._sqlCalendarStore
-
- # Make sure homes are provisioned
- txn = self.transactionUnderTest()
- home_uid1 = yield txn.homeWithUID(ECALENDARTYPE, "user01", create=True)
- home_uid2 = yield txn.homeWithUID(ECALENDARTYPE, "user02", create=True)
- self.assertNotEqual(home_uid1, None)
- self.assertNotEqual(home_uid2, None)
- yield self.commit()
-
- # Create first events in different calendar homes
- txn1 = calendarStore.newTransaction()
- txn2 = calendarStore.newTransaction()
-
- calendar_uid1_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user01")
- calendar_uid2_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user02")
-
- data = """BEGIN:VCALENDAR
-VERSION:2.0
-CALSCALE:GREGORIAN
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:data%(ctr)s
-DTSTART:20130102T140000Z
-DURATION:PT1H
-CREATED:20060102T190000Z
-DTSTAMP:20051222T210507Z
-SUMMARY:data%(ctr)s
-END:VEVENT
-END:VCALENDAR
-"""
-
- component = Component.fromString(data % {"ctr": 1})
- yield calendar_uid1_in_txn1.createCalendarObjectWithName("data1.ics", component)
-
- component = Component.fromString(data % {"ctr": 2})
- yield calendar_uid2_in_txn2.createCalendarObjectWithName("data2.ics", component)
-
- # Setup deferreds to run concurrently and create second events in the calendar homes
- # previously used by the other transaction - this could create the deadlock.
- @inlineCallbacks
- def _defer_uid3():
- calendar_uid1_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user01")
- component = Component.fromString(data % {"ctr": 3})
- yield calendar_uid1_in_txn2.createCalendarObjectWithName("data3.ics", component)
- yield txn2.commit()
- d1 = _defer_uid3()
-
- @inlineCallbacks
- def _defer_uid4():
- calendar_uid2_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user02")
- component = Component.fromString(data % {"ctr": 4})
- yield calendar_uid2_in_txn1.createCalendarObjectWithName("data4.ics", component)
- yield txn1.commit()
- d2 = _defer_uid4()
-
- # Now do the concurrent provision attempt
- yield DeferredList([d1, d2])
-
- # Verify we did not have a deadlock and all resources have been created.
- caldata1 = yield self.calendarObjectUnderTest(name="data1.ics", calendar_name="calendar", home="user01")
- caldata2 = yield self.calendarObjectUnderTest(name="data2.ics", calendar_name="calendar", home="user02")
- caldata3 = yield self.calendarObjectUnderTest(name="data3.ics", calendar_name="calendar", home="user01")
- caldata4 = yield self.calendarObjectUnderTest(name="data4.ics", calendar_name="calendar", home="user02")
- self.assertNotEqual(caldata1, None)
- self.assertNotEqual(caldata2, None)
- self.assertNotEqual(caldata3, None)
- self.assertNotEqual(caldata4, None)
-
-
- @inlineCallbacks
- def test_calendarMissingRevision(self):
- """
- Test that two concurrent attempts to add resources in two separate
- calendar homes does not deadlock on the revision table update.
- """
-
- # Get details
- home = yield self.homeUnderTest(name="user01", create=True)
- self.assertNotEqual(home, None)
- calendar = yield home.childWithName("calendar")
- self.assertNotEqual(calendar, None)
-
- rev = calendar._revisionsSchema
- yield Delete(
- From=rev,
- Where=(
- rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
- rev.COLLECTION_NAME == Parameter("collectionName")
- )
- ).on(self.transactionUnderTest(), homeID=home.id(), collectionName="calendar")
-
- yield self.commit()
-
- home = yield self.homeUnderTest(name="user01")
- children = yield home.loadChildren()
- self.assertEqual(len(children), 3)
- yield self.commit()
-
- calendar = yield self.calendarUnderTest(home="user01", name="calendar")
- token = yield calendar.syncToken()
- self.assertTrue(token is not None)
-
-
- @inlineCallbacks
def test_inboxTransp(self):
"""
Make sure inbox is always transparent no matter what is stored in the DB.
@@ -2319,7 +2208,164 @@
yield self.commit()
+
+class SyncTests(CommonCommonTests, unittest.TestCase):
+ """
+ Revision table/sync report tests.
+ """
+
@inlineCallbacks
+ def setUp(self):
+ yield super(SyncTests, self).setUp()
+ yield self.buildStoreAndDirectory()
+ yield self.populate()
+
+
+ requirements = {
+ "user01": {
+ "calendar": {
+ "1.ics": (cal1Root.child("1.ics").getContent(), CalendarCommonTests.metadata1),
+ "2.ics": (cal1Root.child("2.ics").getContent(), CalendarCommonTests.metadata2),
+ "3.ics": (cal1Root.child("3.ics").getContent(), CalendarCommonTests.metadata3),
+ "4.ics": (cal1Root.child("4.ics").getContent(), CalendarCommonTests.metadata4),
+ "5.ics": (cal1Root.child("5.ics").getContent(), CalendarCommonTests.metadata5),
+ },
+ },
+ }
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateCalendarsFrom(self.requirements, self.storeUnderTest())
+ self.notifierFactory.reset()
+
+
+ def token2revision(self, token):
+ """
+ FIXME: the API names for L{syncToken}() and L{resourceNamesSinceToken}()
+ are slightly inaccurate; one doesn't produce input for the other.
+ Actually it should be resource names since I{revision} and you need to
+ understand the structure of the tokens to extract the revision. Right
+ now that logic lives in the protocol layer, so this testing method
+ replicates it.
+ """
+ _ignore_uuid, rev = token.split("_", 1)
+ rev = int(rev)
+ return rev
+
+
+ @inlineCallbacks
+ def test_calendarRevisionChangeConcurrency(self):
+ """
+ Test that two concurrent attempts to add resources in two separate
+ calendar homes does not deadlock on the revision table update.
+ """
+
+ calendarStore = self._sqlCalendarStore
+
+ # Make sure homes are provisioned
+ txn = self.transactionUnderTest()
+ home_uid1 = yield txn.homeWithUID(ECALENDARTYPE, "user01", create=True)
+ home_uid2 = yield txn.homeWithUID(ECALENDARTYPE, "user02", create=True)
+ self.assertNotEqual(home_uid1, None)
+ self.assertNotEqual(home_uid2, None)
+ yield self.commit()
+
+ # Create first events in different calendar homes
+ txn1 = calendarStore.newTransaction()
+ txn2 = calendarStore.newTransaction()
+
+ calendar_uid1_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user01")
+ calendar_uid2_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user02")
+
+ data = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:data%(ctr)s
+DTSTART:20130102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:data%(ctr)s
+END:VEVENT
+END:VCALENDAR
+"""
+
+ component = Component.fromString(data % {"ctr": 1})
+ yield calendar_uid1_in_txn1.createCalendarObjectWithName("data1.ics", component)
+
+ component = Component.fromString(data % {"ctr": 2})
+ yield calendar_uid2_in_txn2.createCalendarObjectWithName("data2.ics", component)
+
+ # Setup deferreds to run concurrently and create second events in the calendar homes
+ # previously used by the other transaction - this could create the deadlock.
+ @inlineCallbacks
+ def _defer_uid3():
+ calendar_uid1_in_txn2 = yield self.calendarUnderTest(txn2, "calendar", "user01")
+ component = Component.fromString(data % {"ctr": 3})
+ yield calendar_uid1_in_txn2.createCalendarObjectWithName("data3.ics", component)
+ yield txn2.commit()
+ d1 = _defer_uid3()
+
+ @inlineCallbacks
+ def _defer_uid4():
+ calendar_uid2_in_txn1 = yield self.calendarUnderTest(txn1, "calendar", "user02")
+ component = Component.fromString(data % {"ctr": 4})
+ yield calendar_uid2_in_txn1.createCalendarObjectWithName("data4.ics", component)
+ yield txn1.commit()
+ d2 = _defer_uid4()
+
+ # Now do the concurrent provision attempt
+ yield DeferredList([d1, d2])
+
+ # Verify we did not have a deadlock and all resources have been created.
+ caldata1 = yield self.calendarObjectUnderTest(name="data1.ics", calendar_name="calendar", home="user01")
+ caldata2 = yield self.calendarObjectUnderTest(name="data2.ics", calendar_name="calendar", home="user02")
+ caldata3 = yield self.calendarObjectUnderTest(name="data3.ics", calendar_name="calendar", home="user01")
+ caldata4 = yield self.calendarObjectUnderTest(name="data4.ics", calendar_name="calendar", home="user02")
+ self.assertNotEqual(caldata1, None)
+ self.assertNotEqual(caldata2, None)
+ self.assertNotEqual(caldata3, None)
+ self.assertNotEqual(caldata4, None)
+
+
+ @inlineCallbacks
+ def test_calendarMissingRevision(self):
+ """
+ Test that two concurrent attempts to add resources in two separate
+ calendar homes does not deadlock on the revision table update.
+ """
+
+ # Get details
+ home = yield self.homeUnderTest(name="user02", create=True)
+ self.assertNotEqual(home, None)
+ calendar = yield home.childWithName("calendar")
+ self.assertNotEqual(calendar, None)
+
+ rev = calendar._revisionsSchema
+ yield Delete(
+ From=rev,
+ Where=(
+ rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
+ rev.COLLECTION_NAME == Parameter("collectionName")
+ )
+ ).on(self.transactionUnderTest(), homeID=home.id(), collectionName="calendar")
+
+ yield self.commit()
+
+ home = yield self.homeUnderTest(name="user02")
+ children = yield home.loadChildren()
+ self.assertEqual(len(children), 3)
+ yield self.commit()
+
+ calendar = yield self.calendarUnderTest(home="user02", name="calendar")
+ token = yield calendar.syncToken()
+ self.assertTrue(token is not None)
+
+
+ @inlineCallbacks
def test_removeAfterRevisionCleanup(self):
"""
Make sure L{Calendar}'s can be renamed after revision cleanup
@@ -2350,6 +2396,58 @@
@inlineCallbacks
+ def test_revisionModified(self):
+ """
+ Make sure the revision table MODIFIED value changes for an update or delete
+ """
+
+ @inlineCallbacks
+ def _getModified():
+ home = yield self.homeUnderTest(name="user01")
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ rev = calendar._revisionsSchema
+ modified = yield Select(
+ [rev.MODIFIED, ],
+ From=rev,
+ Where=(
+ rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
+ rev.CALENDAR_RESOURCE_ID == Parameter("collectionID")).And(
+ rev.RESOURCE_NAME == Parameter("resourceName")
+ )
+ ).on(
+ home._txn,
+ homeID=home.id(),
+ collectionID=calendar.id(),
+ resourceName="1.ics",
+ )
+ yield self.commit()
+ returnValue(modified[0][0])
+
+ # Get current modified
+ old_modified = yield _getModified()
+ self.assertNotEqual(old_modified, None)
+
+ # Update resource
+ cobj = yield self.calendarObjectUnderTest(home="user01", calendar_name="calendar", name="1.ics")
+ yield cobj.setComponent(Component.fromString(updateToCurrentYear(cal1Root.child("1.ics").getContent())))
+ yield self.commit()
+
+ # Modified changed
+ update_modified = yield _getModified()
+ self.assertGreater(update_modified, old_modified)
+
+ # Delete resource
+ cobj = yield self.calendarObjectUnderTest(home="user01", calendar_name="calendar", name="1.ics")
+ yield cobj.remove()
+ yield self.commit()
+
+ # Modified changed
+ delete_modified = yield _getModified()
+ self.assertGreater(delete_modified, old_modified)
+ self.assertGreater(delete_modified, update_modified)
+
+
+ @inlineCallbacks
def test_homeSyncTokenWithTrash_Visible(self):
"""
L{ICalendarHome.resourceNamesSinceToken} will return the names of
@@ -2360,8 +2458,8 @@
self.patch(config, "EnableTrashCollection", True)
self.patch(config, "ExposeTrashCollection", True)
- home = yield self.homeUnderTest()
- cal = yield self.calendarUnderTest()
+ home = yield self.homeUnderTest(name="user01")
+ cal = yield self.calendarUnderTest(home="user01", name="calendar")
st = yield home.syncToken()
yield cal.createCalendarObjectWithName("new.ics", Component.fromString(
test_event_text
@@ -2373,12 +2471,12 @@
st2 = yield home.syncToken()
self.failIfEquals(st, st2)
- home = yield self.homeUnderTest()
+ home = yield self.homeUnderTest(name="user01")
expected = [
- "calendar_1/",
- "calendar_1/new.ics",
- "calendar_1/2.ics",
+ "calendar/",
+ "calendar/new.ics",
+ "calendar/2.ics",
"other-calendar/"
]
@@ -2394,7 +2492,7 @@
self.token2revision(st), "infinity")
self.assertEquals(set(changed), set(expected))
- self.assertEquals(set(deleted), set(["calendar_1/2.ics"]))
+ self.assertEquals(set(deleted), set(["calendar/2.ics"]))
self.assertEquals(invalid, [])
changed, deleted, invalid = yield home.resourceNamesSinceToken(
@@ -2414,8 +2512,8 @@
self.patch(config, "EnableTrashCollection", True)
- home = yield self.homeUnderTest()
- cal = yield self.calendarUnderTest()
+ home = yield self.homeUnderTest(name="user01")
+ cal = yield self.calendarUnderTest(home="user01", name="calendar")
st = yield home.syncToken()
yield cal.createCalendarObjectWithName("new.ics", Component.fromString(
test_event_text
@@ -2427,12 +2525,12 @@
st2 = yield home.syncToken()
self.failIfEquals(st, st2)
- home = yield self.homeUnderTest()
+ home = yield self.homeUnderTest(name="user01")
expected = [
- "calendar_1/",
- "calendar_1/new.ics",
- "calendar_1/2.ics",
+ "calendar/",
+ "calendar/new.ics",
+ "calendar/2.ics",
"other-calendar/"
]
@@ -2440,7 +2538,7 @@
self.token2revision(st), "infinity")
self.assertEquals(set(changed), set(expected))
- self.assertEquals(set(deleted), set(["calendar_1/2.ics"]))
+ self.assertEquals(set(deleted), set(["calendar/2.ics"]))
self.assertEquals(invalid, [])
changed, deleted, invalid = yield home.resourceNamesSinceToken(
Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -570,7 +570,8 @@
{
rev.REVISION: schema.REVISION_SEQ,
rev.OBJECT_RESOURCE_ID: Parameter("id"),
- rev.DELETED: True
+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
},
Where=(
rev.RESOURCE_ID == Parameter("resourceID")).And(
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -28,20 +28,21 @@
from twisted.trial import unittest
from twistedcaldav import carddavxml
-from twistedcaldav.vcard import Component as VCard
+from twistedcaldav.vcard import Component as VCard, Component
from twistedcaldav.vcard import Component as VComponent
from txdav.base.propertystore.base import PropertyName
from txdav.carddav.datastore.test.common import CommonTests as AddressBookCommonTests, \
- vcard4_text
+ vcard4_text, adbk1Root
from txdav.carddav.datastore.test.test_file import setUpAddressBookStore
from txdav.carddav.datastore.util import _migrateAddressbook, migrateHome
from txdav.common.icommondatastore import NoSuchObjectResourceError
from txdav.common.datastore.sql import EADDRESSBOOKTYPE, CommonObjectResource
from txdav.common.datastore.sql_tables import _ABO_KIND_PERSON, _ABO_KIND_GROUP, schema
-from txdav.common.datastore.test.util import cleanStore
+from txdav.common.datastore.test.util import cleanStore, CommonCommonTests, \
+ populateAddressBooksFrom
from txdav.carddav.datastore.sql import AddressBook
from txdav.xml.rfc2518 import GETContentLanguage, ResourceType
@@ -919,7 +920,38 @@
yield self.commit()
+
+class SyncTests(CommonCommonTests, unittest.TestCase):
+ """
+ Revision table/sync report tests.
+ """
+
@inlineCallbacks
+ def setUp(self):
+ yield super(SyncTests, self).setUp()
+ yield self.buildStoreAndDirectory()
+ yield self.populate()
+
+
+ requirements = {
+ "user01": {
+ "addressbook": {
+ "1.vcf": adbk1Root.child("1.vcf").getContent(),
+ "2.vcf": adbk1Root.child("2.vcf").getContent(),
+ "3.vcf": adbk1Root.child("3.vcf").getContent(),
+ },
+ "not_a_addressbook": None
+ },
+ }
+
+
+ @inlineCallbacks
+ def populate(self):
+ yield populateAddressBooksFrom(self.requirements, self.storeUnderTest())
+ self.notifierFactory.reset()
+
+
+ @inlineCallbacks
def test_updateAfterRevisionCleanup(self):
"""
Make sure L{AddressBookObject}'s can be updated or removed after revision cleanup
@@ -957,26 +989,26 @@
END:VCARD
"""
- yield self.homeUnderTest()
- adbk = yield self.addressbookUnderTest(name="addressbook")
+ yield self.addressbookHomeUnderTest(name="user01")
+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
yield adbk.createAddressBookObjectWithName("person.vcf", VCard.fromString(person))
yield adbk.createAddressBookObjectWithName("group.vcf", VCard.fromString(group))
yield self.commit()
# Remove the revision
- adbk = yield self.addressbookUnderTest(name="addressbook")
+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
yield adbk.syncToken()
yield self.transactionUnderTest().deleteRevisionsBefore(adbk._syncTokenRevision + 1)
yield self.commit()
# Update the object
- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
yield obj.setComponent(VCard.fromString(group_update))
yield self.commit()
- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
self.assertTrue(obj is not None)
- obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook")
+ obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook", home="user01")
self.assertTrue(obj is not None)
yield self.commit()
@@ -1010,26 +1042,76 @@
END:VCARD
"""
- yield self.homeUnderTest()
- adbk = yield self.addressbookUnderTest(name="addressbook")
+ yield self.addressbookHomeUnderTest(name="user01")
+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
yield adbk.createAddressBookObjectWithName("person.vcf", VCard.fromString(person))
yield adbk.createAddressBookObjectWithName("group.vcf", VCard.fromString(group))
yield self.commit()
# Remove the revision
- adbk = yield self.addressbookUnderTest(name="addressbook")
+ adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
yield adbk.syncToken()
yield self.transactionUnderTest().deleteRevisionsBefore(adbk._syncTokenRevision + 1)
yield self.commit()
# Remove the object
- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
self.assertTrue(obj is not None)
yield obj.remove()
yield self.commit()
- obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook")
+ obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
self.assertTrue(obj is None)
- obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook")
+ obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook", home="user01")
self.assertTrue(obj is not None)
yield self.commit()
+
+
+ @inlineCallbacks
+ def test_revisionModified(self):
+ """
+ Make sure the revision table MODIFIED value changes for an update or delete
+ """
+
+ @inlineCallbacks
+ def _getModified():
+ home = yield self.addressbookHomeUnderTest(name="user01")
+ addressbook = yield self.addressbookUnderTest(home="user01", name="addressbook")
+ rev = addressbook._revisionsSchema
+ modified = yield Select(
+ [rev.MODIFIED, ],
+ From=rev,
+ Where=(
+ rev.ADDRESSBOOK_HOME_RESOURCE_ID == Parameter("homeID")).And(
+ rev.RESOURCE_NAME == Parameter("resourceName")
+ )
+ ).on(
+ home._txn,
+ homeID=home.id(),
+ resourceName="1.vcf",
+ )
+ yield self.commit()
+ returnValue(modified[0][0])
+
+ # Get current modified
+ old_modified = yield _getModified()
+ self.assertNotEqual(old_modified, None)
+
+ # Update resource
+ aobj = yield self.addressbookObjectUnderTest(home="user01", addressbook_name="addressbook", name="1.vcf")
+ yield aobj.setComponent(Component.fromString(adbk1Root.child("1.vcf").getContent()))
+ yield self.commit()
+
+ # Modified changed
+ update_modified = yield _getModified()
+ self.assertGreater(update_modified, old_modified)
+
+ # Delete resource
+ aobj = yield self.addressbookObjectUnderTest(home="user01", addressbook_name="addressbook", name="1.vcf")
+ yield aobj.remove()
+ yield self.commit()
+
+ # Modified changed
+ delete_modified = yield _getModified()
+ self.assertGreater(delete_modified, old_modified)
+ self.assertGreater(delete_modified, update_modified)
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -3409,8 +3409,7 @@
)
# Get revisions
- revisions = (yield cls._revisionsForResourceIDs(childResourceIDs).on(home._txn, resourceIDs=childResourceIDs))
- revisions = dict(revisions)
+ revisions = yield cls.childSyncTokenRevisions(home, childResourceIDs)
# Create the actual objects merging in properties
for dataRow in dataRows:
@@ -3421,7 +3420,7 @@
propstore = propertyStores.get(resourceID, None)
child = yield cls.makeClass(home, bindData, additionalBindData, metadataData, propstore)
- child._syncTokenRevision = revisions.get(resourceID, 0)
+ child._syncTokenRevision = revisions.get(resourceID, None)
results.append(child)
returnValue(results)
Modified: CalendarServer/trunk/txdav/common/datastore/sql_sharing.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_sharing.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/sql_sharing.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -19,7 +19,7 @@
from pycalendar.datetime import DateTime
from twext.enterprise.dal.syntax import Insert, Parameter, Update, Delete, \
- Select, Max
+ Select
from twext.python.clsprop import classproperty
from twext.python.log import Logger
@@ -1451,18 +1451,6 @@
)
- @classmethod
- def _revisionsForResourceIDs(cls, resourceIDs):
- rev = cls._revisionsSchema
- return Select(
- [rev.RESOURCE_ID, Max(rev.REVISION)],
- From=rev,
- Where=rev.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs))).And(
- (rev.RESOURCE_NAME != None).Or(rev.DELETED == False)),
- GroupBy=rev.RESOURCE_ID
- )
-
-
@inlineCallbacks
def invalidateQueryCache(self):
queryCacher = self._txn._queryCacher
Modified: CalendarServer/trunk/txdav/common/datastore/sql_util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_util.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/sql_util.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -16,7 +16,7 @@
##
from twext.enterprise.dal.syntax import Max, Select, Parameter, Delete, Insert, \
- Update, ColumnSyntax, TableSyntax, Upper
+ Update, ColumnSyntax, TableSyntax, Upper, utcNowSQL
from twext.python.clsprop import classproperty
from twext.python.log import Logger
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
@@ -66,6 +66,18 @@
Where=rev.RESOURCE_ID == Parameter("resourceID"))
+ @classmethod
+ def _revisionsForResourceIDs(cls, resourceIDs):
+ rev = cls._revisionsSchema
+ return Select(
+ [rev.RESOURCE_ID, Max(rev.REVISION)],
+ From=rev,
+ Where=rev.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs))).And(
+ (rev.RESOURCE_NAME != None).Or(rev.DELETED == False)),
+ GroupBy=rev.RESOURCE_ID
+ )
+
+
def revisionFromToken(self, token):
if token is None:
return 0
@@ -91,6 +103,21 @@
returnValue(revision)
+ @classmethod
+ @inlineCallbacks
+ def childSyncTokenRevisions(cls, home, childResourceIDs):
+ rows = (yield cls._revisionsForResourceIDs(childResourceIDs).on(home._txn, resourceIDs=childResourceIDs))
+ revisions = dict(rows)
+
+ # Add in any that were missing - this assumes that childResourceIDs were all valid to begin with
+ missingIDs = set(childResourceIDs) - set(revisions.keys())
+ if missingIDs:
+ min_revision = int((yield home._txn.calendarserverValue("MIN-VALID-REVISION")))
+ for resourceID in missingIDs:
+ revisions[resourceID] = min_revision
+ returnValue(revisions)
+
+
def objectResourcesSinceToken(self, token):
raise NotImplementedError()
@@ -206,7 +233,8 @@
return Update(
{
rev.REVISION: schema.REVISION_SEQ,
- rev.COLLECTION_NAME: Parameter("name")
+ rev.COLLECTION_NAME: Parameter("name"),
+ rev.MODIFIED: utcNowSQL,
},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And
(rev.RESOURCE_NAME == None),
@@ -233,7 +261,10 @@
"""
rev = cls._revisionsSchema
return Update(
- {rev.REVISION: schema.REVISION_SEQ, },
+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.MODIFIED: utcNowSQL,
+ },
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And
(rev.RESOURCE_NAME == None)
)
@@ -276,7 +307,8 @@
{
rev.RESOURCE_ID: None,
rev.REVISION: schema.REVISION_SEQ,
- rev.DELETED: True
+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
},
Where=(rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
rev.RESOURCE_ID == Parameter("resourceID")).And(
@@ -294,7 +326,8 @@
{
rev.RESOURCE_ID: None,
rev.REVISION: schema.REVISION_SEQ,
- rev.DELETED: True
+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == None),
@@ -346,7 +379,11 @@
def _deleteBumpTokenQuery(cls):
rev = cls._revisionsSchema
return Update(
- {rev.REVISION: schema.REVISION_SEQ, rev.DELETED: True},
+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.DELETED: True,
+ rev.MODIFIED: utcNowSQL,
+ },
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == Parameter("name")),
Return=rev.REVISION
@@ -357,7 +394,10 @@
def _updateBumpTokenQuery(cls):
rev = cls._revisionsSchema
return Update(
- {rev.REVISION: schema.REVISION_SEQ},
+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.MODIFIED: utcNowSQL,
+ },
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == Parameter("name")),
Return=rev.REVISION
@@ -379,7 +419,11 @@
def _updatePreviouslyNamedQuery(cls):
rev = cls._revisionsSchema
return Update(
- {rev.REVISION: schema.REVISION_SEQ, rev.DELETED: False},
+ {
+ rev.REVISION: schema.REVISION_SEQ,
+ rev.DELETED: False,
+ rev.MODIFIED: utcNowSQL,
+ },
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == Parameter("name")),
Return=rev.REVISION
Modified: CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/work/revision_cleanup.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -53,6 +53,10 @@
return float(config.RevisionCleanup.CleanupPeriodDays) * 24 * 60 * 60
+ def dateCutoff(self):
+ return datetime.datetime.utcnow() - datetime.timedelta(days=float(config.RevisionCleanup.SyncTokenLifetimeDays))
+
+
@inlineCallbacks
def doWork(self):
@@ -60,10 +64,7 @@
minValidRevision = int((yield self.transaction.calendarserverValue("MIN-VALID-REVISION")))
# get max revision on table rows before dateLimit
- dateLimit = (
- datetime.datetime.utcnow() -
- datetime.timedelta(days=float(config.RevisionCleanup.SyncTokenLifetimeDays))
- )
+ dateLimit = self.dateCutoff()
maxRevOlderThanDate = 0
# TODO: Use one Select statement
Modified: CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py 2015-06-26 20:56:38 UTC (rev 14924)
+++ CalendarServer/trunk/txdav/common/datastore/work/test/test_revision_cleanup.py 2015-06-26 21:21:39 UTC (rev 14925)
@@ -23,12 +23,14 @@
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.trial.unittest import TestCase
from twistedcaldav.config import config
+from twistedcaldav.ical import Component
from twistedcaldav.vcard import Component as VCard
from txdav.common.datastore.sql_tables import schema, _BIND_MODE_READ
from txdav.common.datastore.test.util import CommonCommonTests, populateCalendarsFrom
from txdav.common.datastore.work.revision_cleanup import FindMinValidRevisionWork, RevisionCleanupWork
from txdav.common.icommondatastore import SyncTokenValidException
import datetime
+import time
@@ -88,6 +90,21 @@
END:VCALENDAR
"""
+ cal1_mod = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:uid1
+DTSTART:20131122T140000
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+SUMMARY:event 1.1
+END:VEVENT
+END:VCALENDAR
+"""
+
cal2 = """BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
@@ -242,6 +259,10 @@
Verify that all extra calendar object revisions are deleted by FindMinValidRevisionWork and RevisionCleanupWork
"""
+ # get home sync token
+ home = yield self.homeUnderTest(name="user01")
+ hometoken = yield home.syncToken()
+
# get sync token
calendar = yield self.calendarUnderTest(home="user01", name="calendar")
token = yield calendar.syncToken()
@@ -267,7 +288,7 @@
# Get the minimum valid revision and check it
minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
- self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]) + 1)
# do RevisionCleanupWork
yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
@@ -280,14 +301,111 @@
[rev.REVISION],
From=rev,
).on(self.transactionUnderTest())
- self.assertEqual(len(revisionRows), 1) # deleteRevisionsBefore() leaves 1 revision behind
+ self.assertEqual(len(revisionRows), 0)
# old sync token fails
calendar = yield self.calendarUnderTest(home="user01", name="calendar")
yield self.failUnlessFailure(calendar.resourceNamesSinceToken(token), SyncTokenValidException)
+ yield self.commit()
+ # old sync token fails
+ home = yield self.homeUnderTest(name="user01")
+ yield self.failUnlessFailure(home.resourceNamesSinceToken(hometoken, 1), SyncTokenValidException)
+ yield self.commit()
+ # calendar sync token changed
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ newtoken = yield calendar.syncToken()
+ self.assertGreater(newtoken, token)
+ yield self.commit()
+
+ # home sync token changed
+ home = yield self.homeUnderTest(name="user01")
+ newhometoken = yield home.syncToken()
+ self.assertGreater(newhometoken, hometoken)
+ yield self.commit()
+
+ # Depth:1 tokens match
+ home = yield self.homeUnderTest(name="user01")
+ yield home.loadChildren()
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ newtoken1 = yield calendar.syncToken()
+ self.assertEqual(newtoken1, newtoken)
+ yield self.commit()
+
+
@inlineCallbacks
+ def test_calendarObjectRevisions_Modified(self):
+ """
+ Verify that a calendar object created before the revision cut-off, but modified after it is correctly reported as changed
+ after revision clean-up
+ """
+
+ # Need to add one non-event change that creates a revision after the last event change revisions in order
+ # for the logic in this test to work correctly
+ home = yield self.homeUnderTest(name="user01")
+ yield home.createCalendarWithName("_ignore_me")
+ yield self.commit()
+
+ # get initial sync token
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ initial_token = yield calendar.syncToken()
+ yield self.commit()
+
+ # Pause to give some space in the modified time
+ time.sleep(1)
+ modified = datetime.datetime.utcnow()
+ time.sleep(1)
+
+ # Patch the work item to use the modified cut-off we need
+ def _dateCutoff(self):
+ return modified
+ self.patch(FindMinValidRevisionWork, "dateCutoff", _dateCutoff)
+
+ # Make a change to get a pre-update token
+ cal2Object = yield self.calendarObjectUnderTest(self.transactionUnderTest(), name="cal2.ics", calendar_name="calendar", home="user01")
+ yield cal2Object.remove()
+ yield self.commit()
+
+ # get changed sync token
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ pre_update_token = yield calendar.syncToken()
+ yield self.commit()
+
+ # make changes
+ cal1Object = yield self.calendarObjectUnderTest(self.transactionUnderTest(), name="cal1.ics", calendar_name="calendar", home="user01")
+ yield cal1Object.setComponent(Component.fromString(self.cal1_mod))
+ yield self.commit()
+
+ # get changed sync token
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ update_token = yield calendar.syncToken()
+ yield self.commit()
+
+ # do FindMinValidRevisionWork and RevisionCleanupWork
+ yield FindMinValidRevisionWork.reschedule(self.transactionUnderTest(), 0)
+ yield self.commit()
+ yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60)
+
+ # initial sync token fails
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ yield self.failUnlessFailure(calendar.resourceNamesSinceToken(initial_token), SyncTokenValidException)
+ yield self.commit()
+
+ # Pre-update sync token returns one item
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ names = yield calendar.resourceNamesSinceToken(pre_update_token)
+ self.assertEqual(names, (['cal1.ics'], [], []))
+ yield self.commit()
+
+ # Post-update sync token returns one item
+ calendar = yield self.calendarUnderTest(home="user01", name="calendar")
+ names = yield calendar.resourceNamesSinceToken(update_token)
+ self.assertEqual(names, ([], [], []))
+ yield self.commit()
+
+
+ @inlineCallbacks
def test_notificationObjectRevisions(self):
"""
Verify that all extra notification object revisions are deleted by FindMinValidRevisionWork and RevisionCleanupWork
@@ -315,7 +433,7 @@
# Get the minimum valid revision and check it
minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
- self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]) + 1)
# do RevisionCleanupWork
yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
@@ -328,7 +446,7 @@
[rev.REVISION],
From=rev,
).on(self.transactionUnderTest())
- self.assertEqual(len(revisionRows), 1) # deleteRevisionsBefore() leaves 1 revision behind
+ self.assertEqual(len(revisionRows), 0)
# old sync token fails
home = yield self.homeUnderTest(name="user01")
@@ -367,7 +485,7 @@
# Get the minimum valid revision and check it
minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
- self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]))
+ self.assertEqual(int(minValidRevision), max([row[0] for row in revisionRows]) + 1)
# do RevisionCleanupWork
yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
@@ -380,7 +498,7 @@
[rev.REVISION],
From=rev,
).on(self.transactionUnderTest())
- self.assertEqual(len(revisionRows), 1) # deleteRevisionsBefore() leaves 1 revision behind
+ self.assertEqual(len(revisionRows), 0)
# old sync token fails
addressbook = yield self.addressbookUnderTest(home="user01", name="addressbook")
@@ -440,7 +558,7 @@
# Get the minimum valid revision and check it
minValidRevision = yield self.transactionUnderTest().calendarserverValue("MIN-VALID-REVISION")
- self.assertEqual(int(minValidRevision), max([row[3] for row in group1Rows + group2Rows]))
+ self.assertEqual(int(minValidRevision), max([row[3] for row in group1Rows + group2Rows]) + 1)
# do RevisionCleanupWork
yield self.transactionUnderTest().enqueue(RevisionCleanupWork, notBefore=datetime.datetime.utcnow())
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150626/731cdaa6/attachment-0001.html>
More information about the calendarserver-changes
mailing list