[CalendarServer-changes] [7269] CalendarServer/trunk/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Tue Mar 29 07:03:54 PDT 2011
Revision: 7269
http://trac.macosforge.org/projects/calendarserver/changeset/7269
Author: cdaboo at apple.com
Date: 2011-03-29 07:03:50 -0700 (Tue, 29 Mar 2011)
Log Message:
-----------
Make sure etags are preserved when migrating file->sql store.
Modified Paths:
--------------
CalendarServer/trunk/txdav/caldav/datastore/sql.py
CalendarServer/trunk/txdav/caldav/datastore/test/common.py
CalendarServer/trunk/txdav/caldav/datastore/util.py
CalendarServer/trunk/txdav/carddav/datastore/sql.py
CalendarServer/trunk/txdav/carddav/datastore/test/common.py
CalendarServer/trunk/txdav/carddav/datastore/util.py
CalendarServer/trunk/txdav/common/datastore/test/test_util.py
CalendarServer/trunk/txdav/common/datastore/test/util.py
Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -475,7 +475,11 @@
self._uid = component.resourceUID()
self._md5 = hashlib.md5(componentText).hexdigest()
self._size = len(componentText)
-
+
+ # Special - if migrating we need to preserve the original md5
+ if self._txn._migrating and hasattr(component, "md5"):
+ self._md5 = component.md5
+
# Determine attachment mode (ignore inbox's) - NB we have to do this
# after setting up other properties as UID at least is needed
self._attachment = _ATTACHMENTS_MODE_NONE
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/common.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/common.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -22,6 +22,7 @@
from twisted.internet.defer import Deferred, inlineCallbacks, returnValue,\
maybeDeferred
from twisted.internet.protocol import Protocol
+from twisted.python import hashlib
from twext.enterprise.ienterprise import AlreadyFinishedError
@@ -174,12 +175,17 @@
"hasPrivateComment": True,
}
+ md5Values = (
+ hashlib.md5("1234").hexdigest(),
+ hashlib.md5("5678").hexdigest(),
+ hashlib.md5("9ABC").hexdigest(),
+ )
requirements = {
"home1": {
"calendar_1": {
- "1.ics": (cal1Root.child("1.ics").getContent(), metadata1,),
- "2.ics": (cal1Root.child("2.ics").getContent(), metadata2,),
- "3.ics": (cal1Root.child("3.ics").getContent(), metadata3,),
+ "1.ics": (cal1Root.child("1.ics").getContent(), metadata1),
+ "2.ics": (cal1Root.child("2.ics").getContent(), metadata2),
+ "3.ics": (cal1Root.child("3.ics").getContent(), metadata3),
},
"calendar_2": {},
"calendar_empty": {},
@@ -187,6 +193,19 @@
},
"not_a_home": None
}
+ md5s = {
+ "home1": {
+ "calendar_1": {
+ "1.ics": md5Values[0],
+ "2.ics": md5Values[1],
+ "3.ics": md5Values[2],
+ },
+ "calendar_2": {},
+ "calendar_empty": {},
+ "not_a_calendar": None
+ },
+ "not_a_home": None
+ }
def storeUnderTest(self):
Modified: CalendarServer/trunk/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/util.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/caldav/datastore/util.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -125,10 +125,11 @@
try:
# Must account for metadata
-
+ component = (yield calendarObject.component()) # XXX WRONG SHOULD CALL getComponent
+ component.md5 = calendarObject.md5()
yield outCalendar.createCalendarObjectWithName(
calendarObject.name(),
- (yield calendarObject.component()), # XXX WRONG SHOULD CALL getComponent
+ component,
metadata=calendarObject.getMetadata(),
)
Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/sql.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -242,6 +242,11 @@
# ADDRESSBOOK_OBJECT table update
self._md5 = hashlib.md5(componentText).hexdigest()
self._size = len(componentText)
+
+ # Special - if migrating we need to preserve the original md5
+ if self._txn._migrating and hasattr(component, "md5"):
+ self._md5 = component.md5
+
if inserting:
self._resourceID, self._created, self._modified = (
yield Insert(
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/common.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/common.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -18,6 +18,7 @@
Tests for common addressbook store API functions.
"""
from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
+from twisted.python import hashlib
from txdav.idav import IPropertyStore, IDataStore
from txdav.base.propertystore.base import PropertyName
@@ -107,6 +108,11 @@
L{txdav.carddav.iaddressbookstore}.
"""
+ md5Values = (
+ hashlib.md5("1234").hexdigest(),
+ hashlib.md5("5678").hexdigest(),
+ hashlib.md5("9ABC").hexdigest(),
+ )
requirements = {
"home1": {
"addressbook_1": {
@@ -120,6 +126,19 @@
},
"not_a_home": None
}
+ md5s = {
+ "home1": {
+ "addressbook_1": {
+ "1.vcf": md5Values[0],
+ "2.vcf": md5Values[1],
+ "3.vcf": md5Values[2],
+ },
+ "addressbook_2": {},
+ "addressbook_empty": {},
+ "not_a_addressbook": None
+ },
+ "not_a_home": None
+ }
def storeUnderTest(self):
"""
Modified: CalendarServer/trunk/txdav/carddav/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/util.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/carddav/datastore/util.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -80,9 +80,12 @@
for addressbookObject in inObjects:
try:
+ component = (yield addressbookObject.component()) # XXX WRONG SHOULD CALL getComponent
+ component.md5 = addressbookObject.md5()
yield outAddressbook.createAddressBookObjectWithName(
addressbookObject.name(),
- (yield addressbookObject.component())) # XXX WRONG SHOULD CALL getComponent
+ component,
+ )
# Only the owner's properties are migrated, since previous releases of
# addressbook server didn't have per-user properties.
Modified: CalendarServer/trunk/txdav/common/datastore/test/test_util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/test_util.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/common/datastore/test/test_util.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -30,9 +30,11 @@
from twistedcaldav.memcacher import Memcacher
from txdav.caldav.datastore.test.common import CommonTests
+from txdav.carddav.datastore.test.common import CommonTests as ABCommonTests
from txdav.common.datastore.file import CommonDataStore
from txdav.common.datastore.test.util import theStoreBuilder, \
- populateCalendarsFrom, StubNotifierFactory
+ populateCalendarsFrom, StubNotifierFactory, resetCalendarMD5s,\
+ populateAddressBooksFrom, resetAddressBookMD5s
from txdav.common.datastore.util import UpgradeToDatabaseService
class HomeMigrationTests(TestCase):
@@ -68,13 +70,24 @@
fileStore, self.sqlStore, self.stubService
)
self.upgrader.setServiceParent(self.topService)
+
requirements = CommonTests.requirements
yield populateCalendarsFrom(requirements, fileStore)
+ md5s = CommonTests.md5s
+ yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
+ requirements = ABCommonTests.requirements
+ yield populateAddressBooksFrom(requirements, fileStore)
+ md5s = ABCommonTests.md5s
+ yield resetAddressBookMD5s(md5s, fileStore)
+ self.filesPath.child("addressbooks").child(
+ "__uids__").child("ho").child("me").child("home1").child(
+ ".some-extra-data").setContent("some extra data")
+
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
@@ -101,13 +114,14 @@
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
- for name, metadata in (
- ("1.ics", CommonTests.metadata1),
- ("2.ics", CommonTests.metadata2),
- ("3.ics", CommonTests.metadata3),
+ for name, metadata, md5 in (
+ ("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
+ ("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
+ ("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
+ self.assertEquals(object.md5(), md5)
@inlineCallbacks
@@ -182,3 +196,38 @@
self.assertEquals(allData, someAttachmentData)
+ @inlineCallbacks
+ def test_upgradeAddressBookHomes(self):
+ """
+ L{UpgradeToDatabaseService.startService} will do the upgrade, then
+ start its dependent service by adding it to its service hierarchy.
+ """
+ self.topService.startService()
+ yield self.subStarted
+ self.assertEquals(self.stubService.running, True)
+ txn = self.sqlStore.newTransaction()
+ self.addCleanup(txn.commit)
+ for uid in ABCommonTests.requirements:
+ if ABCommonTests.requirements[uid] is not None:
+ self.assertNotIdentical(
+ None, (yield txn.addressbookHomeWithUID(uid))
+ )
+ # Un-migrated data should be preserved.
+ self.assertEquals(self.filesPath.child("addressbooks-migrated").child(
+ "__uids__").child("ho").child("me").child("home1").child(
+ ".some-extra-data").getContent(),
+ "some extra data"
+ )
+
+ # Want metadata preserved
+ home = (yield txn.addressbookHomeWithUID("home1"))
+ adbk = (yield home.addressbookWithName("addressbook_1"))
+ for name, md5 in (
+ ("1.vcf", ABCommonTests.md5Values[0]),
+ ("2.vcf", ABCommonTests.md5Values[1]),
+ ("3.vcf", ABCommonTests.md5Values[2]),
+ ):
+ object = (yield adbk.addressbookObjectWithName(name))
+ self.assertEquals(object.md5(), md5)
+
+
Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py 2011-03-28 19:07:32 UTC (rev 7268)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py 2011-03-29 14:03:50 UTC (rev 7269)
@@ -26,6 +26,7 @@
from twext.python.filepath import CachingFilePath
from twext.python.vcomponent import VComponent
+from twext.web2.dav.resource import TwistedGETContentMD5
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
@@ -36,11 +37,14 @@
from txdav.common.datastore.sql import CommonDataStore, v1_schema
from txdav.base.datastore.subpostgres import PostgresService
from txdav.base.datastore.dbapiclient import DiagnosticConnectionWrapper
+from txdav.base.propertystore.base import PropertyName
from txdav.common.icommondatastore import NoSuchHomeChildError
from twext.enterprise.adbapi2 import ConnectionPool
from twisted.internet.defer import returnValue
from twistedcaldav.notify import Notifier, NodeCreationException
+from twistedcaldav.vcard import Component as ABComponent
+md5key = PropertyName.fromElement(TwistedGETContentMD5)
def allInstancesOf(cls):
for o in gc.get_referrers(cls):
@@ -216,7 +220,101 @@
)
yield populateTxn.commit()
+ at inlineCallbacks
+def resetCalendarMD5s(md5s, store):
+ """
+ Change MD5s in C{store} from C{requirements}.
+ @param requirements: a dictionary of the format described by
+ L{txdav.caldav.datastore.test.common.CommonTests.requirements}.
+
+ @param store: the L{IDataStore} to populate with calendar data.
+ """
+ populateTxn = store.newTransaction()
+ for homeUID in md5s:
+ calendars = md5s[homeUID]
+ if calendars is not None:
+ home = yield populateTxn.calendarHomeWithUID(homeUID, True)
+ for calendarName in calendars:
+ calendarObjNames = calendars[calendarName]
+ if calendarObjNames is not None:
+ # XXX should not be yielding! this SQL will be executed
+ # first!
+ calendar = yield home.calendarWithName(calendarName)
+ for objectName in calendarObjNames:
+ md5 = calendarObjNames[objectName]
+ obj = yield calendar.calendarObjectWithName(
+ objectName,
+ )
+ obj.properties()[md5key] = TwistedGETContentMD5.fromString(md5)
+ yield populateTxn.commit()
+
+
+ at inlineCallbacks
+def populateAddressBooksFrom(requirements, store):
+ """
+ Populate C{store} from C{requirements}.
+
+ @param requirements: a dictionary of the format described by
+ L{txdav.caldav.datastore.test.common.CommonTests.requirements}.
+
+ @param store: the L{IDataStore} to populate with addressbook data.
+ """
+ populateTxn = store.newTransaction()
+ for homeUID in requirements:
+ addressbooks = requirements[homeUID]
+ if addressbooks is not None:
+ home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
+ # We don't want the default addressbook
+ try:
+ yield home.removeAddressBookWithName("addressbook")
+ except NoSuchHomeChildError:
+ pass
+ for addressbookName in addressbooks:
+ addressbookObjNames = addressbooks[addressbookName]
+ if addressbookObjNames is not None:
+ # XXX should not be yielding! this SQL will be executed
+ # first!
+ yield home.createAddressBookWithName(addressbookName)
+ addressbook = yield home.addressbookWithName(addressbookName)
+ for objectName in addressbookObjNames:
+ objData = addressbookObjNames[objectName]
+ yield addressbook.createAddressBookObjectWithName(
+ objectName,
+ ABComponent.fromString(objData),
+ )
+ yield populateTxn.commit()
+
+ at inlineCallbacks
+def resetAddressBookMD5s(md5s, store):
+ """
+ Change MD5s in C{store} from C{requirements}.
+
+ @param requirements: a dictionary of the format described by
+ L{txdav.caldav.datastore.test.common.CommonTests.requirements}.
+
+ @param store: the L{IDataStore} to populate with addressbook data.
+ """
+ populateTxn = store.newTransaction()
+ for homeUID in md5s:
+ addressbooks = md5s[homeUID]
+ if addressbooks is not None:
+ home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
+ for addressbookName in addressbooks:
+ addressbookObjNames = addressbooks[addressbookName]
+ if addressbookObjNames is not None:
+ # XXX should not be yielding! this SQL will be executed
+ # first!
+ addressbook = yield home.addressbookWithName(addressbookName)
+ for objectName in addressbookObjNames:
+ md5 = addressbookObjNames[objectName]
+ obj = yield addressbook.addressbookObjectWithName(
+ objectName,
+ )
+ obj.properties()[md5key] = TwistedGETContentMD5.fromString(md5)
+ yield populateTxn.commit()
+
+
def assertProvides(testCase, interface, provider):
"""
Verify that C{provider} properly provides C{interface}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110329/910848de/attachment-0001.html>
More information about the calendarserver-changes
mailing list