[CalendarServer-changes] [6224] CalendarServer/trunk/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Tue Aug 31 16:50:38 PDT 2010
Revision: 6224
http://trac.macosforge.org/projects/calendarserver/changeset/6224
Author: glyph at apple.com
Date: 2010-08-31 16:50:38 -0700 (Tue, 31 Aug 2010)
Log Message:
-----------
address book migration functions
Modified Paths:
--------------
CalendarServer/trunk/txdav/carddav/datastore/test/common.py
CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/carddav/datastore/util.py
CalendarServer/trunk/txdav/common/datastore/file.py
CalendarServer/trunk/txdav/common/datastore/sql.py
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/common.py 2010-08-31 23:49:42 UTC (rev 6223)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/common.py 2010-08-31 23:50:38 UTC (rev 6224)
@@ -83,7 +83,7 @@
-vcard4notCardDAV_text = ( # Missing UID, N and FN
+vcard4notCardDAV_text = (# Missing UID, N and FN
"""BEGIN:VCARD
VERSION:3.0
EMAIL;type=INTERNET;type=WORK;type=pref:lthompson at example.com
@@ -347,12 +347,13 @@
self.assertNotIdentical(home.addressbookWithName(name), None)
def checkProperties():
addressbookProperties = home.addressbookWithName(name).properties()
+ addressbookType = davxml.ResourceType.addressbook #@UndefinedVariable
self.assertEquals(
addressbookProperties[
PropertyName.fromString(davxml.ResourceType.sname())
],
- davxml.ResourceType.addressbook
- ) #@UndefinedVariable
+ addressbookType
+ )
checkProperties()
self.commit()
@@ -879,7 +880,33 @@
addressbook2.addressbookObjectWithUID(obj.uid()), None)
+ def test_eachAddressbookHome(self):
+ """
+ L{IAddressbookTransaction.eachAddressbookHome} returns an iterator that
+ yields 2-tuples of (transaction, home).
+ """
+ # create some additional addressbook homes
+ additionalUIDs = set('alpha-uid home2 home3 beta-uid'.split())
+ txn = self.transactionUnderTest()
+ for name in additionalUIDs:
+ txn.addressbookHomeWithUID(name, create=True)
+ self.commit()
+ foundUIDs = set([])
+ lastTxn = None
+ for txn, home in self.storeUnderTest().eachAddressbookHome():
+ self.addCleanup(txn.commit)
+ foundUIDs.add(home.uid())
+ self.assertNotIdentical(lastTxn, txn)
+ lastTxn = txn
+ requiredUIDs = set([
+ uid for uid in self.requirements
+ if self.requirements[uid] is not None
+ ])
+ expectedUIDs = additionalUIDs.union(requiredUIDs)
+ self.assertEquals(foundUIDs, expectedUIDs)
+
+
class StubNotifierFactory(object):
""" For testing push notifications without an XMPP server """
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2010-08-31 23:49:42 UTC (rev 6223)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2010-08-31 23:50:38 UTC (rev 6224)
@@ -25,6 +25,10 @@
from txdav.common.datastore.sql import EADDRESSBOOKTYPE
from txdav.common.datastore.test.util import buildStore
+from txdav.carddav.datastore.test.test_file import setUpAddressBookStore
+from twext.web2.dav.element.rfc2518 import GETContentLanguage, ResourceType
+from txdav.base.propertystore.base import PropertyName
+from txdav.carddav.datastore.util import _migrateAddressbook, migrateHome
from twisted.trial import unittest
from twisted.internet.defer import inlineCallbacks
@@ -40,11 +44,11 @@
@inlineCallbacks
def setUp(self):
super(AddressBookSQLStorageTests, self).setUp()
- self.addressbookStore = yield buildStore(self, self.notifierFactory)
+ self._sqlStore = yield buildStore(self, self.notifierFactory)
self.populate()
def populate(self):
- populateTxn = self.addressbookStore.newTransaction()
+ populateTxn = self.storeUnderTest().newTransaction()
for homeUID in self.requirements:
addressbooks = self.requirements[homeUID]
if addressbooks is not None:
@@ -72,16 +76,119 @@
"""
Create and return a L{AddressBookStore} for testing.
"""
- return self.addressbookStore
+ return self._sqlStore
+ def assertAddressbooksSimilar(self, a, b, bAddressbookFilter=None):
+ """
+ Assert that two addressbooks have a similar structure (contain the same
+ events).
+ """
+ def namesAndComponents(x, filter=lambda x:x.component()):
+ return dict([(fromObj.name(), filter(fromObj))
+ for fromObj in x.addressbookObjects()])
+ if bAddressbookFilter is not None:
+ extra = [bAddressbookFilter]
+ else:
+ extra = []
+ self.assertEquals(namesAndComponents(a), namesAndComponents(b, *extra))
+
+
+ def assertPropertiesSimilar(self, a, b, disregard=[]):
+ """
+ Assert that two objects with C{properties} methods have similar
+ properties.
+
+ @param disregard: a list of L{PropertyName} keys to discard from both
+ input and output.
+ """
+ def sanitize(x):
+ result = dict(x.properties().items())
+ for key in disregard:
+ result.pop(key, None)
+ return result
+ self.assertEquals(sanitize(a), sanitize(b))
+
+
+ def fileTransaction(self):
+ """
+ Create a file-backed addressbook transaction, for migration testing.
+ """
+ setUpAddressBookStore(self)
+ fileStore = self.addressbookStore
+ txn = fileStore.newTransaction()
+ self.addCleanup(txn.commit)
+ return txn
+
+
+ def test_migrateAddressbookFromFile(self):
+ """
+ C{_migrateAddressbook()} can migrate a file-backed addressbook to a
+ database- backed addressbook.
+ """
+ fromAddressbook = self.fileTransaction().addressbookHomeWithUID(
+ "home1").addressbookWithName("addressbook_1")
+ toHome = self.transactionUnderTest().addressbookHomeWithUID(
+ "new-home", create=True)
+ toAddressbook = toHome.addressbookWithName("addressbook")
+ _migrateAddressbook(fromAddressbook, toAddressbook,
+ lambda x: x.component())
+ self.assertAddressbooksSimilar(fromAddressbook, toAddressbook)
+
+
+ def test_migrateHomeFromFile(self):
+ """
+ L{migrateHome} will migrate an L{IAddressbookHome} provider from one
+ backend to another; in this specific case, from the file-based backend
+ to the SQL-based backend.
+ """
+ fromHome = self.fileTransaction().addressbookHomeWithUID("home1")
+
+ builtinProperties = [PropertyName.fromElement(ResourceType)]
+
+ # Populate an arbitrary / unused dead properties so there's something
+ # to verify against.
+
+ key = PropertyName.fromElement(GETContentLanguage)
+ fromHome.properties()[key] = GETContentLanguage("C")
+ fromHome.addressbookWithName("addressbook_1").properties()[key] = (
+ GETContentLanguage("pig-latin")
+ )
+ toHome = self.transactionUnderTest().addressbookHomeWithUID(
+ "new-home", create=True
+ )
+ migrateHome(fromHome, toHome, lambda x: x.component())
+ self.assertEquals(set([c.name() for c in toHome.addressbooks()]),
+ set([k for k in self.requirements['home1'].keys()
+ if self.requirements['home1'][k] is not None]))
+ for c in fromHome.addressbooks():
+ self.assertPropertiesSimilar(
+ c, toHome.addressbookWithName(c.name()),
+ builtinProperties
+ )
+ self.assertPropertiesSimilar(fromHome, toHome, builtinProperties)
+
+
+ def test_eachAddressbookHome(self):
+ """
+ L{IAddressbookStore.eachAddressbookHome} is currently stubbed out by
+ L{txdav.common.datastore.sql.CommonDataStore}.
+ """
+ return super(AddressBookSQLStorageTests, self).test_eachAddressbookHome()
+
+
+ test_eachAddressbookHome.todo = (
+ "stubbed out, as migration only needs to go from file->sql currently")
+
+
@inlineCallbacks
def test_homeProvisioningConcurrency(self):
"""
- Test that two concurrent attempts to provision an addressbook home do not cause a race-condition
- whereby the second commit results in a second INSERT that violates a unique constraint. Also verify
- that, whilst the two provisioning attempts are happening and doing various lock operations, that we
- do not block other reads of the table.
+ Test that two concurrent attempts to provision an addressbook home do
+ not cause a race-condition whereby the second commit results in a
+ second INSERT that violates a unique constraint. Also verify that,
+ whilst the two provisioning attempts are happening and doing various
+ lock operations, that we do not block other reads of the table.
"""
addressbookStore1 = yield buildStore(self, self.notifierFactory)
@@ -91,7 +198,7 @@
txn1 = addressbookStore1.newTransaction()
txn2 = addressbookStore2.newTransaction()
txn3 = addressbookStore3.newTransaction()
-
+
# Provision one home now - we will use this to later verify we can do reads of
# existing data in the table
home_uid2 = txn3.homeWithUID(EADDRESSBOOKTYPE, "uid2", create=True)
@@ -99,28 +206,28 @@
txn3.commit()
home_uid1_1 = txn1.homeWithUID(EADDRESSBOOKTYPE, "uid1", create=True)
-
+
def _defer_home_uid1_2():
home_uid1_2 = txn2.homeWithUID(EADDRESSBOOKTYPE, "uid1", create=True)
txn2.commit()
return home_uid1_2
d1 = deferToThread(_defer_home_uid1_2)
-
+
def _pause_home_uid1_1():
time.sleep(1)
txn1.commit()
d2 = deferToThread(_pause_home_uid1_1)
-
+
# Verify that we can still get to the existing home - i.e. the lock
# on the table allows concurrent reads
txn4 = addressbookStore3.newTransaction()
home_uid2 = txn4.homeWithUID(EADDRESSBOOKTYPE, "uid2", create=True)
self.assertNotEqual(home_uid2, None)
txn4.commit()
-
+
# Now do the concurrent provision attempt
yield d2
home_uid1_2 = yield d1
-
+
self.assertNotEqual(home_uid1_1, None)
self.assertNotEqual(home_uid1_2, None)
Modified: CalendarServer/trunk/txdav/carddav/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/util.py 2010-08-31 23:49:42 UTC (rev 6223)
+++ CalendarServer/trunk/txdav/carddav/datastore/util.py 2010-08-31 23:50:38 UTC (rev 6224)
@@ -1,3 +1,4 @@
+# -*- test-case-name: txdav.carddav.datastore.test.test_sql -*-
##
# Copyright (c) 2010 Apple Inc. All rights reserved.
#
@@ -21,7 +22,7 @@
from twistedcaldav.vcard import Component as VCard
from twistedcaldav.vcard import InvalidVCardDataError
-from txdav.common.icommondatastore import InvalidObjectResourceError,\
+from txdav.common.icommondatastore import InvalidObjectResourceError, \
NoSuchObjectResourceError
def validateAddressBookComponent(addressbookObject, vcard, component, inserting):
@@ -55,4 +56,39 @@
component.validForCardDAV()
except InvalidVCardDataError, e:
raise InvalidObjectResourceError(e)
-
\ No newline at end of file
+
+
+def _migrateAddressbook(inAddressbook, outAddressbook, getComponent):
+ """
+ Copy all addressbook objects and properties in the given input addressbook
+ to the given output addressbook.
+
+ @param inAddressbook: the L{IAddressbook} to retrieve addressbook objects
+ from.
+ @param outAddressbook: the L{IAddressbook} to store addressbook objects to.
+ @param getComponent: a 1-argument callable; see L{migrateHome}.
+ """
+ outAddressbook.properties().update(inAddressbook.properties())
+ for addressbookObject in inAddressbook.addressbookObjects():
+ outAddressbook.createAddressBookObjectWithName(
+ addressbookObject.name(),
+ addressbookObject.component()) # XXX WRONG SHOULD CALL getComponent
+
+ # Only the owner's properties are migrated, since previous releases of
+ # addressbook server didn't have per-user properties.
+ outAddressbook.addressbookObjectWithName(
+ addressbookObject.name()).properties().update(
+ addressbookObject.properties())
+ # XXX attachments
+
+
+def migrateHome(inHome, outHome, getComponent=lambda x:x.component()):
+ outHome.removeAddressBookWithName("addressbook")
+ outHome.properties().update(inHome.properties())
+ for addressbook in inHome.addressbooks():
+ name = addressbook.name()
+ outHome.createAddressBookWithName(name)
+ outAddressbook = outHome.addressbookWithName(name)
+ _migrateAddressbook(addressbook, outAddressbook, getComponent)
+
+
Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py 2010-08-31 23:49:42 UTC (rev 6223)
+++ CalendarServer/trunk/txdav/common/datastore/file.py 2010-08-31 23:50:38 UTC (rev 6224)
@@ -124,7 +124,11 @@
return self._homesOfType(ECALENDARTYPE)
+ def eachAddressbookHome(self):
+ return self._homesOfType(EADDRESSBOOKTYPE)
+
+
class CommonStoreTransaction(DataStoreTransaction):
"""
In-memory implementation of
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2010-08-31 23:49:42 UTC (rev 6223)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2010-08-31 23:50:38 UTC (rev 6224)
@@ -88,6 +88,13 @@
return []
+ def eachAddressbookHome(self):
+ """
+ @see L{IAddressbookStore.eachAddressbookHome}
+ """
+ return []
+
+
def newTransaction(self, label="unlabeled"):
return CommonStoreTransaction(
self,
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100831/cb4452fc/attachment-0001.html>
More information about the calendarserver-changes
mailing list