[CalendarServer-changes] [10069] CalendarServer/branches/users/glyph/one-home-list-api
source_changes at macosforge.org
source_changes at macosforge.org
Fri Nov 16 14:49:13 PST 2012
Revision: 10069
http://trac.calendarserver.org//changeset/10069
Author: glyph at apple.com
Date: 2012-11-16 14:49:13 -0800 (Fri, 16 Nov 2012)
Log Message:
-----------
Eliminate duplicate API #1; eachCalendarHome/eachAddressbookHome.
Modified Paths:
--------------
CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/common.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/test_sql.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/icalendarstore.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/common.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/test_sql.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/file.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/sql.py
CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/upgrade/migrate.py
Property Changed:
----------------
CalendarServer/branches/users/glyph/one-home-list-api/
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/common.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/common.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -2272,37 +2272,6 @@
@inlineCallbacks
- def test_eachCalendarHome(self):
- """
- L{ICalendarTransaction.eachCalendarHome} returns an iterator that
- yields 2-tuples of (transaction, home).
- """
- # create some additional calendar homes
- additionalUIDs = set('alpha-uid home2 home3 beta-uid'.split())
- txn = self.transactionUnderTest()
- for name in additionalUIDs:
- # maybe it's not actually necessary to yield (i.e. wait) for each
- # one? commit() should wait for all of them.
- yield txn.calendarHomeWithUID(name, create=True)
- yield self.commit()
- foundUIDs = set([])
- lastTxn = None
- for txn, home in (yield self.storeUnderTest().eachCalendarHome()):
- self.addCleanup(txn.commit)
- foundUIDs.add(home.uid())
- self.assertNotIdentical(lastTxn, txn)
- lastTxn = txn
- requiredUIDs = set([
- uid for uid in self.requirements
- if self.requirements[uid] is not None
- ])
- additionalUIDs.add("home_bad")
- additionalUIDs.add("home_attachments")
- expectedUIDs = additionalUIDs.union(requiredUIDs)
- self.assertEquals(foundUIDs, expectedUIDs)
-
-
- @inlineCallbacks
def test_withEachCalendarHomeDo(self):
"""
L{ICalendarStore.withEachCalendarHomeDo} executes its C{action}
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/test_sql.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/datastore/test/test_sql.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -516,21 +516,8 @@
Where=ch.OWNER_UID == "home_version",
).on(txn)[0][0]
self.assertEqual(int(homeVersion, version))
-
-
- def test_eachCalendarHome(self):
- """
- L{ICalendarStore.eachCalendarHome} is currently stubbed out by
- L{txdav.common.datastore.sql.CommonDataStore}.
- """
- return super(CalendarSQLStorageTests, self).test_eachCalendarHome()
-
- test_eachCalendarHome.todo = (
- "stubbed out, as migration only needs to go from file->sql currently")
-
-
@inlineCallbacks
def test_homeProvisioningConcurrency(self):
"""
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/icalendarstore.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/icalendarstore.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/caldav/icalendarstore.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -114,17 +114,6 @@
API root for calendar data storage.
"""
- def eachCalendarHome():
- """
- Enumerate all calendar homes in this store, with each one in an
- accompanying transaction.
-
- @return: an iterator of 2-tuples of C{(transaction, calendar home)}
- where C{transaction} is an L{ITransaction} provider and C{calendar
- home} is an L{ICalendarHome} provider.
- """
-
-
def withEachCalendarHomeDo(action, batchSize=None):
"""
Execute a given action with each calendar home present in this store,
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/common.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/common.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -967,29 +967,3 @@
(yield addressbook2.addressbookObjectWithUID(obj.uid())), None)
- @inlineCallbacks
- def test_eachAddressbookHome(self):
- """
- L{IAddressbookTransaction.eachAddressbookHome} returns an iterator that
- yields 2-tuples of (transaction, home).
- """
- # create some additional addressbook homes
- additionalUIDs = set('alpha-uid home2 home3 beta-uid'.split())
- txn = self.transactionUnderTest()
- for name in additionalUIDs:
- yield txn.addressbookHomeWithUID(name, create=True)
- yield self.commit()
- foundUIDs = set([])
- lastTxn = None
- for txn, home in (yield self.storeUnderTest().eachAddressbookHome()):
- self.addCleanup(txn.commit)
- foundUIDs.add(home.uid())
- self.assertNotIdentical(lastTxn, txn)
- lastTxn = txn
- requiredUIDs = set([
- uid for uid in self.requirements
- if self.requirements[uid] is not None
- ])
- additionalUIDs.add("home_bad")
- expectedUIDs = additionalUIDs.union(requiredUIDs)
- self.assertEquals(foundUIDs, expectedUIDs)
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/test_sql.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/carddav/datastore/test/test_sql.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -206,11 +206,11 @@
The DATAVERSION column for new calendar homes must match the
ADDRESSBOOK-DATAVERSION value.
"""
-
+
home = yield self.transactionUnderTest().addressbookHomeWithUID("home_version")
self.assertTrue(home is not None)
yield self.transactionUnderTest().commit
-
+
txn = yield self.transactionUnderTest()
version = yield txn.calendarserverValue("ADDRESSBOOK-DATAVERSION")[0][0]
ch = schema.ADDRESSBOOK_HOME
@@ -220,21 +220,8 @@
Where=ch.OWNER_UID == "home_version",
).on(txn)[0][0]
self.assertEqual(int(homeVersion, version))
-
-
- def test_eachAddressbookHome(self):
- """
- L{IAddressbookStore.eachAddressbookHome} is currently stubbed out by
- L{txdav.common.datastore.sql.CommonDataStore}.
- """
- return super(AddressBookSQLStorageTests, self).test_eachAddressbookHome()
-
- test_eachAddressbookHome.todo = (
- "stubbed out, as migration only needs to go from file->sql currently")
-
-
@inlineCallbacks
def test_putConcurrency(self):
"""
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/file.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/file.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -29,6 +29,7 @@
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from twisted.python.util import FancyEqMixin
+from twisted.python import log
from twisted.python import hashlib
from twistedcaldav import customxml
@@ -133,15 +134,36 @@
@inlineCallbacks
+ def _withEachHomeDo(self, enumerator, action, batchSize):
+ """
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo} and
+ L{IAddressBookStore.withEachAddressbookHomeDo}.
+ """
+ for txn, home in enumerator():
+ try:
+ yield action(txn, home)
+ except:
+ log.err()
+ yield txn.abort()
+ else:
+ yield txn.commit()
+
+
def withEachCalendarHomeDo(self, action, batchSize=None):
"""
Implementation of L{ICalendarStore.withEachCalendarHomeDo}.
"""
- for txn, home in self.eachCalendarHome():
- yield action(txn, home)
- yield txn.commit()
+ return self._withEachHomeDo(self._eachCalendarHome, action, batchSize)
+ def withEachAddressbookHomeDo(self, action, batchSize=None):
+ """
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo}.
+ """
+ return self._withEachHomeDo(self._eachAddressbookHome, action,
+ batchSize)
+
+
def setMigrating(self, state):
"""
Set the "migrating" state
@@ -159,9 +181,9 @@
def _homesOfType(self, storeType):
"""
- Common implementation of L{ICalendarStore.eachCalendarHome} and
- L{IAddressBookStore.eachAddressbookHome}; see those for a description
- of the return type.
+ Common implementation of L{_eachCalendarHome} and
+ L{_eachAddressbookHome}; see those for a description of the return
+ type.
@param storeType: one of L{EADDRESSBOOKTYPE} or L{ECALENDARTYPE}.
"""
@@ -182,11 +204,11 @@
yield (txn, home)
- def eachCalendarHome(self):
+ def _eachCalendarHome(self):
return self._homesOfType(ECALENDARTYPE)
- def eachAddressbookHome(self):
+ def _eachAddressbookHome(self):
return self._homesOfType(EADDRESSBOOKTYPE)
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/sql.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/sql.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -180,20 +180,6 @@
self.queryCacher = None
- def eachCalendarHome(self):
- """
- @see: L{ICalendarStore.eachCalendarHome}
- """
- return []
-
-
- def eachAddressbookHome(self):
- """
- @see: L{IAddressbookStore.eachAddressbookHome}
- """
- return []
-
-
@inlineCallbacks
def _withEachHomeDo(self, homeTable, homeFromTxn, action, batchSize):
"""
Modified: CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/upgrade/migrate.py
===================================================================
--- CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/upgrade/migrate.py 2012-11-16 22:49:10 UTC (rev 10068)
+++ CalendarServer/branches/users/glyph/one-home-list-api/txdav/common/datastore/upgrade/migrate.py 2012-11-16 22:49:13 UTC (rev 10069)
@@ -223,6 +223,9 @@
lambda fileHome:
self.upgrader.migrateOneHome(fileTxn, homeType, fileHome)
)
+ .addCallbacks(lambda ignored: fileTxn.commit(),
+ lambda err: fileTxn.abort()
+ .addCallback(lambda ign: err))
.addCallback(lambda ignored: {})
)
@@ -343,7 +346,6 @@
"%s home %r already existed not migrating" % (
homeType, uid))
yield sqlTxn.abort()
- yield fileTxn.commit()
returnValue(None)
try:
if sqlHome is None:
@@ -351,11 +353,9 @@
yield migrateFunc(fileHome, sqlHome, merge=self.merge)
except:
f = Failure()
- yield fileTxn.abort()
yield sqlTxn.abort()
f.raiseException()
else:
- yield fileTxn.commit()
yield sqlTxn.commit()
# Remove file home after migration. FIXME: instead, this should be a
# public remove...HomeWithUID() API for de-provisioning. (If we had
@@ -402,27 +402,20 @@
)
self.log_warn("Upgrade helpers ready.")
parallelizer = Parallelizer(drivers)
+ else:
+ parallelizer = None
self.log_warn("Beginning filesystem -> database upgrade.")
+
for homeType, eachFunc in [
- ("calendar", self.fileStore.eachCalendarHome),
- ("addressbook", self.fileStore.eachAddressbookHome),
+ ("calendar", self.fileStore.withEachCalendarHomeDo),
+ ("addressbook", self.fileStore.withEachAddressbookHomeDo),
]:
- for fileTxn, fileHome in eachFunc():
- uid = fileHome.uid()
- self.log_warn("Migrating %s UID %r" % (homeType, uid))
- if parallel:
- # No-op transaction here: make sure everything's unlocked
- # before asking the subprocess to handle it.
- yield fileTxn.commit()
- @inlineCallbacks
- def doOneUpgrade(driver, fileUID=uid, homeType=homeType):
- yield driver.oneUpgrade(fileUID, homeType)
- self.log_warn("Completed migration of %s uid %r" %
- (homeType, fileUID))
- yield parallelizer.do(doOneUpgrade)
- else:
- yield self.migrateOneHome(fileTxn, homeType, fileHome)
+ yield eachFunc(
+ lambda txn, home: self._upgradeAction(
+ txn, home, homeType, parallel, parallelizer
+ )
+ )
if parallel:
yield parallelizer.done()
@@ -458,6 +451,23 @@
reactor.callLater(0, wrapped.setServiceParent, self.parent)
+ @inlineCallbacks
+ def _upgradeAction(self, fileTxn, fileHome, homeType, parallel,
+ parallelizer):
+ uid = fileHome.uid()
+ self.log_warn("Migrating %s UID %r" % (homeType, uid))
+ if parallel:
+ @inlineCallbacks
+ def doOneUpgrade(driver, fileUID=uid, homeType=homeType):
+ yield driver.oneUpgrade(fileUID, homeType)
+ self.log_warn("Completed migration of %s uid %r" %
+ (homeType, fileUID))
+ yield parallelizer.do(doOneUpgrade)
+ else:
+ yield self.migrateOneHome(fileTxn, homeType, fileHome)
+
+
+
def startService(self):
"""
Start the service.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121116/9f02966d/attachment-0001.html>
More information about the calendarserver-changes
mailing list