[CalendarServer-changes] [8385] CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/ datastore
source_changes at macosforge.org
source_changes at macosforge.org
Sat Dec 3 00:38:07 PST 2011
Revision: 8385
http://trac.macosforge.org/projects/calendarserver/changeset/8385
Author: glyph at apple.com
Date: 2011-12-03 00:38:07 -0800 (Sat, 03 Dec 2011)
Log Message:
-----------
disable memcache in the right place, have an explicit signature for spawnWithStore
Modified Paths:
--------------
CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/test/util.py
CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/migrate.py
CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/test/test_migrate.py
Modified: CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/test/util.py 2011-12-03 08:37:58 UTC (rev 8384)
+++ CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/test/util.py 2011-12-03 08:38:07 UTC (rev 8385)
@@ -43,6 +43,7 @@
from txdav.base.propertystore.base import PropertyName
from txdav.common.icommondatastore import NoSuchHomeChildError
from twext.enterprise.adbapi2 import ConnectionPool
+from twisted.trial.unittest import TestCase
from twisted.internet.defer import returnValue
from twistedcaldav.notify import Notifier, NodeCreationException
from twext.enterprise.ienterprise import AlreadyFinishedError
@@ -108,6 +109,7 @@
Create a store suitable for use in a child process, that is hooked up
to the store that a parent test process is managing.
"""
+ disableMemcacheForTest(TestCase())
staticQuota = 3000
attachmentRoot = (CachingFilePath(cls.SHARED_DB_PATH)
.child("attachments"))
Modified: CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/migrate.py
===================================================================
--- CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/migrate.py 2011-12-03 08:37:58 UTC (rev 8384)
+++ CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/migrate.py 2011-12-03 08:38:07 UTC (rev 8385)
@@ -30,8 +30,9 @@
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.defer import maybeDeferred, DeferredList
from twisted.python.runtime import platform
-from twisted.python import log
+
from twext.python.filepath import CachingFilePath
+from twext.internet.spawnsvc import SpawnerService
from twisted.protocols.amp import AMP, Command, String
@@ -39,7 +40,6 @@
from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
-
from txdav.base.propertystore.appledouble_xattr import (PropertyStore
as AppleDoubleStore)
@@ -52,7 +52,43 @@
}
+def swapAMP(oldAMP, newAMP):
+ """
+ Swap delivery of messages from an old L{AMP} instance to a new one.
+ This is useful for implementors of L{StoreSpawnerService} since they will
+ typically want to create one protocol for initializing the store, and
+ another for processing application commands.
+
+ @param oldAMP: An AMP instance currently hooked up to a transport, whose
+ job is done and wants to stop receiving messages.
+
+ @param newAMP: An AMP instance who wants to take over and start receiving
+ messages previously destined for oldAMP.
+
+ @return: C{newAMP}
+ """
+ oldAMP.boxReceiver = newAMP
+ newAMP.startReceivingBoxes(oldAMP)
+ return newAMP
+
+
+
+class StoreSpawnerService(SpawnerService):
+ """
+ Abstract subclass of L{SpawnerService} that describes how to spawn a subclass.
+ """
+
+ def spawnWithStore(self, here, there):
+ """
+ Like L{SpawnerService.spawn}, but instead of instantiating C{there}
+ with 0 arguments, it instantiates it with an L{ICalendarStore} /
+ L{IAddressbookStore}.
+ """
+ raise NotImplementedError("subclasses must implement the specifics")
+
+
+
class Configure(Command):
"""
Configure the upgrade helper process.
@@ -91,40 +127,23 @@
def configure(self, filename):
- return self.callRemote(Configure,
- filename=filename
- # config._provider.getConfigFileName() or ""
- )
+ """
+ Configure the subprocess to examine the file store at the given path
+ name.
+ """
+ return self.callRemote(Configure, filename=filename)
def oneUpgrade(self, uid, homeType):
- return self.callRemote(OneUpgrade, uid=uid, homeType=homeType)
-
-
- @LogIt.responder
- def logIt(self, message):
"""
- Log a message from the subprocess.
+ Upgrade one calendar or addressbook home, with the given uid of the
+ given type, and return a L{Deferred} which will fire when the upgrade
+ is complete.
"""
- log.msg("Subprocess: " + message)
- return {}
+ return self.callRemote(OneUpgrade, uid=uid, homeType=homeType)
-def logFailures(thunk):
- """
- Decorator which logs
- """
- def real(self, *a, **kw):
- d = maybeDeferred(thunk, self, *a, **kw)
- def logit(failure):
- self.callRemote(LogIt, message=failure.getTraceback())
- return failure
- d.addErrback(logit)
- return d
- return real
-
-
class UpgradeHelperProcess(AMP):
"""
Helper protocol which runs in a subprocess to upgrade.
@@ -140,7 +159,6 @@
@Configure.responder
- @logFailures
def configure(self, filename):
subsvc = None
self.upgrader = UpgradeToDatabaseService.wrapService(
@@ -165,7 +183,6 @@
@OneUpgrade.responder
- @logFailures
def oneUpgrade(self, uid, homeType):
"""
Upgrade one calendar home.
@@ -213,7 +230,8 @@
@param parallel: The number of parallel subprocesses that should manage
the upgrade.
- @param spawner: the L{SpawnerService} subclass.
+ @param spawner: a concrete L{StoreSpawnerService} subclass that will be
+ used to spawn helper processes.
@return: a service
@rtype: L{IService}
@@ -319,8 +337,8 @@
spawner.startService()
drivers = []
for value in xrange(parallel):
- driver = yield spawner.spawn(UpgradeDriver(self),
- UpgradeHelperProcess)
+ driver = yield spawner.spawnWithStore(UpgradeDriver(self),
+ UpgradeHelperProcess)
drivers.append(driver)
# Wait for all subprocesses to be fully configured before
Modified: CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/test/test_migrate.py
===================================================================
--- CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/test/test_migrate.py 2011-12-03 08:37:58 UTC (rev 8384)
+++ CalendarServer/branches/users/glyph/parallel-upgrade/txdav/common/datastore/upgrade/test/test_migrate.py 2011-12-03 08:38:07 UTC (rev 8385)
@@ -32,83 +32,51 @@
from txdav.common.datastore.test.util import theStoreBuilder, \
populateCalendarsFrom, StubNotifierFactory, resetCalendarMD5s,\
populateAddressBooksFrom, resetAddressBookMD5s
-from twext.internet.spawnsvc import SpawnerService
+
from txdav.common.datastore.test.util import SQLStoreBuilder
-from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService, LogIt, logFailures
+from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService, \
+ StoreSpawnerService, swapAMP
-class StoreCreateMaster(AMP):
- """
- Helper protocol.
- """
- @LogIt.responder
- def logIt(self, message):
- """
- Log a message from the subprocess.
- """
- print 'LOG:', message
- return {}
-
-
-
class CreateStore(Command):
"""
Create a store in a subprocess.
"""
-
arguments = [('delegateTo', String())]
-
-class StoreCreatorSlave(AMP):
+class StoreCreator(AMP):
"""
Helper protocol.
"""
@CreateStore.responder
- @logFailures
def createStore(self, delegateTo):
"""
Create a store and pass it to the named delegate class.
"""
- from twistedcaldav.memcacher import Memcacher
- from twistedcaldav.config import config
-
- # Normally these would be patched out for an individual test, but in
- # this case, the process lifetime will be shorter than the test.
- config.Memcached.Pools.Default.ClientEnabled = False
- config.Memcached.Pools.Default.ServerEnabled = False
- Memcacher.allowTestCache = True
-
- cls = namedAny(delegateTo)
- store = SQLStoreBuilder.childStore()
- newself = cls(store)
- self.boxReceiver = newself
- newself.startReceivingBoxes(self)
+ swapAMP(self, namedAny(delegateTo)(SQLStoreBuilder.childStore()))
return {}
-class StubSpawner(SpawnerService):
+class StubSpawner(StoreSpawnerService):
"""
Stub spawner service which populates the store forcibly.
"""
@inlineCallbacks
- def spawn(self, here, there):
+ def spawnWithStore(self, here, there):
"""
- 'here' and 'there' are the helper protocols; 'there' will expect to
- have 'storeCreated' called on it.
+ 'here' and 'there' are the helper protocols; in a slight modification
+ of the signature, 'there' will expect to be created with an instance of
+ a store.
"""
- master = yield super(StubSpawner, self).spawn(
- StoreCreateMaster(),
- StoreCreatorSlave)
+ master = yield self.spawn(AMP(), StoreCreator)
yield master.callRemote(CreateStore, delegateTo=qual(there))
- master.boxReceiver = here
- here.startReceivingBoxes(master)
- returnValue(here)
+ returnValue(swapAMP(master, here))
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20111203/3e618552/attachment-0001.html>
More information about the calendarserver-changes
mailing list