[CalendarServer-changes] [8401] CalendarServer/trunk

[source_changes at macosforge.org]

Revision: 8401
          http://trac.macosforge.org/projects/calendarserver/changeset/8401
Author:   glyph at apple.com
Date:     2011-12-03 00:43:59 -0800 (Sat, 03 Dec 2011)
Log Message:
-----------
Land parallel-upgrade branch.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/caldav.py
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/txdav/common/datastore/test/util.py
    CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py
    CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py

Added Paths:
-----------
    CalendarServer/trunk/bin/calendarserver_upgrade
    CalendarServer/trunk/calendarserver/tap/cfgchild.py
    CalendarServer/trunk/calendarserver/tools/upgrade.py
    CalendarServer/trunk/twext/internet/spawnsvc.py

Property Changed:
----------------
    CalendarServer/trunk/
    CalendarServer/trunk/support/build.sh
    CalendarServer/trunk/txdav/caldav/datastore/index_file.py
    CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py
    CalendarServer/trunk/txdav/carddav/datastore/index_file.py
    CalendarServer/trunk/txdav/carddav/datastore/test/test_index_file.py


Property changes on: CalendarServer/trunk
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
   + /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593

Copied: CalendarServer/trunk/bin/calendarserver_upgrade (from rev 8400, CalendarServer/branches/users/glyph/parallel-upgrade/bin/calendarserver_upgrade)
===================================================================
--- CalendarServer/trunk/bin/calendarserver_upgrade	                        (rev 0)
+++ CalendarServer/trunk/bin/calendarserver_upgrade	2011-12-03 08:43:59 UTC (rev 8401)
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+##
+# Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import sys
+
+#PYTHONPATH
+
+if __name__ == "__main__":
+    if "PYTHONPATH" in globals():
+        sys.path.insert(0, PYTHONPATH)
+    else:
+        try:
+            import _calendarserver_preamble
+        except ImportError:
+            sys.exc_clear()
+
+    from calendarserver.tools.upgrade import main
+    main()

Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py	2011-12-03 08:40:33 UTC (rev 8400)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -34,15 +34,18 @@
 
 from zope.interface import implements
 
+from twisted.plugin import IPlugin
+
 from twisted.python.log import FileLogObserver, ILogObserver
 from twisted.python.logfile import LogFile
 from twisted.python.usage import Options, UsageError
-from twisted.plugin import IPlugin
+
 from twisted.internet.defer import gatherResults, Deferred
-from twisted.internet import reactor as _reactor
+
 from twisted.internet.process import ProcessExitedAlready
 from twisted.internet.protocol import Protocol, Factory
 from twisted.internet.protocol import ProcessProtocol
+
 from twisted.application.internet import TCPServer, UNIXServer
 from twisted.application.service import MultiService, IServiceMaker
 from twisted.application.service import Service
@@ -57,9 +60,10 @@
 from twext.web2.channel.http import LimitingHTTPFactory, SSLRedirectRequest
 from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
 
+from txdav.common.datastore.upgrade.sql.upgrade import (
+    UpgradeDatabaseSchemaService, UpgradeDatabaseDataService,
+)
 from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService
-from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService,\
-    UpgradeDatabaseDataService
 
 from twistedcaldav.config import ConfigurationError
 from twistedcaldav.config import config
@@ -74,7 +78,6 @@
 from twext.enterprise.ienterprise import POSTGRES_DIALECT
 from twext.enterprise.ienterprise import ORACLE_DIALECT
 from twext.enterprise.adbapi2 import ConnectionPool
-from twext.enterprise.adbapi2 import ConnectionPoolConnection
 
 try:
     from twistedcaldav.authkerb import NegotiateCredentialFactory
@@ -82,14 +85,17 @@
 except ImportError:
     NegotiateCredentialFactory = None
 
+from calendarserver.tap.util import ConnectionDispenser
+
 from calendarserver.accesslog import AMPCommonAccessLoggingObserver
 from calendarserver.accesslog import AMPLoggingFactory
 from calendarserver.accesslog import RotatingFileAccessLoggingObserver
 from calendarserver.tap.util import getRootResource, computeProcessCount
-from calendarserver.tap.util import ConnectionWithPeer
+
 from calendarserver.tap.util import storeFromConfig
 from calendarserver.tap.util import pgConnectorFromConfig
 from calendarserver.tap.util import oracleConnectorFromConfig
+from calendarserver.tap.cfgchild import ConfiguredChildSpawner
 from calendarserver.tools.util import checkDirectory
 
 try:
@@ -208,6 +214,7 @@
 
         self.overrides = {}
 
+
     @staticmethod
     def coerceOption(configDict, key, value):
         """
@@ -233,6 +240,7 @@
 
         return value
 
+
     @classmethod
     def setOverride(cls, configDict, path, value, overrideDict):
         """
@@ -893,6 +901,47 @@
 
         @rtype: L{IService}
         """
+        def createSubServiceFactory(dialect=POSTGRES_DIALECT,
+                                    paramstyle='pyformat'):
+            def subServiceFactory(connectionFactory):
+                ms = MultiService()
+                cp = ConnectionPool(connectionFactory, dialect=dialect,
+                                    paramstyle=paramstyle,
+                                    maxConnections=config.MaxDBConnectionsPerPool)
+                cp.setServiceParent(ms)
+                store = storeFromConfig(config, cp.connection)
+                mainService = createMainService(cp, store)
+                if config.ParallelUpgrades:
+                    parallel = config.MultiProcess.ProcessCount
+                else:
+                    parallel = 0
+                upgradeSvc = UpgradeFileSystemFormatService(
+                    config,
+                    UpgradeDatabaseSchemaService.wrapService(
+                        UpgradeDatabaseDataService.wrapService(
+                            UpgradeToDatabaseService.wrapService(
+                                CachingFilePath(config.DocumentRoot),
+                                PostDBImportService(config, store, mainService),
+                                store, uid=overrideUID, gid=overrideGID,
+                                spawner=ConfiguredChildSpawner(
+                                    self, ConnectionDispenser(cp), config
+                                ),
+                                parallel=parallel
+                            ),
+                            store, uid=overrideUID, gid=overrideGID,
+                        ),
+                        store, uid=overrideUID, gid=overrideGID,
+                    )
+                )
+                upgradeSvc.setServiceParent(ms)
+                return ms
+            return subServiceFactory
+
+        # FIXME: this is replicating the logic of getDBPool(), except for the
+        # part where the pgServiceFromConfig service is actually started here,
+        # and discarded in that function.  This should be refactored to simply
+        # use getDBPool.
+
         if config.UseDatabase:
 
             if os.getuid() == 0: # Only override if root
@@ -907,22 +956,19 @@
                 # to it.
                 pgserv = pgServiceFromConfig(
                     config,
-                    self.subServiceFactoryFactory(createMainService,
-                        uid=overrideUID, gid=overrideGID),
+                    createSubServiceFactory(),
                     uid=overrideUID, gid=overrideGID
                 )
                 return pgserv
             elif config.DBType == 'postgres':
                 # Connect to a postgres database that is already running.
-                return self.subServiceFactoryFactory(createMainService,
-                    uid=overrideUID, gid=overrideGID)(
-                            pgConnectorFromConfig(config))
+                return createSubServiceFactory()(pgConnectorFromConfig(config))
             elif config.DBType == 'oracle':
                 # Connect to an Oracle database that is already running.
-                return self.subServiceFactoryFactory(createMainService,
-                    uid=overrideUID, gid=overrideGID,
-                    dialect=ORACLE_DIALECT, paramstyle='numeric')(
-                            oracleConnectorFromConfig(config))
+                return createSubServiceFactory(dialect=ORACLE_DIALECT,
+                                               paramstyle='numeric')(
+                    oracleConnectorFromConfig(config)
+                )
             else:
                 raise UsageError("Unknown database type %r" (config.DBType,))
         else:
@@ -930,35 +976,6 @@
             return createMainService(None, store)
 
 
-    def subServiceFactoryFactory(self, createMainService, uid=None, gid=None,
-                                 dialect=POSTGRES_DIALECT,
-                                 paramstyle='pyformat'):
-        def subServiceFactory(connectionFactory):
-            ms = MultiService()
-            cp = ConnectionPool(connectionFactory, dialect=dialect,
-                                paramstyle=paramstyle,
-                                maxConnections=config.MaxDBConnectionsPerPool)
-            cp.setServiceParent(ms)
-            store = storeFromConfig(config, cp.connection)
-            mainService = createMainService(cp, store)
-            upgradeSvc = UpgradeFileSystemFormatService(config,
-                UpgradeDatabaseSchemaService.wrapService(
-                    UpgradeDatabaseDataService.wrapService(
-                        UpgradeToDatabaseService.wrapService(
-                            CachingFilePath(config.DocumentRoot),
-                            PostDBImportService(config, store, mainService),
-                            store, uid=uid, gid=gid
-                        ),
-                        store, uid=uid, gid=gid
-                    ),
-                    store, uid=uid, gid=gid
-                )
-            )
-            upgradeSvc.setServiceParent(ms)
-            return ms
-        return subServiceFactory
-
-
     def makeService_Combined(self, options):
         """
         Create a master service to coordinate a multi-process configuration,
@@ -1032,6 +1049,7 @@
         # Calculate the number of processes to spawn
         #
         if config.MultiProcess.ProcessCount == 0:
+            # TODO: this should probably be happening in a configuration hook.
             processCount = computeProcessCount(
                 config.MultiProcess.MinProcessCount,
                 config.MultiProcess.PerCPU,
@@ -1150,28 +1168,6 @@
 
 
 
-class ConnectionDispenser(object):
-
-    def __init__(self, connectionPool):
-        self.pool = connectionPool
-
-
-    def dispense(self):
-        """
-        Dispense a file descriptor, already connected to a server, for a
-        client.
-        """
-        # FIXME: these sockets need to be re-dispensed when the process is
-        # respawned, and they currently won't be.
-        c, s = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
-        protocol = ConnectionPoolConnection(self.pool)
-        transport = ConnectionWithPeer(s, protocol)
-        protocol.makeConnection(transport)
-        transport.startReading()
-        return c
-
-
-
 class TwistdSlaveProcess(object):
     """
     A L{TwistdSlaveProcess} is information about how to start a slave process
@@ -1315,6 +1311,7 @@
         return args
 
 
+
 class ControlPortTCPServer(TCPServer):
     """ This TCPServer retrieves the port number that was actually assigned
         when the service was started, and stores that into config.ControlPort
@@ -1354,8 +1351,10 @@
     minRestartDelay = 1
     maxRestartDelay = 3600
 
-    def __init__(self, reactor=_reactor):
+    def __init__(self, reactor=None):
         super(DelayedStartupProcessMonitor, self).__init__()
+        if reactor is None:
+            from twisted.internet import reactor
         self._reactor = reactor
         self.processes = {}
         self.protocols = {}

Copied: CalendarServer/trunk/calendarserver/tap/cfgchild.py (from rev 8400, CalendarServer/branches/users/glyph/parallel-upgrade/calendarserver/tap/cfgchild.py)
===================================================================
--- CalendarServer/trunk/calendarserver/tap/cfgchild.py	                        (rev 0)
+++ CalendarServer/trunk/calendarserver/tap/cfgchild.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -0,0 +1,188 @@
+##
+# Copyright (c) 2005-2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Tools for spawning general-purpose child processes that have a store derived
+from an existing configuration.
+"""
+
+__all__ = [
+    # Only the spawner service is really interesting; the other parts are
+    # internal implementation details which shouldn't be needed outside this
+    # file.
+    'ConfiguredChildSpawner',
+]
+
+from calendarserver.tools.util import setupMemcached
+from twisted.python.reflect import namedAny, qual
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.protocols.amp import AMP, Command, String, Integer#, ListOf
+from txdav.common.datastore.upgrade.migrate import (
+    StoreSpawnerService, swapAMP
+)
+
+from calendarserver.tap.util import getDBPool, storeFromConfig
+
+class ConfigureChild(Command):
+    """
+    Configure a child process, most especially with all the information that it
+    needs in order to construct a data store.
+    """
+
+    arguments = [
+        # The name of the class to delegate to once configuration is complete.
+        ("delegateTo", String()),
+        ("pidFile", String()),
+        ("logID", String()),
+        ("configFile", String()),
+
+        # computed value determined only in master, so needs to be propagated
+        # to be correct.
+        ("processCount", Integer()),
+
+        ## only needed for request processing, and we're not using this
+        ## facility for that work (yet)
+        # ("inheritFDs", ListOf(Integer())),
+        # ("inheritSSLFDs", ListOf(Integer())),
+        # ("metaFD", String(optional=True)),
+
+        ## shared connection pool!
+        ("connectionPoolFD", Integer(optional=True)),
+    ]
+
+
+
+class ChildConfigurator(AMP):
+    """
+    Protocol which can configure a child process.
+    """
+
+    def __init__(self, config=None):
+        """
+        Optionally accept a configuration for testing, but normally created in
+        the subprocess configuration-free.
+        """
+        super(ChildConfigurator, self).__init__()
+        if config is None:
+            from twistedcaldav.config import config
+        self.config = config
+
+
+    @ConfigureChild.responder
+    def conf(self, delegateTo, pidFile, logID, configFile, processCount,
+             connectionPoolFD=None):
+        """
+        Load the current config file into this child process, create a store
+        based on it, and delegate to the upgrade logic.
+        """
+        # Load the configuration file.
+        self.config.load(configFile)
+
+        # Adjust the child's configuration to add all the relevant options for
+        # the store that won't be mentioned in the config file.
+        changedConfig = dict(
+            EnableCalDAV  = True,
+            EnableCardDAV = True,
+            LogID         = logID,
+            PIDFile       = pidFile,
+            MultiProcess  = dict(
+                ProcessCount = processCount
+            )
+        )
+        setupMemcached(self.config)
+        if connectionPoolFD is not None:
+            changedConfig.update(DBAMPFD=connectionPoolFD)
+        self.config.updateDefaults(changedConfig)
+
+        # Construct and start database pool and store.
+        pool, txnf = getDBPool(self.config)
+        if pool is not None:
+            from twisted.internet import reactor
+            pool.startService()
+            reactor.addSystemEventTrigger(
+                "before", "shutdown", pool.stopService
+            )
+        dbstore = storeFromConfig(self.config, txnf)
+
+        # Finally, construct the class we're supposed to delegate to.
+        delegateClass = namedAny(delegateTo)
+        swapAMP(self, delegateClass(dbstore))
+        return {}
+
+
+
+class ConfiguredChildSpawner(StoreSpawnerService):
+    """
+    L{StoreSpawnerService} that will load a full configuration into each child.
+    """
+
+    def __init__(self, maker, dispenser, config):
+        """
+        Create a L{ConfiguredChildSpawner}.
+
+        @param maker: a L{CalDAVServiceMaker} instance that supplies the
+            configuration.
+
+        @param dispenser: a L{calendarserver.tap.ConnectionDispenser} or C{None}.
+
+        @param config: the L{twistedcaldav.config.Config} to use to configure
+            the subprocess.
+        """
+        super(ConfiguredChildSpawner, self).__init__()
+        self.nextID = 0
+        self.maker = maker
+        self.dispenser = dispenser
+        self.config = config
+
+
+    @inlineCallbacks
+    def spawnWithStore(self, here, there):
+        """
+        Spawn the child with a store based on a configuration.
+        """
+        thisID = self.nextID
+        self.nextID += 1
+        if self.dispenser is not None:
+            # NOTE: important and super subtle, 'poolskt' must not be GC'd
+            # until the call to spawn (and hence spawnProcess) below; otherwise
+            # that end of the socket will be closed and there will be nothing
+            # to inherit.
+            poolskt  = self.dispenser.dispense()
+            poolfd   = poolskt.fileno()
+            childFDs = {
+                0: "w", 1: "r", 2: "r", # behave like normal, but
+                poolfd: poolfd          # bonus FD
+            }
+            extra    = dict(connectionPoolFD=poolfd)
+        else:
+            childFDs = None
+            extra    = {}
+        controller = yield self.spawn(
+            AMP(), ChildConfigurator, childFDs=childFDs
+        )
+        yield controller.callRemote(
+            ConfigureChild,
+            delegateTo=qual(there),
+            pidFile="%s-migrator-%s" % (self.maker.tapname, thisID),
+            logID="migrator-%s" % (thisID,),
+            configFile=self.config.getProvider().getConfigFileName(),
+            processCount=self.config.MultiProcess.ProcessCount,
+            **extra
+        )
+        returnValue(swapAMP(controller, here))
+
+
+

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2011-12-03 08:40:33 UTC (rev 8400)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -28,7 +28,7 @@
 import errno
 import os
 from time import sleep
-from socket import fromfd, AF_UNIX, SOCK_STREAM
+from socket import fromfd, AF_UNIX, SOCK_STREAM, socketpair
 
 from twext.python.filepath import CachingFilePath as FilePath
 from twext.python.log import Logger
@@ -67,7 +67,7 @@
 from twistedcaldav.util import getMemorySize, getNCPU
 from twext.enterprise.ienterprise import POSTGRES_DIALECT
 from twext.enterprise.ienterprise import ORACLE_DIALECT
-from twext.enterprise.adbapi2 import ConnectionPool
+from twext.enterprise.adbapi2 import ConnectionPool, ConnectionPoolConnection
 
 try:
     from twistedcaldav.authkerb import NegotiateCredentialFactory
@@ -163,7 +163,8 @@
 
 def transactionFactoryFromFD(dbampfd, dialect, paramstyle):
     """
-    Create a transaction factory from an inherited file descriptor.
+    Create a transaction factory from an inherited file descriptor, such as one
+    created by L{ConnectionDispenser}.
     """
     skt = fromfd(dbampfd, AF_UNIX, SOCK_STREAM)
     os.close(dbampfd)
@@ -175,6 +176,35 @@
 
 
 
+class ConnectionDispenser(object):
+    """
+    A L{ConnectionDispenser} can dispense already-connected file descriptors,
+    for use with subprocess spawning.
+    """
+    # Very long term FIXME: this mechanism should ideally be eliminated, by
+    # making all subprocesses have a single stdio AMP connection that
+    # multiplexes between multiple protocols.
+
+    def __init__(self, connectionPool):
+        self.pool = connectionPool
+
+
+    def dispense(self):
+        """
+        Dispense a socket object, already connected to a server, for a client
+        in a subprocess.
+        """
+        # FIXME: these sockets need to be re-dispensed when the process is
+        # respawned, and they currently won't be.
+        c, s = socketpair(AF_UNIX, SOCK_STREAM)
+        protocol = ConnectionPoolConnection(self.pool)
+        transport = ConnectionWithPeer(s, protocol)
+        protocol.makeConnection(transport)
+        transport.startReading()
+        return c
+
+
+
 def storeFromConfig(config, txnFactory):
     """
     Produce an L{IDataStore} from the given configuration, transaction factory,

Copied: CalendarServer/trunk/calendarserver/tools/upgrade.py (from rev 8400, CalendarServer/branches/users/glyph/parallel-upgrade/calendarserver/tools/upgrade.py)
===================================================================
--- CalendarServer/trunk/calendarserver/tools/upgrade.py	                        (rev 0)
+++ CalendarServer/trunk/calendarserver/tools/upgrade.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+# -*- test-case-name: calendarserver.tools.test.test_upgrade -*-
+##
+# Copyright (c) 2006-2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+This tool allows any necessary upgrade to complete, then exits.
+"""
+
+import os
+import sys
+
+from twisted.python.text import wordWrap
+from twisted.python import log
+from twisted.python.usage import Options, UsageError
+
+from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
+from calendarserver.tools.cmdline import utilityMain
+from twisted.application.service import Service
+
+
+from twext.python.log import setLogLevelForNamespace
+
+def usage(e=None):
+    if e:
+        print e
+        print ""
+    try:
+        UpgradeOptions().opt_help()
+    except SystemExit:
+        pass
+    if e:
+        sys.exit(64)
+    else:
+        sys.exit(0)
+
+
+description = '\n'.join(
+    wordWrap(
+        """
+        Usage: calendarserver_upgrade [options] [input specifiers]\n
+        """ + __doc__,
+        int(os.environ.get('COLUMNS', '80'))
+    )
+)
+
+class UpgradeOptions(Options):
+    """
+    Command-line options for 'calendarserver_upgrade'
+
+    @ivar upgradeers: a list of L{DirectoryUpgradeer} objects which can identify the
+        calendars to upgrade, given a directory service.  This list is built by
+        parsing --record and --collection options.
+    """
+
+    synopsis = description
+
+    optParameters = [['config', 'f', DEFAULT_CONFIG_FILE,
+                      "Specify caldavd.plist configuration path."]]
+
+    def __init__(self):
+        super(UpgradeOptions, self).__init__()
+        self.upgradeers = []
+        self.outputName = '-'
+
+
+    def opt_output(self, filename):
+        """
+        Specify output file path (default: '-', meaning stdout).
+        """
+        self.outputName = filename
+
+    opt_o = opt_output
+
+
+    def openOutput(self):
+        """
+        Open the appropriate output file based on the '--output' option.
+        """
+        if self.outputName == '-':
+            return sys.stdout
+        else:
+            return open(self.outputName, 'wb')
+
+
+
+class UpgraderService(Service, object):
+    """
+    Service which runs, exports the appropriate records, then stops the reactor.
+    """
+
+    def __init__(self, store, options, output, reactor, config):
+        super(UpgraderService, self).__init__()
+        self.store   = store
+        self.options = options
+        self.output  = output
+        self.reactor = reactor
+        self.config = config
+        self._directory = None
+
+
+    def startService(self):
+        """
+        Immediately stop.  The upgrade will have been run before this.
+        """
+        self.output.write("Upgrade complete, shutting down.\n")
+        from twisted.internet import reactor
+        from twisted.internet.error import ReactorNotRunning
+        try:
+            reactor.stop()
+        except ReactorNotRunning:
+            # I don't care.
+            pass
+
+
+    def stopService(self):
+        """
+        Stop the service.  Nothing to do; everything should be finished by this
+        time.
+        """
+
+from txdav.common.datastore import upgrade
+from txdav.common.datastore.upgrade import migrate
+
+
+def main(argv=sys.argv, stderr=sys.stderr, reactor=None):
+    """
+    Do the export.
+    """
+    if reactor is None:
+        from twisted.internet import reactor
+
+    options = UpgradeOptions()
+    try:
+        options.parseOptions(argv[1:])
+    except UsageError, e:
+        usage(e)
+
+    try:
+        output = options.openOutput()
+    except IOError, e:
+        stderr.write("Unable to open output file for writing: %s\n" %
+                     (e))
+        sys.exit(1)
+
+    def makeService(store):
+        from twistedcaldav.config import config
+        return UpgraderService(store, options, output, reactor, config)
+
+    def onlyUpgradeEvents(event):
+        if event.get("namespace", "").startswith(upgrade.__name__):
+            output.write(log.textFromEventDict(event)+"\n")
+
+    setLogLevelForNamespace(upgrade.__name__, "info")
+    setLogLevelForNamespace(migrate.__name__, "info")
+    log.addObserver(onlyUpgradeEvents)
+    utilityMain(options["config"], makeService, reactor)


Property changes on: CalendarServer/trunk/support/build.sh
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation/support/build.sh:4379-4443
/CalendarServer/branches/egg-info-351/support/build.sh:4589-4615
/CalendarServer/branches/generic-sqlstore/support/build.sh:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/support/build.sh:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/support/build.sh:5911-5935
/CalendarServer/branches/new-store/support/build.sh:5594-5934
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/support/build.sh:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/support/build.sh:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/support/build.sh:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace/support/build.sh:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591/support/build.sh:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/support/build.sh:4465-4957
/CalendarServer/branches/users/cdaboo/pods/support/build.sh:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/support/build.sh:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/support/build.sh:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/support/build.sh:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/support/build.sh:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/support/build.sh:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/support/build.sh:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/support/build.sh:4971-5080
/CalendarServer/branches/users/glyph/dalify/support/build.sh:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect/support/build.sh:6824-6876
/CalendarServer/branches/users/glyph/deploybuild/support/build.sh:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/support/build.sh:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/support/build.sh:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/support/build.sh:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/support/build.sh:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/support/build.sh:6369-6445
/CalendarServer/branches/users/glyph/new-export/support/build.sh:7444-7485
/CalendarServer/branches/users/glyph/oracle-nulls/support/build.sh:7340-7351
/CalendarServer/branches/users/glyph/sendfdport/support/build.sh:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/support/build.sh:6490-6550
/CalendarServer/branches/users/glyph/sql-store/support/build.sh:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/support/build.sh:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/support/build.sh:5084-5149
/CalendarServer/branches/users/sagen/applepush/support/build.sh:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/support/build.sh:7380-7381
/CalendarServer/branches/users/sagen/locations-resources-2/support/build.sh:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/support/build.sh:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/support/build.sh:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/support/build.sh:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/support/build.sh:4068-4075
/CalendarServer/branches/users/sagen/resources-2/support/build.sh:5084-5093
/CalendarServer/branches/users/wsanchez/transations/support/build.sh:5515-5593
   + /CalendarServer/branches/config-separation/support/build.sh:4379-4443
/CalendarServer/branches/egg-info-351/support/build.sh:4589-4615
/CalendarServer/branches/generic-sqlstore/support/build.sh:6167-6191
/CalendarServer/branches/new-store/support/build.sh:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/support/build.sh:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/support/build.sh:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/support/build.sh:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/support/build.sh:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/support/build.sh:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace/support/build.sh:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591/support/build.sh:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/support/build.sh:4465-4957
/CalendarServer/branches/users/cdaboo/pods/support/build.sh:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/support/build.sh:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/support/build.sh:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/support/build.sh:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/support/build.sh:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/support/build.sh:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/support/build.sh:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/support/build.sh:4971-5080
/CalendarServer/branches/users/glyph/dalify/support/build.sh:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect/support/build.sh:6824-6876
/CalendarServer/branches/users/glyph/deploybuild/support/build.sh:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/support/build.sh:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/support/build.sh:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/support/build.sh:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/support/build.sh:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/support/build.sh:6369-6445
/CalendarServer/branches/users/glyph/new-export/support/build.sh:7444-7485
/CalendarServer/branches/users/glyph/oracle-nulls/support/build.sh:7340-7351
/CalendarServer/branches/users/glyph/parallel-upgrade/support/build.sh:8376-8400
/CalendarServer/branches/users/glyph/sendfdport/support/build.sh:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/support/build.sh:6490-6550
/CalendarServer/branches/users/glyph/sql-store/support/build.sh:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/support/build.sh:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/support/build.sh:5084-5149
/CalendarServer/branches/users/sagen/applepush/support/build.sh:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/support/build.sh:7380-7381
/CalendarServer/branches/users/sagen/locations-resources/support/build.sh:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/support/build.sh:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/support/build.sh:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/support/build.sh:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/support/build.sh:4068-4075
/CalendarServer/branches/users/sagen/resources-2/support/build.sh:5084-5093
/CalendarServer/branches/users/wsanchez/transations/support/build.sh:5515-5593

Copied: CalendarServer/trunk/twext/internet/spawnsvc.py (from rev 8400, CalendarServer/branches/users/glyph/parallel-upgrade/twext/internet/spawnsvc.py)
===================================================================
--- CalendarServer/trunk/twext/internet/spawnsvc.py	                        (rev 0)
+++ CalendarServer/trunk/twext/internet/spawnsvc.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -0,0 +1,261 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Utility service that can spawn subprocesses.
+"""
+
+import os
+import sys
+
+from twisted.python import log
+
+from twisted.python.reflect import namedAny
+from twisted.internet.stdio import StandardIO
+from twisted.internet.error import ReactorNotRunning
+
+if __name__ == '__main__':
+
+    sys.stdout = sys.stderr
+    there = sys.argv[1]
+    protocolClass = namedAny(there)
+    proto = protocolClass()
+    origLost = proto.connectionLost
+    def goodbye(reason):
+        """
+        Stop the process if stdin is closed.
+        """
+        try:
+            reactor.stop()
+        except ReactorNotRunning:
+            pass
+        return origLost(reason)
+    proto.connectionLost = goodbye
+    StandardIO(proto)
+    from twisted.internet import reactor
+    reactor.run()
+    os._exit(0)
+
+
+import sys
+
+from zope.interface import implements
+
+from twisted.internet.interfaces import ITransport, IPushProducer, IConsumer
+
+from twisted.application.service import Service
+from twisted.python.reflect import qual
+from twisted.internet.protocol import ProcessProtocol
+from twisted.internet.defer import Deferred, succeed
+
+
+class BridgeTransport(object):
+    """
+    ITransport implementation for the protocol in the parent process running a
+    L{SpawnerService}.
+    """
+
+    implements(ITransport, IPushProducer, IConsumer)
+
+    def __init__(self, processTransport):
+        """
+        Create this bridge transport connected to an L{IProcessTransport}.
+        """
+        self.transport = processTransport
+
+
+    def __getattr__(self, name):
+        """
+        Delegate all attribute accesses to the process traansport.
+        """
+        return getattr(self.transport, name)
+
+
+    def getPeer(self):
+        """
+        Get a fake peer address indicating the subprocess's pid.
+        """
+        return "Peer:PID:" + str(self.transport.pid)
+
+
+    def getHost(self):
+        """
+        Get a fake host address indicating the subprocess's pid.
+        """
+        return "Host:PID:" + str(self.transport.pid)
+
+
+
+class BridgeProtocol(ProcessProtocol, object):
+    """
+    Process protocol implementation that delivers data to the C{hereProto}
+    associated with an invocation of L{SpawnerService.spawn}.
+
+    @ivar service: a L{SpawnerService} that created this L{BridgeProtocol}
+
+    @ivar protocol: a reference to the L{IProtocol}.
+
+    @ivar killTimeout: number of seconds after sending SIGINT that this process
+        will send SIGKILL.
+    """
+
+    def __init__(self, service, protocol, killTimeout=15.0):
+        self.service = service
+        self.protocol = protocol
+        self.killTimeout = killTimeout
+        self.service.addBridge(self)
+
+
+    def connectionMade(self):
+        """
+        The subprocess was started.
+        """
+        self.protocol.makeConnection(BridgeTransport(self.transport))
+
+
+    def outReceived(self, data):
+        """
+        Some data was received to standard output; relay it to the protocol.
+        """
+        self.protocol.dataReceived(data)
+
+
+    def errReceived(self, data):
+        """
+        Some standard error was received from the subprocess.
+        """
+        log.msg("Error output from process: " + data,
+                isError=True)
+
+
+    _killTimeout = None
+    def eventuallyStop(self):
+        """
+        Eventually stop this subprocess.  Send it a SIGTERM, and if it hasn't
+        stopped by C{self.killTimeout} seconds, send it a SIGKILL.
+        """
+        self.transport.signalProcess('TERM')
+        def reallyStop():
+            self.transport.signalProcess("KILL")
+            self._killTimeout = None
+        self._killTimeout = (
+            self.service.reactor.callLater(self.killTimeout, reallyStop)
+        )
+
+
+    def processEnded(self, reason):
+        """
+        The process has ended; notify the L{SpawnerService} that this bridge
+        has stopped.
+        """
+        if self._killTimeout is not None:
+            self._killTimeout.cancel()
+        self.protocol.connectionLost(reason)
+        self.service.removeBridge(self)
+
+
+
+class SpawnerService(Service, object):
+    """
+    Process to spawn services and then shut them down.
+
+    @ivar reactor: an L{IReactorProcess}/L{IReactorTime}
+
+    @ivar pendingSpawns: a C{list} of 2-C{tuple}s of hereProto, thereProto.
+
+    @ivar bridges: a C{list} of L{BridgeProtocol} instances.
+    """
+
+    def __init__(self, reactor=None):
+        if reactor is None:
+            from twisted.internet import reactor
+        self.reactor = reactor
+        self.pendingSpawns = []
+        self.bridges = []
+        self._stopAllDeferred = None
+
+
+    def spawn(self, hereProto, thereProto, childFDs=None):
+        """
+        Spawn a subprocess with a connected pair of protocol objects, one in
+        the current process, one in the subprocess.
+
+        @param hereProto: a L{Protocol} instance to listen in this process.
+
+        @param thereProto: a top-level class or function that will be imported
+            and called in the spawned subprocess.
+
+        @param childFDs: File descriptors to share with the subprocess; same
+            format as L{IReactorProcess.spawnProcess}.
+
+        @return: a L{Deferred} that fires when C{hereProto} is ready.
+        """
+        if not self.running:
+            self.pendingSpawns.append((hereProto, thereProto))
+            return
+        name = qual(thereProto)
+        argv = [sys.executable, '-u', '-m', __name__, name]
+        self.reactor.spawnProcess(
+            BridgeProtocol(self, hereProto), sys.executable,
+            argv, os.environ, childFDs=childFDs
+        )
+        return succeed(hereProto)
+
+
+    def startService(self):
+        """
+        Start the service; spawn any processes previously started with spawn().
+        """
+        super(SpawnerService, self).startService()
+        for spawn in self.pendingSpawns:
+            self.spawn(*spawn)
+
+
+    def addBridge(self, bridge):
+        """
+        Add a L{BridgeProtocol} to the list to be tracked.
+        """
+        self.bridges.append(bridge)
+
+
+    def removeBridge(self, bridge):
+        """
+        The process controlled by a L{BridgeProtocol} has terminated; remove it
+        from the active list, and fire any outstanding Deferred.
+
+        @param bridge: the protocol which has ended.
+        """
+        self.bridges.remove(bridge)
+        if self._stopAllDeferred is not None:
+            if len(self.bridges) == 0:
+                self._stopAllDeferred.callback(None)
+                self._stopAllDeferred = False
+
+
+    def stopService(self):
+        """
+        Stop the service.
+        """
+        super(SpawnerService, self).stopService()
+        if self.bridges:
+            self._stopAllDeferred = Deferred()
+            for bridge in self.bridges:
+                bridge.eventuallyStop()
+            return self._stopAllDeferred
+        return succeed(None)
+
+
+

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2011-12-03 08:40:33 UTC (rev 8400)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -530,7 +530,12 @@
                                                    # split existing calendars into multiples based on component type.
                                                    # If on, it will also cause new accounts to provision with separate
                                                    # calendars for events and tasks.
-    
+
+    "ParallelUpgrades" : False,    # Perform upgrades - currently only the
+                                   # database -> filesystem migration - but in
+                                   # the future, hopefully all relevant
+                                   # upgrades - in parallel in subprocesses.
+
     "EnableDefaultAlarms" : True, # Support for default alarms generated by the server
 
     # CardDAV Features


Property changes on: CalendarServer/trunk/txdav/caldav/datastore/index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation/txdav/caldav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/caldav/datastore/index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/caldav/datastore/index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace/txdav/caldav/datastore/index_file.py:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/caldav/datastore/index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/caldav/datastore/index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/caldav/datastore/index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/caldav/datastore/index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/caldav/datastore/index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/caldav/datastore/index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/caldav/datastore/index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/caldav/datastore/index_file.py:7340-7351
/CalendarServer/branches/users/glyph/oracle/txdav/caldav/datastore/index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/caldav/datastore/index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/caldav/datastore/index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/caldav/datastore/index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/index.py:6322-6394
   + /CalendarServer/branches/config-separation/txdav/caldav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/caldav/datastore/index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace/txdav/caldav/datastore/index_file.py:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/caldav/datastore/index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/caldav/datastore/index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/caldav/datastore/index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/caldav/datastore/index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/caldav/datastore/index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/caldav/datastore/index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/caldav/datastore/index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle/txdav/caldav/datastore/index_file.py:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/caldav/datastore/index_file.py:7340-7351
/CalendarServer/branches/users/glyph/parallel-upgrade/txdav/caldav/datastore/index_file.py:8376-8400
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/caldav/datastore/index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/caldav/datastore/index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/caldav/datastore/index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/index.py:6322-6394


Property changes on: CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation/txdav/caldav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/caldav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/caldav/datastore/test/test_index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/test/test_index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/caldav/datastore/test/test_index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/caldav/datastore/test/test_index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/caldav/datastore/test/test_index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/caldav/datastore/test/test_index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/caldav/datastore/test/test_index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/caldav/datastore/test/test_index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/test/test_index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/caldav/datastore/test/test_index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/caldav/datastore/test/test_index_file.py:7340-7351
/CalendarServer/branches/users/glyph/oracle/txdav/caldav/datastore/test/test_index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/caldav/datastore/test/test_index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/caldav/datastore/test/test_index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/caldav/datastore/test/test_index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_index.py:6322-6394
   + /CalendarServer/branches/config-separation/txdav/caldav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/caldav/datastore/test/test_index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/test/test_index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/caldav/datastore/test/test_index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/caldav/datastore/test/test_index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/caldav/datastore/test/test_index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/caldav/datastore/test/test_index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/caldav/datastore/test/test_index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/caldav/datastore/test/test_index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/test/test_index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/caldav/datastore/test/test_index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle/txdav/caldav/datastore/test/test_index_file.py:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/caldav/datastore/test/test_index_file.py:7340-7351
/CalendarServer/branches/users/glyph/parallel-upgrade/txdav/caldav/datastore/test/test_index_file.py:8376-8400
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/caldav/datastore/test/test_index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/caldav/datastore/test/test_index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/caldav/datastore/test/test_index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_index.py:6322-6394


Property changes on: CalendarServer/trunk/txdav/carddav/datastore/index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation/txdav/carddav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/carddav/datastore/index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/carddav/datastore/index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/carddav/datastore/index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace/txdav/carddav/datastore/index_file.py:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/carddav/datastore/index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/carddav/datastore/index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/carddav/datastore/index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/carddav/datastore/index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/carddav/datastore/index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/carddav/datastore/index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/carddav/datastore/index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/carddav/datastore/index_file.py:7340-7351
/CalendarServer/branches/users/glyph/oracle/txdav/carddav/datastore/index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/carddav/datastore/index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/carddav/datastore/index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/carddav/datastore/index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/vcardindex.py:6322-6394
   + /CalendarServer/branches/config-separation/txdav/carddav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/carddav/datastore/index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/carddav/datastore/index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace/txdav/carddav/datastore/index_file.py:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/carddav/datastore/index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/carddav/datastore/index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/carddav/datastore/index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/carddav/datastore/index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/carddav/datastore/index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/carddav/datastore/index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/carddav/datastore/index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle/txdav/carddav/datastore/index_file.py:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/carddav/datastore/index_file.py:7340-7351
/CalendarServer/branches/users/glyph/parallel-upgrade/txdav/carddav/datastore/index_file.py:8376-8400
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/carddav/datastore/index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/carddav/datastore/index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/carddav/datastore/index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/vcardindex.py:6322-6394


Property changes on: CalendarServer/trunk/txdav/carddav/datastore/test/test_index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
   - /CalendarServer/branches/config-separation/txdav/carddav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/carddav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/carddav/datastore/test/test_index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/carddav/datastore/test/test_index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/carddav/datastore/test/test_index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/carddav/datastore/test/test_index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/carddav/datastore/test/test_index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/carddav/datastore/test/test_index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/carddav/datastore/test/test_index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/carddav/datastore/test/test_index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/test/test_index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/carddav/datastore/test/test_index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/carddav/datastore/test/test_index_file.py:7340-7351
/CalendarServer/branches/users/glyph/oracle/txdav/carddav/datastore/test/test_index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/carddav/datastore/test/test_index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/carddav/datastore/test/test_index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/carddav/datastore/test/test_index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_vcardindex.py:6322-6394
   + /CalendarServer/branches/config-separation/txdav/carddav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/carddav/datastore/test/test_index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/carddav/datastore/test/test_index_file.py:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/pods/txdav/carddav/datastore/test/test_index_file.py:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_index_file.py:7085-7206
/CalendarServer/branches/users/cdaboo/pycard/txdav/carddav/datastore/test/test_index_file.py:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes/txdav/carddav/datastore/test/test_index_file.py:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/cdaboo/timezones/txdav/carddav/datastore/test/test_index_file.py:7443-7699
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/deploybuild/txdav/carddav/datastore/test/test_index_file.py:7563-7572
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/misc-portability-fixes/txdav/carddav/datastore/test/test_index_file.py:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/test/test_index_file.py:6369
/CalendarServer/branches/users/glyph/new-export/txdav/carddav/datastore/test/test_index_file.py:7444-7485
/CalendarServer/branches/users/glyph/oracle/txdav/carddav/datastore/test/test_index_file.py:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls/txdav/carddav/datastore/test/test_index_file.py:7340-7351
/CalendarServer/branches/users/glyph/parallel-upgrade/txdav/carddav/datastore/test/test_index_file.py:8376-8400
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/subtransactions/txdav/carddav/datastore/test/test_index_file.py:7248-7258
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/applepush/txdav/carddav/datastore/test/test_index_file.py:8126-8184
/CalendarServer/branches/users/sagen/inboxitems/txdav/carddav/datastore/test/test_index_file.py:7380-7381
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_vcardindex.py:6322-6394

Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py	2011-12-03 08:40:33 UTC (rev 8400)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -43,6 +43,7 @@
 from txdav.base.propertystore.base import PropertyName
 from txdav.common.icommondatastore import NoSuchHomeChildError
 from twext.enterprise.adbapi2 import ConnectionPool
+from twisted.trial.unittest import TestCase
 from twisted.internet.defer import returnValue
 from twistedcaldav.notify import Notifier, NodeCreationException
 from twext.enterprise.ienterprise import AlreadyFinishedError
@@ -83,6 +84,48 @@
 
     SHARED_DB_PATH = "_test_sql_db"
 
+
+    @classmethod
+    def createService(cls, serviceFactory):
+        """
+        Create a L{PostgresService} to use for building a store.
+        """
+        dbRoot = CachingFilePath(cls.SHARED_DB_PATH)
+        return PostgresService(
+            dbRoot, serviceFactory, current_sql_schema, resetSchema=True,
+            databaseName="caldav",
+            options = [
+                "-c log_lock_waits=TRUE",
+                "-c log_statement=all",
+                "-c log_line_prefix='%p.%x '",
+            ],
+            testMode=True
+        )
+
+
+    @classmethod
+    def childStore(cls):
+        """
+        Create a store suitable for use in a child process, that is hooked up
+        to the store that a parent test process is managing.
+        """
+        disableMemcacheForTest(TestCase())
+        staticQuota = 3000
+        attachmentRoot = (CachingFilePath(cls.SHARED_DB_PATH)
+                          .child("attachments"))
+        stubsvc = cls.createService(lambda cf: Service())
+
+        cp = ConnectionPool(stubsvc.produceConnection, maxConnections=1)
+        # Attach the service to the running reactor.
+        cp.startService()
+        reactor.addSystemEventTrigger("before", "shutdown", cp.stopService)
+        cds = CommonDataStore(
+            cp.connection, StubNotifierFactory(),
+            attachmentRoot, quota=staticQuota
+        )
+        return cds
+
+
     def buildStore(self, testCase, notifierFactory):
         """
         Do the necessary work to build a store for a particular test case.
@@ -99,16 +142,7 @@
                     testCase, notifierFactory, attachmentRoot
                 ).chainDeferred(ready)
                 return Service()
-            self.sharedService = PostgresService(
-                dbRoot, getReady, current_sql_schema, resetSchema=True,
-                databaseName="caldav",
-                options = [
-                    "-c log_lock_waits=TRUE",
-                    "-c log_statement=all",
-                    "-c log_line_prefix='%p.%x '",
-                ],
-                testMode=True
-            )
+            self.sharedService = self.createService(getReady)
             self.sharedService.startService()
             def startStopping():
                 log.msg("Starting stopping.")
@@ -145,7 +179,8 @@
         except OSError:
             pass
         currentTestID = testCase.id()
-        cp = ConnectionPool(self.sharedService.produceConnection)
+        cp = ConnectionPool(self.sharedService.produceConnection,
+                            maxConnections=5)
         quota = deriveQuota(testCase)
         store = CommonDataStore(
             cp.connection, notifierFactory, attachmentRoot, quota=quota

Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py	2011-12-03 08:40:33 UTC (rev 8400)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -24,25 +24,175 @@
 import xattr
 
 from twext.python.log import LoggingMixIn
+
 from twisted.application.service import Service
 from twisted.internet import reactor
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.internet.defer import maybeDeferred, DeferredList
 from twisted.python.runtime import platform
 
+from twext.python.filepath import CachingFilePath
+from twext.internet.spawnsvc import SpawnerService
+
+from twisted.protocols.amp import AMP, Command, String
+
 from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
 from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
 from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
 from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
-from txdav.base.propertystore.appledouble_xattr import (
-    PropertyStore as AppleDoubleStore)
+from txdav.base.propertystore.appledouble_xattr import (PropertyStore
+                                                        as AppleDoubleStore)
 
+
+homeTypeLookup = {
+    "calendar": (migrateCalendarHome,
+                 lambda txn: txn.calendarHomeWithUID),
+    "addressbook": (migrateAddressbookHome,
+                    lambda txn: txn.addressbookHomeWithUID)
+}
+
+
+def swapAMP(oldAMP, newAMP):
+    """
+    Swap delivery of messages from an old L{AMP} instance to a new one.
+
+    This is useful for implementors of L{StoreSpawnerService} since they will
+    typically want to create one protocol for initializing the store, and
+    another for processing application commands.
+
+    @param oldAMP: An AMP instance currently hooked up to a transport, whose
+        job is done and wants to stop receiving messages.
+
+    @param newAMP: An AMP instance who wants to take over and start receiving
+        messages previously destined for oldAMP.
+
+    @return: C{newAMP}
+    """
+    oldAMP.boxReceiver = newAMP
+    newAMP.startReceivingBoxes(oldAMP)
+    return newAMP
+
+
+
+class StoreSpawnerService(SpawnerService):
+    """
+    Abstract subclass of L{SpawnerService} that describes how to spawn a subclass.
+    """
+
+    def spawnWithStore(self, here, there):
+        """
+        Like L{SpawnerService.spawn}, but instead of instantiating C{there}
+        with 0 arguments, it instantiates it with an L{ICalendarStore} /
+        L{IAddressbookStore}.
+        """
+        raise NotImplementedError("subclasses must implement the specifics")
+
+
+
+class Configure(Command):
+    """
+    Configure the upgrade helper process.
+    """
+
+    arguments = [("filename", String())]
+
+
+    
+class OneUpgrade(Command):
+    """
+    Upgrade a single calendar home.
+    """
+
+    arguments = [("uid", String()),
+                 ("homeType", String())]
+
+
+
+class LogIt(Command):
+    """
+    Log a message.
+    """
+    arguments = [("message", String())]
+
+
+
+class UpgradeDriver(AMP):
+    """
+    Helper protocol which runs in the master process doing the upgrade.
+    """
+
+    def __init__(self, upgradeService):
+        super(UpgradeDriver, self).__init__()
+        self.service = upgradeService
+
+
+    def configure(self, filename):
+        """
+        Configure the subprocess to examine the file store at the given path
+        name.
+        """
+        return self.callRemote(Configure, filename=filename)
+
+
+    def oneUpgrade(self, uid, homeType):
+        """
+        Upgrade one calendar or addressbook home, with the given uid of the
+        given type, and return a L{Deferred} which will fire when the upgrade
+        is complete.
+        """
+        return self.callRemote(OneUpgrade, uid=uid, homeType=homeType)
+
+
+
+class UpgradeHelperProcess(AMP):
+    """
+    Helper protocol which runs in a subprocess to upgrade.
+    """
+
+    def __init__(self, store):
+        """
+        Create with a reference to an SQL store.
+        """
+        super(UpgradeHelperProcess, self).__init__()
+        self.store = store
+        self.store.setMigrating(True)
+        
+
+    @Configure.responder
+    def configure(self, filename):
+        subsvc = None
+        self.upgrader = UpgradeToDatabaseService.wrapService(
+            CachingFilePath(filename), subsvc, self.store
+        )
+        return {}
+
+
+    @OneUpgrade.responder
+    def oneUpgrade(self, uid, homeType):
+        """
+        Upgrade one calendar home.
+        """
+        migrateFunc, destFunc = homeTypeLookup[homeType]
+        fileTxn = self.upgrader.fileStore.newTransaction()
+        return (
+            maybeDeferred(destFunc(fileTxn), uid)
+            .addCallback(
+                lambda fileHome:
+                self.upgrader.migrateOneHome(fileTxn, homeType, fileHome)
+            )
+            .addCallback(lambda ignored: {})
+        )
+
+
+
 class UpgradeToDatabaseService(Service, LoggingMixIn, object):
     """
     Upgrade resources from a filesystem store to a database store.
     """
 
     @classmethod
-    def wrapService(cls, path, service, store, uid=None, gid=None):
+    def wrapService(cls, path, service, store, uid=None, gid=None,
+                    parallel=0, spawner=None):
         """
         Create an L{UpgradeToDatabaseService} if there are still file-based
         calendar or addressbook homes remaining in the given path.
@@ -62,6 +212,12 @@
 
         @type service: L{IService}
 
+        @param parallel: The number of parallel subprocesses that should manage
+            the upgrade.
+
+        @param spawner: a concrete L{StoreSpawnerService} subclass that will be
+            used to spawn helper processes.
+
         @return: a service
         @rtype: L{IService}
         """
@@ -103,12 +259,14 @@
                     FileStore(path, None, True, True,
                               propertyStoreClass=appropriateStoreClass),
                     store, service, uid=uid, gid=gid,
+                    parallel=parallel, spawner=spawner,
                 )
                 return self
         return service
 
 
-    def __init__(self, fileStore, sqlStore, service, uid=None, gid=None):
+    def __init__(self, fileStore, sqlStore, service, uid=None, gid=None,
+                 parallel=0, spawner=None):
         """
         Initialize the service.
         """
@@ -117,9 +275,40 @@
         self.sqlStore = sqlStore
         self.uid = uid
         self.gid = gid
+        self.parallel = parallel
+        self.spawner = spawner
 
 
     @inlineCallbacks
+    def migrateOneHome(self, fileTxn, homeType, fileHome):
+        """
+        Migrate an individual calendar or addressbook home.
+        """
+        migrateFunc, destFunc = homeTypeLookup.get(homeType)
+        uid = fileHome.uid()
+        self.log_warn("Starting migration transaction %s UID %r" %
+                      (homeType, uid))
+        sqlTxn = self.sqlStore.newTransaction()
+        homeGetter = destFunc(sqlTxn)
+        if (yield homeGetter(uid, create=False)) is not None:
+            self.log_warn(
+                "%s home %r already existed not migrating" % (
+                    homeType, uid))
+            yield sqlTxn.abort()
+            yield fileTxn.commit()
+            returnValue(None)
+        sqlHome = yield homeGetter(uid, create=True)
+        yield migrateFunc(fileHome, sqlHome)
+        yield fileTxn.commit()
+        yield sqlTxn.commit()
+        # Remove file home after migration. FIXME: instead, this should be a
+        # public remove...HomeWithUID() API for de-provisioning.  (If we had
+        # this, this would simply be a store-to-store migrator rather than a
+        # filesystem-to-database upgrade.)
+        fileHome._path.remove()
+
+
+    @inlineCallbacks
     def doMigration(self):
         """
         Do the migration.  Called by C{startService}, but a different method
@@ -127,43 +316,61 @@
 
         @return: a Deferred which fires when the migration is complete.
         """
-        self.log_warn("Beginning filesystem -> database upgrade.")
-
         self.sqlStore.setMigrating(True)
+        parallel = self.parallel
+        if parallel:
+            self.log_warn("Starting %d upgrade helper processes." %
+                          (parallel,))
+            spawner = self.spawner
+            spawner.startService()
+            drivers = []
+            for value in xrange(parallel):
+                driver = yield spawner.spawnWithStore(UpgradeDriver(self),
+                                                      UpgradeHelperProcess)
+                drivers.append(driver)
 
-        for homeType, migrateFunc, eachFunc, destFunc, _ignore_topPathName in [
-            ("calendar", migrateCalendarHome,
-                self.fileStore.eachCalendarHome,
-                lambda txn: txn.calendarHomeWithUID,
-                "calendars"),
-            ("addressbook", migrateAddressbookHome,
-                self.fileStore.eachAddressbookHome,
-                lambda txn: txn.addressbookHomeWithUID,
-                "addressbooks")
+            # Wait for all subprocesses to be fully configured before
+            # continuing, but let them configure in any order.
+            self.log_warn("Configuring upgrade helper processes.")
+            yield DeferredList([driver.configure(self.fileStore._path.path)
+                                for driver in drivers])
+            self.log_warn("Upgrade helpers ready.")
+
+        self.log_warn("Beginning filesystem -> database upgrade.")
+        inParallel = []
+        for homeType, eachFunc in [
+                ("calendar", self.fileStore.eachCalendarHome),
+                ("addressbook", self.fileStore.eachAddressbookHome),
             ]:
             for fileTxn, fileHome in eachFunc():
                 uid = fileHome.uid()
                 self.log_warn("Migrating %s UID %r" % (homeType, uid))
-                sqlTxn = self.sqlStore.newTransaction()
-                homeGetter = destFunc(sqlTxn)
-                if (yield homeGetter(uid, create=False)) is not None:
-                    self.log_warn(
-                        "%s home %r already existed not migrating" % (
-                            homeType, uid))
-                    yield sqlTxn.abort()
+                if parallel:
+                    # No-op transaction here: make sure everything's unlocked
+                    # before asking the subprocess to handle it.
                     yield fileTxn.commit()
-                    continue
-                sqlHome = yield homeGetter(uid, create=True)
-                if sqlHome is None:
-                    raise RuntimeError("THIS SHOULD NOT BE POSSIBLE.")
-                yield migrateFunc(fileHome, sqlHome)
-                yield fileTxn.commit()
-                yield sqlTxn.commit()
-                # FIXME: need a public remove...HomeWithUID() for de-
-                # provisioning
+                    if not drivers:
+                        # All the subprocesses are currently busy processing an
+                        # upgrade.  Wait for one to become available.
+                        yield DeferredList(inParallel, fireOnOneCallback=True,
+                                           fireOnOneErrback=True)
+                    busy = drivers.pop(0)
+                    d = busy.oneUpgrade(fileHome.uid(), homeType)
+                    inParallel.append(d)
+                    def freeUp(result, d=d, busy=busy, uid=uid,
+                               homeType=homeType):
+                        inParallel.remove(d)
+                        drivers.append(busy)
+                        self.log_warn("Completed migration of %s uid %r" %
+                                      (homeType, uid))
+                        return result
+                    d.addBoth(freeUp)
+                else:
+                    yield self.migrateOneHome(fileTxn, homeType, fileHome)
 
-                # Remove file home after migration
-                fileHome._path.remove()
+        if inParallel:
+            yield DeferredList(inParallel)
+
         for homeType in TOPPATHS:
             homesPath = self.fileStore._path.child(homeType)
             if homesPath.isdir():
@@ -180,15 +387,21 @@
             for fp in sqlAttachmentsPath.walk():
                 os.chown(fp.path, uid, gid)
 
-        self.sqlStore.setMigrating(False) 
+        self.sqlStore.setMigrating(False)
+
+        if parallel:
+            self.log_warn("Stopping upgrade helper processes.")
+            yield spawner.stopService()
+            self.log_warn("Upgrade helpers all stopped.")
         self.log_warn(
             "Filesystem upgrade complete, launching database service."
         )
+        wrapped = self.wrappedService
+        if wrapped is not None:
+            # see http://twistedmatrix.com/trac/ticket/4649
+            reactor.callLater(0, wrapped.setServiceParent, self.parent)
 
-        # see http://twistedmatrix.com/trac/ticket/4649
-        reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
 
-
     def startService(self):
         """
         Start the service.

Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py	2011-12-03 08:40:33 UTC (rev 8400)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py	2011-12-03 08:43:59 UTC (rev 8401)
@@ -23,6 +23,8 @@
 from twisted.application.service import Service, MultiService
 from twisted.internet.defer import inlineCallbacks, Deferred, returnValue
 from twisted.internet.protocol import Protocol
+from twisted.protocols.amp import AMP, Command, String
+from twisted.python.reflect import qual, namedAny
 from twisted.trial.unittest import TestCase
 from txdav.caldav.datastore.test.common import CommonTests
 from txdav.carddav.datastore.test.common import CommonTests as ABCommonTests
@@ -30,13 +32,68 @@
 from txdav.common.datastore.test.util import theStoreBuilder, \
     populateCalendarsFrom, StubNotifierFactory, resetCalendarMD5s,\
     populateAddressBooksFrom, resetAddressBookMD5s
-from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService
 
+from txdav.common.datastore.test.util import SQLStoreBuilder
+from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService, \
+    StoreSpawnerService, swapAMP
+
+
+
+class CreateStore(Command):
+    """
+    Create a store in a subprocess.
+    """
+    arguments = [('delegateTo', String())]
+
+
+
+class StoreCreator(AMP):
+    """
+    Helper protocol.
+    """
+
+    @CreateStore.responder
+    def createStore(self, delegateTo):
+        """
+        Create a store and pass it to the named delegate class.
+        """
+        swapAMP(self, namedAny(delegateTo)(SQLStoreBuilder.childStore()))
+        return {}
+
+
+
+class StubSpawner(StoreSpawnerService):
+    """
+    Stub spawner service which populates the store forcibly.
+    """
+
+    @inlineCallbacks
+    def spawnWithStore(self, here, there):
+        """
+        'here' and 'there' are the helper protocols; in a slight modification
+        of the signature, 'there' will expect to be created with an instance of
+        a store.
+        """
+        master = yield self.spawn(AMP(), StoreCreator)
+        yield master.callRemote(CreateStore, delegateTo=qual(there))
+        returnValue(swapAMP(master, here))
+
+
+
 class HomeMigrationTests(TestCase):
     """
     Tests for L{UpgradeToDatabaseService}.
     """
 
+    def createUpgradeService(self):
+        """
+        Create an upgrade service.
+        """
+        return UpgradeToDatabaseService(
+            self.fileStore, self.sqlStore, self.stubService
+        )
+
+
     @inlineCallbacks
     def setUp(self):
         """
@@ -59,9 +116,7 @@
                 subStarted.callback(None)
         self.stubService = StubService()
         self.topService = MultiService()
-        self.upgrader = UpgradeToDatabaseService(
-            fileStore, self.sqlStore, self.stubService
-        )
+        self.upgrader = self.createUpgradeService()
         self.upgrader.setServiceParent(self.topService)
 
         requirements = CommonTests.requirements
@@ -216,3 +271,21 @@
         ):
             object = (yield adbk.addressbookObjectWithName(name))
             self.assertEquals(object.md5(), md5)
+
+
+
+class ParallelHomeMigrationTests(HomeMigrationTests):
+    """
+    Tests for home migrations running in parallel.  Functionally this should be
+    the same, so it's just a store created slightly differently.
+    """
+
+    def createUpgradeService(self):
+        """
+        Create an upgrade service.
+        """
+        return UpgradeToDatabaseService(
+            self.fileStore, self.sqlStore, self.stubService,
+            parallel=2, spawner=StubSpawner()
+        )
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20111203/23452248/attachment-0001.html>