[CalendarServer-changes] [6615] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Wed Nov 10 23:47:41 PST 2010
Revision: 6615
http://trac.macosforge.org/projects/calendarserver/changeset/6615
Author: glyph at apple.com
Date: 2010-11-10 23:47:38 -0800 (Wed, 10 Nov 2010)
Log Message:
-----------
Allow connecting to a database without starting it up.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/__init__.py
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/twext/internet/sendfdport.py
CalendarServer/trunk/twext/python/sendmsg.c
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/base/datastore/subpostgres.py
CalendarServer/trunk/txdav/common/datastore/test/util.py
CalendarServer/trunk/txdav/common/datastore/util.py
Added Paths:
-----------
CalendarServer/trunk/twext/internet/test/
CalendarServer/trunk/twext/internet/test/__init__.py
CalendarServer/trunk/twext/internet/test/test_sendfdport.py
CalendarServer/trunk/txdav/base/datastore/dbapiclient.py
Removed Paths:
-------------
CalendarServer/trunk/twext/internet/test/__init__.py
CalendarServer/trunk/twext/internet/test/test_sendfdport.py
Property Changed:
----------------
CalendarServer/trunk/
CalendarServer/trunk/txdav/caldav/datastore/index_file.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py
CalendarServer/trunk/txdav/carddav/datastore/index_file.py
CalendarServer/trunk/txdav/carddav/datastore/test/test_index_file.py
Property changes on: CalendarServer/trunk
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
Modified: CalendarServer/trunk/calendarserver/tap/__init__.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/__init__.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/calendarserver/tap/__init__.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -17,3 +17,13 @@
"""
CalendarServer TAP plugin support.
"""
+
+from calendarserver.tap import profiling # Pre-imported for side-effect
+
+__all__ = [
+ "caldav",
+ "carddav",
+ "profiling",
+ "util"
+]
+
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -69,14 +69,13 @@
from twistedcaldav.mail import IMIPReplyInboxResource
from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twistedcaldav.upgrade import upgradeData
-from txdav.base.datastore.subpostgres import PostgresService
-import calendarserver.tap.profiling # Imported for side-effect
-
from calendarserver.tap.util import pgServiceFromConfig
from txdav.base.datastore.asyncsqlpool import ConnectionPool
from txdav.base.datastore.asyncsqlpool import ConnectionPoolConnection
+from txdav.base.datastore.dbapiclient import DBAPIConnector
+from txdav.base.datastore.dbapiclient import postgresPreflight
try:
from twistedcaldav.authkerb import NegotiateCredentialFactory
@@ -89,6 +88,8 @@
from calendarserver.accesslog import RotatingFileAccessLoggingObserver
from calendarserver.tap.util import getRootResource, computeProcessCount
from calendarserver.tap.util import ConnectionWithPeer
+from calendarserver.tap.util import storeFromConfig
+from calendarserver.tap.util import transactionFactoryFromFD
from calendarserver.tools.util import checkDirectory
try:
@@ -113,6 +114,17 @@
gid = gidFromString(gid)
return (uid, gid)
+
+PARENT_ENVIRONMENT = {
+ "PATH": os.environ.get("PATH", ""),
+ "PYTHONPATH": os.environ.get("PYTHONPATH", ""),
+ "LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", ""),
+ "DYLD_LIBRARY_PATH": os.environ.get("DYLD_LIBRARY_PATH", ""),
+}
+
+if "KRB5_KTNAME" in os.environ:
+ PARENT_ENVIRONMENT["KRB5_KTNAME"] = os.environ["KRB5_KTNAME"]
+
class CalDAVStatisticsProtocol (Protocol):
def connectionMade(self):
@@ -380,6 +392,110 @@
+class SlaveSpawnerService(Service):
+ """
+ Service to add all Python subprocesses that need to do work to a
+ L{DelayedStartupProcessMonitor}:
+
+ - regular slave processes (CalDAV workers)
+ - task sidecar
+ - notifier
+ - mail gateway
+ """
+
+ def __init__(self, maker, monitor, dispenser, dispatcher, configPath,
+ inheritFDs=None, inheritSSLFDs=None):
+ self.maker = maker
+ self.monitor = monitor
+ self.dispenser = dispenser
+ self.dispatcher = dispatcher
+ self.configPath = configPath
+ self.inheritFDs = inheritFDs
+ self.inheritSSLFDs = inheritSSLFDs
+
+
+ def startService(self):
+ for slaveNumber in xrange(0, config.MultiProcess.ProcessCount):
+ if config.UseMetaFD:
+ extraArgs = dict(metaSocket=self.dispatcher.addSocket())
+ else:
+ extraArgs = dict(inheritFDs=self.inheritFDs,
+ inheritSSLFDs=self.inheritSSLFDs)
+ if self.dispenser is not None:
+ extraArgs.update(ampSQLDispenser=self.dispenser)
+ process = TwistdSlaveProcess(
+ sys.argv[0], self.maker.tapname, self.configPath, slaveNumber,
+ config.BindAddresses, **extraArgs
+ )
+ self.monitor.addProcessObject(process, PARENT_ENVIRONMENT)
+
+
+ if (
+ config.Notifications.Enabled and
+ config.Notifications.InternalNotificationHost == "localhost"
+ ):
+ self.maker.log_info("Adding notification service")
+
+ notificationsArgv = [
+ sys.executable,
+ sys.argv[0],
+ ]
+ if config.UserName:
+ notificationsArgv.extend(("-u", config.UserName))
+ if config.GroupName:
+ notificationsArgv.extend(("-g", config.GroupName))
+ notificationsArgv.extend((
+ "--reactor=%s" % (config.Twisted.reactor,),
+ "-n", self.maker.notifierTapName,
+ "-f", self.configPath,
+ ))
+ self.monitor.addProcess("notifications", notificationsArgv,
+ env=PARENT_ENVIRONMENT)
+
+ if (
+ config.Scheduling.iMIP.Enabled and
+ config.Scheduling.iMIP.MailGatewayServer == "localhost"
+ ):
+ self.maker.log_info("Adding mail gateway service")
+
+ mailGatewayArgv = [
+ sys.executable,
+ sys.argv[0],
+ ]
+ if config.UserName:
+ mailGatewayArgv.extend(("-u", config.UserName))
+ if config.GroupName:
+ mailGatewayArgv.extend(("-g", config.GroupName))
+ mailGatewayArgv.extend((
+ "--reactor=%s" % (config.Twisted.reactor,),
+ "-n", self.maker.mailGatewayTapName,
+ "-f", self.configPath,
+ ))
+
+ self.monitor.addProcess("mailgateway", mailGatewayArgv,
+ env=PARENT_ENVIRONMENT)
+
+ self.maker.log_info("Adding task service")
+ taskArgv = [
+ sys.executable,
+ sys.argv[0],
+ ]
+ if config.UserName:
+ taskArgv.extend(("-u", config.UserName))
+ if config.GroupName:
+ taskArgv.extend(("-g", config.GroupName))
+ taskArgv.extend((
+ "--reactor=%s" % (config.Twisted.reactor,),
+ "-n", "caldav_task",
+ "-f", self.configPath,
+ ))
+
+ self.monitor.addProcess(
+ "caldav_task", taskArgv, env=PARENT_ENVIRONMENT
+ )
+
+
+
class CalDAVServiceMaker (LoggingMixIn):
implements(IPlugin, IServiceMaker)
@@ -502,6 +618,26 @@
L{makeService_Combined}, which does the work of actually handling
CalDAV and CardDAV requests.
"""
+ if config.DBAMPFD:
+ txnFactory = transactionFactoryFromFD(int(config.DBAMPFD))
+ elif not config.UseDatabase:
+ txnFactory = None
+ else:
+ raise UsageError(
+ "trying to use DB in slave, but no connection info from parent"
+ )
+ store = storeFromConfig(config, txnFactory)
+ return self.requestProcessingService(options, store)
+
+
+ def requestProcessingService(self, options, store):
+ """
+ Make a service that will actually process HTTP requests.
+
+ This may be a 'Slave' service, which runs as a worker subprocess of the
+ 'Combined' configuration, or a 'Single' service, which is a stand-alone
+ process that answers CalDAV/CardDAV requests by itself.
+ """
#
# Change default log level to "info" as its useful to have
# that during startup
@@ -524,7 +660,7 @@
id = config.ControlSocket
self.log_info("Logging via AF_UNIX: %s" % (id,))
else:
- mode = "IF_INET"
+ mode = "AF_INET"
id = int(config.ControlPort)
self.log_info("Logging via AF_INET: %d" % (id,))
@@ -542,12 +678,13 @@
service = CalDAVService(logObserver)
- rootResource = getRootResource(config, service, additional)
+ rootResource = getRootResource(config, store, additional)
underlyingSite = Site(rootResource)
requestFactory = underlyingSite
if config.RedirectHTTPToHTTPS:
+ self.log_info("Redirecting to HTTPS port %s" % (config.SSLPort,))
def requestFactory(*args, **kw):
return SSLRedirectRequest(site=underlyingSite, *args, **kw)
@@ -569,29 +706,21 @@
# Inherit sockets to call accept() on them individually.
if config.EnableSSL:
- for fd in config.InheritSSLFDs:
- fd = int(fd)
-
+ for fdAsStr in config.InheritSSLFDs:
try:
contextFactory = self.createContextFactory()
except SSLError, e:
log.error("Unable to set up SSL context factory: %s" % (e,))
else:
MaxAcceptSSLServer(
- fd, httpFactory,
+ int(fdAsStr), httpFactory,
contextFactory,
backlog=config.ListenBacklog,
inherit=True
).setServiceParent(service)
-
- for fd in config.InheritFDs:
- fd = int(fd)
-
- if config.RedirectHTTPToHTTPS:
- self.log_info("Redirecting to HTTPS port %s" % (config.SSLPort,))
-
+ for fdAsStr in config.InheritFDs:
MaxAcceptTCPServer(
- fd, httpFactory,
+ int(fdAsStr), httpFactory,
backlog=config.ListenBacklog,
inherit=True
).setServiceParent(service)
@@ -600,8 +729,6 @@
# Inherit a single socket to receive accept()ed connections via
# recvmsg() and SCM_RIGHTS.
- fd = int(config.MetaFD)
-
try:
contextFactory = self.createContextFactory()
except SSLError, e:
@@ -612,31 +739,12 @@
contextFactory = None
ReportingHTTPService(
- requestFactory, fd, contextFactory
+ requestFactory, int(config.MetaFD), contextFactory
).setServiceParent(service)
else: # Not inheriting, therefore we open our own:
-
- if not config.BindAddresses:
- config.BindAddresses = [""]
-
- for bindAddress in config.BindAddresses:
- if config.BindHTTPPorts:
- if config.HTTPPort == 0:
- raise UsageError(
- "HTTPPort required if BindHTTPPorts is not empty"
- )
- elif config.HTTPPort != 0:
- config.BindHTTPPorts = [config.HTTPPort]
-
- if config.BindSSLPorts:
- if config.SSLPort == 0:
- raise UsageError(
- "SSLPort required if BindSSLPorts is not empty"
- )
- elif config.SSLPort != 0:
- config.BindSSLPorts = [config.SSLPort]
-
+ for bindAddress in self._allBindAddresses():
+ self._validatePortConfig()
if config.EnableSSL:
for port in config.BindSSLPorts:
self.log_info("Adding SSL server at %s:%s"
@@ -658,20 +766,6 @@
httpsService.setServiceParent(service)
for port in config.BindHTTPPorts:
-
- if config.RedirectHTTPToHTTPS:
- #
- # Redirect non-SSL ports to the configured SSL port.
- #
- self.log_info("Redirecting HTTP port %s to HTTPS port %s"
- % (port, config.SSLPort)
- )
- else:
- self.log_info(
- "Adding server at %s:%s"
- % (bindAddress, port)
- )
-
MaxAcceptTCPServer(
int(port), httpFactory,
interface=bindAddress,
@@ -682,10 +776,46 @@
# Change log level back to what it was before
setLogLevelForNamespace(None, oldLogLevel)
-
return service
+ def _validatePortConfig(self):
+ """
+ If BindHTTPPorts is specified, HTTPPort must also be specified to
+ indicate which is the preferred port (the one to be used in URL
+ generation, etc). If only HTTPPort is specified, BindHTTPPorts should
+ be set to a list containing only that port number. Similarly for
+ BindSSLPorts/SSLPort.
+
+ @raise UsageError: if configuration is not valid.
+ """
+ if config.BindHTTPPorts:
+ if config.HTTPPort == 0:
+ raise UsageError(
+ "HTTPPort required if BindHTTPPorts is not empty"
+ )
+ elif config.HTTPPort != 0:
+ config.BindHTTPPorts = [config.HTTPPort]
+ if config.BindSSLPorts:
+ if config.SSLPort == 0:
+ raise UsageError(
+ "SSLPort required if BindSSLPorts is not empty"
+ )
+ elif config.SSLPort != 0:
+ config.BindSSLPorts = [config.SSLPort]
+
+
+ def _allBindAddresses(self):
+ """
+ An empty array for the config value of BindAddresses should be
+ equivalent a BindAddresses with a single empty string, meaning "bind
+ everything".
+ """
+ if not config.BindAddresses:
+ config.BindAddresses = [""]
+ return config.BindAddresses
+
+
def scheduleOnDiskUpgrade(self):
"""
Schedule any on disk upgrades we might need. Note that this will only
@@ -703,10 +833,12 @@
"""
self.scheduleOnDiskUpgrade()
- return self.storageService(self.makeService_Slave(options))
+ def slaveSvcCreator(pool, store):
+ return self.requestProcessingService(options, store)
+ return self.storageService(slaveSvcCreator)
- def storageService(self, mainService, uid=None, gid=None):
+ def storageService(self, createMainService, uid=None, gid=None):
"""
If necessary, create a service to be started used for storage; for
example, starting a database backend. This service will then start the
@@ -716,12 +848,13 @@
standalone port-binding until the backing for the selected data store
implementation is ready to process requests.
- @param mainService: This is the service that will be doing the main
+ @param createMainService: This is the service that will be doing the main
work of the current process. If the configured storage mode does
not require any particular setup, then this may return the
C{mainService} argument.
- @type mainService: L{IService}
+ @type mainService: C{callable} that takes C{(connectionPool, store)}
+ and returns L{IService}
@param uid: the user ID to run the backend as, if this process is
running as root.
@@ -736,34 +869,42 @@
@rtype: L{IService}
"""
if config.UseDatabase:
- dbRoot = CachingFilePath(config.DatabaseRoot)
def subServiceFactory(connectionFactory):
- # The database server is running at this point, so do the
- # filesystem->database upgrade.
- attachmentsRoot = dbRoot.child("attachments")
- return UpgradeToDatabaseService.wrapService(
+ ms = ErrorLoggingMultiService()
+ cp = ConnectionPool(connectionFactory)
+ cp.setServiceParent(ms)
+ store = storeFromConfig(config, cp.connection)
+ mainService = createMainService(cp, store)
+ maybeUpgradeSvc = UpgradeToDatabaseService.wrapService(
CachingFilePath(config.DocumentRoot), mainService,
- # FIXME: somehow, this should be a connection pool too, not
- # unpooled connections; this only runs in the master
- # process, so this would be a good point to bootstrap that
- # whole process. However, it's somewhat tricky to do that
- # right. The upgrade needs to run in the master, before
- # any other things have run.
- pgserv.produceLocalTransaction, attachmentsRoot,
- uid=postgresUID, gid=postgresGID
+ store, uid=postgresUID, gid=postgresGID
)
- if os.getuid() == 0: # Only override if root
- postgresUID = uid
- postgresGID = gid
+ maybeUpgradeSvc.setServiceParent(ms)
+ return ms
+ if config.DBType == '':
+ # Spawn our own database as an inferior process, then connect
+ # to it.
+ if os.getuid() == 0: # Only override if root
+ postgresUID = uid
+ postgresGID = gid
+ else:
+ postgresUID = None
+ postgresGID = None
+ pgserv = pgServiceFromConfig(
+ config, subServiceFactory, postgresUID, postgresGID
+ )
+ return pgserv
+ elif config.DBType == 'postgres':
+ # Connect to a postgres database that is already running.
+ import pgdb
+ return subServiceFactory(
+ DBAPIConnector(
+ pgdb, postgresPreflight, config.DSN).connect)
else:
- postgresUID = None
- postgresGID = None
- pgserv = pgServiceFromConfig(
- config, subServiceFactory, postgresUID, postgresGID
- )
- return pgserv
+ raise UsageError("Unknown database type %r" (config.DBType,))
else:
- return mainService
+ store = storeFromConfig(config, None)
+ return createMainService(None, store)
def makeService_Combined(self, options):
@@ -814,28 +955,29 @@
monitor = DelayedStartupProcessMonitor()
s.processMonitor = monitor
+ monitor.setServiceParent(s)
- ssvc = self.storageService(monitor, uid, gid)
- ssvc.setServiceParent(s)
+ for name, pool in config.Memcached.Pools.items():
+ if pool.ServerEnabled:
+ self.log_info(
+ "Adding memcached service for pool: %s" % (name,)
+ )
+ memcachedArgv = [
+ config.Memcached.memcached,
+ "-p", str(pool.Port),
+ "-l", pool.BindAddress,
+ "-U", "0",
+ ]
+ if config.Memcached.MaxMemory is not 0:
+ memcachedArgv.extend(
+ ["-m", str(config.Memcached.MaxMemory)]
+ )
+ if config.UserName:
+ memcachedArgv.extend(["-u", config.UserName])
+ memcachedArgv.extend(config.Memcached.Options)
+ monitor.addProcess('memcached-%s' % (name,), memcachedArgv,
+ env=PARENT_ENVIRONMENT)
- if isinstance(ssvc, PostgresService):
- # TODO: better way of doing this conditional. Look at the config
- # again, possibly?
- pool = ConnectionPool(ssvc.produceConnection)
- pool.setServiceParent(s)
- dispenser = ConnectionDispenser(pool)
- else:
- dispenser = None
-
- parentEnv = {
- "PATH": os.environ.get("PATH", ""),
- "PYTHONPATH": os.environ.get("PYTHONPATH", ""),
- "LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", ""),
- "DYLD_LIBRARY_PATH": os.environ.get("DYLD_LIBRARY_PATH", ""),
- }
- if "KRB5_KTNAME" in os.environ:
- parentEnv["KRB5_KTNAME"] = os.environ["KRB5_KTNAME"]
-
#
# Calculate the number of processes to spawn
#
@@ -850,10 +992,6 @@
# Open the socket(s) to be inherited by the slaves
-
- if not config.BindAddresses:
- config.BindAddresses = [""]
-
inheritFDs = []
inheritSSLFDs = []
@@ -865,23 +1003,8 @@
else:
s._inheritedSockets = [] # keep a reference to these so they don't close
- for bindAddress in config.BindAddresses:
- if config.BindHTTPPorts:
- if config.HTTPPort == 0:
- raise UsageError(
- "HTTPPort required if BindHTTPPorts is not empty"
- )
- elif config.HTTPPort != 0:
- config.BindHTTPPorts = [config.HTTPPort]
-
- if config.BindSSLPorts:
- if config.SSLPort == 0:
- raise UsageError(
- "SSLPort required if BindSSLPorts is not empty"
- )
- elif config.SSLPort != 0:
- config.BindSSLPorts = [config.SSLPort]
-
+ for bindAddress in self._allBindAddresses():
+ self._validatePortConfig()
if config.UseMetaFD:
portsList = [(config.BindHTTPPorts, "TCP")]
if config.EnableSSL:
@@ -909,106 +1032,8 @@
sock = _openSocket(bindAddress, int(portNum))
inheritSSLFDs.append(sock.fileno())
- for p in xrange(0, config.MultiProcess.ProcessCount):
- if config.UseMetaFD:
- extraArgs = dict(metaSocket=cl.dispatcher.addSocket())
- else:
- extraArgs = dict(inheritFDs=inheritFDs,
- inheritSSLFDs=inheritSSLFDs)
- if dispenser is not None:
- extraArgs.update(ampSQLDispenser=dispenser)
- process = TwistdSlaveProcess(
- sys.argv[0],
- self.tapname,
- options["config"],
- p,
- config.BindAddresses,
- **extraArgs
- )
- monitor.addProcessObject(process, parentEnv)
-
- for name, pool in config.Memcached.Pools.items():
- if pool.ServerEnabled:
- self.log_info("Adding memcached service for pool: %s" % (name,))
-
- memcachedArgv = [
- config.Memcached.memcached,
- "-p", str(pool.Port),
- "-l", pool.BindAddress,
- "-U", "0",
- ]
-
- if config.Memcached.MaxMemory is not 0:
- memcachedArgv.extend(["-m", str(config.Memcached.MaxMemory)])
- if config.UserName:
- memcachedArgv.extend(["-u", config.UserName])
-
- memcachedArgv.extend(config.Memcached.Options)
-
- monitor.addProcess('memcached-%s' % (name,), memcachedArgv, env=parentEnv)
-
- if (
- config.Notifications.Enabled and
- config.Notifications.InternalNotificationHost == "localhost"
- ):
- self.log_info("Adding notification service")
-
- notificationsArgv = [
- sys.executable,
- sys.argv[0],
- ]
- if config.UserName:
- notificationsArgv.extend(("-u", config.UserName))
- if config.GroupName:
- notificationsArgv.extend(("-g", config.GroupName))
- notificationsArgv.extend((
- "--reactor=%s" % (config.Twisted.reactor,),
- "-n", self.notifierTapName,
- "-f", options["config"],
- ))
- monitor.addProcess("notifications", notificationsArgv,
- env=parentEnv)
-
- if (
- config.Scheduling.iMIP.Enabled and
- config.Scheduling.iMIP.MailGatewayServer == "localhost"
- ):
- self.log_info("Adding mail gateway service")
-
- mailGatewayArgv = [
- sys.executable,
- sys.argv[0],
- ]
- if config.UserName:
- mailGatewayArgv.extend(("-u", config.UserName))
- if config.GroupName:
- mailGatewayArgv.extend(("-g", config.GroupName))
- mailGatewayArgv.extend((
- "--reactor=%s" % (config.Twisted.reactor,),
- "-n", self.mailGatewayTapName,
- "-f", options["config"],
- ))
-
- monitor.addProcess("mailgateway", mailGatewayArgv, env=parentEnv)
-
- self.log_info("Adding task service")
- taskArgv = [
- sys.executable,
- sys.argv[0],
- ]
- if config.UserName:
- taskArgv.extend(("-u", config.UserName))
- if config.GroupName:
- taskArgv.extend(("-g", config.GroupName))
- taskArgv.extend((
- "--reactor=%s" % (config.Twisted.reactor,),
- "-n", "caldav_task",
- "-f", options["config"],
- ))
-
- monitor.addProcess("caldav_task", taskArgv, env=parentEnv)
-
-
+ # Start listening on the stats socket, for administrators to inspect
+ # the current stats on the server.
stats = CalDAVStatisticsServer(logger)
statsService = GroupOwnedUNIXServer(
gid, config.GlobalStatsSocket, stats, mode=0440
@@ -1016,6 +1041,25 @@
statsService.setName("stats")
statsService.setServiceParent(s)
+ # Finally, let's get the real show on the road. Create a service that
+ # will spawn all of our worker processes when started, and wrap that
+ # service in zero to two necessary layers before it's started: first,
+ # the service which spawns a subsidiary database (if that's necessary,
+ # and we don't have an external, already-running database to connect
+ # to), and second, the service which does an upgrade from the
+ # filesystem to the database (if that's necessary, and there is
+ # filesystem data in need of upgrading).
+ def spawnerSvcCreator(pool, store):
+ if pool is not None:
+ dispenser = ConnectionDispenser(pool)
+ else:
+ dispenser = None
+ return SlaveSpawnerService(
+ self, monitor, dispenser, cl.dispatcher, options["config"],
+ inheritFDs=inheritFDs, inheritSSLFDs=inheritSSLFDs
+ )
+ ssvc = self.storageService(spawnerSvcCreator, uid, gid)
+ ssvc.setServiceParent(s)
return s
Modified: CalendarServer/trunk/calendarserver/tap/test/test_caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/calendarserver/tap/test/test_caldav.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -34,7 +34,7 @@
from twisted.internet.defer import Deferred
from twisted.internet.task import Clock
-from twisted.application.service import IService
+from twisted.application.service import IService, IServiceCollection
from twisted.application import internet
from twext.web2.dav import auth
@@ -114,8 +114,10 @@
if self.processTransports:
return self.processTransports.pop(0)
else:
- print 'wth', self.calls
- raise AssertionError("There were no process transports available.")
+ raise AssertionError(
+ "There were no process transports available. Calls: " +
+ repr(self.calls)
+ )
def spawnProcess(self, processProtocol, executable, args=(), env={},
@@ -310,9 +312,10 @@
"type": "twistedcaldav.directory.augment.AugmentXMLDB"
}
+ self.config.UseDatabase = False
self.config.ServerRoot = self.mktemp()
self.config.ConfigRoot = "config"
- self.config.ProcessType = "Slave"
+ self.config.ProcessType = "Single"
self.config.SSLPrivateKey = pemFile
self.config.SSLCertificate = pemFile
self.config.EnableSSL = True
@@ -334,16 +337,19 @@
self.writeConfig()
+
def tearDown(self):
config.setDefaults(DEFAULT_CONFIG)
config.reset()
+
def writeConfig(self):
"""
Flush self.config out to self.configFile
"""
writePlist(self.config, self.configFile)
+
def makeService(self):
"""
Create a service by calling into CalDAVServiceMaker with
@@ -353,17 +359,39 @@
return CalDAVServiceMaker().makeService(self.options)
+
def getSite(self):
"""
- Get the server.Site from the service by finding the HTTPFactory
+ Get the server.Site from the service by finding the HTTPFactory.
"""
service = self.makeService()
+ for listeningService in inServiceHierarchy(
+ service,
+ # FIXME: need a better predicate for 'is this really an HTTP
+ # factory' but this works for now.
+ # NOTE: in a database 'single' configuration, PostgresService
+ # will prevent the HTTP services from actually getting added to
+ # the hierarchy until the hierarchy has started.
+ lambda x: hasattr(x, 'args')
+ ):
+ return listeningService.args[1].protocolArgs['requestFactory']
+ raise RuntimeError("No site found.")
- # FIXME: should at least use service name, not index
- return service.services[2].args[1].protocolArgs["requestFactory"]
+def inServiceHierarchy(svc, predicate):
+ """
+ Find services in the service collection which satisfy the given predicate.
+ """
+ for subsvc in svc.services:
+ if IServiceCollection.providedBy(subsvc):
+ for value in inServiceHierarchy(subsvc, predicate):
+ yield value
+ if predicate(subsvc):
+ yield subsvc
+
+
def determineAppropriateGroupID():
"""
Determine a secondary group ID which can be used for testing.
@@ -451,8 +479,26 @@
self.assertEquals(socketService.gid, alternateGroup)
+ def test_processMonitor(self):
+ """
+ In the master, there should be exactly one
+ L{DelayedStartupProcessMonitor} in the service hierarchy so that it
+ will be started by startup.
+ """
+ self.config["ProcessType"] = "Combined"
+ self.writeConfig()
+ self.assertEquals(
+ 1,
+ len(
+ list(inServiceHierarchy(
+ self.makeService(),
+ lambda x: isinstance(x, DelayedStartupProcessMonitor)))
+ )
+ )
+
+
class SlaveServiceTest(BaseServiceMakerTests):
"""
Test various configurations of the Slave service
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -68,7 +68,6 @@
NegotiateCredentialFactory # pacify pyflakes
except ImportError:
NegotiateCredentialFactory = None
-from txdav.base.datastore.asyncsqlpool import ConnectionPool
from txdav.base.datastore.asyncsqlpool import ConnectionPoolClient
from calendarserver.accesslog import DirectoryLogWrapperResource
@@ -130,33 +129,45 @@
def getHost(self):
return "<host: %r %r>" % (self.socket.fileno(), id(self))
-def storeFromConfig(config, serviceParent, notifierFactory=None):
+
+def transactionFactoryFromFD(dbampfd):
"""
- Produce an L{IDataStore} from the given configuration and notifier factory.
+ Create a transaction factory from an inherited file descriptor.
"""
- if config.UseDatabase:
- postgresService = pgServiceFromConfig(config, None)
- if config.DBAMPFD == 0:
- cp = ConnectionPool(postgresService.produceConnection)
- cp.setServiceParent(serviceParent)
- txnFactory = cp.connection
- else:
- # TODO: something to do with loseConnection here, maybe? I don't
- # think it actually needs to be shut down, though.
- skt = fromfd(int(config.DBAMPFD), AF_UNIX, SOCK_STREAM)
- os.close(config.DBAMPFD)
- protocol = ConnectionPoolClient()
- transport = ConnectionWithPeer(skt, protocol)
- protocol.makeConnection(transport)
- transport.startReading()
- txnFactory = protocol.newTransaction
- dataStore = CommonSQLDataStore(
- txnFactory, notifierFactory,
- postgresService.dataStoreDirectory.child("attachments"),
+ skt = fromfd(dbampfd, AF_UNIX, SOCK_STREAM)
+ os.close(dbampfd)
+ protocol = ConnectionPoolClient()
+ transport = ConnectionWithPeer(skt, protocol)
+ protocol.makeConnection(transport)
+ transport.startReading()
+ return protocol.newTransaction
+
+
+# txnFacSub(int(config.DBAMPFD))
+
+def storeFromConfig(config, txnFactory):
+ """
+ Produce an L{IDataStore} from the given configuration, transaction factory,
+ and notifier factory.
+
+ If the transaction factory is C{None}, we will create a filesystem
+ store. Otherwise, a SQL store, using that connection information.
+ """
+ #
+ # Configure NotifierFactory
+ #
+ if config.Notifications.Enabled:
+ notifierFactory = NotifierFactory(
+ config.Notifications.InternalNotificationHost,
+ config.Notifications.InternalNotificationPort,
+ )
+ else:
+ notifierFactory = None
+ if txnFactory is not None:
+ return CommonSQLDataStore(
+ txnFactory, notifierFactory, FilePath(config.AttachmentsRoot),
config.EnableCalDAV, config.EnableCardDAV
)
- dataStore.setServiceParent(serviceParent)
- return dataStore
else:
return CommonFileDataStore(FilePath(config.DocumentRoot),
notifierFactory, config.EnableCalDAV, config.EnableCardDAV)
@@ -253,7 +264,6 @@
directory.userRecordTypes.insert(0,
SudoDirectoryService.recordType_sudoers)
-
#
# Use system-wide realm on OSX
#
@@ -269,7 +279,7 @@
return directory
-def getRootResource(config, serviceParent, resources=None):
+def getRootResource(config, newStore, resources=None):
"""
Set up directory service and resource hierarchy based on config.
Return root resource.
@@ -277,8 +287,14 @@
Additional resources can be added to the hierarchy by passing a list of
tuples containing: path, resource class, __init__ args list, and optional
authentication scheme ("basic" or "digest").
+
+ If the store is specified, then it has already been constructed, so use it.
+ Otherwise build one with L{storeFromConfig}.
"""
+ if newStore is None:
+ raise RuntimeError("Internal error, 'newStore' must be specified.")
+
# FIXME: this is only here to workaround circular imports
doBind()
@@ -386,19 +402,6 @@
principalCollection = directory.principalCollection
- #
- # Configure NotifierFactory
- #
- if config.Notifications.Enabled:
- notifierFactory = NotifierFactory(
- config.Notifications.InternalNotificationHost,
- config.Notifications.InternalNotificationPort,
- )
- else:
- notifierFactory = None
-
- newStore = storeFromConfig(config, serviceParent, notifierFactory)
-
if config.EnableCalDAV:
log.info("Setting up calendar collection: %r" % (calendarResourceClass,))
calendarCollection = calendarResourceClass(
Modified: CalendarServer/trunk/twext/internet/sendfdport.py
===================================================================
--- CalendarServer/trunk/twext/internet/sendfdport.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/twext/internet/sendfdport.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -1,6 +1,6 @@
# -*- test-case-name: twext.internet.test.test_sendfdport -*-
##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+# Copyright (c) 2010 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -162,6 +162,7 @@
self.statusWatcher = statusWatcher
from twisted.internet import reactor
self.reactor = reactor
+ self._isDispatching = False
@property
@@ -202,6 +203,7 @@
"""
Start listening on all subprocess sockets.
"""
+ self._isDispatching = True
for subSocket in self._subprocessSockets:
subSocket.startReading()
@@ -218,6 +220,8 @@
i, o = socketpair(AF_UNIX, SOCK_DGRAM)
a = _SubprocessSocket(self, o)
self._subprocessSockets.append(a)
+ if self._isDispatching:
+ a.startReading()
return i
Deleted: CalendarServer/trunk/twext/internet/test/__init__.py
===================================================================
Copied: CalendarServer/trunk/twext/internet/test/__init__.py (from rev 6614, CalendarServer/branches/users/glyph/dont-start-postgres/twext/internet/test/__init__.py)
===================================================================
Deleted: CalendarServer/trunk/twext/internet/test/test_sendfdport.py
===================================================================
--- CalendarServer/branches/users/glyph/dont-start-postgres/twext/internet/test/test_sendfdport.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/twext/internet/test/test_sendfdport.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -1,57 +0,0 @@
-# -*- test-case-name: twext.internet.test.test_sendfdport -*-
-##
-# Copyright (c) 2010 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-Tests for L{twext.internet.sendfdport}.
-"""
-
-from twisted.trial.unittest import TestCase
-from twext.internet.sendfdport import InheritedSocketDispatcher
-from twisted.internet.interfaces import IReactorFDSet
-from zope.interface.declarations import implements
-
-
-class ReaderAdder(object):
- implements(IReactorFDSet)
-
- def __init__(self):
- self.readers = []
-
- def addReader(self, reader):
- self.readers.append(reader)
-
- def getReaders(self):
- return self.readers[:]
-
-
-
-class InheritedSocketDispatcherTests(TestCase):
- """
- Inherited socket dispatcher tests.
- """
-
- def test_addAfterStart(self):
- """
- Adding a socket to an L{InheritedSocketDispatcher} after it has already
- been started results in it immediately starting reading.
- """
- reactor = ReaderAdder()
- dispatcher = InheritedSocketDispatcher(None)
- dispatcher.reactor = reactor
- dispatcher.startDispatching()
- dispatcher.addSocket()
- self.assertEquals(reactor.getReaders(), dispatcher._subprocessSockets)
Copied: CalendarServer/trunk/twext/internet/test/test_sendfdport.py (from rev 6614, CalendarServer/branches/users/glyph/dont-start-postgres/twext/internet/test/test_sendfdport.py)
===================================================================
--- CalendarServer/trunk/twext/internet/test/test_sendfdport.py (rev 0)
+++ CalendarServer/trunk/twext/internet/test/test_sendfdport.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -0,0 +1,57 @@
+# -*- test-case-name: twext.internet.test.test_sendfdport -*-
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Tests for L{twext.internet.sendfdport}.
+"""
+
+from twisted.trial.unittest import TestCase
+from twext.internet.sendfdport import InheritedSocketDispatcher
+from twisted.internet.interfaces import IReactorFDSet
+from zope.interface.declarations import implements
+
+
+class ReaderAdder(object):
+ implements(IReactorFDSet)
+
+ def __init__(self):
+ self.readers = []
+
+ def addReader(self, reader):
+ self.readers.append(reader)
+
+ def getReaders(self):
+ return self.readers[:]
+
+
+
+class InheritedSocketDispatcherTests(TestCase):
+ """
+ Inherited socket dispatcher tests.
+ """
+
+ def test_addAfterStart(self):
+ """
+ Adding a socket to an L{InheritedSocketDispatcher} after it has already
+ been started results in it immediately starting reading.
+ """
+ reactor = ReaderAdder()
+ dispatcher = InheritedSocketDispatcher(None)
+ dispatcher.reactor = reactor
+ dispatcher.startDispatching()
+ dispatcher.addSocket()
+ self.assertEquals(reactor.getReaders(), dispatcher._subprocessSockets)
Modified: CalendarServer/trunk/twext/python/sendmsg.c
===================================================================
--- CalendarServer/trunk/twext/python/sendmsg.c 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/twext/python/sendmsg.c 2010-11-11 07:47:38 UTC (rev 6615)
@@ -158,11 +158,16 @@
Py_DECREF(iterator);
iterator = NULL;
- /* Allocate the buffer for all of the ancillary elements. */
- message_header.msg_control = malloc(all_data_len);
- if (!message_header.msg_control) {
- PyErr_NoMemory();
- return NULL;
+ /* Allocate the buffer for all of the ancillary elements, if we have
+ * any. */
+ if (all_data_len) {
+ message_header.msg_control = malloc(all_data_len);
+ if (!message_header.msg_control) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ } else {
+ message_header.msg_control = NULL;
}
message_header.msg_controllen = all_data_len;
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -153,8 +153,18 @@
"UseMetaFD" : True, # Use a 'meta' FD, i.e. an FD to transmit other FDs
# to slave processes.
- "UseDatabase" : True, # True: postgres; False: files
+ "UseDatabase" : True, # True: database; False: files
+ "DBType" : "", # 2 possible values: empty, meaning 'spawn postgres
+ # yourself', or 'postgres', meaning 'connect to a
+ # postgres database as specified by the 'DSN'
+ # configuration key. Will support more values in
+ # the future.
+
+ "DSN" : "", # Data Source Name. Used to connect to an external
+ # database if DBType is non-empty. Format varies
+ # depending on database type.
+
"DBAMPFD" : 0, # Internally used by database to tell slave
# processes to inherit a file descriptor and use it
# as an AMP connection over a UNIX socket; see
@@ -172,6 +182,7 @@
"ServerRoot" : "/var/db/caldavd",
"DataRoot" : "Data",
"DatabaseRoot" : "Database",
+ "AttachmentsRoot" : "Attachments",
"DocumentRoot" : "Documents",
"ConfigRoot" : "/etc/caldavd",
"LogRoot" : "/var/log/caldavd",
@@ -666,6 +677,7 @@
("ServerRoot", "LogRoot"),
("ServerRoot", "RunRoot"),
("DataRoot", "DatabaseRoot"),
+ ("DataRoot", "AttachmentsRoot"),
("ConfigRoot", "SudoersFile"),
("LogRoot", "AccessLogFile"),
("LogRoot", "ErrorLogFile"),
Copied: CalendarServer/trunk/txdav/base/datastore/dbapiclient.py (from rev 6614, CalendarServer/branches/users/glyph/dont-start-postgres/txdav/base/datastore/dbapiclient.py)
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/dbapiclient.py (rev 0)
+++ CalendarServer/trunk/txdav/base/datastore/dbapiclient.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -0,0 +1,149 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+General utility client code for interfacing with DB-API 2.0 modules.
+"""
+
+class DiagnosticCursorWrapper(object):
+ """
+ Diagnostic wrapper around a DB-API 2.0 cursor for debugging connection
+ status.
+ """
+
+ def __init__(self, realCursor, connectionWrapper):
+ self.realCursor = realCursor
+ self.connectionWrapper = connectionWrapper
+
+
+ @property
+ def rowcount(self):
+ return self.realCursor.rowcount
+
+
+ @property
+ def description(self):
+ return self.realCursor.description
+
+
+ def execute(self, sql, args=()):
+ self.connectionWrapper.state = 'executing %r' % (sql,)
+# Use log.debug
+# sys.stdout.write(
+# "Really executing SQL %r in thread %r\n" %
+# ((sql % tuple(args)), thread.get_ident())
+# )
+ self.realCursor.execute(sql, args)
+
+
+ def close(self):
+ self.realCursor.close()
+
+
+ def fetchall(self):
+ results = self.realCursor.fetchall()
+# Use log.debug
+# sys.stdout.write(
+# "Really fetching results %r thread %r\n" %
+# (results, thread.get_ident())
+# )
+ return results
+
+
+
+class DiagnosticConnectionWrapper(object):
+ """
+ Diagnostic wrapper around a DB-API 2.0 connection for debugging connection
+ status.
+ """
+
+ def __init__(self, realConnection, label):
+ self.realConnection = realConnection
+ self.label = label
+ self.state = 'idle (start)'
+
+
+ def cursor(self):
+ return DiagnosticCursorWrapper(self.realConnection.cursor(), self)
+
+
+ def close(self):
+ self.realConnection.close()
+ self.state = 'closed'
+
+
+ def commit(self):
+ self.realConnection.commit()
+ self.state = 'idle (after commit)'
+
+
+ def rollback(self):
+ self.realConnection.rollback()
+ self.state = 'idle (after rollback)'
+
+
+
+class DBAPIConnector(object):
+ """
+ A simple wrapper for DB-API connectors.
+
+ @ivar dbModule: the DB-API module to use.
+ """
+
+ def __init__(self, dbModule, preflight, *connectArgs, **connectKw):
+ self.dbModule = dbModule
+ self.connectArgs = connectArgs
+ self.connectKw = connectKw
+ self.preflight = preflight
+
+
+ def connect(self, label="<unlabeled>"):
+ connection = self.dbModule.connect(*self.connectArgs, **self.connectKw)
+ w = DiagnosticConnectionWrapper(connection, label)
+ self.preflight(w)
+ return w
+
+
+
+def postgresPreflight(connection):
+ """
+ Pre-flight function for PostgreSQL connections: enable standard conforming
+ strings, and set a non-infinite statement timeout.
+ """
+ c = connection.cursor()
+
+ # Turn on standard conforming strings. This option is _required_ if
+ # you want to get correct behavior out of parameter-passing with the
+ # pgdb module. If it is not set then the server is potentially
+ # vulnerable to certain types of SQL injection.
+ c.execute("set standard_conforming_strings=on")
+
+ # Abort any second that takes more than 30 seconds (30000ms) to
+ # execute. This is necessary as a temporary workaround since it's
+ # hypothetically possible that different database operations could
+ # block each other, while executing SQL in the same process (in the
+ # same thread, since SQL executes in the main thread now). It's
+ # preferable to see some exceptions while we're in this state than to
+ # have the entire worker process hang.
+ c.execute("set statement_timeout=30000")
+
+ # pgdb (as per DB-API 2.0) automatically puts the connection into a
+ # 'executing a transaction' state when _any_ statement is executed on
+ # it (even these not-touching-any-data statements); make sure to commit
+ # first so that the application sees a fresh transaction, and the
+ # connection can safely be pooled without executing anything on it.
+ connection.commit()
+ c.close()
Modified: CalendarServer/trunk/txdav/base/datastore/subpostgres.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -36,7 +36,8 @@
from twisted.protocols.basic import LineReceiver
from twisted.internet import reactor
from twisted.internet.defer import Deferred
-from txdav.base.datastore.asyncsqlpool import BaseSqlTxn
+from txdav.base.datastore.dbapiclient import DBAPIConnector
+from txdav.base.datastore.dbapiclient import postgresPreflight
from twisted.application.service import MultiService
@@ -47,84 +48,6 @@
_MAGIC_READY_COOKIE = "database system is ready to accept connections"
-class DiagnosticCursorWrapper(object):
- """
- Diagnostic wrapper around a DB-API 2.0 cursor for debugging connection
- status.
- """
-
- def __init__(self, realCursor, connectionWrapper):
- self.realCursor = realCursor
- self.connectionWrapper = connectionWrapper
-
-
- @property
- def rowcount(self):
- return self.realCursor.rowcount
-
-
- @property
- def description(self):
- return self.realCursor.description
-
-
- def execute(self, sql, args=()):
- self.connectionWrapper.state = 'executing %r' % (sql,)
-# Use log.debug
-# sys.stdout.write(
-# "Really executing SQL %r in thread %r\n" %
-# ((sql % tuple(args)), thread.get_ident())
-# )
- self.realCursor.execute(sql, args)
-
-
- def close(self):
- self.realCursor.close()
-
-
- def fetchall(self):
- results = self.realCursor.fetchall()
-# Use log.debug
-# sys.stdout.write(
-# "Really fetching results %r thread %r\n" %
-# (results, thread.get_ident())
-# )
- return results
-
-
-
-class DiagnosticConnectionWrapper(object):
- """
- Diagnostic wrapper around a DB-API 2.0 connection for debugging connection
- status.
- """
-
- def __init__(self, realConnection, label):
- self.realConnection = realConnection
- self.label = label
- self.state = 'idle (start)'
-
-
- def cursor(self):
- return DiagnosticCursorWrapper(self.realConnection.cursor(), self)
-
-
- def close(self):
- self.realConnection.close()
- self.state = 'closed'
-
-
- def commit(self):
- self.realConnection.commit()
- self.state = 'idle (after commit)'
-
-
- def rollback(self):
- self.realConnection.rollback()
- self.state = 'idle (after rollback)'
-
-
-
class _PostgresMonitor(ProcessProtocol):
"""
A monitoring protocol which watches the postgres subprocess.
@@ -231,23 +154,6 @@
-class UnpooledSqlTxn(BaseSqlTxn):
- """
- Unpooled variant (releases thread immediately on commit or abort),
- currently exclusively for testing.
- """
- def commit(self):
- result = super(UnpooledSqlTxn, self).commit()
- self.stop()
- return result
-
- def abort(self):
- result = super(UnpooledSqlTxn, self).abort()
- self.stop()
- return result
-
-
-
class PostgresService(MultiService):
def __init__(self, dataStoreDirectory, subServiceFactory,
@@ -335,10 +241,7 @@
self.shutdownDeferred.callback(None)
- def produceConnection(self, label="<unlabeled>", databaseName=None):
- """
- Produce a DB-API 2.0 connection pointed at this database.
- """
+ def _connectorFor(self, databaseName=None):
if databaseName is None:
databaseName = self.databaseName
@@ -347,41 +250,14 @@
pwd.getpwuid(self.uid).pw_name)
else:
dsn = "%s:dbname=%s" % (self.host, databaseName)
- connection = pgdb.connect(dsn)
+ return DBAPIConnector(pgdb, postgresPreflight, dsn)
- w = DiagnosticConnectionWrapper(connection, label)
- c = w.cursor()
- # Turn on standard conforming strings. This option is _required_ if
- # you want to get correct behavior out of parameter-passing with the
- # pgdb module. If it is not set then the server is potentially
- # vulnerable to certain types of SQL injection.
- c.execute("set standard_conforming_strings=on")
-
- # Abort any second that takes more than 30 seconds (30000ms) to
- # execute. This is necessary as a temporary workaround since it's
- # hypothetically possible that different database operations could
- # block each other, while executing SQL in the same process (in the
- # same thread, since SQL executes in the main thread now). It's
- # preferable to see some exceptions while we're in this state than to
- # have the entire worker process hang.
- c.execute("set statement_timeout=30000")
-
- # pgdb (as per DB-API 2.0) automatically puts the connection into a
- # 'executing a transaction' state when _any_ statement is executed on
- # it (even these not-touching-any-data statements); make sure to commit
- # first so that the application sees a fresh transaction, and the
- # connection can safely be pooled without executing anything on it.
- w.commit()
- c.close()
- return w
-
-
- def produceLocalTransaction(self, label="<unlabeled>"):
+ def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
- Create a L{IAsyncTransaction} based on a thread in the current process.
+ Produce a DB-API 2.0 connection pointed at this database.
"""
- return UnpooledSqlTxn(lambda : self.produceConnection(label))
+ return self._connectorFor(databaseName).connect(label)
def ready(self):
Property changes on: CalendarServer/trunk/txdav/caldav/datastore/index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/caldav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/index.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/caldav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/index.py:6322-6394
Property changes on: CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/caldav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/test/test_index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_index.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/caldav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/test/test_index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_index.py:6322-6394
Property changes on: CalendarServer/trunk/txdav/carddav/datastore/index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/carddav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/vcardindex.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/carddav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/vcardindex.py:6322-6394
Property changes on: CalendarServer/trunk/txdav/carddav/datastore/test/test_index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/carddav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/test/test_index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_vcardindex.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/carddav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/test/test_index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_vcardindex.py:6322-6394
Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -34,8 +34,8 @@
from twisted.application.service import Service
from txdav.common.datastore.sql import CommonDataStore, v1_schema
-from txdav.base.datastore.subpostgres import PostgresService,\
- DiagnosticConnectionWrapper
+from txdav.base.datastore.subpostgres import PostgresService
+from txdav.base.datastore.dbapiclient import DiagnosticConnectionWrapper
from txdav.common.icommondatastore import NoSuchHomeChildError
from txdav.base.datastore.asyncsqlpool import ConnectionPool
from twisted.internet.defer import returnValue
Modified: CalendarServer/trunk/txdav/common/datastore/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/util.py 2010-11-11 05:58:24 UTC (rev 6614)
+++ CalendarServer/trunk/txdav/common/datastore/util.py 2010-11-11 07:47:38 UTC (rev 6615)
@@ -19,7 +19,6 @@
from twext.python.log import LoggingMixIn
from twisted.application.service import Service
from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
-from txdav.common.datastore.sql import CommonDataStore as SqlStore
from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
from twisted.internet.defer import inlineCallbacks
@@ -31,20 +30,12 @@
Upgrade resources from a filesystem store to a database store.
"""
-
@classmethod
- def wrapService(cls, path, service, connectionFactory, sqlAttachmentsPath,
- uid=None, gid=None):
+ def wrapService(cls, path, service, store, uid=None, gid=None):
"""
Create an L{UpgradeToDatabaseService} if there are still file-based
calendar or addressbook homes remaining in the given path.
- Maintenance note: we may want to pass a SQL store in directly rather
- than the combination of connection factory and attachments path, since
- there always must be a SQL store, but the path should remain a path
- because there may not I{be} a file-backed store present and we should
- not create it as a result of checking for it.
-
@param path: a path pointing at the document root.
@type path: L{CachingFilePath}
@@ -55,35 +46,32 @@
service parent of the resulting service will be set to a
L{MultiService} or similar.)
+ @param store: the SQL storage service.
+
@type service: L{IService}
@return: a service
@rtype: L{IService}
"""
+ # TODO: TOPPATHS should be computed based on enabled flags in 'store',
+ # not hard coded.
for homeType in TOPPATHS:
if path.child(homeType).exists():
self = cls(
FileStore(path, None, True, True),
- SqlStore(connectionFactory, None, sqlAttachmentsPath,
- True, True),
- service,
- sqlAttachmentsPath=sqlAttachmentsPath,
- uid=uid,
- gid=gid,
+ store, service, uid=uid, gid=gid,
)
return self
return service
- def __init__(self, fileStore, sqlStore, service, sqlAttachmentsPath=None,
- uid=None, gid=None):
+ def __init__(self, fileStore, sqlStore, service, uid=None, gid=None):
"""
Initialize the service.
"""
self.wrappedService = service
self.fileStore = fileStore
self.sqlStore = sqlStore
- self.sqlAttachmentsPath = sqlAttachmentsPath
self.uid = uid
self.gid = gid
@@ -144,9 +132,11 @@
if homesPath.isdir():
homesPath.remove()
- # Set attachment directory ownership
- if (self.sqlAttachmentsPath and
- self.sqlAttachmentsPath.exists() and
+ # Set attachment directory ownership. FIXME: is this still necessary
+ # since attachments started living outside the database directory
+ # created by initdb? default permissions might be correct now.
+ sqlAttachmentsPath = self.sqlStore.attachmentsPath
+ if (sqlAttachmentsPath and sqlAttachmentsPath.exists() and
(self.uid or self.gid)):
uid = self.uid or -1
gid = self.gid or -1
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20101110/92c2b952/attachment-0001.html>
More information about the calendarserver-changes
mailing list