[CalendarServer-changes] [6610] CalendarServer/branches/users/glyph/dont-start-postgres
source_changes at macosforge.org
source_changes at macosforge.org
Wed Nov 10 18:56:46 PST 2010
Revision: 6610
http://trac.macosforge.org/projects/calendarserver/changeset/6610
Author: glyph at apple.com
Date: 2010-11-10 18:56:44 -0800 (Wed, 10 Nov 2010)
Log Message:
-----------
make connection pool creation solely the responsibility of storageService; make storeFromConfig require a transactionFactory so that the configuration is only examined once.
Modified Paths:
--------------
CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/caldav.py
CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/util.py
CalendarServer/branches/users/glyph/dont-start-postgres/txdav/common/datastore/util.py
Modified: CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/caldav.py 2010-11-11 02:56:27 UTC (rev 6609)
+++ CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/caldav.py 2010-11-11 02:56:44 UTC (rev 6610)
@@ -69,7 +69,6 @@
from twistedcaldav.mail import IMIPReplyInboxResource
from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twistedcaldav.upgrade import upgradeData
-from txdav.base.datastore.subpostgres import PostgresService
import calendarserver.tap.profiling # Imported for side-effect
@@ -77,6 +76,8 @@
from txdav.base.datastore.asyncsqlpool import ConnectionPool
from txdav.base.datastore.asyncsqlpool import ConnectionPoolConnection
+from txdav.base.datastore.dbapiclient import DBAPIConnector
+from txdav.base.datastore.dbapiclient import postgresPreflight
try:
from twistedcaldav.authkerb import NegotiateCredentialFactory
@@ -89,6 +90,8 @@
from calendarserver.accesslog import RotatingFileAccessLoggingObserver
from calendarserver.tap.util import getRootResource, computeProcessCount
from calendarserver.tap.util import ConnectionWithPeer
+from calendarserver.tap.util import storeFromConfig
+from calendarserver.tap.util import transactionFactoryFromFD
from calendarserver.tools.util import checkDirectory
try:
@@ -617,6 +620,26 @@
L{makeService_Combined}, which does the work of actually handling
CalDAV and CardDAV requests.
"""
+ if config.DBAMPFD:
+ txnFactory = transactionFactoryFromFD(int(config.DBAMPFD))
+ elif not config.UseDatabase:
+ txnFactory = None
+ else:
+ raise UsageError(
+ "trying to use DB in slave, but no connection info from parent"
+ )
+ store = storeFromConfig(config, txnFactory)
+ return self.requestProcessingService(options, store)
+
+
+ def requestProcessingService(self, options, store):
+ """
+ Make a service that will actually process HTTP requests.
+
+ This may be a 'Slave' service, which runs as a worker subprocess of the
+ 'Combined' configuration, or a 'Single' service, which is a stand-alone
+ process that answers CalDAV/CardDAV requests by itself.
+ """
#
# Change default log level to "info" as its useful to have
# that during startup
@@ -657,7 +680,7 @@
service = CalDAVService(logObserver)
- rootResource = getRootResource(config, service, additional)
+ rootResource = getRootResource(config, service, additional, store)
underlyingSite = Site(rootResource)
requestFactory = underlyingSite
@@ -813,10 +836,12 @@
"""
self.scheduleOnDiskUpgrade()
- return self.storageService(self.makeService_Slave(options))
+ def slaveSvcCreator(pool, store):
+ return self.requestProcessingService(options, store)
+ return self.storageService(slaveSvcCreator)
- def storageService(self, mainService, uid=None, gid=None):
+ def storageService(self, createMainService, uid=None, gid=None):
"""
If necessary, create a service to be started used for storage; for
example, starting a database backend. This service will then start the
@@ -826,12 +851,13 @@
standalone port-binding until the backing for the selected data store
implementation is ready to process requests.
- @param mainService: This is the service that will be doing the main
+ @param createMainService: This is the service that will be doing the main
work of the current process. If the configured storage mode does
not require any particular setup, then this may return the
C{mainService} argument.
- @type mainService: L{IService}
+ @type mainService: C{callable} that takes C{(connectionPool, store)}
+ and returns L{IService}
@param uid: the user ID to run the backend as, if this process is
running as root.
@@ -847,30 +873,41 @@
"""
if config.UseDatabase:
def subServiceFactory(connectionFactory):
- # The database server is running at this point, so do the
- # filesystem->database upgrade.
ms = ErrorLoggingMultiService()
cp = ConnectionPool(connectionFactory)
cp.setServiceParent(ms)
+ store = storeFromConfig(config, cp.connection)
+ mainService = createMainService(cp, store)
maybeUpgradeSvc = UpgradeToDatabaseService.wrapService(
CachingFilePath(config.DocumentRoot), mainService,
- cp.connection, CachingFilePath(config.AttachmentsRoot),
- uid=postgresUID, gid=postgresGID
+ store, uid=postgresUID, gid=postgresGID
)
maybeUpgradeSvc.setServiceParent(ms)
return ms
- if os.getuid() == 0: # Only override if root
- postgresUID = uid
- postgresGID = gid
+ if config.DBType == '':
+ # Spawn our own database as an inferior process, then connect
+ # to it.
+ if os.getuid() == 0: # Only override if root
+ postgresUID = uid
+ postgresGID = gid
+ else:
+ postgresUID = None
+ postgresGID = None
+ pgserv = pgServiceFromConfig(
+ config, subServiceFactory, postgresUID, postgresGID
+ )
+ return pgserv
+ elif config.DBType == 'postgres':
+ # Connect to a postgres database that is already running.
+ import pgdb
+ return subServiceFactory(
+ DBAPIConnector(
+ pgdb, postgresPreflight, config.DSN).connect)
else:
- postgresUID = None
- postgresGID = None
- pgserv = pgServiceFromConfig(
- config, subServiceFactory, postgresUID, postgresGID
- )
- return pgserv
+ raise UsageError("Unknown database type %r" (config.DBType,))
else:
- return mainService
+ store = storeFromConfig(config, None)
+ return createMainService(None, store)
def makeService_Combined(self, options):
@@ -1014,22 +1051,17 @@
# to), and second, the service which does an upgrade from the
# filesystem to the database (if that's necessary, and there is
# filesystem data in need of upgrading).
- ssvc = self.storageService(monitor, uid, gid)
+ def spawnerSvcCreator(pool, store):
+ if pool is not None:
+ dispenser = ConnectionDispenser(pool)
+ else:
+ dispenser = None
+ return SlaveSpawnerService(
+ self, monitor, dispenser, cl.dispatcher, options["config"],
+ inheritFDs=inheritFDs, inheritSSLFDs=inheritSSLFDs
+ )
+ ssvc = self.storageService(spawnerSvcCreator, uid, gid)
ssvc.setServiceParent(s)
-
- if isinstance(ssvc, PostgresService):
- # TODO: better way of doing this conditional. Look at the config
- # again, possibly?
- pool = ConnectionPool(ssvc.produceConnection)
- pool.setServiceParent(s)
- dispenser = ConnectionDispenser(pool)
- else:
- dispenser = None
-
- SlaveSpawnerService(
- self, monitor, dispenser, cl.dispatcher, options["config"],
- inheritFDs=inheritFDs, inheritSSLFDs=inheritSSLFDs
- ).setServiceParent(s)
return s
Modified: CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/util.py 2010-11-11 02:56:27 UTC (rev 6609)
+++ CalendarServer/branches/users/glyph/dont-start-postgres/calendarserver/tap/util.py 2010-11-11 02:56:44 UTC (rev 6610)
@@ -68,7 +68,6 @@
NegotiateCredentialFactory # pacify pyflakes
except ImportError:
NegotiateCredentialFactory = None
-from txdav.base.datastore.asyncsqlpool import ConnectionPool
from txdav.base.datastore.asyncsqlpool import ConnectionPoolClient
from calendarserver.accesslog import DirectoryLogWrapperResource
@@ -130,34 +129,45 @@
def getHost(self):
return "<host: %r %r>" % (self.socket.fileno(), id(self))
-def storeFromConfig(config, serviceParent, notifierFactory=None):
+
+def transactionFactoryFromFD(dbampfd):
"""
- Produce an L{IDataStore} from the given configuration and notifier factory.
+ Create a transaction factory from an inherited file descriptor.
"""
- if config.UseDatabase:
- if config.DBAMPFD == 0:
- cp = ConnectionPool(
- pgServiceFromConfig(config, None).produceConnection
- )
- cp.setServiceParent(serviceParent)
- txnFactory = cp.connection
- else:
- # TODO: something to do with loseConnection here, maybe? I don't
- # think it actually needs to be shut down, though.
- skt = fromfd(int(config.DBAMPFD), AF_UNIX, SOCK_STREAM)
- os.close(config.DBAMPFD)
- protocol = ConnectionPoolClient()
- transport = ConnectionWithPeer(skt, protocol)
- protocol.makeConnection(transport)
- transport.startReading()
- txnFactory = protocol.newTransaction
- dataStore = CommonSQLDataStore(
- txnFactory, notifierFactory,
- FilePath(config.AttachmentsRoot),
+ skt = fromfd(dbampfd, AF_UNIX, SOCK_STREAM)
+ os.close(dbampfd)
+ protocol = ConnectionPoolClient()
+ transport = ConnectionWithPeer(skt, protocol)
+ protocol.makeConnection(transport)
+ transport.startReading()
+ return protocol.newTransaction
+
+
+# txnFacSub(int(config.DBAMPFD))
+
+def storeFromConfig(config, txnFactory):
+ """
+ Produce an L{IDataStore} from the given configuration, transaction factory,
+ and notifier factory.
+
+ If the transaction factory is C{None}, we will create a filesystem
+ store. Otherwise, a SQL store, using that connection information.
+ """
+ #
+ # Configure NotifierFactory
+ #
+ if config.Notifications.Enabled:
+ notifierFactory = NotifierFactory(
+ config.Notifications.InternalNotificationHost,
+ config.Notifications.InternalNotificationPort,
+ )
+ else:
+ notifierFactory = None
+ if txnFactory is not None:
+ return CommonSQLDataStore(
+ txnFactory, notifierFactory, FilePath(config.AttachmentsRoot),
config.EnableCalDAV, config.EnableCardDAV
)
- dataStore.setServiceParent(serviceParent)
- return dataStore
else:
return CommonFileDataStore(FilePath(config.DocumentRoot),
notifierFactory, config.EnableCalDAV, config.EnableCardDAV)
@@ -254,7 +264,6 @@
directory.userRecordTypes.insert(0,
SudoDirectoryService.recordType_sudoers)
-
#
# Use system-wide realm on OSX
#
@@ -270,7 +279,7 @@
return directory
-def getRootResource(config, serviceParent, resources=None, store=None):
+def getRootResource(config, newStore, resources=None):
"""
Set up directory service and resource hierarchy based on config.
Return root resource.
@@ -283,6 +292,9 @@
Otherwise build one with L{storeFromConfig}.
"""
+ if newStore is None:
+ raise RuntimeError("Internal error, 'newStore' must be specified.")
+
# FIXME: this is only here to workaround circular imports
doBind()
@@ -390,22 +402,6 @@
principalCollection = directory.principalCollection
- #
- # Configure NotifierFactory
- #
- if config.Notifications.Enabled:
- notifierFactory = NotifierFactory(
- config.Notifications.InternalNotificationHost,
- config.Notifications.InternalNotificationPort,
- )
- else:
- notifierFactory = None
-
- if store is not None:
- newStore = store
- else:
- newStore = storeFromConfig(config, serviceParent, notifierFactory)
-
if config.EnableCalDAV:
log.info("Setting up calendar collection: %r" % (calendarResourceClass,))
calendarCollection = calendarResourceClass(
Modified: CalendarServer/branches/users/glyph/dont-start-postgres/txdav/common/datastore/util.py
===================================================================
--- CalendarServer/branches/users/glyph/dont-start-postgres/txdav/common/datastore/util.py 2010-11-11 02:56:27 UTC (rev 6609)
+++ CalendarServer/branches/users/glyph/dont-start-postgres/txdav/common/datastore/util.py 2010-11-11 02:56:44 UTC (rev 6610)
@@ -19,7 +19,6 @@
from twext.python.log import LoggingMixIn
from twisted.application.service import Service
from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
-from txdav.common.datastore.sql import CommonDataStore as SqlStore
from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
from twisted.internet.defer import inlineCallbacks
@@ -31,20 +30,12 @@
Upgrade resources from a filesystem store to a database store.
"""
-
@classmethod
- def wrapService(cls, path, service, connectionFactory, sqlAttachmentsPath,
- uid=None, gid=None):
+ def wrapService(cls, path, service, store, uid=None, gid=None):
"""
Create an L{UpgradeToDatabaseService} if there are still file-based
calendar or addressbook homes remaining in the given path.
- Maintenance note: we may want to pass a SQL store in directly rather
- than the combination of connection factory and attachments path, since
- there always must be a SQL store, but the path should remain a path
- because there may not I{be} a file-backed store present and we should
- not create it as a result of checking for it.
-
@param path: a path pointing at the document root.
@type path: L{CachingFilePath}
@@ -55,35 +46,32 @@
service parent of the resulting service will be set to a
L{MultiService} or similar.)
+ @param store: the SQL storage service.
+
@type service: L{IService}
@return: a service
@rtype: L{IService}
"""
+ # TODO: TOPPATHS should be computed based on enabled flags in 'store',
+ # not hard coded.
for homeType in TOPPATHS:
if path.child(homeType).exists():
self = cls(
FileStore(path, None, True, True),
- SqlStore(connectionFactory, None, sqlAttachmentsPath,
- True, True),
- service,
- sqlAttachmentsPath=sqlAttachmentsPath,
- uid=uid,
- gid=gid,
+ store, service, uid=uid, gid=gid,
)
return self
return service
- def __init__(self, fileStore, sqlStore, service, sqlAttachmentsPath=None,
- uid=None, gid=None):
+ def __init__(self, fileStore, sqlStore, service, uid=None, gid=None):
"""
Initialize the service.
"""
self.wrappedService = service
self.fileStore = fileStore
self.sqlStore = sqlStore
- self.sqlAttachmentsPath = sqlAttachmentsPath
self.uid = uid
self.gid = gid
@@ -144,9 +132,11 @@
if homesPath.isdir():
homesPath.remove()
- # Set attachment directory ownership
- if (self.sqlAttachmentsPath and
- self.sqlAttachmentsPath.exists() and
+ # Set attachment directory ownership. FIXME: is this still necessary
+ # since attachments started living outside the database directory
+ # created by initdb? default permissions might be correct now.
+ sqlAttachmentsPath = self.sqlStore.attachmentsPath
+ if (sqlAttachmentsPath and sqlAttachmentsPath.exists() and
(self.uid or self.gid)):
uid = self.uid or -1
gid = self.gid or -1
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20101110/ec0262a9/attachment-0001.html>
More information about the calendarserver-changes
mailing list