[CalendarServer-changes] [6255] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Sep 7 19:51:18 PDT 2010
Revision: 6255
http://trac.macosforge.org/projects/calendarserver/changeset/6255
Author: cdaboo at apple.com
Date: 2010-09-07 19:51:18 -0700 (Tue, 07 Sep 2010)
Log Message:
-----------
Put some postgres configuration into config.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/base/datastore/subpostgres.py
CalendarServer/trunk/txdav/base/datastore/test/test_subpostgres.py
CalendarServer/trunk/txdav/common/datastore/test/util.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2010-09-08 02:51:18 UTC (rev 6255)
@@ -722,8 +722,14 @@
postgresUID = None
postgresGID = None
pgserv = PostgresService(
- dbRoot, subServiceFactory, v1_schema, "caldav",
- logFile=config.PostgresLogFile,
+ dbRoot, subServiceFactory, v1_schema,
+ databaseName=config.Postgres.DatabaseName,
+ logFile=config.Postgres.LogFile,
+ socketDir=config.RunRoot if config.Postgres.UnixSocket else None,
+ listenAddresses=config.Postgres.ListenAddresses,
+ sharedBuffers=config.Postgres.SharedBuffers,
+ maxConnections=config.Postgres.MaxConnections,
+ options=config.Postgres.Options,
uid=postgresUID, gid=postgresGID
)
return pgserv
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2010-09-08 02:51:18 UTC (rev 6255)
@@ -84,8 +84,16 @@
"""
if config.UseDatabase:
dbRoot = CachingFilePath(config.DatabaseRoot)
- postgresService = PostgresService(dbRoot, None, v1_schema, "caldav",
- logFile=config.PostgresLogFile)
+ postgresService = PostgresService(
+ dbRoot, None, v1_schema,
+ databaseName=config.Postgres.DatabaseName,
+ logFile=config.Postgres.LogFile,
+ socketDir=config.RunRoot if config.Postgres.UnixSocket else None,
+ listenAddresses=config.Postgres.ListenAddresses,
+ sharedBuffers=config.Postgres.SharedBuffers,
+ maxConnections=config.Postgres.MaxConnections,
+ options=config.Postgres.Options,
+ )
return CommonSQLDataStore(postgresService.produceConnection,
notifierFactory, dbRoot.child("attachments"),
config.EnableCalDAV, config.EnableCardDAV)
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2010-09-08 02:51:18 UTC (rev 6255)
@@ -751,6 +751,17 @@
</array>
</dict>
+ <!-- Support for Postgres -->
+ <key>Postgres</key>
+ <dict>
+ <key>Options</key>
+ <array>
+ <!-- Optional extra logging for posgres -->
+ <!-- <string>-c log_lock_waits=TRUE</string> -->
+ <!-- <string>-c log_statement=all</string> -->
+ <!-- <string>-c log_line_prefix='%p.%x '</string> -->
+ </array>
+ </dict>
<!--
Twisted
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2010-09-08 02:51:18 UTC (rev 6255)
@@ -145,12 +145,12 @@
"BindSSLPorts" : [], # List of port numbers to bind to for SSL [empty = same as "SSLPort"]
"InheritFDs" : [], # File descriptors to inherit for HTTP requests (empty = don't inherit)
"InheritSSLFDs": [], # File descriptors to inherit for HTTPS requests (empty = don't inherit)
- "MetaFD": 0, # Inherited file descriptor to call recvmsg() on to recive sockets (none = don't inherit)
+ "MetaFD": 0, # Inherited file descriptor to call recvmsg() on to receive sockets (none = don't inherit)
"UseMetaFD": True, # Use a 'meta' FD, i.e. an FD to transmit other
# FDs to slave processes.
- "UseDatabase" : True, # True: postgress; False: files
+ "UseDatabase" : True, # True: postgres; False: files
#
# Types of service provided
@@ -270,7 +270,6 @@
"ErrorLogEnabled" : True, # True = use log file, False = stdout
"ErrorLogRotateMB" : 10, # Rotate error log after so many megabytes
"ErrorLogMaxRotatedFiles" : 5, # Retain this many error log files
- "PostgresLogFile" : "postgres.log", # Postgres log
"PIDFile" : "caldavd.pid",
"RotateAccessLog" : False,
"EnableExtendedAccessLog": True,
@@ -573,6 +572,18 @@
"Options": [],
},
+ "Postgres": {
+ "DatabaseName": "caldav",
+ "LogFile": "postgres.log",
+ "UnixSocket": True,
+ "ListenAddresses": [],
+ "SharedBuffers": 30,
+ "MaxConnections": 20,
+ "Options": [
+ "-c standard_conforming_strings=on",
+ ],
+ },
+
"EnableKeepAlive": True,
"Includes": [], # Other plists to parse after this one
@@ -628,20 +639,22 @@
return configDict
-RELATIVE_PATHS = [("ServerRoot", "DataRoot"),
- ("ServerRoot", "DocumentRoot"),
- ("ServerRoot", "ConfigRoot"),
- ("ServerRoot", "LogRoot"),
- ("ServerRoot", "RunRoot"),
- ("DataRoot", "DatabaseRoot"),
- ("ConfigRoot", "SudoersFile"),
- ("LogRoot", "AccessLogFile"),
- ("LogRoot", "ErrorLogFile"),
- ("LogRoot", "PostgresLogFile"),
- ("LogRoot", "AccountingLogRoot"),
- ("RunRoot", "PIDFile"),
- ("RunRoot", "GlobalStatsSocket"),
- ("RunRoot", "ControlSocket")]
+RELATIVE_PATHS = [
+ ("ServerRoot", "DataRoot"),
+ ("ServerRoot", "DocumentRoot"),
+ ("ServerRoot", "ConfigRoot"),
+ ("ServerRoot", "LogRoot"),
+ ("ServerRoot", "RunRoot"),
+ ("DataRoot", "DatabaseRoot"),
+ ("ConfigRoot", "SudoersFile"),
+ ("LogRoot", "AccessLogFile"),
+ ("LogRoot", "ErrorLogFile"),
+ ("LogRoot", ("Postgres", "LogFile",)),
+ ("LogRoot", "AccountingLogRoot"),
+ ("RunRoot", "PIDFile"),
+ ("RunRoot", "GlobalStatsSocket"),
+ ("RunRoot", "ControlSocket"),
+]
def _updateDataStore(configDict):
@@ -650,30 +663,42 @@
their respective root directories rather than the current working directory.
"""
for root, relativePath in RELATIVE_PATHS:
- if root in configDict and relativePath in configDict:
- previousAbsoluteName = ".absolute." + relativePath
- previousRelativeName = ".relative." + relativePath
+ if root in configDict:
+ if isinstance(relativePath, str):
+ relativePath = (relativePath,)
+
+ inDict = configDict
+ for segment in relativePath[:-1]:
+ if segment not in inDict:
+ inDict = None
+ break
+ inDict = inDict[segment]
+ lastPath = relativePath[-1]
+ relativePath = ".".join(relativePath)
+ if lastPath in inDict:
+ previousAbsoluteName = ".absolute." + relativePath
+ previousRelativeName = ".relative." + relativePath
+
+ # If we previously made the name absolute, and the name in the
+ # config is still the same absolute name that we made it, let's
+ # change it to be the relative name again. (This is necessary
+ # because the config data is actually updated several times before
+ # the config *file* has been read, so these keys will be made
+ # absolute based on default values, and need to be made relative to
+ # non-default values later.) -glyph
+ if previousAbsoluteName in configDict and (
+ configDict[previousAbsoluteName] == inDict[lastPath]
+ ):
+ userSpecifiedPath = configDict[previousRelativeName]
+ else:
+ userSpecifiedPath = inDict[lastPath]
+ configDict[previousRelativeName] = inDict[lastPath]
+ newAbsolutePath = fullServerPath(configDict[root],
+ userSpecifiedPath)
+ inDict[lastPath] = newAbsolutePath
+ configDict[previousAbsoluteName] = newAbsolutePath
- # If we previously made the name absolute, and the name in the
- # config is still the same absolute name that we made it, let's
- # change it to be the relative name again. (This is necessary
- # because the config data is actually updated several times before
- # the config *file* has been read, so these keys will be made
- # absolute based on default values, and need to be made relative to
- # non-default values later.) -glyph
- if previousAbsoluteName in configDict and (
- configDict[previousAbsoluteName] == configDict[relativePath]
- ):
- userSpecifiedPath = configDict[previousRelativeName]
- else:
- userSpecifiedPath = configDict[relativePath]
- configDict[previousRelativeName] = configDict[relativePath]
- newAbsolutePath = fullServerPath(configDict[root],
- userSpecifiedPath)
- configDict[relativePath] = newAbsolutePath
- configDict[previousAbsoluteName] = newAbsolutePath
-
def _updateHostName(configDict):
if not configDict.ServerHostName:
from socket import getfqdn
Modified: CalendarServer/trunk/txdav/base/datastore/subpostgres.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2010-09-08 02:51:18 UTC (rev 6255)
@@ -216,8 +216,11 @@
class PostgresService(MultiService):
def __init__(self, dataStoreDirectory, subServiceFactory,
- schema, databaseName='subpostgres', resetSchema=False,
- logFile="postgres.log", testMode=False,
+ schema, resetSchema=False, databaseName='subpostgres',
+ logFile="postgres.log", socketDir="/tmp",
+ listenAddresses=[], sharedBuffers=30,
+ maxConnections=20, options=[],
+ testMode=False,
uid=None, gid=None):
"""
Initialize a L{PostgresService} pointed at a data store directory.
@@ -229,38 +232,45 @@
1-arg callable which returns a DB-API cursor.
@type subServiceFactory: C{callable}
"""
+
+ # FIXME: By default there is very little (4MB) shared memory available,
+ # so at the moment I am lowering these postgres config options to allow
+ # multiple servers to run. We might want to look into raising
+ # kern.sysv.shmmax.
+ # See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
+
MultiService.__init__(self)
self.subServiceFactory = subServiceFactory
self.dataStoreDirectory = dataStoreDirectory
self.resetSchema = resetSchema
- if os.getuid() == 0:
- socketRoot = "/var/run"
- else:
- socketRoot = "/tmp"
- self.socketDir = CachingFilePath("%s/ccs_postgres_%s/" %
- (socketRoot, md5(dataStoreDirectory.path).hexdigest()))
+ # Options from config
self.databaseName = databaseName
self.logFile = logFile
+ if socketDir:
+ # Unix socket length path limit
+ self.socketDir = CachingFilePath("%s/ccs_postgres_%s/" %
+ (socketDir, md5(dataStoreDirectory.path).hexdigest()))
+ if len(self.socketDir.path) > 64:
+ socketDir = "/tmp"
+ self.socketDir = CachingFilePath("/tmp/ccs_postgres_%s/" %
+ (md5(dataStoreDirectory.path).hexdigest()))
+ self.host = self.socketDir.path
+ else:
+ self.socketDir = None
+ self.host = "localhost"
+ self.listenAddresses = listenAddresses
+ self.sharedBuffers = sharedBuffers if not testMode else 16
+ self.maxConnections = maxConnections if not testMode else 4
+ self.options = options
+
self.uid = uid
self.gid = gid
self.schema = schema
self.monitor = None
self.openConnections = []
- # FIXME: By default there is very little (4MB) shared memory available,
- # so at the moment I am lowering these postgres config options to allow
- # multiple servers to run. We might want to look into raising
- # kern.sysv.shmmax.
- # See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
- if testMode:
- self.sharedBuffers = 16
- self.maxConnections = 4
- else:
- self.sharedBuffers = 30
- self.maxConnections = 20
-
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
@@ -269,10 +279,10 @@
databaseName = self.databaseName
if self.uid is not None:
- dsn = "%s:dbname=%s:%s" % (self.socketDir.path, databaseName,
+ dsn = "%s:dbname=%s:%s" % (self.host, databaseName,
pwd.getpwuid(self.uid).pw_name)
else:
- dsn = "%s:dbname=%s" % (self.socketDir.path, databaseName)
+ dsn = "%s:dbname=%s" % (self.host, databaseName)
connection = pgdb.connect(dsn)
w = DiagnosticConnectionWrapper(connection, label)
@@ -367,6 +377,17 @@
monitor = _PostgresMonitor(self)
pg_ctl = which("pg_ctl")[0]
# check consistency of initdb and postgres?
+
+ options = []
+ if self.listenAddresses:
+ options.append("-c listen_addresses='%s'" % (",".join(self.listenAddresses)))
+ if self.socketDir:
+ options.append("-k '%s'" % (self.socketDir.path,))
+ options.append("-c shared_buffers=%d" % (self.sharedBuffers,))
+ options.append("-c max_connections=%d" % (self.maxConnections,))
+ options.append("-c standard_conforming_strings=on")
+ options.extend(self.options)
+
reactor.spawnProcess(
monitor, pg_ctl,
[
@@ -376,8 +397,8 @@
"-w",
# XXX what are the quoting rules for '-o'? do I need to repr()
# the path here?
- "-o", "-c listen_addresses='' -k '%s' -c standard_conforming_strings=on -c shared_buffers=%d -c max_connections=%d"
- % (self.socketDir.path, self.sharedBuffers, self.maxConnections),
+ "-o",
+ " ".join(options),
],
self.env,
uid=self.uid, gid=self.gid,
@@ -397,12 +418,13 @@
workingDir = self.dataStoreDirectory.child("working")
env = self.env = os.environ.copy()
env.update(PGDATA=clusterDir.path,
- PGHOST=self.socketDir.path)
+ PGHOST=self.host)
initdb = which("initdb")[0]
- if not self.socketDir.isdir():
- self.socketDir.createDirectory()
- if self.uid and self.gid:
- os.chown(self.socketDir.path, self.uid, self.gid)
+ if self.socketDir:
+ if not self.socketDir.isdir():
+ self.socketDir.createDirectory()
+ if self.uid and self.gid:
+ os.chown(self.socketDir.path, self.uid, self.gid)
if clusterDir.isdir():
self.startDatabase()
else:
Modified: CalendarServer/trunk/txdav/base/datastore/test/test_subpostgres.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/test/test_subpostgres.py 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/txdav/base/datastore/test/test_subpostgres.py 2010-09-08 02:51:18 UTC (rev 6255)
@@ -32,7 +32,7 @@
"""
@inlineCallbacks
- def test_startService(self):
+ def test_startService_Unix(self):
"""
Assuming a properly configured environment ($PATH points at an 'initdb'
and 'postgres', $PYTHONPATH includes pgdb), starting a
@@ -41,10 +41,9 @@
"""
test = self
- class SimpleService(Service):
+ class SimpleService1(Service):
instances = []
- rows = []
ready = Deferred()
def __init__(self, connectionFactory):
@@ -66,21 +65,71 @@
finally:
cursor.close()
+ svc = PostgresService(
+ CachingFilePath("../_postgres_test_db1"),
+ SimpleService1,
+ "create table TEST_DUMMY_TABLE (stub varchar)",
+ databaseName="dummy_db",
+ testMode=True
+ )
+ svc.startService()
+ self.addCleanup(svc.stopService)
+ yield SimpleService1.ready
+ connection = SimpleService1.instances[0].connection
+ cursor = connection.cursor()
+ cursor.execute("select * from test_dummy_table")
+ values = cursor.fetchall()
+ self.assertEquals(values, [["dummy"]])
- dbPath = "../_postgres_test_db"
+ @inlineCallbacks
+ def test_startService_Socket(self):
+ """
+ Assuming a properly configured environment ($PATH points at an 'initdb'
+ and 'postgres', $PYTHONPATH includes pgdb), starting a
+ L{PostgresService} will start the service passed to it, after executing
+ the schema.
+ """
+
+ test = self
+ class SimpleService2(Service):
+
+ instances = []
+ ready = Deferred()
+
+ def __init__(self, connectionFactory):
+ self.connection = connectionFactory()
+ test.addCleanup(self.connection.close)
+ self.instances.append(self)
+
+
+ def startService(self):
+ cursor = self.connection.cursor()
+ try:
+ cursor.execute(
+ "insert into test_dummy_table values ('dummy')"
+ )
+ except:
+ self.ready.errback()
+ else:
+ self.ready.callback(None)
+ finally:
+ cursor.close()
+
svc = PostgresService(
- CachingFilePath(dbPath),
- SimpleService,
- "create table TEST_DUMMY_TABLE (stub varchar)",
- "dummy_db",
- testMode=True
+ CachingFilePath("../_postgres_test_db2"),
+ SimpleService2,
+ "create table TEST_DUMMY_TABLE (stub varchar)",
+ databaseName="dummy_db",
+ socketDir=None,
+ listenAddresses=['127.0.0.1',],
+ testMode=True
)
-
svc.startService()
self.addCleanup(svc.stopService)
- yield SimpleService.ready
- connection = SimpleService.instances[0].connection
+ yield SimpleService2.ready
+ connection = SimpleService2.instances[0].connection
cursor = connection.cursor()
cursor.execute("select * from test_dummy_table")
values = cursor.fetchall()
self.assertEquals(values, [["dummy"]])
+
Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py 2010-09-08 02:03:29 UTC (rev 6254)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py 2010-09-08 02:51:18 UTC (rev 6255)
@@ -86,7 +86,8 @@
ready.callback(self.store)
return self.store
self.sharedService = PostgresService(
- dbRoot, getReady, v1_schema, "caldav", resetSchema=True,
+ dbRoot, getReady, v1_schema, resetSchema=True,
+ databaseName="caldav",
testMode=True
)
self.sharedService.startService()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100907/ae85380f/attachment-0001.html>
More information about the calendarserver-changes
mailing list