[CalendarServer-changes] [14663] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Mon Apr 6 14:13:15 PDT 2015
Revision: 14663
http://trac.calendarserver.org//changeset/14663
Author: cdaboo at apple.com
Date: 2015-04-06 14:13:15 -0700 (Mon, 06 Apr 2015)
Log Message:
-----------
Replace config.DSN with a set of attributes for the various fields needed to connect to a DB.
Modified Paths:
--------------
CalendarServer/trunk/.project
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/conf/caldavd-apple.plist
CalendarServer/trunk/conf/caldavd-test.plist
CalendarServer/trunk/conf/caldavd.plist
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/txdav/base/datastore/dbapiclient.py
CalendarServer/trunk/txdav/base/datastore/suboracle.py
CalendarServer/trunk/txdav/base/datastore/subpostgres.py
Modified: CalendarServer/trunk/.project
===================================================================
--- CalendarServer/trunk/.project 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/.project 2015-04-06 21:13:15 UTC (rev 14663)
@@ -5,6 +5,7 @@
<projects>
<project>CalDAVClientLibrary</project>
<project>cffi</project>
+ <project>cx_Oracle</project>
<project>kerberos</project>
<project>pg8000</project>
<project>psutil</project>
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2015-04-06 21:13:15 UTC (rev 14663)
@@ -80,6 +80,7 @@
from txweb2.metafd import ConnectionLimiter, ReportingHTTPService
from txweb2.server import Site
+from txdav.base.datastore.dbapiclient import DBAPIConnector
from txdav.caldav.datastore.scheduling.imip.inbound import MailRetriever
from txdav.caldav.datastore.scheduling.imip.inbound import scheduleNextMailPoll
from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseStep
@@ -114,7 +115,6 @@
from calendarserver.tap.util import (
ConnectionDispenser, Stepper,
checkDirectories, getRootResource,
- oracleConnectorFromConfig, pgConnectorFromConfig,
pgServiceFromConfig, getDBPool, MemoryLimitService,
storeFromConfig, getSSLPassphrase, preFlightChecks,
storeFromConfigWithDPSClient, storeFromConfigWithoutDPS,
@@ -1500,9 +1500,17 @@
@rtype: L{IService}
"""
- def createSubServiceFactory(
- dialect=POSTGRES_DIALECT, paramstyle='pyformat'
- ):
+ def createSubServiceFactory(dbtype):
+ if dbtype == "":
+ dialect = POSTGRES_DIALECT
+ paramstyle = "pyformat"
+ elif dbtype == "postgres":
+ dialect = POSTGRES_DIALECT
+ paramstyle = "pyformat"
+ elif dbtype == "oracle":
+ dialect = ORACLE_DIALECT
+ paramstyle = "numeric"
+
def subServiceFactory(connectionFactory, storageService):
ms = MultiService()
cp = ConnectionPool(
@@ -1606,26 +1614,14 @@
# to it.
pgserv = pgServiceFromConfig(
config,
- createSubServiceFactory(),
+ createSubServiceFactory(""),
uid=overrideUID, gid=overrideGID
)
return pgserv
- elif config.DBType == "postgres":
- # Connect to a postgres database that is already running.
- return createSubServiceFactory()(
- pgConnectorFromConfig(config), None
- )
- elif config.DBType == "oracle":
- # Connect to an Oracle database that is already running.
- return createSubServiceFactory(
- dialect=ORACLE_DIALECT,
- paramstyle="numeric"
- )(
- oracleConnectorFromConfig(config), None
- )
else:
- raise UsageError(
- "Unknown database type {}".format(config.DBType)
+ # Connect to a database that is already running.
+ return createSubServiceFactory(config.DBType)(
+ DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect, None
)
else:
store = storeFromConfig(config, None, None)
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2015-04-06 21:13:15 UTC (rev 14663)
@@ -75,8 +75,7 @@
from twistedcaldav.timezoneservice import TimezoneServiceResource
from twistedcaldav.timezonestdservice import TimezoneStdServiceResource
-from txdav.base.datastore.dbapiclient import DBAPIConnector, OracleConnector
-from txdav.base.datastore.dbapiclient import postgresPreflight
+from txdav.base.datastore.dbapiclient import DBAPIConnector
from txdav.base.datastore.subpostgres import PostgresService
from txdav.caldav.datastore.scheduling.ischedule.dkim import DKIMUtils, DomainKeyResource
from txdav.caldav.datastore.scheduling.ischedule.localservers import buildServersDB
@@ -157,23 +156,6 @@
-def pgConnectorFromConfig(config):
- """
- Create a postgres DB-API connector from the given configuration.
- """
- from txdav.base.datastore.subpostgres import postgres
- return DBAPIConnector(postgres, postgresPreflight, config.DSN).connect
-
-
-
-def oracleConnectorFromConfig(config):
- """
- Create a postgres DB-API connector from the given configuration.
- """
- return OracleConnector(config.DSN).connect
-
-
-
class ConnectionWithPeer(Connection):
connected = True
@@ -828,14 +810,10 @@
# get a PostgresService to tell us what the local connection
# info is, but *don't* start it (that would start one postgres
# master per slave, resulting in all kinds of mayhem...)
- connectionFactory = pgServiceFromConfig(
- config, None).produceConnection
- elif config.DBType == 'postgres':
- connectionFactory = pgConnectorFromConfig(config)
- elif config.DBType == 'oracle':
- connectionFactory = oracleConnectorFromConfig(config)
+ connectionFactory = pgServiceFromConfig(config, None).produceConnection
else:
- raise UsageError("unknown DB type: %r" % (config.DBType,))
+ connectionFactory = DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect
+
pool = ConnectionPool(connectionFactory, dialect=dialect,
paramstyle=paramstyle,
maxConnections=config.MaxDBConnectionsPerPool)
Modified: CalendarServer/trunk/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-apple.plist 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/conf/caldavd-apple.plist 2015-04-06 21:13:15 UTC (rev 14663)
@@ -97,8 +97,7 @@
<!-- Database connection -->
<key>DBType</key>
<string></string>
- <key>DSN</key>
- <string></string>
+
<key>Postgres</key>
<dict>
<key>Ctl</key>
Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/conf/caldavd-test.plist 2015-04-06 21:13:15 UTC (rev 14663)
@@ -98,8 +98,17 @@
<!--
<key>DBType</key>
<string>postgres</string>
- <key>DSN</key>
- <string>:caldav:caldav:::</string>
+ <key>DatabaseConnection</key>
+ <dict>
+ <key>endpoint</key>
+ <string>tcp:localhost</string>
+ <key>database</key>
+ <string>caldav</string>
+ <key>user</key>
+ <string>caldav</string>
+ <key>password</key>
+ <string></string>
+ </dict>
-->
<!-- Data root -->
Modified: CalendarServer/trunk/conf/caldavd.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd.plist 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/conf/caldavd.plist 2015-04-06 21:13:15 UTC (rev 14663)
@@ -84,8 +84,17 @@
<!--
<key>DBType</key>
<string>postgres</string>
- <key>DSN</key>
- <string>:caldav:caldav:::</string>
+ <key>DatabaseConnection</key>
+ <dict>
+ <key>endpoint</key>
+ <string>tcp:localhost</string>
+ <key>database</key>
+ <string>caldav</string>
+ <key>user</key>
+ <string>caldav</string>
+ <key>password</key>
+ <string></string>
+ </dict>
-->
<!-- Data root -->
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2015-04-06 21:13:15 UTC (rev 14663)
@@ -202,9 +202,12 @@
"SpawnedDBUser": "caldav", # The username to use when DBType is empty
- "DSN": "", # Data Source Name. Used to connect to an external
- # database if DBType is non-empty. Format varies
- # depending on database type.
+ "DatabaseConnection": { # Used to connect to an external database if DBType is non-empty
+ "endpoint": "", # Database connection endpoint
+ "database": "", # Name of database or Oracle SID
+ "user": "", # User name to connect as
+ "password": "", # Password to use
+ },
"DBAMPFD": 0, # Internally used by database to tell slave
# processes to inherit a file descriptor and use it
Modified: CalendarServer/trunk/txdav/base/datastore/dbapiclient.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/dbapiclient.py 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/txdav/base/datastore/dbapiclient.py 2015-04-06 21:13:15 UTC (rev 14663)
@@ -14,11 +14,17 @@
# limitations under the License.
##
+
"""
General utility client code for interfacing with DB-API 2.0 modules.
"""
from twext.enterprise.util import mapOracleOutputType
+from twext.python.filepath import CachingFilePath
+from txdav.common.icommondatastore import InternalDataStoreError
+
+import pg8000 as postgres
+
try:
import os
# In order to encode and decode values going to and from the database,
@@ -178,6 +184,45 @@
+class DBAPIParameters(object):
+ """
+ Object that holds the parameters needed to configure a DBAPIConnector. Since this varies based on
+ the actual DB module in use, this class abstracts the parameters into separate properties that
+ are then used to create the actual parameters for each module.
+ """
+
+ def __init__(self, endpoint=None, user=None, password=None, database=None):
+ """
+ @param endpoint: endpoint string describing the connection
+ @type endpoint: L{str}
+ @param user: user name to connect as
+ @type user: L{str}
+ @param password: password to use
+ @type password: L{str}
+ @param database: database name to connect to
+ @type database: L{str}
+ """
+ self.endpoint = endpoint
+ if self.endpoint.startswith("unix:"):
+ self.unixsocket = self.endpoint[5:]
+ if ":" in self.unixsocket:
+ self.unixsocket, self.port = self.unixsocket.split(":")
+ else:
+ self.port = None
+ self.host = None
+ elif self.endpoint.startswith("tcp:"):
+ self.unixsocket = None
+ self.host = self.endpoint[4:]
+ if ":" in self.host:
+ self.host, self.port = self.host.split(":")
+ else:
+ self.port = None
+ self.user = user
+ self.password = password
+ self.database = database
+
+
+
class DBAPIConnector(object):
"""
A simple wrapper for DB-API connectors.
@@ -201,7 +246,87 @@
return w
+ @staticmethod
+ def connectorFor(dbtype, **kwargs):
+ if dbtype == "postgres":
+ return DBAPIConnector._connectorFor_module(postgres, **kwargs)
+ elif dbtype == "oracle":
+ return DBAPIConnector._connectorFor_module(cx_Oracle, **kwargs)
+ else:
+ raise InternalDataStoreError(
+ "Unknown database type: {}".format(dbtype)
+ )
+
+ @staticmethod
+ def _connectorFor_module(dbmodule, **kwargs):
+ m = getattr(DBAPIConnector, "_connectorFor_{}".format(dbmodule.__name__), None)
+ if m is None:
+ raise InternalDataStoreError(
+ "Unknown DBAPI module: {}".format(dbmodule)
+ )
+
+ return m(dbmodule, **kwargs)
+
+
+ @staticmethod
+ def _connectorFor_pgdb(dbmodule, **kwargs):
+ """
+ Turn properties into pgdb kwargs
+ """
+ params = DBAPIParameters(**kwargs)
+
+ dsn = "{0.host}:dbname={0.database}:{0.user}:{0.password}::".format(params)
+
+ dbkwargs = {}
+ if params.port:
+ dbkwargs["host"] = "{}:{}".format(params.host, params.port)
+ return DBAPIConnector(postgres, postgresPreflight, dsn, **dbkwargs)
+
+
+ @staticmethod
+ def _connectorFor_pg8000(dbmodule, **kwargs):
+ """
+ Turn properties into pg8000 kwargs
+ """
+ params = DBAPIParameters(**kwargs)
+ dbkwargs = {
+ "user": params.user,
+ "password": params.password,
+ "database": params.database,
+ }
+ if params.unixsocket:
+ dbkwargs["unix_sock"] = params.unixsocket
+
+ # We're using a socket file
+ socketFP = CachingFilePath(dbkwargs["unix_sock"])
+
+ if socketFP.isdir():
+ # We have been given the directory, not the actual socket file
+ socketFP = socketFP.child(".s.PGSQL.{}".format(params.port if params.port else "5432"))
+ dbkwargs["unix_sock"] = socketFP.path
+
+ if not socketFP.isSocket():
+ raise InternalDataStoreError(
+ "No such socket file: {}".format(socketFP.path)
+ )
+ else:
+ dbkwargs["host"] = params.host
+ if params.port:
+ dbkwargs["port"] = int(params.port)
+ return DBAPIConnector(dbmodule, postgresPreflight, **dbkwargs)
+
+
+ @staticmethod
+ def _connectorFor_cx_Oracle(self, **kwargs):
+ """
+ Turn properties into DSN string
+ """
+ dsn = "{0.user}/{0.password}@{0.host}:{0.port}/{0.database}".format(DBAPIParameters(**kwargs))
+ return OracleConnector(dsn)
+
+
+
class OracleConnectionWrapper(DiagnosticConnectionWrapper):
wrapper = OracleCursorWrapper
Modified: CalendarServer/trunk/txdav/base/datastore/suboracle.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/suboracle.py 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/txdav/base/datastore/suboracle.py 2015-04-06 21:13:15 UTC (rev 14663)
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# #
-from twisted.internet.defer import inlineCallbacks
"""
Stub service for Oracle.
@@ -22,10 +21,10 @@
from twext.python.log import Logger
-from txdav.base.datastore.dbapiclient import OracleConnector
-from txdav.common.icommondatastore import InternalDataStoreError
+from txdav.base.datastore.dbapiclient import DBAPIConnector
from twisted.application.service import MultiService
+from twisted.internet.defer import inlineCallbacks
log = Logger()
@@ -70,26 +69,16 @@
def _connectorFor(self):
- m = getattr(self, "_connectorFor_{}".format("cx_oracle"), None)
- if m is None:
- raise InternalDataStoreError(
- "Unknown Oracle DBM module: {}".format("cx_oracle")
- )
+ kwargs = {
+ "endpoint": "tcp:192.168.56.101:1521",
+ "database": "orcl",
+ "user": self.dsnUser if self.dsnUser else "hr",
+ "password": "oracle",
+ }
- return m()
+ return DBAPIConnector.connectorFor("oracle", **kwargs)
- def _connectorFor_cx_oracle(self):
- dsn = "{}/oracle at 192.168.56.101:1521/orcl".format(self.dsnUser if self.dsnUser else "hr")
-
- log.info(
- "Connecting to Oracle with dsn={dsn!r}",
- dsn=dsn,
- )
-
- return OracleConnector(dsn)
-
-
def produceConnection(self, label="<unlabeled>"):
"""
Produce a DB-API 2.0 connection pointed at this database.
Modified: CalendarServer/trunk/txdav/base/datastore/subpostgres.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/txdav/base/datastore/subpostgres.py 2015-04-06 21:13:15 UTC (rev 14663)
@@ -26,7 +26,6 @@
from hashlib import md5
from pipes import quote as shell_quote
-import pg8000 as postgres
from twisted.python.procutils import which
from twisted.internet.protocol import ProcessProtocol
@@ -38,7 +37,7 @@
from twisted.protocols.basic import LineReceiver
from twisted.internet.defer import Deferred
from txdav.base.datastore.dbapiclient import DBAPIConnector
-from txdav.base.datastore.dbapiclient import postgresPreflight
+from txdav.base.datastore.dbapiclient import postgres
from txdav.common.icommondatastore import InternalDataStoreError
from twisted.application.service import MultiService
@@ -320,73 +319,24 @@
if databaseName is None:
databaseName = self.databaseName
- m = getattr(self, "_connectorFor_{}".format(postgres.__name__), None)
- if m is None:
- raise InternalDataStoreError(
- "Unknown Postgres DBM module: {}".format(postgres)
- )
+ kwargs = {
+ "database": databaseName,
+ }
- return m(databaseName)
-
-
- def _connectorFor_pgdb(self, databaseName):
- dsn = "{}:dbname={}".format(self.host, databaseName)
-
- if self.spawnedDBUser:
- dsn = "{}:{}".format(dsn, self.spawnedDBUser)
- elif self.uid is not None:
- dsn = "{}:{}".format(dsn, pwd.getpwuid(self.uid).pw_name)
-
- kwargs = {}
- if self.port:
- kwargs["host"] = "{}:{}".format(self.host, self.port)
-
- log.info(
- "Connecting to Postgres with dsn={dsn!r} args={args}",
- dsn=dsn, args=kwargs
- )
-
- return DBAPIConnector(postgres, postgresPreflight, dsn, **kwargs)
-
-
- def _connectorFor_pg8000(self, databaseName):
- kwargs = dict(database=databaseName)
-
if self.host.startswith("/"):
- # We're using a socket file
- socketFP = CachingFilePath(self.host)
-
- if socketFP.isdir():
- # We have been given the directory, not the actual socket file
- nameFormat = self.socketName if self.socketName else ".s.PGSQL.{}"
- socketFP = socketFP.child(
- nameFormat.format(self.port if self.port else 5432)
- )
-
- if not socketFP.isSocket():
- raise InternalDataStoreError(
- "No such socket file: {}".format(socketFP.path)
- )
-
- kwargs["host"] = None
- kwargs["unix_sock"] = socketFP.path
+ kwargs["endpoint"] = "unix:{}".format(self.host)
else:
- kwargs["host"] = self.host
- kwargs["unix_sock"] = None
-
- if self.port:
- kwargs["port"] = int(self.port)
-
+ kwargs["endpoint"] = "tcp:{}".format(self.host)
+ if self.port:
+ kwargs["endpoint"] = "{}:{}".format(kwargs["endpoint"], self.port)
if self.spawnedDBUser:
kwargs["user"] = self.spawnedDBUser
elif self.uid is not None:
kwargs["user"] = pwd.getpwuid(self.uid).pw_name
- log.info("Connecting to Postgres with args={args}", args=kwargs)
+ return DBAPIConnector.connectorFor("postgres", **kwargs)
- return DBAPIConnector(postgres, postgresPreflight, **kwargs)
-
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150406/efb4af2d/attachment-0001.html>
More information about the calendarserver-changes
mailing list