[CalendarServer-changes] [14663] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Mon Apr 6 14:13:15 PDT 2015


Revision: 14663
          http://trac.calendarserver.org//changeset/14663
Author:   cdaboo at apple.com
Date:     2015-04-06 14:13:15 -0700 (Mon, 06 Apr 2015)
Log Message:
-----------
Replace config.DSN with a set of attributes for the various fields needed to connect to a DB.

Modified Paths:
--------------
    CalendarServer/trunk/.project
    CalendarServer/trunk/calendarserver/tap/caldav.py
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/conf/caldavd-apple.plist
    CalendarServer/trunk/conf/caldavd-test.plist
    CalendarServer/trunk/conf/caldavd.plist
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/txdav/base/datastore/dbapiclient.py
    CalendarServer/trunk/txdav/base/datastore/suboracle.py
    CalendarServer/trunk/txdav/base/datastore/subpostgres.py

Modified: CalendarServer/trunk/.project
===================================================================
--- CalendarServer/trunk/.project	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/.project	2015-04-06 21:13:15 UTC (rev 14663)
@@ -5,6 +5,7 @@
 	<projects>
 		<project>CalDAVClientLibrary</project>
 		<project>cffi</project>
+		<project>cx_Oracle</project>
 		<project>kerberos</project>
 		<project>pg8000</project>
 		<project>psutil</project>

Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py	2015-04-06 21:13:15 UTC (rev 14663)
@@ -80,6 +80,7 @@
 from txweb2.metafd import ConnectionLimiter, ReportingHTTPService
 from txweb2.server import Site
 
+from txdav.base.datastore.dbapiclient import DBAPIConnector
 from txdav.caldav.datastore.scheduling.imip.inbound import MailRetriever
 from txdav.caldav.datastore.scheduling.imip.inbound import scheduleNextMailPoll
 from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseStep
@@ -114,7 +115,6 @@
 from calendarserver.tap.util import (
     ConnectionDispenser, Stepper,
     checkDirectories, getRootResource,
-    oracleConnectorFromConfig, pgConnectorFromConfig,
     pgServiceFromConfig, getDBPool, MemoryLimitService,
     storeFromConfig, getSSLPassphrase, preFlightChecks,
     storeFromConfigWithDPSClient, storeFromConfigWithoutDPS,
@@ -1500,9 +1500,17 @@
         @rtype: L{IService}
         """
 
-        def createSubServiceFactory(
-            dialect=POSTGRES_DIALECT, paramstyle='pyformat'
-        ):
+        def createSubServiceFactory(dbtype):
+            if dbtype == "":
+                dialect = POSTGRES_DIALECT
+                paramstyle = "pyformat"
+            elif dbtype == "postgres":
+                dialect = POSTGRES_DIALECT
+                paramstyle = "pyformat"
+            elif dbtype == "oracle":
+                dialect = ORACLE_DIALECT
+                paramstyle = "numeric"
+
             def subServiceFactory(connectionFactory, storageService):
                 ms = MultiService()
                 cp = ConnectionPool(
@@ -1606,26 +1614,14 @@
                 # to it.
                 pgserv = pgServiceFromConfig(
                     config,
-                    createSubServiceFactory(),
+                    createSubServiceFactory(""),
                     uid=overrideUID, gid=overrideGID
                 )
                 return pgserv
-            elif config.DBType == "postgres":
-                # Connect to a postgres database that is already running.
-                return createSubServiceFactory()(
-                    pgConnectorFromConfig(config), None
-                )
-            elif config.DBType == "oracle":
-                # Connect to an Oracle database that is already running.
-                return createSubServiceFactory(
-                    dialect=ORACLE_DIALECT,
-                    paramstyle="numeric"
-                )(
-                    oracleConnectorFromConfig(config), None
-                )
             else:
-                raise UsageError(
-                    "Unknown database type {}".format(config.DBType)
+                # Connect to a database that is already running.
+                return createSubServiceFactory(config.DBType)(
+                    DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect, None
                 )
         else:
             store = storeFromConfig(config, None, None)

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2015-04-06 21:13:15 UTC (rev 14663)
@@ -75,8 +75,7 @@
 from twistedcaldav.timezoneservice import TimezoneServiceResource
 from twistedcaldav.timezonestdservice import TimezoneStdServiceResource
 
-from txdav.base.datastore.dbapiclient import DBAPIConnector, OracleConnector
-from txdav.base.datastore.dbapiclient import postgresPreflight
+from txdav.base.datastore.dbapiclient import DBAPIConnector
 from txdav.base.datastore.subpostgres import PostgresService
 from txdav.caldav.datastore.scheduling.ischedule.dkim import DKIMUtils, DomainKeyResource
 from txdav.caldav.datastore.scheduling.ischedule.localservers import buildServersDB
@@ -157,23 +156,6 @@
 
 
 
-def pgConnectorFromConfig(config):
-    """
-    Create a postgres DB-API connector from the given configuration.
-    """
-    from txdav.base.datastore.subpostgres import postgres
-    return DBAPIConnector(postgres, postgresPreflight, config.DSN).connect
-
-
-
-def oracleConnectorFromConfig(config):
-    """
-    Create a postgres DB-API connector from the given configuration.
-    """
-    return OracleConnector(config.DSN).connect
-
-
-
 class ConnectionWithPeer(Connection):
 
     connected = True
@@ -828,14 +810,10 @@
             # get a PostgresService to tell us what the local connection
             # info is, but *don't* start it (that would start one postgres
             # master per slave, resulting in all kinds of mayhem...)
-            connectionFactory = pgServiceFromConfig(
-                config, None).produceConnection
-        elif config.DBType == 'postgres':
-            connectionFactory = pgConnectorFromConfig(config)
-        elif config.DBType == 'oracle':
-            connectionFactory = oracleConnectorFromConfig(config)
+            connectionFactory = pgServiceFromConfig(config, None).produceConnection
         else:
-            raise UsageError("unknown DB type: %r" % (config.DBType,))
+            connectionFactory = DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect
+
         pool = ConnectionPool(connectionFactory, dialect=dialect,
                               paramstyle=paramstyle,
                               maxConnections=config.MaxDBConnectionsPerPool)

Modified: CalendarServer/trunk/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-apple.plist	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/conf/caldavd-apple.plist	2015-04-06 21:13:15 UTC (rev 14663)
@@ -97,8 +97,7 @@
     <!-- Database connection -->
     <key>DBType</key>
     <string></string>
-    <key>DSN</key>
-    <string></string>
+
     <key>Postgres</key>
     <dict>
         <key>Ctl</key>

Modified: CalendarServer/trunk/conf/caldavd-test.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-test.plist	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/conf/caldavd-test.plist	2015-04-06 21:13:15 UTC (rev 14663)
@@ -98,8 +98,17 @@
     <!--
     <key>DBType</key>
     <string>postgres</string>
-    <key>DSN</key>
-    <string>:caldav:caldav:::</string>
+    <key>DatabaseConnection</key>
+    <dict>
+    	<key>endpoint</key>
+    	<string>tcp:localhost</string>
+    	<key>database</key>
+    	<string>caldav</string>
+    	<key>user</key>
+    	<string>caldav</string>
+    	<key>password</key>
+    	<string></string>
+    </dict>
      -->
 
     <!-- Data root -->

Modified: CalendarServer/trunk/conf/caldavd.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd.plist	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/conf/caldavd.plist	2015-04-06 21:13:15 UTC (rev 14663)
@@ -84,8 +84,17 @@
     <!--
     <key>DBType</key>
     <string>postgres</string>
-    <key>DSN</key>
-    <string>:caldav:caldav:::</string>
+    <key>DatabaseConnection</key>
+    <dict>
+    	<key>endpoint</key>
+    	<string>tcp:localhost</string>
+    	<key>database</key>
+    	<string>caldav</string>
+    	<key>user</key>
+    	<string>caldav</string>
+    	<key>password</key>
+    	<string></string>
+    </dict>
      -->
 
     <!-- Data root -->

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2015-04-06 21:13:15 UTC (rev 14663)
@@ -202,9 +202,12 @@
 
     "SpawnedDBUser": "caldav", # The username to use when DBType is empty
 
-    "DSN": "", # Data Source Name.  Used to connect to an external
-               # database if DBType is non-empty.  Format varies
-               # depending on database type.
+    "DatabaseConnection": { # Used to connect to an external database if DBType is non-empty
+        "endpoint": "",     # Database connection endpoint
+        "database": "",     # Name of database or Oracle SID
+        "user": "",         # User name to connect as
+        "password": "",     # Password to use
+    },
 
     "DBAMPFD": 0, # Internally used by database to tell slave
                   # processes to inherit a file descriptor and use it

Modified: CalendarServer/trunk/txdav/base/datastore/dbapiclient.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/dbapiclient.py	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/txdav/base/datastore/dbapiclient.py	2015-04-06 21:13:15 UTC (rev 14663)
@@ -14,11 +14,17 @@
 # limitations under the License.
 ##
 
+
 """
 General utility client code for interfacing with DB-API 2.0 modules.
 """
 from twext.enterprise.util import mapOracleOutputType
+from twext.python.filepath import CachingFilePath
 
+from txdav.common.icommondatastore import InternalDataStoreError
+
+import pg8000 as postgres
+
 try:
     import os
     # In order to encode and decode values going to and from the database,
@@ -178,6 +184,45 @@
 
 
 
+class DBAPIParameters(object):
+    """
+    Object that holds the parameters needed to configure a DBAPIConnector. Since this varies based on
+    the actual DB module in use, this class abstracts the parameters into separate properties that
+    are then used to create the actual parameters for each module.
+    """
+
+    def __init__(self, endpoint=None, user=None, password=None, database=None):
+        """
+        @param endpoint: endpoint string describing the connection
+        @type endpoint: L{str}
+        @param user: user name to connect as
+        @type user: L{str}
+        @param password: password to use
+        @type password: L{str}
+        @param database: database name to connect to
+        @type database: L{str}
+        """
+        self.endpoint = endpoint
+        if self.endpoint.startswith("unix:"):
+            self.unixsocket = self.endpoint[5:]
+            if ":" in self.unixsocket:
+                self.unixsocket, self.port = self.unixsocket.split(":")
+            else:
+                self.port = None
+            self.host = None
+        elif self.endpoint.startswith("tcp:"):
+            self.unixsocket = None
+            self.host = self.endpoint[4:]
+            if ":" in self.host:
+                self.host, self.port = self.host.split(":")
+            else:
+                self.port = None
+        self.user = user
+        self.password = password
+        self.database = database
+
+
+
 class DBAPIConnector(object):
     """
     A simple wrapper for DB-API connectors.
@@ -201,7 +246,87 @@
         return w
 
 
+    @staticmethod
+    def connectorFor(dbtype, **kwargs):
+        if dbtype == "postgres":
+            return DBAPIConnector._connectorFor_module(postgres, **kwargs)
+        elif dbtype == "oracle":
+            return DBAPIConnector._connectorFor_module(cx_Oracle, **kwargs)
+        else:
+            raise InternalDataStoreError(
+                "Unknown database type: {}".format(dbtype)
+            )
 
+
+    @staticmethod
+    def _connectorFor_module(dbmodule, **kwargs):
+        m = getattr(DBAPIConnector, "_connectorFor_{}".format(dbmodule.__name__), None)
+        if m is None:
+            raise InternalDataStoreError(
+                "Unknown DBAPI module: {}".format(dbmodule)
+            )
+
+        return m(dbmodule, **kwargs)
+
+
+    @staticmethod
+    def _connectorFor_pgdb(dbmodule, **kwargs):
+        """
+        Turn properties into pgdb kwargs
+        """
+        params = DBAPIParameters(**kwargs)
+
+        dsn = "{0.host}:dbname={0.database}:{0.user}:{0.password}::".format(params)
+
+        dbkwargs = {}
+        if params.port:
+            dbkwargs["host"] = "{}:{}".format(params.host, params.port)
+        return DBAPIConnector(postgres, postgresPreflight, dsn, **dbkwargs)
+
+
+    @staticmethod
+    def _connectorFor_pg8000(dbmodule, **kwargs):
+        """
+        Turn properties into pg8000 kwargs
+        """
+        params = DBAPIParameters(**kwargs)
+        dbkwargs = {
+            "user": params.user,
+            "password": params.password,
+            "database": params.database,
+        }
+        if params.unixsocket:
+            dbkwargs["unix_sock"] = params.unixsocket
+
+            # We're using a socket file
+            socketFP = CachingFilePath(dbkwargs["unix_sock"])
+
+            if socketFP.isdir():
+                # We have been given the directory, not the actual socket file
+                socketFP = socketFP.child(".s.PGSQL.{}".format(params.port if params.port else "5432"))
+                dbkwargs["unix_sock"] = socketFP.path
+
+            if not socketFP.isSocket():
+                raise InternalDataStoreError(
+                    "No such socket file: {}".format(socketFP.path)
+                )
+        else:
+            dbkwargs["host"] = params.host
+            if params.port:
+                dbkwargs["port"] = int(params.port)
+        return DBAPIConnector(dbmodule, postgresPreflight, **dbkwargs)
+
+
+    @staticmethod
+    def _connectorFor_cx_Oracle(self, **kwargs):
+        """
+        Turn properties into DSN string
+        """
+        dsn = "{0.user}/{0.password}@{0.host}:{0.port}/{0.database}".format(DBAPIParameters(**kwargs))
+        return OracleConnector(dsn)
+
+
+
 class OracleConnectionWrapper(DiagnosticConnectionWrapper):
 
     wrapper = OracleCursorWrapper

Modified: CalendarServer/trunk/txdav/base/datastore/suboracle.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/suboracle.py	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/txdav/base/datastore/suboracle.py	2015-04-06 21:13:15 UTC (rev 14663)
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # #
-from twisted.internet.defer import inlineCallbacks
 
 """
 Stub service for Oracle.
@@ -22,10 +21,10 @@
 
 from twext.python.log import Logger
 
-from txdav.base.datastore.dbapiclient import OracleConnector
-from txdav.common.icommondatastore import InternalDataStoreError
+from txdav.base.datastore.dbapiclient import DBAPIConnector
 
 from twisted.application.service import MultiService
+from twisted.internet.defer import inlineCallbacks
 
 log = Logger()
 
@@ -70,26 +69,16 @@
 
 
     def _connectorFor(self):
-        m = getattr(self, "_connectorFor_{}".format("cx_oracle"), None)
-        if m is None:
-            raise InternalDataStoreError(
-                "Unknown Oracle DBM module: {}".format("cx_oracle")
-            )
+        kwargs = {
+            "endpoint": "tcp:192.168.56.101:1521",
+            "database": "orcl",
+            "user": self.dsnUser if self.dsnUser else "hr",
+            "password": "oracle",
+        }
 
-        return m()
+        return DBAPIConnector.connectorFor("oracle", **kwargs)
 
 
-    def _connectorFor_cx_oracle(self):
-        dsn = "{}/oracle at 192.168.56.101:1521/orcl".format(self.dsnUser if self.dsnUser else "hr")
-
-        log.info(
-            "Connecting to Oracle with dsn={dsn!r}",
-            dsn=dsn,
-        )
-
-        return OracleConnector(dsn)
-
-
     def produceConnection(self, label="<unlabeled>"):
         """
         Produce a DB-API 2.0 connection pointed at this database.

Modified: CalendarServer/trunk/txdav/base/datastore/subpostgres.py
===================================================================
--- CalendarServer/trunk/txdav/base/datastore/subpostgres.py	2015-04-06 16:44:14 UTC (rev 14662)
+++ CalendarServer/trunk/txdav/base/datastore/subpostgres.py	2015-04-06 21:13:15 UTC (rev 14663)
@@ -26,7 +26,6 @@
 from hashlib import md5
 from pipes import quote as shell_quote
 
-import pg8000 as postgres
 
 from twisted.python.procutils import which
 from twisted.internet.protocol import ProcessProtocol
@@ -38,7 +37,7 @@
 from twisted.protocols.basic import LineReceiver
 from twisted.internet.defer import Deferred
 from txdav.base.datastore.dbapiclient import DBAPIConnector
-from txdav.base.datastore.dbapiclient import postgresPreflight
+from txdav.base.datastore.dbapiclient import postgres
 from txdav.common.icommondatastore import InternalDataStoreError
 
 from twisted.application.service import MultiService
@@ -320,73 +319,24 @@
         if databaseName is None:
             databaseName = self.databaseName
 
-        m = getattr(self, "_connectorFor_{}".format(postgres.__name__), None)
-        if m is None:
-            raise InternalDataStoreError(
-                "Unknown Postgres DBM module: {}".format(postgres)
-            )
+        kwargs = {
+            "database": databaseName,
+        }
 
-        return m(databaseName)
-
-
-    def _connectorFor_pgdb(self, databaseName):
-        dsn = "{}:dbname={}".format(self.host, databaseName)
-
-        if self.spawnedDBUser:
-            dsn = "{}:{}".format(dsn, self.spawnedDBUser)
-        elif self.uid is not None:
-            dsn = "{}:{}".format(dsn, pwd.getpwuid(self.uid).pw_name)
-
-        kwargs = {}
-        if self.port:
-            kwargs["host"] = "{}:{}".format(self.host, self.port)
-
-        log.info(
-            "Connecting to Postgres with dsn={dsn!r} args={args}",
-            dsn=dsn, args=kwargs
-        )
-
-        return DBAPIConnector(postgres, postgresPreflight, dsn, **kwargs)
-
-
-    def _connectorFor_pg8000(self, databaseName):
-        kwargs = dict(database=databaseName)
-
         if self.host.startswith("/"):
-            # We're using a socket file
-            socketFP = CachingFilePath(self.host)
-
-            if socketFP.isdir():
-                # We have been given the directory, not the actual socket file
-                nameFormat = self.socketName if self.socketName else ".s.PGSQL.{}"
-                socketFP = socketFP.child(
-                    nameFormat.format(self.port if self.port else 5432)
-                )
-
-            if not socketFP.isSocket():
-                raise InternalDataStoreError(
-                    "No such socket file: {}".format(socketFP.path)
-                )
-
-            kwargs["host"] = None
-            kwargs["unix_sock"] = socketFP.path
+            kwargs["endpoint"] = "unix:{}".format(self.host)
         else:
-            kwargs["host"] = self.host
-            kwargs["unix_sock"] = None
-
-        if self.port:
-            kwargs["port"] = int(self.port)
-
+            kwargs["endpoint"] = "tcp:{}".format(self.host)
+            if self.port:
+                kwargs["endpoint"] = "{}:{}".format(kwargs["endpoint"], self.port)
         if self.spawnedDBUser:
             kwargs["user"] = self.spawnedDBUser
         elif self.uid is not None:
             kwargs["user"] = pwd.getpwuid(self.uid).pw_name
 
-        log.info("Connecting to Postgres with args={args}", args=kwargs)
+        return DBAPIConnector.connectorFor("postgres", **kwargs)
 
-        return DBAPIConnector(postgres, postgresPreflight, **kwargs)
 
-
     def produceConnection(self, label="<unlabeled>", databaseName=None):
         """
         Produce a DB-API 2.0 connection pointed at this database.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150406/efb4af2d/attachment-0001.html>


More information about the calendarserver-changes mailing list