[CalendarServer-changes] [7136] CalendarServer/branches/users/glyph/oracle
source_changes at macosforge.org
source_changes at macosforge.org
Mon Mar 7 19:04:08 PST 2011
Revision: 7136
http://trac.macosforge.org/projects/calendarserver/changeset/7136
Author: glyph at apple.com
Date: 2011-03-07 19:04:08 -0800 (Mon, 07 Mar 2011)
Log Message:
-----------
Tweak things towards oracle a bit more, allow for an actual cx_Oracle configuration in the config file.
Modified Paths:
--------------
CalendarServer/branches/users/glyph/oracle/calendarserver/tap/caldav.py
CalendarServer/branches/users/glyph/oracle/calendarserver/tap/util.py
CalendarServer/branches/users/glyph/oracle/twext/enterprise/adbapi2.py
CalendarServer/branches/users/glyph/oracle/twistedcaldav/dateops.py
CalendarServer/branches/users/glyph/oracle/txdav/base/datastore/dbapiclient.py
CalendarServer/branches/users/glyph/oracle/txdav/common/datastore/sql.py
Added Paths:
-----------
CalendarServer/branches/users/glyph/oracle/twext/enterprise/util.py
Modified: CalendarServer/branches/users/glyph/oracle/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/calendarserver/tap/caldav.py 2011-03-08 03:03:54 UTC (rev 7135)
+++ CalendarServer/branches/users/glyph/oracle/calendarserver/tap/caldav.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -73,6 +73,8 @@
from calendarserver.tap.util import pgServiceFromConfig
+from twext.enterprise.ienterprise import POSTGRES_DIALECT
+from twext.enterprise.ienterprise import ORACLE_DIALECT
from twext.enterprise.adbapi2 import ConnectionPool
from twext.enterprise.adbapi2 import ConnectionPoolConnection
@@ -90,6 +92,7 @@
from calendarserver.tap.util import storeFromConfig
from calendarserver.tap.util import transactionFactoryFromFD
from calendarserver.tap.util import pgConnectorFromConfig
+from calendarserver.tap.util import oracleConnectorFromConfig
from calendarserver.tools.util import checkDirectory
try:
@@ -641,6 +644,8 @@
elif not config.UseDatabase:
txnFactory = None
elif not config.SharedConnectionPool:
+ dialect = POSTGRES_DIALECT
+ paramstyle = 'pyformat'
if config.DBType == '':
# get a PostgresService to tell us what the local connection
# info is, but *don't* start it (that would start one postgres
@@ -649,9 +654,14 @@
config, None).produceConnection
elif config.DBType == 'postgres':
connectionFactory = pgConnectorFromConfig(config)
+ elif config.DBType == 'oracle':
+ dialect = ORACLE_DIALECT
+ paramstyle = 'numeric'
+ connectionFactory = oracleConnectorFromConfig(config)
else:
raise UsageError("unknown DB type: %r" % (config.DBType,))
- pool = ConnectionPool(connectionFactory)
+ pool = ConnectionPool(connectionFactory, dialect=dialect,
+ paramstyle=paramstyle)
txnFactory = pool.connection
else:
raise UsageError(
@@ -885,6 +895,7 @@
return self.storageService(toolServiceCreator)
+
def storageService(self, createMainService, uid=None, gid=None):
"""
If necessary, create a service to be started used for storage; for
@@ -939,6 +950,12 @@
return self.subServiceFactoryFactory(createMainService,
uid=overrideUID, gid=overrideGID)(
pgConnectorFromConfig(config))
+ elif config.DBType == 'oracle':
+ # Connect to an Oracle database that is already running.
+ return self.subServiceFactoryFactory(createMainService,
+ uid=overrideUID, gid=overrideGID)(
+ oracleConnectorFromConfig(config),
+ dialect=ORACLE_DIALECT, paramstyle='numeric')
else:
raise UsageError("Unknown database type %r" (config.DBType,))
else:
@@ -946,11 +963,13 @@
return createMainService(None, store)
- def subServiceFactoryFactory(self, createMainService,
- uid=None, gid=None):
+ def subServiceFactoryFactory(self, createMainService, uid=None, gid=None,
+ dialect=POSTGRES_DIALECT,
+ paramstyle='pyformat'):
def subServiceFactory(connectionFactory):
ms = MultiService()
- cp = ConnectionPool(connectionFactory)
+ cp = ConnectionPool(connectionFactory, dialect=dialect,
+ paramstyle=paramstyle)
cp.setServiceParent(ms)
store = storeFromConfig(config, cp.connection)
mainService = createMainService(cp, store)
Modified: CalendarServer/branches/users/glyph/oracle/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/calendarserver/tap/util.py 2011-03-08 03:03:54 UTC (rev 7135)
+++ CalendarServer/branches/users/glyph/oracle/calendarserver/tap/util.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -69,7 +69,7 @@
NegotiateCredentialFactory = None
from twext.enterprise.adbapi2 import ConnectionPoolClient
-from txdav.base.datastore.dbapiclient import DBAPIConnector
+from txdav.base.datastore.dbapiclient import DBAPIConnector, OracleConnector
from txdav.base.datastore.dbapiclient import postgresPreflight
from txdav.base.datastore.subpostgres import PostgresService
@@ -132,6 +132,14 @@
+def oracleConnectorFromConfig(config):
+ """
+ Create a postgres DB-API connector from the given configuration.
+ """
+ return OracleConnector(config.DSN).connect
+
+
+
class ConnectionWithPeer(Connection):
connected = True
@@ -139,10 +147,12 @@
def getPeer(self):
return "<peer: %r %r>" % (self.socket.fileno(), id(self))
+
def getHost(self):
return "<host: %r %r>" % (self.socket.fileno(), id(self))
+
def transactionFactoryFromFD(dbampfd):
"""
Create a transaction factory from an inherited file descriptor.
Modified: CalendarServer/branches/users/glyph/oracle/twext/enterprise/adbapi2.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/twext/enterprise/adbapi2.py 2011-03-08 03:03:54 UTC (rev 7135)
+++ CalendarServer/branches/users/glyph/oracle/twext/enterprise/adbapi2.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -452,13 +452,14 @@
RETRY_TIMEOUT = 10.0
- def __init__(self, connectionFactory, maxConnections=10):
+ def __init__(self, connectionFactory, maxConnections=10,
+ paramstyle=DEFAULT_PARAM_STYLE, dialect=DEFAULT_DIALECT):
super(ConnectionPool, self).__init__()
self.connectionFactory = connectionFactory
self.maxConnections = maxConnections
- self.paramstyle = DEFAULT_PARAM_STYLE
- self.dialect = DEFAULT_DIALECT
+ self.paramstyle = paramstyle
+ self.dialect = dialect
self._free = []
self._busy = []
Added: CalendarServer/branches/users/glyph/oracle/twext/enterprise/util.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/twext/enterprise/util.py (rev 0)
+++ CalendarServer/branches/users/glyph/oracle/twext/enterprise/util.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -0,0 +1,55 @@
+
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Utilities for dealing with different databases.
+"""
+
+from datetime import datetime
+from twistedcaldav.dateops import SQL_TIMESTAMP_FORMAT
+
+def mapOracleOutputType(column):
+ """
+ Map a single output value from cx_Oracle based on some rules and
+ expectations that we have based on the pgdb bindings.
+
+ @param column: a single value from a column.
+
+ @return: a converted value based on the type of the input; oracle CLOBs and
+ datetime timestamps will be converted to strings, all other types will
+ be left alone.
+ """
+ if hasattr(column, 'read'):
+ # Try to detect large objects and format convert them to
+ # strings on the fly. We need to do this as we read each
+ # row, due to the issue described here -
+ # http://cx-oracle.sourceforge.net/html/lob.html - in
+ # particular, the part where it says "In particular, do not
+ # use the fetchall() method".
+ return column.read()
+ elif isinstance(column, datetime):
+ # cx_Oracle properly maps the type of timestamps to datetime
+ # objects. However, our code is mostly written against
+ # PyGreSQL, which just emits strings as results and expects
+ # to have to convert them itself.. Since it's easier to
+ # just detect the datetimes and stringify them, for now
+ # we'll do that.
+ return column.strftime(SQL_TIMESTAMP_FORMAT)
+ else:
+ return column
+
+
Modified: CalendarServer/branches/users/glyph/oracle/twistedcaldav/dateops.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/twistedcaldav/dateops.py 2011-03-08 03:03:54 UTC (rev 7135)
+++ CalendarServer/branches/users/glyph/oracle/twistedcaldav/dateops.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -239,18 +239,19 @@
else:
return (start, end)
+SQL_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
+
def parseSQLTimestamp(ts):
-
# Handle case where fraction seconds may not be present
if len(ts) < 20:
ts += ".0"
- return datetime.datetime.strptime(ts, "%Y-%m-%d %H:%M:%S.%f")
+ return datetime.datetime.strptime(ts, SQL_TIMESTAMP_FORMAT)
def datetimeMktime(dt):
assert isinstance(dt, datetime.date)
-
+
if dt.tzinfo is None:
dt.replace(tzinfo=utc)
return calendar.timegm(dt.utctimetuple())
-
\ No newline at end of file
+
Modified: CalendarServer/branches/users/glyph/oracle/txdav/base/datastore/dbapiclient.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/txdav/base/datastore/dbapiclient.py 2011-03-08 03:03:54 UTC (rev 7135)
+++ CalendarServer/branches/users/glyph/oracle/txdav/base/datastore/dbapiclient.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -17,7 +17,14 @@
"""
General utility client code for interfacing with DB-API 2.0 modules.
"""
+from twext.enterprise.util import mapOracleOutputType
+try:
+ import cx_Oracle
+except ImportError:
+ cx_Oracle = None
+
+
class DiagnosticCursorWrapper(object):
"""
Diagnostic wrapper around a DB-API 2.0 cursor for debugging connection
@@ -41,11 +48,11 @@
def execute(self, sql, args=()):
self.connectionWrapper.state = 'executing %r' % (sql,)
-# Use log.debug
-# sys.stdout.write(
-# "Really executing SQL %r in thread %r\n" %
-# ((sql % tuple(args)), thread.get_ident())
-# )
+ # Use log.debug
+ # sys.stdout.write(
+ # "Really executing SQL %r in thread %r\n" %
+ # ((sql % tuple(args)), thread.get_ident())
+ # )
self.realCursor.execute(sql, args)
@@ -55,20 +62,14 @@
def fetchall(self):
results = self.realCursor.fetchall()
-# Use log.debug
-# sys.stdout.write(
-# "Really fetching results %r thread %r\n" %
-# (results, thread.get_ident())
-# )
+ # Use log.debug
+ # sys.stdout.write(
+ # "Really fetching results %r thread %r\n" %
+ # (results, thread.get_ident())
+ # )
return results
-try:
- import cx_Oracle
-except ImportError:
- pass
-
-
class OracleCursorWrapper(DiagnosticCursorWrapper):
"""
Wrapper for cx_Oracle DB-API connections which implements fetchall() to read
@@ -80,10 +81,7 @@
for row in self.realCursor:
newRow = []
for column in row:
- if hasattr(column, 'read'):
- newRow.append(column.read())
- else:
- newRow.append(column.read())
+ newRow.append(mapOracleOutputType(column))
return accum
@@ -91,6 +89,13 @@
realArgs = []
for arg in args:
if isinstance(arg, (str, unicode)) and len(arg) > 1024:
+ # This *may* cause a type mismatch, but none of the non-CLOB
+ # strings that we're passing would allow a value this large
+ # anyway. Smaller strings will be automatically converted by
+ # the bindings; larger ones will generate an error. I'm not
+ # sure why cx_Oracle itself doesn't just do the following hack
+ # automatically and internally for larger values too, but, here
+ # it is:
v = self.realCursor.var(cx_Oracle.CLOB, len(arg) + 1)
v.setvalue(0, arg)
realArgs.append(v)
@@ -104,6 +109,8 @@
status.
"""
+ wrapper = DiagnosticCursorWrapper
+
def __init__(self, realConnection, label):
self.realConnection = realConnection
self.label = label
@@ -111,7 +118,7 @@
def cursor(self):
- return DiagnosticCursorWrapper(self.realConnection.cursor(), self)
+ return self.wrapper(self.realConnection.cursor(), self)
def close(self):
@@ -137,6 +144,8 @@
@ivar dbModule: the DB-API module to use.
"""
+ wrapper = DiagnosticConnectionWrapper
+
def __init__(self, dbModule, preflight, *connectArgs, **connectKw):
self.dbModule = dbModule
self.connectArgs = connectArgs
@@ -146,12 +155,35 @@
def connect(self, label="<unlabeled>"):
connection = self.dbModule.connect(*self.connectArgs, **self.connectKw)
- w = DiagnosticConnectionWrapper(connection, label)
+ w = self.wrapper(connection, label)
self.preflight(w)
return w
+class OracleConnectionWrapper(DBAPIConnector):
+
+ wrapper = OracleCursorWrapper
+
+
+
+class OracleConnector(DBAPIConnector):
+ """
+ A connector for cx_Oracle connections, with some special-cased behavior to
+ make it work more like other DB-API bindings.
+
+ Note: this is currently necessary to make our usage of twext.enterprise.dal
+ work with cx_Oracle, and should be factored somewhere higher-level.
+ """
+
+ wrapper = OracleConnectionWrapper
+
+ def __init__(self, dsn):
+ super(OracleConnector, self).__init__(
+ cx_Oracle, lambda whatever: None, dsn)
+
+
+
def postgresPreflight(connection):
"""
Pre-flight function for PostgreSQL connections: enable standard conforming
Modified: CalendarServer/branches/users/glyph/oracle/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle/txdav/common/datastore/sql.py 2011-03-08 03:03:54 UTC (rev 7135)
+++ CalendarServer/branches/users/glyph/oracle/txdav/common/datastore/sql.py 2011-03-08 03:04:08 UTC (rev 7136)
@@ -1081,8 +1081,7 @@
rev.DELETED: True},
Where=(rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
rev.RESOURCE_ID == Parameter("resourceID")).And(
- rev.RESOURCE_NAME == None),
- #Return=rev.REVISION
+ rev.RESOURCE_NAME == None)
)
@@ -1097,7 +1096,6 @@
rev.DELETED: True},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == None),
- # Return=rev.REVISION,
)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110307/cb545848/attachment-0001.html>
More information about the calendarserver-changes
mailing list