[CalendarServer-changes] [7346] CalendarServer/branches/users/glyph/oracle-nulls
source_changes at macosforge.org
source_changes at macosforge.org
Thu Apr 21 09:52:59 PDT 2011
Revision: 7346
http://trac.macosforge.org/projects/calendarserver/changeset/7346
Author: glyph at apple.com
Date: 2011-04-21 09:52:59 -0700 (Thu, 21 Apr 2011)
Log Message:
-----------
oracle schema adaptations
Modified Paths:
--------------
CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/syntax.py
CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/test/test_sqlsyntax.py
CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/util.py
CalendarServer/branches/users/glyph/oracle-nulls/txdav/base/datastore/dbapiclient.py
CalendarServer/branches/users/glyph/oracle-nulls/txdav/common/datastore/sql_tables.py
Modified: CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/syntax.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/syntax.py 2011-04-21 16:52:50 UTC (rev 7345)
+++ CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/syntax.py 2011-04-21 16:52:59 UTC (rev 7346)
@@ -850,7 +850,7 @@
def preQuery(self, cursor):
self.columnSyntax
typeMap = {'integer': cx_Oracle.NUMBER,
- 'text': cx_Oracle.CLOB,
+ 'text': cx_Oracle.NCLOB,
'varchar': cx_Oracle.STRING,
'timestamp': cx_Oracle.TIMESTAMP}
typeID = self.columnSyntax.model.type.name.lower()
Modified: CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/test/test_sqlsyntax.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/test/test_sqlsyntax.py 2011-04-21 16:52:50 UTC (rev 7345)
+++ CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/dal/test/test_sqlsyntax.py 2011-04-21 16:52:59 UTC (rev 7346)
@@ -520,7 +520,7 @@
class FakeCXOracleModule(object):
NUMBER = 'the NUMBER type'
STRING = 'a string type (for varchars)'
- CLOB = 'the clob type. (for text)'
+ NCLOB = 'the NCLOB type. (for text)'
TIMESTAMP = 'for timestamps!'
self.patch(syntax, 'cx_Oracle', FakeCXOracleModule)
factory = ConnectionFactory()
Modified: CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/util.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/util.py 2011-04-21 16:52:50 UTC (rev 7345)
+++ CalendarServer/branches/users/glyph/oracle-nulls/twext/enterprise/util.py 2011-04-21 16:52:59 UTC (rev 7346)
@@ -30,8 +30,10 @@
@param column: a single value from a column.
@return: a converted value based on the type of the input; oracle CLOBs and
- datetime timestamps will be converted to strings, all other types will
- be left alone.
+ datetime timestamps will be converted to strings, unicode values will be
+ converted to UTF-8 encoded byte sequences (C{str}s), and floating point
+ numbers will be converted to integer types if they are integers. Any
+ other types will be left alone.
"""
if hasattr(column, 'read'):
# Try to detect large objects and format convert them to
@@ -40,7 +42,7 @@
# http://cx-oracle.sourceforge.net/html/lob.html - in
# particular, the part where it says "In particular, do not
# use the fetchall() method".
- return column.read()
+ column = column.read()
elif isinstance(column, datetime):
# cx_Oracle properly maps the type of timestamps to datetime
# objects. However, our code is mostly written against
@@ -50,12 +52,20 @@
# we'll do that.
return column.strftime(SQL_TIMESTAMP_FORMAT)
elif isinstance(column, float):
+ # cx_Oracle maps _all_ nubmers to float types, which is more consistent,
+ # but we expect the database to be able to store integers as integers
+ # (in fact almost all the values in our schema are integers), so we map
+ # those values which exactly match back into integers.
if int(column) == column:
return int(column)
else:
return column
- else:
- return column
+ if isinstance(column, unicode):
+ # Finally, we process all data as UTF-8 bytestrings in order to reduce
+ # memory consumption. Pass any unicode string values back to the
+ # application as unicode.
+ column = column.encode('utf-8')
+ return column
Modified: CalendarServer/branches/users/glyph/oracle-nulls/txdav/base/datastore/dbapiclient.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle-nulls/txdav/base/datastore/dbapiclient.py 2011-04-21 16:52:50 UTC (rev 7345)
+++ CalendarServer/branches/users/glyph/oracle-nulls/txdav/base/datastore/dbapiclient.py 2011-04-21 16:52:59 UTC (rev 7346)
@@ -99,7 +99,13 @@
def execute(self, sql, args=()):
realArgs = []
for arg in args:
- if isinstance(arg, (str, unicode)) and len(arg) > 1024:
+ if isinstance(arg, str):
+ # We use NCLOB everywhere, so cx_Oracle requires a unicode-type
+ # input. But we mostly pass around utf-8 encoded bytes at the
+ # application layer as they consume less memory, so do the
+ # conversion here.
+ arg = arg.decode('utf-8')
+ if isinstance(arg, unicode) and len(arg) > 1024:
# This *may* cause a type mismatch, but none of the non-CLOB
# strings that we're passing would allow a value this large
# anyway. Smaller strings will be automatically converted by
@@ -107,7 +113,7 @@
# sure why cx_Oracle itself doesn't just do the following hack
# automatically and internally for larger values too, but, here
# it is:
- v = self.var(cx_Oracle.CLOB, len(arg) + 1)
+ v = self.var(cx_Oracle.NCLOB, len(arg) + 1)
v.setvalue(0, arg)
else:
v = arg
Modified: CalendarServer/branches/users/glyph/oracle-nulls/txdav/common/datastore/sql_tables.py
===================================================================
--- CalendarServer/branches/users/glyph/oracle-nulls/txdav/common/datastore/sql_tables.py 2011-04-21 16:52:50 UTC (rev 7345)
+++ CalendarServer/branches/users/glyph/oracle-nulls/txdav/common/datastore/sql_tables.py 2011-04-21 16:52:59 UTC (rev 7346)
@@ -222,7 +222,7 @@
out.write(",\n")
typeName = column.model.type.name
if typeName == 'text':
- typeName = 'clob'
+ typeName = 'nclob'
if typeName == 'boolean':
typeName = 'integer'
out.write(' "%s" %s' % (column.model.name, typeName))
@@ -247,7 +247,11 @@
elif default is False:
default = 0
out.write(" " + repr(default))
- if not column.model.canBeNull():
+ if ( (not column.model.canBeNull())
+ # Oracle treats empty strings as NULLs, so we have to accept
+ # NULL values in columns of a string type. Other types should
+ # be okay though.
+ and typeName not in ('text', 'varchar') ):
out.write(' not null')
if set([column.model]) in list(table.model.uniques()):
out.write(' unique')
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110421/d1415175/attachment.html>
More information about the calendarserver-changes
mailing list