[CalendarServer-changes] [14531] twext/trunk/twext/enterprise
source_changes at macosforge.org
source_changes at macosforge.org
Mon Mar 9 12:43:14 PDT 2015
Revision: 14531
http://trac.calendarserver.org//changeset/14531
Author: cdaboo at apple.com
Date: 2015-03-09 12:43:14 -0700 (Mon, 09 Mar 2015)
Log Message:
-----------
Support for pg8000.
Modified Paths:
--------------
twext/trunk/twext/enterprise/dal/parseschema.py
twext/trunk/twext/enterprise/test/test_jobqueue.py
twext/trunk/twext/enterprise/test/test_queue.py
twext/trunk/twext/enterprise/util.py
Modified: twext/trunk/twext/enterprise/dal/parseschema.py
===================================================================
--- twext/trunk/twext/enterprise/dal/parseschema.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/dal/parseschema.py 2015-03-09 19:43:14 UTC (rev 14531)
@@ -34,6 +34,7 @@
]
from itertools import chain
+from re import compile
from sqlparse import parse, keywords
from sqlparse.tokens import (
@@ -752,3 +753,41 @@
here. The only quoting syntax respected is "''".)
"""
return strval[1:-1].replace("''", "'")
+
+
+
+def splitSQLString(sqlString):
+ """
+ Strings which mix zero or more sql statements with zero or more pl/sql
+ statements need to be split into individual sql statements for execution.
+ This function was written to allow execution of pl/sql during Oracle schema
+ upgrades.
+ """
+ aggregated = ''
+ inPlSQL = None
+ parsed = parse(sqlString)
+ for stmt in parsed:
+ while stmt.tokens and not significant(stmt.tokens[0]):
+ stmt.tokens.pop(0)
+ if not stmt.tokens:
+ continue
+ if inPlSQL is not None:
+ agg = str(stmt).strip()
+ if "end;".lower() in agg.lower():
+ inPlSQL = None
+ aggregated += agg
+ rex = compile("\n +")
+ aggregated = rex.sub('\n', aggregated)
+ yield aggregated.strip()
+ continue
+ aggregated += agg
+ continue
+ if inPlSQL is None:
+ # if 'begin'.lower() in str(stmt).split()[0].lower():
+ if str(stmt).lower().strip().startswith('begin'):
+ inPlSQL = True
+ aggregated += str(stmt)
+ continue
+ else:
+ continue
+ yield str(stmt).rstrip().rstrip(";")
Modified: twext/trunk/twext/enterprise/test/test_jobqueue.py
===================================================================
--- twext/trunk/twext/enterprise/test/test_jobqueue.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/test/test_jobqueue.py 2015-03-09 19:43:14 UTC (rev 14531)
@@ -33,6 +33,7 @@
from twisted.application.service import Service, MultiService
from twext.enterprise.dal.syntax import SchemaSyntax, Delete
+from twext.enterprise.dal.parseschema import splitSQLString
from twext.enterprise.dal.record import fromTable
from twext.enterprise.dal.test.test_parseschema import SchemaTestHelper
from twext.enterprise.fixtures import buildConnectionPool
@@ -708,7 +709,6 @@
self.checkPerformer(LocalPerformer)
-
def test_choosingPerformerWithLocalCapacity(self):
"""
If L{PeerConnectionPool.choosePerformer} is invoked when some workers
@@ -1152,8 +1152,10 @@
"""
self.store = yield buildStore(self, None)
+ @inlineCallbacks
def doit(txn):
- return txn.execSQL(schemaText)
+ for statement in splitSQLString(schemaText):
+ yield txn.execSQL(statement)
yield inTransaction(
self.store.newTransaction,
Modified: twext/trunk/twext/enterprise/test/test_queue.py
===================================================================
--- twext/trunk/twext/enterprise/test/test_queue.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/test/test_queue.py 2015-03-09 19:43:14 UTC (rev 14531)
@@ -33,6 +33,7 @@
from twisted.application.service import Service, MultiService
from twext.enterprise.dal.syntax import SchemaSyntax, Select
+from twext.enterprise.dal.parseschema import splitSQLString
from twext.enterprise.dal.record import fromTable, Record
from twext.enterprise.dal.test.test_parseschema import SchemaTestHelper
from twext.enterprise.fixtures import buildConnectionPool
@@ -772,8 +773,10 @@
"""
self.store = yield buildStore(self, None)
+ @inlineCallbacks
def doit(txn):
- return txn.execSQL(schemaText)
+ for statement in splitSQLString(schemaText):
+ yield txn.execSQL(statement)
yield inTransaction(
lambda: self.store.newTransaction("bonus schema"), doit
@@ -907,7 +910,7 @@
).on(txn)
rows = yield inTransaction(self.store.newTransaction, op2)
- self.assertEquals(rows, [])
+ self.assertEquals(list(rows), [])
def op3(txn):
return Select(
@@ -919,7 +922,7 @@
).on(txn)
rows = yield inTransaction(self.store.newTransaction, op3)
- self.assertEquals(rows, [])
+ self.assertEquals(list(rows), [])
Modified: twext/trunk/twext/enterprise/util.py
===================================================================
--- twext/trunk/twext/enterprise/util.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/util.py 2015-03-09 19:43:14 UTC (rev 14531)
@@ -30,9 +30,12 @@
Parse an SQL timestamp string.
"""
# Handle case where fraction seconds may not be present
- if len(ts) < len(SQL_TIMESTAMP_FORMAT):
- ts += ".0"
- return datetime.strptime(ts, SQL_TIMESTAMP_FORMAT)
+ if not isinstance(ts, datetime):
+ if len(ts) < len(SQL_TIMESTAMP_FORMAT):
+ ts += ".0"
+ return datetime.strptime(ts, SQL_TIMESTAMP_FORMAT)
+ else:
+ return ts
@@ -58,17 +61,8 @@
# use the fetchall() method".
column = column.read()
- elif isinstance(column, datetime):
- # cx_Oracle properly maps the type of timestamps to datetime
- # objects. However, our code is mostly written against
- # PyGreSQL, which just emits strings as results and expects
- # to have to convert them itself. Since it's easier to
- # just detect the datetimes and stringify them, for now
- # we'll do that.
- return column.strftime(SQL_TIMESTAMP_FORMAT)
-
elif isinstance(column, float):
- # cx_Oracle maps _all_ nubmers to float types, which is more
+ # cx_Oracle maps _all_ numbers to float types, which is more
# consistent, but we expect the database to be able to store integers
# as integers (in fact almost all the values in our schema are
# integers), so we map those values which exactly match back into
@@ -79,7 +73,7 @@
return column
if isinstance(column, unicode):
- # Finally, we process all data as UTF-8 bytestrings in order to reduce
+ # Finally, we process all data as UTF-8 byte strings in order to reduce
# memory consumption. Pass any unicode string values back to the
# application as unicode.
column = column.encode("utf-8")
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150309/cadb5fe1/attachment.html>
More information about the calendarserver-changes
mailing list