<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[14531] twext/trunk/twext/enterprise</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/14531">14531</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2015-03-09 12:43:14 -0700 (Mon, 09 Mar 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Support for pg8000.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#twexttrunktwextenterprisedalparseschemapy">twext/trunk/twext/enterprise/dal/parseschema.py</a></li>
<li><a href="#twexttrunktwextenterprisetesttest_jobqueuepy">twext/trunk/twext/enterprise/test/test_jobqueue.py</a></li>
<li><a href="#twexttrunktwextenterprisetesttest_queuepy">twext/trunk/twext/enterprise/test/test_queue.py</a></li>
<li><a href="#twexttrunktwextenterpriseutilpy">twext/trunk/twext/enterprise/util.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="twexttrunktwextenterprisedalparseschemapy"></a>
<div class="modfile"><h4>Modified: twext/trunk/twext/enterprise/dal/parseschema.py (14530 => 14531)</h4>
<pre class="diff"><span>
<span class="info">--- twext/trunk/twext/enterprise/dal/parseschema.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/dal/parseschema.py 2015-03-09 19:43:14 UTC (rev 14531)
</span><span class="lines">@@ -34,6 +34,7 @@
</span><span class="cx"> ]
</span><span class="cx">
</span><span class="cx"> from itertools import chain
</span><ins>+from re import compile
</ins><span class="cx">
</span><span class="cx"> from sqlparse import parse, keywords
</span><span class="cx"> from sqlparse.tokens import (
</span><span class="lines">@@ -752,3 +753,41 @@
</span><span class="cx"> here. The only quoting syntax respected is "''".)
</span><span class="cx"> """
</span><span class="cx"> return strval[1:-1].replace("''", "'")
</span><ins>+
+
+
+def splitSQLString(sqlString):
+ """
+ Strings which mix zero or more sql statements with zero or more pl/sql
+ statements need to be split into individual sql statements for execution.
+ This function was written to allow execution of pl/sql during Oracle schema
+ upgrades.
+ """
+ aggregated = ''
+ inPlSQL = None
+ parsed = parse(sqlString)
+ for stmt in parsed:
+ while stmt.tokens and not significant(stmt.tokens[0]):
+ stmt.tokens.pop(0)
+ if not stmt.tokens:
+ continue
+ if inPlSQL is not None:
+ agg = str(stmt).strip()
+ if "end;".lower() in agg.lower():
+ inPlSQL = None
+ aggregated += agg
+ rex = compile("\n +")
+ aggregated = rex.sub('\n', aggregated)
+ yield aggregated.strip()
+ continue
+ aggregated += agg
+ continue
+ if inPlSQL is None:
+ # if 'begin'.lower() in str(stmt).split()[0].lower():
+ if str(stmt).lower().strip().startswith('begin'):
+ inPlSQL = True
+ aggregated += str(stmt)
+ continue
+ else:
+ continue
+ yield str(stmt).rstrip().rstrip(";")
</ins></span></pre></div>
<a id="twexttrunktwextenterprisetesttest_jobqueuepy"></a>
<div class="modfile"><h4>Modified: twext/trunk/twext/enterprise/test/test_jobqueue.py (14530 => 14531)</h4>
<pre class="diff"><span>
<span class="info">--- twext/trunk/twext/enterprise/test/test_jobqueue.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/test/test_jobqueue.py 2015-03-09 19:43:14 UTC (rev 14531)
</span><span class="lines">@@ -33,6 +33,7 @@
</span><span class="cx"> from twisted.application.service import Service, MultiService
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.syntax import SchemaSyntax, Delete
</span><ins>+from twext.enterprise.dal.parseschema import splitSQLString
</ins><span class="cx"> from twext.enterprise.dal.record import fromTable
</span><span class="cx"> from twext.enterprise.dal.test.test_parseschema import SchemaTestHelper
</span><span class="cx"> from twext.enterprise.fixtures import buildConnectionPool
</span><span class="lines">@@ -708,7 +709,6 @@
</span><span class="cx"> self.checkPerformer(LocalPerformer)
</span><span class="cx">
</span><span class="cx">
</span><del>-
</del><span class="cx"> def test_choosingPerformerWithLocalCapacity(self):
</span><span class="cx"> """
</span><span class="cx"> If L{PeerConnectionPool.choosePerformer} is invoked when some workers
</span><span class="lines">@@ -1152,8 +1152,10 @@
</span><span class="cx"> """
</span><span class="cx"> self.store = yield buildStore(self, None)
</span><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def doit(txn):
</span><del>- return txn.execSQL(schemaText)
</del><ins>+ for statement in splitSQLString(schemaText):
+ yield txn.execSQL(statement)
</ins><span class="cx">
</span><span class="cx"> yield inTransaction(
</span><span class="cx"> self.store.newTransaction,
</span></span></pre></div>
<a id="twexttrunktwextenterprisetesttest_queuepy"></a>
<div class="modfile"><h4>Modified: twext/trunk/twext/enterprise/test/test_queue.py (14530 => 14531)</h4>
<pre class="diff"><span>
<span class="info">--- twext/trunk/twext/enterprise/test/test_queue.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/test/test_queue.py 2015-03-09 19:43:14 UTC (rev 14531)
</span><span class="lines">@@ -33,6 +33,7 @@
</span><span class="cx"> from twisted.application.service import Service, MultiService
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.syntax import SchemaSyntax, Select
</span><ins>+from twext.enterprise.dal.parseschema import splitSQLString
</ins><span class="cx"> from twext.enterprise.dal.record import fromTable, Record
</span><span class="cx"> from twext.enterprise.dal.test.test_parseschema import SchemaTestHelper
</span><span class="cx"> from twext.enterprise.fixtures import buildConnectionPool
</span><span class="lines">@@ -772,8 +773,10 @@
</span><span class="cx"> """
</span><span class="cx"> self.store = yield buildStore(self, None)
</span><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def doit(txn):
</span><del>- return txn.execSQL(schemaText)
</del><ins>+ for statement in splitSQLString(schemaText):
+ yield txn.execSQL(statement)
</ins><span class="cx">
</span><span class="cx"> yield inTransaction(
</span><span class="cx"> lambda: self.store.newTransaction("bonus schema"), doit
</span><span class="lines">@@ -907,7 +910,7 @@
</span><span class="cx"> ).on(txn)
</span><span class="cx">
</span><span class="cx"> rows = yield inTransaction(self.store.newTransaction, op2)
</span><del>- self.assertEquals(rows, [])
</del><ins>+ self.assertEquals(list(rows), [])
</ins><span class="cx">
</span><span class="cx"> def op3(txn):
</span><span class="cx"> return Select(
</span><span class="lines">@@ -919,7 +922,7 @@
</span><span class="cx"> ).on(txn)
</span><span class="cx">
</span><span class="cx"> rows = yield inTransaction(self.store.newTransaction, op3)
</span><del>- self.assertEquals(rows, [])
</del><ins>+ self.assertEquals(list(rows), [])
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="twexttrunktwextenterpriseutilpy"></a>
<div class="modfile"><h4>Modified: twext/trunk/twext/enterprise/util.py (14530 => 14531)</h4>
<pre class="diff"><span>
<span class="info">--- twext/trunk/twext/enterprise/util.py 2015-03-09 17:01:19 UTC (rev 14530)
+++ twext/trunk/twext/enterprise/util.py 2015-03-09 19:43:14 UTC (rev 14531)
</span><span class="lines">@@ -30,9 +30,12 @@
</span><span class="cx"> Parse an SQL timestamp string.
</span><span class="cx"> """
</span><span class="cx"> # Handle case where fraction seconds may not be present
</span><del>- if len(ts) < len(SQL_TIMESTAMP_FORMAT):
- ts += ".0"
- return datetime.strptime(ts, SQL_TIMESTAMP_FORMAT)
</del><ins>+ if not isinstance(ts, datetime):
+ if len(ts) < len(SQL_TIMESTAMP_FORMAT):
+ ts += ".0"
+ return datetime.strptime(ts, SQL_TIMESTAMP_FORMAT)
+ else:
+ return ts
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -58,17 +61,8 @@
</span><span class="cx"> # use the fetchall() method".
</span><span class="cx"> column = column.read()
</span><span class="cx">
</span><del>- elif isinstance(column, datetime):
- # cx_Oracle properly maps the type of timestamps to datetime
- # objects. However, our code is mostly written against
- # PyGreSQL, which just emits strings as results and expects
- # to have to convert them itself. Since it's easier to
- # just detect the datetimes and stringify them, for now
- # we'll do that.
- return column.strftime(SQL_TIMESTAMP_FORMAT)
-
</del><span class="cx"> elif isinstance(column, float):
</span><del>- # cx_Oracle maps _all_ nubmers to float types, which is more
</del><ins>+ # cx_Oracle maps _all_ numbers to float types, which is more
</ins><span class="cx"> # consistent, but we expect the database to be able to store integers
</span><span class="cx"> # as integers (in fact almost all the values in our schema are
</span><span class="cx"> # integers), so we map those values which exactly match back into
</span><span class="lines">@@ -79,7 +73,7 @@
</span><span class="cx"> return column
</span><span class="cx">
</span><span class="cx"> if isinstance(column, unicode):
</span><del>- # Finally, we process all data as UTF-8 bytestrings in order to reduce
</del><ins>+ # Finally, we process all data as UTF-8 byte strings in order to reduce
</ins><span class="cx"> # memory consumption. Pass any unicode string values back to the
</span><span class="cx"> # application as unicode.
</span><span class="cx"> column = column.encode("utf-8")
</span></span></pre>
</div>
</div>
</body>
</html>