<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[15735] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/15735">15735</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2016-07-05 12:37:44 -0700 (Tue, 05 Jul 2016)</dd>
</dl>
<h3>Log Message</h3>
<pre>Wrap database parameters into a single class that also supports optional features. Allow SKIP LOCKED optional feature with postgres. postgres 9.5.3 is now the default. Updated pip and virtualenv packages.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkbin_buildsh">CalendarServer/trunk/bin/_build.sh</a></li>
<li><a href="#CalendarServertrunkcalendarservertapcaldavpy">CalendarServer/trunk/calendarserver/tap/caldav.py</a></li>
<li><a href="#CalendarServertrunkcalendarservertaputilpy">CalendarServer/trunk/calendarserver/tap/util.py</a></li>
<li><a href="#CalendarServertrunkconfcaldavdappleplist">CalendarServer/trunk/conf/caldavd-apple.plist</a></li>
<li><a href="#CalendarServertrunkconfcaldavdstdconfigplist">CalendarServer/trunk/conf/caldavd-stdconfig.plist</a></li>
<li><a href="#CalendarServertrunkrequirementscstxt">CalendarServer/trunk/requirements-cs.txt</a></li>
<li><a href="#CalendarServertrunktwistedcaldavdumpconfigpy">CalendarServer/trunk/twistedcaldav/dumpconfig.py</a></li>
<li><a href="#CalendarServertrunktwistedcaldavstdconfigpy">CalendarServer/trunk/twistedcaldav/stdconfig.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresqlpy">CalendarServer/trunk/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_dumppy">CalendarServer/trunk/txdav/common/datastore/sql_dump.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_tablespy">CalendarServer/trunk/txdav/common/datastore/sql_tables.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoretesttest_sql_dumppy">CalendarServer/trunk/txdav/common/datastore/test/test_sql_dump.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoretesttest_trashpy">CalendarServer/trunk/txdav/common/datastore/test/test_trash.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoretestutilpy">CalendarServer/trunk/txdav/common/datastore/test/util.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoreupgradesqltesttest_upgradepy">CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoreupgradesqlupgradepy">CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkbin_buildsh"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/bin/_build.sh (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/bin/_build.sh        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/bin/_build.sh        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -602,7 +602,7 @@
</span><span class="cx"> if command -v postgres > /dev/null; then
</span><span class="cx"> using_system "Postgres";
</span><span class="cx"> else
</span><del>- local v="9.3.10";
</del><ins>+ local v="9.5.3";
</ins><span class="cx"> local n="postgresql";
</span><span class="cx"> local p="${n}-${v}";
</span><span class="cx">
</span><span class="lines">@@ -612,7 +612,7 @@
</span><span class="cx"> local enable_dtrace="";
</span><span class="cx"> fi;
</span><span class="cx">
</span><del>- c_dependency -m "ec2365548d08f69c8023eddd4f2d1a28" \
</del><ins>+ c_dependency -m "3f0c388566c688c82b01a0edf1e6b7a0" \
</ins><span class="cx"> "PostgreSQL" "${p}" \
</span><span class="cx"> "http://ftp.postgresql.org/pub/source/v${v}/${p}.tar.bz2" \
</span><span class="cx"> --with-python ${enable_dtrace};
</span><span class="lines">@@ -740,8 +740,8 @@
</span><span class="cx">
</span><span class="cx"> for pkg in \
</span><span class="cx"> setuptools-18.5 \
</span><del>- pip-7.1.2 \
- virtualenv-13.1.2 \
</del><ins>+ pip-8.1.2 \
+ virtualenv-15.0.2 \
</ins><span class="cx"> ; do
</span><span class="cx"> local name="${pkg%-*}";
</span><span class="cx"> local version="${pkg#*-}";
</span></span></pre></div>
<a id="CalendarServertrunkcalendarservertapcaldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tap/caldav.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tap/caldav.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -66,8 +66,7 @@
</span><span class="cx"> from twext.internet.ssl import ChainingOpenSSLContextFactory
</span><span class="cx"> from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
</span><span class="cx"> from twext.enterprise.adbapi2 import ConnectionPool
</span><del>-from twext.enterprise.ienterprise import ORACLE_DIALECT
-from twext.enterprise.ienterprise import POSTGRES_DIALECT
</del><ins>+from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT, DatabaseType
</ins><span class="cx"> from twext.enterprise.jobs.queue import NonPerformingQueuer
</span><span class="cx"> from twext.enterprise.jobs.queue import ControllerQueue
</span><span class="cx"> from twext.enterprise.jobs.queue import WorkerFactory as QueueWorkerFactory
</span><span class="lines">@@ -1565,7 +1564,7 @@
</span><span class="cx"> @rtype: L{IService}
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def createSubServiceFactory(dbtype):
</del><ins>+ def createSubServiceFactory(dbtype, dbfeatures=()):
</ins><span class="cx"> if dbtype == "":
</span><span class="cx"> dialect = POSTGRES_DIALECT
</span><span class="cx"> paramstyle = "pyformat"
</span><span class="lines">@@ -1579,8 +1578,8 @@
</span><span class="cx"> def subServiceFactory(connectionFactory, storageService):
</span><span class="cx"> ms = MultiService()
</span><span class="cx"> cp = ConnectionPool(
</span><del>- connectionFactory, dialect=dialect,
- paramstyle=paramstyle,
</del><ins>+ connectionFactory,
+ dbtype=DatabaseType(dialect, paramstyle, dbfeatures),
</ins><span class="cx"> maxConnections=config.MaxDBConnectionsPerPool
</span><span class="cx"> )
</span><span class="cx"> cp.setServiceParent(ms)
</span><span class="lines">@@ -1680,13 +1679,13 @@
</span><span class="cx"> # to it.
</span><span class="cx"> pgserv = pgServiceFromConfig(
</span><span class="cx"> config,
</span><del>- createSubServiceFactory(""),
</del><ins>+ createSubServiceFactory("", config.DBFeatures),
</ins><span class="cx"> uid=overrideUID, gid=overrideGID
</span><span class="cx"> )
</span><span class="cx"> return pgserv
</span><span class="cx"> else:
</span><span class="cx"> # Connect to a database that is already running.
</span><del>- return createSubServiceFactory(config.DBType)(
</del><ins>+ return createSubServiceFactory(config.DBType, config.DBFeatures)(
</ins><span class="cx"> DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect, None
</span><span class="cx"> )
</span><span class="cx"> else:
</span></span></pre></div>
<a id="CalendarServertrunkcalendarservertaputilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tap/util.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tap/util.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/calendarserver/tap/util.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -44,8 +44,7 @@
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.adbapi2 import ConnectionPool, ConnectionPoolConnection
</span><span class="cx"> from twext.enterprise.adbapi2 import ConnectionPoolClient
</span><del>-from twext.enterprise.ienterprise import ORACLE_DIALECT
-from twext.enterprise.ienterprise import POSTGRES_DIALECT
</del><ins>+from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT, DatabaseType
</ins><span class="cx"> from twext.internet.ssl import ChainingOpenSSLContextFactory
</span><span class="cx"> from twext.python.filepath import CachingFilePath
</span><span class="cx"> from twext.python.filepath import CachingFilePath as FilePath
</span><span class="lines">@@ -179,14 +178,14 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-def transactionFactoryFromFD(dbampfd, dialect, paramstyle):
</del><ins>+def transactionFactoryFromFD(dbampfd, dbtype):
</ins><span class="cx"> """
</span><span class="cx"> Create a transaction factory from an inherited file descriptor, such as one
</span><span class="cx"> created by L{ConnectionDispenser}.
</span><span class="cx"> """
</span><span class="cx"> skt = fromfd(dbampfd, AF_UNIX, SOCK_STREAM)
</span><span class="cx"> os.close(dbampfd)
</span><del>- protocol = ConnectionPoolClient(dialect=dialect, paramstyle=paramstyle)
</del><ins>+ protocol = ConnectionPoolClient(dbtype=dbtype)
</ins><span class="cx"> transport = ConnectionWithPeer(skt, protocol)
</span><span class="cx"> protocol.makeConnection(transport)
</span><span class="cx"> transport.startReading()
</span><span class="lines">@@ -830,7 +829,7 @@
</span><span class="cx"> pool = None
</span><span class="cx"> if config.DBAMPFD:
</span><span class="cx"> txnFactory = transactionFactoryFromFD(
</span><del>- int(config.DBAMPFD), dialect, paramstyle
</del><ins>+ int(config.DBAMPFD), DatabaseType(dialect, paramstyle, config.DBFeatures)
</ins><span class="cx"> )
</span><span class="cx"> elif not config.UseDatabase:
</span><span class="cx"> txnFactory = None
</span><span class="lines">@@ -843,8 +842,7 @@
</span><span class="cx"> else:
</span><span class="cx"> connectionFactory = DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect
</span><span class="cx">
</span><del>- pool = ConnectionPool(connectionFactory, dialect=dialect,
- paramstyle=paramstyle,
</del><ins>+ pool = ConnectionPool(connectionFactory, dbtype=DatabaseType(dialect, paramstyle, config.DBFeatures),
</ins><span class="cx"> maxConnections=config.MaxDBConnectionsPerPool)
</span><span class="cx"> txnFactory = pool.connection
</span><span class="cx"> else:
</span></span></pre></div>
<a id="CalendarServertrunkconfcaldavdappleplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/conf/caldavd-apple.plist (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/conf/caldavd-apple.plist        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/conf/caldavd-apple.plist        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -108,6 +108,11 @@
</span><span class="cx"> <!-- Database connection -->
</span><span class="cx"> <key>DBType</key>
</span><span class="cx"> <string></string>
</span><ins>+
+ <!-- No features - when macOS supports postgres 9.5 and up change
+          this to include 'skip-locked'. -->
+        <key>DBFeatures</key>
+        <array></array>
</ins><span class="cx">
</span><span class="cx"> <key>Postgres</key>
</span><span class="cx"> <dict>
</span></span></pre></div>
<a id="CalendarServertrunkconfcaldavdstdconfigplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/conf/caldavd-stdconfig.plist (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/conf/caldavd-stdconfig.plist        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/conf/caldavd-stdconfig.plist        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -118,35 +118,43 @@
</span><span class="cx">         <array>
</span><span class="cx">         </array>
</span><span class="cx">
</span><ins>+        <!-- Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes. -->
+        <key>UseMetaFD</key>
+        <true/>
+
</ins><span class="cx">         <!-- Inherited file descriptor to call recvmsg() on to receive sockets (none =
</span><span class="cx">          don't inherit) -->
</span><span class="cx">         <key>MetaFD</key>
</span><span class="cx">         <integer>0</integer>
</span><span class="cx">
</span><del>-        <!-- Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes. -->
-        <key>UseMetaFD</key>
-        <true/>
</del><ins>+        <!-- Database configuration information.
</ins><span class="cx">
</span><del>-        <!-- True: database; False: files -->
</del><ins>+         Defines what kind of database to use: file (deprecated) or SQL. File-based
+         DB is only supported for migration purposes - it cannot be used for a real
+         service.
+
+         For an SQL-based DB, configuration of connection parameters and various
+         timeouts is provided. -->
+
+        <!-- True: database; False: files (deprecated) -->
</ins><span class="cx">         <key>UseDatabase</key>
</span><span class="cx">         <true/>
</span><span class="cx">
</span><del>-        <!-- Timeout transactions that take longer than the specified number of
-         seconds. Zero means no timeouts. 5 minute default. -->
-        <key>TransactionTimeoutSeconds</key>
-        <integer>300</integer>
-
-        <!-- When a transactions times out tell HTTP clients clients to retry after
-         this amount of time -->
-        <key>TransactionHTTPRetrySeconds</key>
-        <integer>300</integer>
-
-        <!-- 2 possible values: empty, meaning 'spawn postgres yourself', or
-         'postgres', meaning 'connect to a postgres database as specified by the
-         'DSN' configuration key. Will support more values in the future. -->
</del><ins>+        <!-- Possible values: empty, meaning 'spawn postgres yourself', or 'postgres'
+         or 'oracle', meaning 'connect to a postgres or Oracle database as
+         specified by the 'DSN' configuration key. -->
</ins><span class="cx">         <key>DBType</key>
</span><span class="cx">         <string></string>
</span><span class="cx">
</span><ins>+        <!-- Features supported by the database
+
+         'skip-locked': SKIP LOCKED available with SELECT (remove if using postgres
+         &lt; v9.5) -->
+        <key>DBFeatures</key>
+        <array>
+                <string>skip-locked</string>
+        </array>
+
</ins><span class="cx">         <!-- The username to use when DBType is empty -->
</span><span class="cx">         <key>SpawnedDBUser</key>
</span><span class="cx">         <string>caldav</string>
</span><span class="lines">@@ -175,17 +183,17 @@
</span><span class="cx">                 <false/>
</span><span class="cx">         </dict>
</span><span class="cx">
</span><ins>+        <!-- Use a shared database connection pool in the master process, rather than
+         having each client make its connections directly. -->
+        <key>SharedConnectionPool</key>
+        <false/>
+
</ins><span class="cx">         <!-- Internally used by database to tell slave processes to inherit a file
</span><span class="cx">          descriptor and use it as an AMP connection over a UNIX socket; see
</span><span class="cx">          twext.enterprise.adbapi2.ConnectionPoolConnection -->
</span><span class="cx">         <key>DBAMPFD</key>
</span><span class="cx">         <integer>0</integer>
</span><span class="cx">
</span><del>-        <!-- Use a shared database connection pool in the master process, rather than
-         having each client make its connections directly. -->
-        <key>SharedConnectionPool</key>
-        <false/>
-
</del><span class="cx">         <!-- Set to True to prevent the server or utility tools from running if the
</span><span class="cx">          database needs a schema upgrade. -->
</span><span class="cx">         <key>FailIfUpgradeNeeded</key>
</span><span class="lines">@@ -203,6 +211,16 @@
</span><span class="cx">         <key>UpgradeHomePrefix</key>
</span><span class="cx">         <string></string>
</span><span class="cx">
</span><ins>+        <!-- Timeout transactions that take longer than the specified number of
+         seconds. Zero means no timeouts. 5 minute default. -->
+        <key>TransactionTimeoutSeconds</key>
+        <integer>300</integer>
+
+        <!-- When a transactions times out tell HTTP clients clients to retry after
+         this amount of time -->
+        <key>TransactionHTTPRetrySeconds</key>
+        <integer>300</integer>
+
</ins><span class="cx">         <!-- Work queue configuration information -->
</span><span class="cx">
</span><span class="cx">         <key>WorkQueue</key>
</span></span></pre></div>
<a id="CalendarServertrunkrequirementscstxt"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/requirements-cs.txt (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/requirements-cs.txt        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/requirements-cs.txt        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -7,7 +7,7 @@
</span><span class="cx"> zope.interface==4.1.3
</span><span class="cx">          setuptools==18.5
</span><span class="cx">
</span><del>- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@15700#egg=twextpy
</del><ins>+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/trunk@15734#egg=twextpy
</ins><span class="cx"> cffi==1.3.0
</span><span class="cx"> pycparser==2.14
</span><span class="cx"> #twisted
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavdumpconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/dumpconfig.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/dumpconfig.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/twistedcaldav/dumpconfig.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -23,6 +23,8 @@
</span><span class="cx"> import re
</span><span class="cx"> import textwrap
</span><span class="cx">
</span><ins>+DEBUG = False
+
</ins><span class="cx"> COPYRIGHT = """
</span><span class="cx"> <!--
</span><span class="cx"> Copyright (c) 2006-2016 Apple Inc. All rights reserved.
</span><span class="lines">@@ -313,12 +315,14 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> for item in maps.keys():
</span><ins>+ if DEBUG:
+ print(item)
</ins><span class="cx"> lines = parseConfigItem(item)
</span><del>- maps[item] = processConfig(lines, with_comments=True, verbose=False)
</del><ins>+ maps[item] = processConfig(lines, with_comments=True, verbose=DEBUG)
</ins><span class="cx">
</span><span class="cx"> # Generate the plist for the default config, substituting for the *_PARAMS items
</span><span class="cx"> lines = parseConfigItem("DEFAULT_CONFIG")
</span><del>- j = processConfig(lines, with_comments=True, verbose=False, substitutions=maps)
</del><ins>+ j = processConfig(lines, with_comments=True, verbose=DEBUG, substitutions=maps)
</ins><span class="cx"> return writeOrderedPlistToString(j)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavstdconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/stdconfig.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -195,24 +195,36 @@
</span><span class="cx"> "BindSSLPorts": [], # List of port numbers to bind to for SSL [empty = same as "SSLPort"]
</span><span class="cx"> "InheritFDs": [], # File descriptors to inherit for HTTP requests [empty = don't inherit]
</span><span class="cx"> "InheritSSLFDs": [], # File descriptors to inherit for HTTPS requests [empty = don't inherit]
</span><del>- "MetaFD": 0, # Inherited file descriptor to call recvmsg() on to receive sockets (none = don't inherit)
</del><span class="cx">
</span><span class="cx"> "UseMetaFD": True, # Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes.
</span><ins>+ "MetaFD": 0, # Inherited file descriptor to call recvmsg() on to receive sockets (none = don't inherit)
</ins><span class="cx">
</span><del>- "UseDatabase": True, # True: database; False: files
</del><ins>+ #
+ # Database configuration information.
+ #
+ # Defines what kind of database to use: file (deprecated) or SQL.
+ # File-based DB is only supported for migration purposes - it
+ # cannot be used for a real service.
+ #
+ # For an SQL-based DB, configuration of connection parameters and various
+ # timeouts is provided.
+ #
</ins><span class="cx">
</span><del>- "TransactionTimeoutSeconds": 300, # Timeout transactions that take longer than
- # the specified number of seconds. Zero means
- # no timeouts. 5 minute default.
- "TransactionHTTPRetrySeconds": 300, # When a transactions times out tell HTTP clients
- # clients to retry after this amount of time
</del><ins>+ "UseDatabase": True, # True: database; False: files (deprecated)
</ins><span class="cx">
</span><del>- "DBType": "", # 2 possible values: empty, meaning 'spawn postgres
- # yourself', or 'postgres', meaning 'connect to a
- # postgres database as specified by the 'DSN'
- # configuration key. Will support more values in
- # the future.
</del><ins>+ "DBType": "", # Possible values: empty, meaning 'spawn postgres
+ # yourself', or 'postgres' or 'oracle', meaning
+ # 'connect to a postgres or Oracle database as
+ # specified by the 'DSN' configuration key.
</ins><span class="cx">
</span><ins>+
+ "DBFeatures": [ # Features supported by the database
+ #
+ # 'skip-locked': SKIP LOCKED available with SELECT (remove if using postgres < v9.5)
+ #
+ "skip-locked",
+ ],
+
</ins><span class="cx"> "SpawnedDBUser": "caldav", # The username to use when DBType is empty
</span><span class="cx">
</span><span class="cx"> "DatabaseConnection": { # Used to connect to an external database if DBType is non-empty
</span><span class="lines">@@ -223,16 +235,16 @@
</span><span class="cx"> "ssl": False, # Set to True to require SSL (pg8000 only).
</span><span class="cx"> },
</span><span class="cx">
</span><ins>+ "SharedConnectionPool": False, # Use a shared database connection pool in
+ # the master process, rather than having
+ # each client make its connections directly.
+
</ins><span class="cx"> "DBAMPFD": 0, # Internally used by database to tell slave
</span><span class="cx"> # processes to inherit a file descriptor and use it
</span><span class="cx"> # as an AMP connection over a UNIX socket; see
</span><span class="cx"> # twext.enterprise.adbapi2.ConnectionPoolConnection
</span><span class="cx">
</span><del>- "SharedConnectionPool": False, # Use a shared database connection pool in
- # the master process, rather than having
- # each client make its connections directly.
-
- "FailIfUpgradeNeeded": True, # Set to True to prevent the server or utility
</del><ins>+ "FailIfUpgradeNeeded": True, # Set to True to prevent the server or utility
</ins><span class="cx"> # tools from running if the database needs a schema
</span><span class="cx"> # upgrade.
</span><span class="cx">
</span><span class="lines">@@ -246,6 +258,12 @@
</span><span class="cx"> # need to be run again without this prefix set to complete the overall
</span><span class="cx"> # upgrade.
</span><span class="cx">
</span><ins>+ "TransactionTimeoutSeconds": 300, # Timeout transactions that take longer than
+ # the specified number of seconds. Zero means
+ # no timeouts. 5 minute default.
+ "TransactionHTTPRetrySeconds": 300, # When a transactions times out tell HTTP clients
+ # clients to retry after this amount of time
+
</ins><span class="cx"> #
</span><span class="cx"> # Work queue configuration information
</span><span class="cx"> #
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -643,8 +643,7 @@
</span><span class="cx"> directlyProvides(self, *extraInterfaces)
</span><span class="cx">
</span><span class="cx"> self._sqlTxn = sqlTxn
</span><del>- self.paramstyle = sqlTxn.paramstyle
- self.dialect = sqlTxn.dialect
</del><ins>+ self.dbtype = sqlTxn.dbtype
</ins><span class="cx">
</span><span class="cx"> self._stats = (
</span><span class="cx"> TransactionStatsCollector(self._label, self._store.logStatsLogFile)
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_dumppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_dump.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_dump.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/sql_dump.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -54,9 +54,9 @@
</span><span class="cx"> Generate the L{Schema}.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- if txn.dialect == POSTGRES_DIALECT:
</del><ins>+ if txn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> return dumpSchema_postgres(txn, title, schemaname)
</span><del>- elif txn.dialect == ORACLE_DIALECT:
</del><ins>+ elif txn.dbtype.dialect == ORACLE_DIALECT:
</ins><span class="cx"> return dumpSchema_oracle(txn, title, schemaname)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_tablespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_tables.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_tables.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/sql_tables.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -25,7 +25,8 @@
</span><span class="cx"> from twext.enterprise.dal.model import Sequence, ProcedureCall
</span><span class="cx"> from twext.enterprise.dal.parseschema import schemaFromPath
</span><span class="cx"> from twext.enterprise.dal.syntax import FixedPlaceholder
</span><del>-from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT
</del><ins>+from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT, \
+ DatabaseType
</ins><span class="cx"> from twext.enterprise.dal.syntax import Insert
</span><span class="cx"> from twext.enterprise.ienterprise import ORACLE_TABLE_NAME_MAX
</span><span class="cx"> import hashlib
</span><span class="lines">@@ -314,7 +315,7 @@
</span><span class="cx"> @return: the generated SQL string.
</span><span class="cx"> @rtype: C{str}
</span><span class="cx"> """
</span><del>- qgen = QueryGenerator(ORACLE_DIALECT, FixedPlaceholder('%s'))
</del><ins>+ qgen = QueryGenerator(DatabaseType(ORACLE_DIALECT, "pyformat"), FixedPlaceholder('%s'))
</ins><span class="cx"> if doquote:
</span><span class="cx"> qgen.shouldQuote = lambda name: True
</span><span class="cx"> if hasattr(sql, 'subSQL'):
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoretesttest_sql_dumppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/test/test_sql_dump.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/test/test_sql_dump.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/test/test_sql_dump.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -53,7 +53,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def cleanUp(self):
</span><span class="cx"> startTxn = self.store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("set search_path to public")
</span><span class="cx"> yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
</span><span class="cx"> else:
</span><span class="lines">@@ -69,7 +69,7 @@
</span><span class="cx"> in postgres that we can quickly wipe clean afterwards.
</span><span class="cx"> """
</span><span class="cx"> startTxn = self.store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("create schema test_dbUpgrades")
</span><span class="cx"> yield startTxn.execSQL("set search_path to test_dbUpgrades")
</span><span class="cx"> yield startTxn.execSQLBlock(schema)
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoretesttest_trashpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/test/test_trash.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/test/test_trash.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/test/test_trash.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -1814,6 +1814,7 @@
</span><span class="cx">
</span><span class="cx"> yield txn.commit()
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def test_trashCalendarRestoreWithFailures(self):
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoretestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/test/util.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -36,7 +36,7 @@
</span><span class="cx"> from twext.python.filepath import CachingFilePath as FilePath
</span><span class="cx"> from twext.enterprise.adbapi2 import ConnectionPool
</span><span class="cx"> from twext.enterprise.ienterprise import AlreadyFinishedError, POSTGRES_DIALECT, \
</span><del>- ORACLE_DIALECT
</del><ins>+ ORACLE_DIALECT, DatabaseType
</ins><span class="cx"> from twext.enterprise.jobs.jobitem import JobItem
</span><span class="cx"> from twext.enterprise.jobs.queue import ControllerQueue
</span><span class="cx"> from twext.who.directory import DirectoryRecord
</span><span class="lines">@@ -162,8 +162,7 @@
</span><span class="cx"> cp = ConnectionPool(
</span><span class="cx"> stubsvc.produceConnection,
</span><span class="cx"> maxConnections=1,
</span><del>- dialect=DB_TYPE[0],
- paramstyle=DB_TYPE[1],
</del><ins>+ dbtype=DatabaseType(DB_TYPE[0], DB_TYPE[1]),
</ins><span class="cx"> )
</span><span class="cx"> # Attach the service to the running reactor.
</span><span class="cx"> cp.startService()
</span><span class="lines">@@ -251,8 +250,7 @@
</span><span class="cx"> cp = ConnectionPool(
</span><span class="cx"> self.sharedService.produceConnection,
</span><span class="cx"> maxConnections=4,
</span><del>- dialect=DB_TYPE[0],
- paramstyle=DB_TYPE[1],
</del><ins>+ dbtype=DatabaseType(DB_TYPE[0], DB_TYPE[1]),
</ins><span class="cx"> )
</span><span class="cx"> quota = deriveQuota(testCase)
</span><span class="cx"> store = CommonDataStore(
</span><span class="lines">@@ -340,10 +338,10 @@
</span><span class="cx"> # Change the starting values of sequences to random values
</span><span class="cx"> for sequence in schema.model.sequences: #@UndefinedVariable
</span><span class="cx"> try:
</span><del>- if cleanupTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if cleanupTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> curval = (yield cleanupTxn.execSQL("select nextval('{}')".format(sequence.name), []))[0][0]
</span><span class="cx"> yield cleanupTxn.execSQL("select setval('{}', {})".format(sequence.name, curval + randint(1, 10000)), [])
</span><del>- elif cleanupTxn.dialect == ORACLE_DIALECT:
</del><ins>+ elif cleanupTxn.dbtype.dialect == ORACLE_DIALECT:
</ins><span class="cx"> yield cleanupTxn.execSQL("alter sequence {} increment by {}".format(sequence.name, randint(1, 10000)), [])
</span><span class="cx"> yield cleanupTxn.execSQL("select {}.nextval from dual".format(sequence.name), [])
</span><span class="cx"> yield cleanupTxn.execSQL("alter sequence {} increment by {}".format(sequence.name, 1), [])
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoreupgradesqltesttest_upgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/test/test_upgrade.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -171,7 +171,7 @@
</span><span class="cx"> in postgres that we can quickly wipe clean afterwards.
</span><span class="cx"> """
</span><span class="cx"> startTxn = store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("create schema test_dbUpgrades")
</span><span class="cx"> yield startTxn.execSQL("set search_path to test_dbUpgrades")
</span><span class="cx"> yield startTxn.execSQLBlock(path.getContent())
</span><span class="lines">@@ -194,20 +194,20 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _unloadOldSchema():
</span><span class="cx"> startTxn = store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("set search_path to public")
</span><span class="cx"> yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
</span><del>- elif startTxn.dialect == ORACLE_DIALECT:
</del><ins>+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
</ins><span class="cx"> yield cleanDatabase(startTxn)
</span><span class="cx"> yield startTxn.commit()
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _cleanupOldSchema():
</span><span class="cx"> startTxn = store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("set search_path to public")
</span><span class="cx"> yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade")
</span><del>- elif startTxn.dialect == ORACLE_DIALECT:
</del><ins>+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
</ins><span class="cx"> yield cleanDatabase(startTxn)
</span><span class="cx"> yield startTxn.commit()
</span><span class="cx">
</span><span class="lines">@@ -286,7 +286,7 @@
</span><span class="cx"> in postgres that we can quickly wipe clean afterwards.
</span><span class="cx"> """
</span><span class="cx"> startTxn = store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("create schema test_dbUpgrades")
</span><span class="cx"> yield startTxn.execSQL("set search_path to test_dbUpgrades")
</span><span class="cx"> yield startTxn.execSQLBlock(path.getContent())
</span><span class="lines">@@ -303,20 +303,20 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _unloadOldData():
</span><span class="cx"> startTxn = store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("set search_path to public")
</span><span class="cx"> yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
</span><del>- elif startTxn.dialect == ORACLE_DIALECT:
</del><ins>+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
</ins><span class="cx"> yield cleanDatabase(startTxn)
</span><span class="cx"> yield startTxn.commit()
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _cleanupOldData():
</span><span class="cx"> startTxn = store.newTransaction("test_dbUpgrades")
</span><del>- if startTxn.dialect == POSTGRES_DIALECT:
</del><ins>+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
</ins><span class="cx"> yield startTxn.execSQL("set search_path to public")
</span><span class="cx"> yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade")
</span><del>- elif startTxn.dialect == ORACLE_DIALECT:
</del><ins>+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
</ins><span class="cx"> yield cleanDatabase(startTxn)
</span><span class="cx"> yield startTxn.commit()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoreupgradesqlupgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py (15734 => 15735)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py        2016-07-05 19:32:22 UTC (rev 15734)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/sql/upgrade.py        2016-07-05 19:37:44 UTC (rev 15735)
</span><span class="lines">@@ -190,7 +190,7 @@
</span><span class="cx">
</span><span class="cx"> # Get the schema version in the current database
</span><span class="cx"> sqlTxn = self.sqlStore.newTransaction(label="UpgradeDatabaseCoreStep.getVersions")
</span><del>- dialect = sqlTxn.dialect
</del><ins>+ dialect = sqlTxn.dbtype.dialect
</ins><span class="cx"> try:
</span><span class="cx"> actual_version = yield sqlTxn.calendarserverValue(self.versionKey)
</span><span class="cx"> actual_version = int(actual_version)
</span></span></pre>
</div>
</div>
</body>
</html>