<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[11861] CalendarServer/branches/users/gaya/sharedgroupfixes</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/11861">11861</a></dd>
<dt>Author</dt> <dd>gaya@apple.com</dd>
<dt>Date</dt> <dd>2013-10-31 15:23:31 -0700 (Thu, 31 Oct 2013)</dd>
</dl>
<h3>Log Message</h3>
<pre>merge in <a href="http://trac.calendarserver.org//changeset/11779">r11779</a> through <a href="http://trac.calendarserver.org//changeset/11860">r11860</a></pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarserveraccesslogpy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/accesslog.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarserverpushamppushpy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/amppush.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarserverpushnotifierpy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/notifier.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarserverpushtesttest_notifierpy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/test/test_notifier.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarservertapcaldavpy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/caldav.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarservertaputilpy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/util.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescalendarservertoolsupgradepy">CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tools/upgrade.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixesconfauthaccountstestxml">CalendarServer/branches/users/gaya/sharedgroupfixes/conf/auth/accounts-test.xml</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestconfigdistplist">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.dist.plist</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestconfigplist">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.plist</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestpopulationpy">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/population.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestsimpy">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/sim.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtesttest_simpy">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/test_sim.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformancesqlusagerequestshttpTestspy">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/requests/httpTests.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformancesqlusagesqlusagepy">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/sqlusage.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribtoolsfix_calendar">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/fix_calendar</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribtoolsprotocolanalysispy">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/protocolanalysis.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixessupportversionpy">CalendarServer/branches/users/gaya/sharedgroupfixes/support/version.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextenterprisedalsyntaxpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/syntax.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextenterprisedaltesttest_sqlsyntaxpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/test/test_sqlsyntax.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextpatchespy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/patches.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhoaggregatepy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/aggregate.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhodirectorypy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/directory.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhoexpressionpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/expression.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhoidirectorypy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/idirectory.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhoindexpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/index.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhoutilpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/util.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwextwhoxmlpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/xml.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavdirectorydirectorypy">CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/directory.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavdirectoryldapdirectorypy">CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/ldapdirectory.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavdirectorytesttest_directorypy">CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/test/test_directory.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavresourcepy">CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavscheduling_storecaldavresourcepy">CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/scheduling_store/caldav/resource.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavstdconfigpy">CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/stdconfig.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavbasedatastoresubpostgrespy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/subpostgres.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavbasedatastoretesttest_subpostgrespy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/test/test_subpostgres.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavbasedatastoreutilpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/util.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastorefilepy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/file.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulepy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/schedule.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingimipinboundpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/inbound.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingimiptesttest_inboundpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/test/test_inbound.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingimplicitpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/implicit.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingutilspy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/utils.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoresqlpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoretestcommonpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/common.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoretesttest_utilpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/test_util.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcarddavdatastoretestcommonpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/carddav/datastore/test/common.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastorefilepy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/file.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoresqlpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_19_to_20sql">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_19_to_20.sql</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_13_to_14sql">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_13_to_14.sql</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoretestutilpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/test/util.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqltesttest_upgradepy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradepy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrade.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradesaddressbook_upgrade_from_1_to_2py">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/addressbook_upgrade_from_1_to_2.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_1_to_2py">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_3_to_4py">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_4_to_5py">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradestesttest_upgrade_from_3_to_4py">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradestesttest_upgrade_from_4_to_5py">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py</a></li>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradesutilpy">CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/util.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestclientsplist">CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/clients.plist</a></li>
<li>CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/standard-configs/</li>
</ul>
<h3>Property Changed</h3>
<ul>
<li><a href="#CalendarServerbranchesusersgayasharedgroupfixes">CalendarServer/branches/users/gaya/sharedgroupfixes/</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesusersgayasharedgroupfixes"></a>
<div class="propset"><h4>Property changes: CalendarServer/branches/users/gaya/sharedgroupfixes</h4>
<pre class="diff"><span>
</span></pre></div>
<a id="svnmergeinfo"></a>
<div class="modfile"><h4>Modified: svn:mergeinfo</h4></div>
<span class="cx">/CalendarServer/branches/config-separation:4379-4443
</span><span class="cx">/CalendarServer/branches/egg-info-351:4589-4625
</span><span class="cx">/CalendarServer/branches/generic-sqlstore:6167-6191
</span><span class="cx">/CalendarServer/branches/new-store:5594-5934
</span><span class="cx">/CalendarServer/branches/new-store-no-caldavfile:5911-5935
</span><span class="cx">/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
</span><span class="cx">/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
</span><span class="cx">/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
</span><span class="cx">/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
</span><span class="cx">/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
</span><span class="cx">/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
</span><span class="cx">/CalendarServer/branches/users/cdaboo/fix-no-ischedule:11612
</span><span class="cx">/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
</span><span class="cx">/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
</span><span class="cx">/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
</span><span class="cx">/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
</span><span class="cx">/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pods:7297-7377
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pycard:7227-7237
</span><span class="cx">/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
</span><span class="cx">/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
</span><span class="cx">/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
</span><span class="cx">/CalendarServer/branches/users/cdaboo/store-scheduling:10876-11129
</span><span class="cx">/CalendarServer/branches/users/cdaboo/timezones:7443-7699
</span><span class="cx">/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
</span><span class="cx">/CalendarServer/branches/users/gaya/sharedgroups-3:11088-11204
</span><span class="cx">/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
</span><span class="cx">/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
</span><span class="cx">/CalendarServer/branches/users/glyph/conn-limit:6574-6577
</span><span class="cx">/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
</span><span class="cx">/CalendarServer/branches/users/glyph/dalify:6932-7023
</span><span class="cx">/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
</span><span class="cx">/CalendarServer/branches/users/glyph/deploybuild:7563-7572
</span><span class="cx">/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
</span><span class="cx">/CalendarServer/branches/users/glyph/disable-quota:7718-7727
</span><span class="cx">/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
</span><span class="cx">/CalendarServer/branches/users/glyph/enforce-max-requests:11640-11643
</span><span class="cx">/CalendarServer/branches/users/glyph/hang-fix:11465-11491
</span><span class="cx">/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
</span><span class="cx">/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
</span><span class="cx">/CalendarServer/branches/users/glyph/launchd-wrapper-bis:11413-11436
</span><span class="cx">/CalendarServer/branches/users/glyph/linux-tests:6893-6900
</span><span class="cx">/CalendarServer/branches/users/glyph/log-cleanups:11691-11731
</span><span class="cx">/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
</span><span class="cx">/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
</span><span class="cx">/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
</span><span class="cx">/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
</span><span class="cx">/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
</span><span class="cx">/CalendarServer/branches/users/glyph/new-export:7444-7485
</span><span class="cx">/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
</span><span class="cx">/CalendarServer/branches/users/glyph/oracle:7106-7155
</span><span class="cx">/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
</span><span class="cx">/CalendarServer/branches/users/glyph/other-html:8062-8091
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
</span><span class="cx">/CalendarServer/branches/users/glyph/q:9560-9688
</span><span class="cx">/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
</span><span class="cx">/CalendarServer/branches/users/glyph/quota:7604-7637
</span><span class="cx">/CalendarServer/branches/users/glyph/sendfdport:5388-5424
</span><span class="cx">/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
</span><span class="cx">/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
</span><span class="cx">/CalendarServer/branches/users/glyph/sharedpool:6490-6550
</span><span class="cx">/CalendarServer/branches/users/glyph/sharing-api:9192-9205
</span><span class="cx">/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
</span><span class="cx">/CalendarServer/branches/users/glyph/sql-store:5929-6073
</span><span class="cx">/CalendarServer/branches/users/glyph/start-service-start-loop:11060-11065
</span><span class="cx">/CalendarServer/branches/users/glyph/subtransactions:7248-7258
</span><span class="cx">/CalendarServer/branches/users/glyph/table-alias:8651-8664
</span><span class="cx">/CalendarServer/branches/users/glyph/uidexport:7673-7676
</span><span class="cx">/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
</span><span class="cx">/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
</span><span class="cx">/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
</span><span class="cx">/CalendarServer/branches/users/glyph/warning-cleanups:11347-11357
</span><span class="cx">/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
</span><span class="cx">/CalendarServer/branches/users/sagen/applepush:8126-8184
</span><span class="cx">/CalendarServer/branches/users/sagen/inboxitems:7380-7381
</span><span class="cx">/CalendarServer/branches/users/sagen/locations-resources:5032-5051
</span><span class="cx">/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
</span><span class="cx">/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
</span><span class="cx">/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
</span><span class="cx">/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
</span><span class="cx">/CalendarServer/branches/users/sagen/resources-2:5084-5093
</span><span class="cx">/CalendarServer/branches/users/sagen/testing:10827-10851,10853-10855
</span><span class="cx">/CalendarServer/branches/users/wsanchez/transations:5515-5593
</span><span class="cx">/CalendarServer/trunk:11632-11778
</span><span class="cx"> + /CalDAVTester/trunk:11193-11198
</span><span class="cx">/CalendarServer/branches/config-separation:4379-4443
</span><span class="cx">/CalendarServer/branches/egg-info-351:4589-4625
</span><span class="cx">/CalendarServer/branches/generic-sqlstore:6167-6191
</span><span class="cx">/CalendarServer/branches/new-store:5594-5934
</span><span class="cx">/CalendarServer/branches/new-store-no-caldavfile:5911-5935
</span><span class="cx">/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
</span><span class="cx">/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
</span><span class="cx">/CalendarServer/branches/release/CalendarServer-5.1-dev:11846
</span><span class="cx">/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
</span><span class="cx">/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
</span><span class="cx">/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
</span><span class="cx">/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
</span><span class="cx">/CalendarServer/branches/users/cdaboo/fix-no-ischedule:11612
</span><span class="cx">/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
</span><span class="cx">/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
</span><span class="cx">/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
</span><span class="cx">/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
</span><span class="cx">/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
</span><span class="cx">/CalendarServer/branches/users/cdaboo/performance-tweaks:11824-11836
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pods:7297-7377
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pycard:7227-7237
</span><span class="cx">/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
</span><span class="cx">/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
</span><span class="cx">/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
</span><span class="cx">/CalendarServer/branches/users/cdaboo/store-scheduling:10876-11129
</span><span class="cx">/CalendarServer/branches/users/cdaboo/timezones:7443-7699
</span><span class="cx">/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
</span><span class="cx">/CalendarServer/branches/users/gaya/sharedgroups-3:11088-11204
</span><span class="cx">/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
</span><span class="cx">/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
</span><span class="cx">/CalendarServer/branches/users/glyph/conn-limit:6574-6577
</span><span class="cx">/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
</span><span class="cx">/CalendarServer/branches/users/glyph/dalify:6932-7023
</span><span class="cx">/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
</span><span class="cx">/CalendarServer/branches/users/glyph/deploybuild:7563-7572
</span><span class="cx">/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
</span><span class="cx">/CalendarServer/branches/users/glyph/disable-quota:7718-7727
</span><span class="cx">/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
</span><span class="cx">/CalendarServer/branches/users/glyph/enforce-max-requests:11640-11643
</span><span class="cx">/CalendarServer/branches/users/glyph/hang-fix:11465-11491
</span><span class="cx">/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
</span><span class="cx">/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
</span><span class="cx">/CalendarServer/branches/users/glyph/launchd-wrapper-bis:11413-11436
</span><span class="cx">/CalendarServer/branches/users/glyph/linux-tests:6893-6900
</span><span class="cx">/CalendarServer/branches/users/glyph/log-cleanups:11691-11731
</span><span class="cx">/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
</span><span class="cx">/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
</span><span class="cx">/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
</span><span class="cx">/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
</span><span class="cx">/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
</span><span class="cx">/CalendarServer/branches/users/glyph/new-export:7444-7485
</span><span class="cx">/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
</span><span class="cx">/CalendarServer/branches/users/glyph/oracle:7106-7155
</span><span class="cx">/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
</span><span class="cx">/CalendarServer/branches/users/glyph/other-html:8062-8091
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
</span><span class="cx">/CalendarServer/branches/users/glyph/q:9560-9688
</span><span class="cx">/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
</span><span class="cx">/CalendarServer/branches/users/glyph/quota:7604-7637
</span><span class="cx">/CalendarServer/branches/users/glyph/sendfdport:5388-5424
</span><span class="cx">/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
</span><span class="cx">/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
</span><span class="cx">/CalendarServer/branches/users/glyph/sharedpool:6490-6550
</span><span class="cx">/CalendarServer/branches/users/glyph/sharing-api:9192-9205
</span><span class="cx">/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
</span><span class="cx">/CalendarServer/branches/users/glyph/sql-store:5929-6073
</span><span class="cx">/CalendarServer/branches/users/glyph/start-service-start-loop:11060-11065
</span><span class="cx">/CalendarServer/branches/users/glyph/subtransactions:7248-7258
</span><span class="cx">/CalendarServer/branches/users/glyph/table-alias:8651-8664
</span><span class="cx">/CalendarServer/branches/users/glyph/uidexport:7673-7676
</span><span class="cx">/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
</span><span class="cx">/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
</span><span class="cx">/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
</span><span class="cx">/CalendarServer/branches/users/glyph/warning-cleanups:11347-11357
</span><span class="cx">/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
</span><span class="cx">/CalendarServer/branches/users/sagen/applepush:8126-8184
</span><span class="cx">/CalendarServer/branches/users/sagen/inboxitems:7380-7381
</span><span class="cx">/CalendarServer/branches/users/sagen/locations-resources:5032-5051
</span><span class="cx">/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
</span><span class="cx">/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
</span><span class="cx">/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
</span><span class="cx">/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
</span><span class="cx">/CalendarServer/branches/users/sagen/resources-2:5084-5093
</span><span class="cx">/CalendarServer/branches/users/sagen/testing:10827-10851,10853-10855
</span><span class="cx">/CalendarServer/branches/users/wsanchez/transations:5515-5593
</span><span class="cx">/CalendarServer/trunk:11632-11860
</span><a id="CalendarServerbranchesusersgayasharedgroupfixescalendarserveraccesslogpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/accesslog.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/accesslog.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/accesslog.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -173,7 +173,7 @@
</span><span class="cx"> formatArgs["t"] = (nowtime - request.timeStamps[0][1]) * 1000
</span><span class="cx">
</span><span class="cx"> if hasattr(request, "extendedLogItems"):
</span><del>- for k, v in request.extendedLogItems.iteritems():
</del><ins>+ for k, v in sorted(request.extendedLogItems.iteritems(), key=lambda x: x[0]):
</ins><span class="cx"> k = str(k).replace('"', "%22")
</span><span class="cx"> v = str(v).replace('"', "%22")
</span><span class="cx"> if " " in v:
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescalendarserverpushamppushpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/amppush.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/amppush.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/amppush.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -48,7 +48,8 @@
</span><span class="cx"> # AMP Commands sent to client (and forwarded to Master)
</span><span class="cx">
</span><span class="cx"> class NotificationForID(amp.Command):
</span><del>- arguments = [('id', amp.String()), ('dataChangedTimestamp', amp.Integer())]
</del><ins>+ arguments = [('id', amp.String()),
+ ('dataChangedTimestamp', amp.Integer(optional=True))]
</ins><span class="cx"> response = [('status', amp.String())]
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescalendarserverpushnotifierpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/notifier.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/notifier.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/notifier.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -84,10 +84,13 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def notify(self):
</del><ins>+ def notify(self, txn):
</ins><span class="cx"> """
</span><span class="cx"> Send the notification. For a home object we just push using the home id. For a home
</span><span class="cx"> child we push both the owner home id and the owned home child id.
</span><ins>+
+ @param txn: The transaction to create the work item with
+ @type txn: L{CommonStoreTransaction}
</ins><span class="cx"> """
</span><span class="cx"> # Push ids from the store objects are a tuple of (prefix, name,) and we need to compose that
</span><span class="cx"> # into a single token.
</span><span class="lines">@@ -100,7 +103,7 @@
</span><span class="cx"> for prefix, id in ids:
</span><span class="cx"> if self._notify:
</span><span class="cx"> self.log.debug("Notifications are enabled: %s %s/%s" % (self._storeObject, prefix, id,))
</span><del>- yield self._notifierFactory.send(prefix, id)
</del><ins>+ yield self._notifierFactory.send(prefix, id, txn)
</ins><span class="cx"> else:
</span><span class="cx"> self.log.debug("Skipping notification for: %s %s/%s" % (self._storeObject, prefix, id,))
</span><span class="cx">
</span><span class="lines">@@ -147,11 +150,12 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def send(self, prefix, id):
- txn = self.store.newTransaction()
</del><ins>+ def send(self, prefix, id, txn):
+ """
+ Enqueue a push notification work item on the provided transaction.
+ """
</ins><span class="cx"> notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=self.coalesceSeconds)
</span><span class="cx"> yield txn.enqueue(PushNotificationWork, pushID=self.pushKeyForId(prefix, id), notBefore=notBefore)
</span><del>- yield txn.commit()
</del><span class="cx">
</span><span class="cx">
</span><span class="cx"> def newNotifier(self, storeObject):
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescalendarserverpushtesttest_notifierpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/test/test_notifier.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/test/test_notifier.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/push/test/test_notifier.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -169,8 +169,8 @@
</span><span class="cx">
</span><span class="cx"> home = yield self.homeUnderTest()
</span><span class="cx"> yield home.notifyChanged()
</span><ins>+ self.assertEquals(self.notifierFactory.history, ["/CalDAV/example.com/home1/"])
</ins><span class="cx"> yield self.commit()
</span><del>- self.assertEquals(self.notifierFactory.history, ["/CalDAV/example.com/home1/"])
</del><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -178,11 +178,11 @@
</span><span class="cx">
</span><span class="cx"> calendar = yield self.calendarUnderTest()
</span><span class="cx"> yield calendar.notifyChanged()
</span><del>- yield self.commit()
</del><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set(["/CalDAV/example.com/home1/", "/CalDAV/example.com/home1/calendar_1/"])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -191,7 +191,6 @@
</span><span class="cx"> calendar = yield self.calendarUnderTest()
</span><span class="cx"> home2 = yield self.homeUnderTest(name="home2")
</span><span class="cx"> yield calendar.shareWith(home2, _BIND_MODE_WRITE)
</span><del>- yield self.commit()
</del><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -200,11 +199,11 @@
</span><span class="cx"> "/CalDAV/example.com/home2/"
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx"> calendar = yield self.calendarUnderTest()
</span><span class="cx"> home2 = yield self.homeUnderTest(name="home2")
</span><span class="cx"> yield calendar.unshareWith(home2)
</span><del>- yield self.commit()
</del><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -213,6 +212,7 @@
</span><span class="cx"> "/CalDAV/example.com/home2/"
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -226,11 +226,11 @@
</span><span class="cx">
</span><span class="cx"> shared = yield self.calendarUnderTest(home="home2", name=shareName)
</span><span class="cx"> yield shared.notifyChanged()
</span><del>- yield self.commit()
</del><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set(["/CalDAV/example.com/home1/", "/CalDAV/example.com/home1/calendar_1/"])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -238,8 +238,8 @@
</span><span class="cx">
</span><span class="cx"> notifications = yield self.transactionUnderTest().notificationsWithUID("home1")
</span><span class="cx"> yield notifications.notifyChanged()
</span><del>- yield self.commit()
</del><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set(["/CalDAV/example.com/home1/", "/CalDAV/example.com/home1/notification/"])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescalendarservertapcaldavpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/caldav.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/caldav.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/caldav.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -1212,6 +1212,28 @@
</span><span class="cx"> else:
</span><span class="cx"> groupCacher = None
</span><span class="cx">
</span><ins>+ # Optionally enable Manhole access
+ if config.Manhole.Enabled:
+ try:
+ from twisted.conch.manhole_tap import makeService as manholeMakeService
+ portString = "tcp:%d:interface=127.0.0.1" % (config.Manhole.StartingPortNumber,)
+ manholeService = manholeMakeService({
+ "sshPort" : None,
+ "telnetPort" : portString,
+ "namespace" : {
+ "config" : config,
+ "service" : result,
+ "store" : store,
+ "directory" : directory,
+ },
+ "passwd" : config.Manhole.PasswordFilePath,
+ })
+ manholeService.setServiceParent(result)
+ # Using print(because logging isn't ready at this point)
+ print("Manhole access enabled: %s" % (portString,))
+ except ImportError:
+ print("Manhole access could not enabled because manhole_tap could not be imported")
+
</ins><span class="cx"> def decorateTransaction(txn):
</span><span class="cx"> txn._pushDistributor = pushDistributor
</span><span class="cx"> txn._rootResource = result.rootResource
</span><span class="lines">@@ -1407,7 +1429,9 @@
</span><span class="cx">
</span><span class="cx"> # Conditionally stop after upgrade at this point
</span><span class="cx"> pps.addStep(
</span><del>- QuitAfterUpgradeStep(config.StopAfterUpgradeTriggerFile)
</del><ins>+ QuitAfterUpgradeStep(
+ config.StopAfterUpgradeTriggerFile or config.UpgradeHomePrefix
+ )
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> pps.addStep(
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescalendarservertaputilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/util.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/util.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tap/util.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -95,6 +95,7 @@
</span><span class="cx"> from txdav.common.datastore.sql import CommonDataStore as CommonSQLDataStore
</span><span class="cx"> from txdav.common.datastore.file import CommonDataStore as CommonFileDataStore
</span><span class="cx"> from txdav.common.datastore.sql import current_sql_schema
</span><ins>+from txdav.common.datastore.upgrade.sql.upgrade import NotAllowedToUpgrade
</ins><span class="cx"> from twext.python.filepath import CachingFilePath
</span><span class="cx"> from urllib import quote
</span><span class="cx"> from twisted.python.usage import UsageError
</span><span class="lines">@@ -1077,7 +1078,8 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def defaultStepWithFailure(self, failure):
</span><del>- log.failure("Step failure", failure=failure)
</del><ins>+ if failure.type != NotAllowedToUpgrade:
+ log.failure("Step failure", failure=failure)
</ins><span class="cx"> return failure
</span><span class="cx">
</span><span class="cx"> # def protectStep(self, callback):
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescalendarservertoolsupgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tools/upgrade.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tools/upgrade.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/calendarserver/tools/upgrade.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -82,6 +82,7 @@
</span><span class="cx">
</span><span class="cx"> optParameters = [
</span><span class="cx"> ['config', 'f', DEFAULT_CONFIG_FILE, "Specify caldavd.plist configuration path."],
</span><ins>+ ['prefix', 'x', "", "Only upgrade homes with the specified GUID prefix - partial upgrade only."],
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx"> def __init__(self):
</span><span class="lines">@@ -142,11 +143,17 @@
</span><span class="cx"> """
</span><span class="cx"> Immediately stop. The upgrade will have been run before this.
</span><span class="cx"> """
</span><del>- # If we get this far the database is OK
- if self.options["status"]:
- self.output.write("Database OK.\n")
</del><ins>+ if self.store is None:
+ if self.options["status"]:
+ self.output.write("Upgrade needed.\n")
+ else:
+ self.output.write("Upgrade failed.\n")
</ins><span class="cx"> else:
</span><del>- self.output.write("Upgrade complete, shutting down.\n")
</del><ins>+ # If we get this far the database is OK
+ if self.options["status"]:
+ self.output.write("Database OK.\n")
+ else:
+ self.output.write("Upgrade complete, shutting down.\n")
</ins><span class="cx"> UpgraderService.started = True
</span><span class="cx">
</span><span class="cx"> from twisted.internet import reactor
</span><span class="lines">@@ -191,9 +198,11 @@
</span><span class="cx"> data.MergeUpgrades = True
</span><span class="cx"> config.addPostUpdateHooks([setMerge])
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def makeService(store):
</span><span class="cx"> return UpgraderService(store, options, output, reactor, config)
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def onlyUpgradeEvents(eventDict):
</span><span class="cx"> text = formatEvent(eventDict)
</span><span class="cx"> output.write(logDateString() + " " + text + "\n")
</span><span class="lines">@@ -203,14 +212,19 @@
</span><span class="cx"> log.publisher.levels.setLogLevelForNamespace(None, LogLevel.debug)
</span><span class="cx"> addObserver(onlyUpgradeEvents)
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def customServiceMaker():
</span><span class="cx"> customService = CalDAVServiceMaker()
</span><span class="cx"> customService.doPostImport = options["postprocess"]
</span><span class="cx"> return customService
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def _patchConfig(config):
</span><span class="cx"> config.FailIfUpgradeNeeded = options["status"]
</span><ins>+ if options["prefix"]:
+ config.UpgradeHomePrefix = options["prefix"]
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def _onShutdown():
</span><span class="cx"> if not UpgraderService.started:
</span><span class="cx"> print("Failed to start service.")
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixesconfauthaccountstestxml"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/conf/auth/accounts-test.xml (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/conf/auth/accounts-test.xml 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/conf/auth/accounts-test.xml 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -89,7 +89,7 @@
</span><span class="cx"> <first-name>ま</first-name>
</span><span class="cx"> <last-name>だ</last-name>
</span><span class="cx"> </user>
</span><del>- <user repeat="99">
</del><ins>+ <user repeat="101">
</ins><span class="cx"> <uid>user%02d</uid>
</span><span class="cx"> <uid>User %02d</uid>
</span><span class="cx"> <guid>user%02d</guid>
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestclientsplistfromrev11860CalendarServertrunkcontribperformanceloadtestclientsplist"></a>
<div class="copfile"><h4>Copied: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/clients.plist (from rev 11860, CalendarServer/trunk/contrib/performance/loadtest/clients.plist) (0 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/clients.plist (rev 0)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/clients.plist 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -0,0 +1,445 @@
</span><ins>+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Copyright (c) 2011-2013 Apple Inc. All rights reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+ <dict>
+ <!-- Define the kinds of software and user behavior the load simulation
+ will simulate. -->
+ <key>clients</key>
+
+ <!-- Have as many different kinds of software and user behavior configurations
+ as you want. Each is a dict -->
+ <array>
+
+ <dict>
+
+ <!-- Here is a OS X client simulator. -->
+ <key>software</key>
+ <string>contrib.performance.loadtest.ical.OS_X_10_7</string>
+
+ <!-- Arguments to use to initialize the OS_X_10_7 instance. -->
+ <key>params</key>
+ <dict>
+ <!-- Name that appears in logs. -->
+ <key>title</key>
+ <string>10.7</string>
+
+ <!-- OS_X_10_7 can poll the calendar home at some interval. This is
+ in seconds. -->
+ <key>calendarHomePollInterval</key>
+ <integer>30</integer>
+
+ <!-- If the server advertises xmpp push, OS_X_10_7 can wait for notifications
+ about calendar home changes instead of polling for them periodically. If
+ this option is true, then look for the server advertisement for xmpp push
+ and use it if possible. Still fall back to polling if there is no xmpp push
+ advertised. -->
+ <key>supportPush</key>
+ <false />
+
+ <key>supportAmpPush</key>
+ <true/>
+ <key>ampPushHost</key>
+ <string>localhost</string>
+ <key>ampPushPort</key>
+ <integer>62311</integer>
+ </dict>
+
+ <!-- The profiles define certain types of user behavior on top of the
+ client software being simulated. -->
+ <key>profiles</key>
+ <array>
+
+ <!-- First an event-creating profile, which will periodically create
+ new events at a random time on a random calendar. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Eventer</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to create a new event. -->
+ <key>interval</key>
+ <integer>60</integer>
+
+ <!-- Define how start times (DTSTART) for the randomly generated events
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>eventStartDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+
+ <!-- Define how recurrences are created. -->
+ <key>recurrenceDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. We have a fixed set of
+ RRULEs defined for this distribution and pick each based on a
+ weight. -->
+ <key>type</key>
+ <string>contrib.performance.stats.RecurrenceDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- False to disable RRULEs -->
+ <key>allowRecurrence</key>
+ <true/>
+
+ <!-- These are the weights for the specific set of RRULEs. -->
+ <key>weights</key>
+ <dict>
+ <!-- Half of all events will be non-recurring -->
+ <key>none</key>
+ <integer>50</integer>
+
+ <!-- Daily and weekly are pretty common -->
+ <key>daily</key>
+ <integer>10</integer>
+ <key>weekly</key>
+ <integer>20</integer>
+
+ <!-- Monthly, yearly, daily & weekly limit not so common -->
+ <key>monthly</key>
+ <integer>2</integer>
+ <key>yearly</key>
+ <integer>1</integer>
+ <key>dailylimit</key>
+ <integer>2</integer>
+ <key>weeklylimit</key>
+ <integer>5</integer>
+
+ <!-- Work days pretty common -->
+ <key>workdays</key>
+ <integer>10</integer>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- This profile invites some number of new attendees to new events. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.RealisticInviter</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the frequency at which new invitations will be sent out. -->
+ <key>sendInvitationDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.NormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mu gives the mean of the normal distribution (in seconds). -->
+ <key>mu</key>
+ <integer>60</integer>
+
+ <!-- and sigma gives its standard deviation. -->
+ <key>sigma</key>
+ <integer>5</integer>
+ </dict>
+ </dict>
+
+ <!-- Define the distribution of who will be invited to an event.
+
+ When inviteeClumping is turned on each invitee is based on a sample of
+ users "close to" the organizer based on account index. If the clumping
+ is too "tight" for the requested number of attendees, then invites for
+ those larger numbers will simply fail (the sim will report that situation).
+
+ When inviteeClumping is off invitees will be sampled across an entire
+ range of account indexes. In this case the distribution ought to be a
+ UniformIntegerDistribution with min=0 and max set to the number of accounts.
+ -->
+ <key>inviteeDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.UniformIntegerDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- The minimum value (inclusive) of the uniform distribution. -->
+ <key>min</key>
+ <integer>0</integer>
+ <!-- The maximum value (exclusive) of the uniform distribution. -->
+ <key>max</key>
+ <integer>99</integer>
+ </dict>
+ </dict>
+
+ <key>inviteeClumping</key>
+ <true/>
+
+ <!-- Define the distribution of how many attendees will be invited to an event.
+
+ LogNormal is the best fit to observed data.
+
+
+ For LogNormal "mode" is the peak, "mean" is the mean value. For invites,
+ mode should typically be 1, and mean whatever matches the user behavior.
+ Our typical mean is 6.
+ -->
+ <key>inviteeCountDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.LogNormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mode - peak-->
+ <key>mode</key>
+ <integer>1</integer>
+ <!-- mean - average-->
+ <key>median</key>
+ <integer>6</integer>
+ <!-- maximum -->
+ <key>maximum</key>
+ <real>60</real>
+ </dict>
+ </dict>
+
+ <!-- Define how start times (DTSTART) for the randomly generated events
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>eventStartDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+
+ <!-- Define how recurrences are created. -->
+ <key>recurrenceDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. We have a fixed set of
+ RRULEs defined for this distribution and pick each based on a
+ weight. -->
+ <key>type</key>
+ <string>contrib.performance.stats.RecurrenceDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- False to disable RRULEs -->
+ <key>allowRecurrence</key>
+ <true/>
+
+ <!-- These are the weights for the specific set of RRULEs. -->
+ <key>weights</key>
+ <dict>
+ <!-- Half of all events will be non-recurring -->
+ <key>none</key>
+ <integer>50</integer>
+
+ <!-- Daily and weekly are pretty common -->
+ <key>daily</key>
+ <integer>10</integer>
+ <key>weekly</key>
+ <integer>20</integer>
+
+ <!-- Monthly, yearly, daily & weekly limit not so common -->
+ <key>monthly</key>
+ <integer>2</integer>
+ <key>yearly</key>
+ <integer>1</integer>
+ <key>dailylimit</key>
+ <integer>2</integer>
+ <key>weeklylimit</key>
+ <integer>5</integer>
+
+ <!-- Work days pretty common -->
+ <key>workdays</key>
+ <integer>10</integer>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- This profile accepts invitations to events, handles cancels, and
+ handles replies received. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Accepter</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define how long to wait after seeing a new invitation before
+ accepting it.
+
+ For LogNormal "mode" is the peak, "median" is the 50% cummulative value
+ (i.e., half of the user have accepted by that time).
+ -->
+ <key>acceptDelayDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.LogNormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mode - peak-->
+ <key>mode</key>
+ <integer>300</integer>
+ <!-- median - 50% done-->
+ <key>median</key>
+ <integer>1800</integer>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- A task-creating profile, which will periodically create
+ new tasks at a random time on a random calendar. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Tasker</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to create a new task. -->
+ <key>interval</key>
+ <integer>300</integer>
+
+ <!-- Define how due times (DUE) for the randomly generated tasks
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>taskDueDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+ </array>
+
+ <!-- Determine the frequency at which this client configuration will
+ appear in the clients which are created by the load tester. -->
+ <key>weight</key>
+ <integer>1</integer>
+ </dict>
+ </array>
+ </dict>
+</plist>
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestconfigdistplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.dist.plist (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.dist.plist 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.dist.plist 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -50,10 +50,19 @@
</span><span class="cx"> <integer>8080</integer>
</span><span class="cx"> </dict>
</span><span class="cx">
</span><del>- <!-- Define whether client data should be saved and re-used. -->
</del><ins>+ <!-- Define whether server supports stats socket. -->
+ <key>serverStats</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+ <key>Port</key>
+ <integer>8100</integer>
+ </dict>
+
+ <!-- Define whether client data should be re-used. It will always be saved to the specified path.-->
</ins><span class="cx"> <key>clientDataSerialization</key>
</span><span class="cx"> <dict>
</span><del>- <key>Enabled</key>
</del><ins>+ <key>UseOldData</key>
</ins><span class="cx"> <true/>
</span><span class="cx"> <key>Path</key>
</span><span class="cx"> <string>/tmp/sim</string>
</span><span class="lines">@@ -119,471 +128,6 @@
</span><span class="cx">
</span><span class="cx"> </dict>
</span><span class="cx">
</span><del>- <!-- Define the kinds of software and user behavior the load simulation
- will simulate. -->
- <key>clients</key>
-
- <!-- Have as many different kinds of software and user behavior configurations
- as you want. Each is a dict -->
- <array>
-
- <dict>
-
- <!-- Here is a OS X client simulator. -->
- <key>software</key>
- <string>contrib.performance.loadtest.ical.OS_X_10_7</string>
-
- <!-- Arguments to use to initialize the OS_X_10_7 instance. -->
- <key>params</key>
- <dict>
- <!-- Name that appears in logs. -->
- <key>title</key>
- <string>10.7</string>
-
- <!-- OS_X_10_7 can poll the calendar home at some interval. This is
- in seconds. -->
- <key>calendarHomePollInterval</key>
- <integer>30</integer>
-
- <!-- If the server advertises xmpp push, OS_X_10_7 can wait for notifications
- about calendar home changes instead of polling for them periodically. If
- this option is true, then look for the server advertisement for xmpp push
- and use it if possible. Still fall back to polling if there is no xmpp push
- advertised. -->
- <key>supportPush</key>
- <false />
- </dict>
-
- <!-- The profiles define certain types of user behavior on top of the
- client software being simulated. -->
- <key>profiles</key>
- <array>
-
- <!-- First an event-creating profile, which will periodically create
- new events at a random time on a random calendar. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Eventer</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define the interval (in seconds) at which this profile will use
- its client to create a new event. -->
- <key>interval</key>
- <integer>60</integer>
-
- <!-- Define how start times (DTSTART) for the randomly generated events
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>eventStartDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
-
- <!-- Define how recurrences are created. -->
- <key>recurrenceDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. We have a fixed set of
- RRULEs defined for this distribution and pick each based on a
- weight. -->
- <key>type</key>
- <string>contrib.performance.stats.RecurrenceDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- False to disable RRULEs -->
- <key>allowRecurrence</key>
- <true/>
-
- <!-- These are the weights for the specific set of RRULEs. -->
- <key>weights</key>
- <dict>
- <!-- Half of all events will be non-recurring -->
- <key>none</key>
- <integer>50</integer>
-
- <!-- Daily and weekly are pretty common -->
- <key>daily</key>
- <integer>10</integer>
- <key>weekly</key>
- <integer>20</integer>
-
- <!-- Monthly, yearly, daily & weekly limit not so common -->
- <key>monthly</key>
- <integer>2</integer>
- <key>yearly</key>
- <integer>1</integer>
- <key>dailylimit</key>
- <integer>2</integer>
- <key>weeklylimit</key>
- <integer>5</integer>
-
- <!-- Work days pretty common -->
- <key>workdays</key>
- <integer>10</integer>
- </dict>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile invites new attendees to existing events.
- This profile should no longer be used - use RealisticInviter instead. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Inviter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define the frequency at which new invitations will be sent out. -->
- <key>sendInvitationDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.NormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mu gives the mean of the normal distribution (in seconds). -->
- <key>mu</key>
- <integer>60</integer>
-
- <!-- and sigma gives its standard deviation. -->
- <key>sigma</key>
- <integer>5</integer>
- </dict>
- </dict>
-
- <!-- Define the distribution of who will be invited to an event. Each
- set of credentials loaded by the load tester has an index; samples from this
- distribution will be added to that index to arrive at the index of some other
- credentials, which will be the target of the invitation. -->
- <key>inviteeDistanceDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.UniformIntegerDistribution</string>
- <key>params</key>
- <dict>
- <!-- The minimum value (inclusive) of the uniform distribution. -->
- <key>min</key>
- <integer>-100</integer>
- <!-- The maximum value (exclusive) of the uniform distribution. -->
- <key>max</key>
- <integer>101</integer>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile invites some number of new attendees to new events. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.RealisticInviter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define the frequency at which new invitations will be sent out. -->
- <key>sendInvitationDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.NormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mu gives the mean of the normal distribution (in seconds). -->
- <key>mu</key>
- <integer>60</integer>
-
- <!-- and sigma gives its standard deviation. -->
- <key>sigma</key>
- <integer>5</integer>
- </dict>
- </dict>
-
- <!-- Define the distribution of who will be invited to an event.
-
- When inviteeClumping is turned on each invitee is based on a sample of
- users "close to" the organizer based on account index. If the clumping
- is too "tight" for the requested number of attendees, then invites for
- those larger numbers will simply fail (the sim will report that situation).
-
- When inviteeClumping is off invitees will be sampled across an entire
- range of account indexes. In this case the distribution ought to be a
- UniformIntegerDistribution with min=0 and max set to the number of accounts.
- -->
- <key>inviteeDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.UniformIntegerDistribution</string>
- <key>params</key>
- <dict>
- <!-- The minimum value (inclusive) of the uniform distribution. -->
- <key>min</key>
- <integer>-100</integer>
- <!-- The maximum value (exclusive) of the uniform distribution. -->
- <key>max</key>
- <integer>101</integer>
- </dict>
- </dict>
-
- <key>inviteeClumping</key>
- <true/>
-
- <!-- Define the distribution of how many attendees will be invited to an event.
-
- LogNormal is the best fit to observed data.
-
-
- For LogNormal "mode" is the peak, "mean" is the mean value. For invites,
- mode should typically be 1, and mean whatever matches the user behavior.
- Our typical mean is 6.
- -->
- <key>inviteeCountDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>1</integer>
- <!-- mean - average-->
- <key>median</key>
- <integer>6</integer>
- <!-- maximum -->
- <key>maximum</key>
- <real>100</real>
- </dict>
- </dict>
-
- <!-- Define how start times (DTSTART) for the randomly generated events
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>eventStartDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
-
- <!-- Define how recurrences are created. -->
- <key>recurrenceDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. We have a fixed set of
- RRULEs defined for this distribution and pick each based on a
- weight. -->
- <key>type</key>
- <string>contrib.performance.stats.RecurrenceDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- False to disable RRULEs -->
- <key>allowRecurrence</key>
- <true/>
-
- <!-- These are the weights for the specific set of RRULEs. -->
- <key>weights</key>
- <dict>
- <!-- Half of all events will be non-recurring -->
- <key>none</key>
- <integer>50</integer>
-
- <!-- Daily and weekly are pretty common -->
- <key>daily</key>
- <integer>10</integer>
- <key>weekly</key>
- <integer>20</integer>
-
- <!-- Monthly, yearly, daily & weekly limit not so common -->
- <key>monthly</key>
- <integer>2</integer>
- <key>yearly</key>
- <integer>1</integer>
- <key>dailylimit</key>
- <integer>2</integer>
- <key>weeklylimit</key>
- <integer>5</integer>
-
- <!-- Work days pretty common -->
- <key>workdays</key>
- <integer>10</integer>
- </dict>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile accepts invitations to events, handles cancels, and
- handles replies received. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Accepter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define how long to wait after seeing a new invitation before
- accepting it.
-
- For LogNormal "mode" is the peak, "median" is the 50% cummulative value
- (i.e., half of the user have accepted by that time).
- -->
- <key>acceptDelayDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>300</integer>
- <!-- median - 50% done-->
- <key>median</key>
- <integer>1800</integer>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- A task-creating profile, which will periodically create
- new tasks at a random time on a random calendar. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Tasker</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define the interval (in seconds) at which this profile will use
- its client to create a new task. -->
- <key>interval</key>
- <integer>300</integer>
-
- <!-- Define how due times (DUE) for the randomly generated tasks
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>taskDueDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
- </dict>
- </dict>
-
- </array>
-
- <!-- Determine the frequency at which this client configuration will
- appear in the clients which are created by the load tester. -->
- <key>weight</key>
- <integer>1</integer>
- </dict>
- </array>
-
</del><span class="cx"> <!-- Define some log observers to report on the load test. -->
</span><span class="cx"> <key>observers</key>
</span><span class="cx"> <array>
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestconfigplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.plist (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.plist 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/config.plist 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -37,10 +37,19 @@
</span><span class="cx"> <integer>8080</integer>
</span><span class="cx"> </dict>
</span><span class="cx">
</span><del>- <!-- Define whether client data should be saved and re-used. -->
</del><ins>+ <!-- Define whether server supports stats socket. -->
+ <key>serverStats</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+ <key>Port</key>
+ <integer>8100</integer>
+ </dict>
+
+ <!-- Define whether client data should be re-used. It will always be saved to the specified path.-->
</ins><span class="cx"> <key>clientDataSerialization</key>
</span><span class="cx"> <dict>
</span><del>- <key>Enabled</key>
</del><ins>+ <key>UseOldData</key>
</ins><span class="cx"> <true/>
</span><span class="cx"> <key>Path</key>
</span><span class="cx"> <string>/tmp/sim</string>
</span><span class="lines">@@ -106,429 +115,6 @@
</span><span class="cx">
</span><span class="cx"> </dict>
</span><span class="cx">
</span><del>- <!-- Define the kinds of software and user behavior the load simulation
- will simulate. -->
- <key>clients</key>
-
- <!-- Have as many different kinds of software and user behavior configurations
- as you want. Each is a dict -->
- <array>
-
- <dict>
-
- <!-- Here is a OS X client simulator. -->
- <key>software</key>
- <string>contrib.performance.loadtest.ical.OS_X_10_7</string>
-
- <!-- Arguments to use to initialize the OS_X_10_7 instance. -->
- <key>params</key>
- <dict>
- <!-- Name that appears in logs. -->
- <key>title</key>
- <string>10.7</string>
-
- <!-- OS_X_10_7 can poll the calendar home at some interval. This is
- in seconds. -->
- <key>calendarHomePollInterval</key>
- <integer>30</integer>
-
- <!-- If the server advertises xmpp push, OS_X_10_7 can wait for notifications
- about calendar home changes instead of polling for them periodically. If
- this option is true, then look for the server advertisement for xmpp push
- and use it if possible. Still fall back to polling if there is no xmpp push
- advertised. -->
- <key>supportPush</key>
- <false />
-
- <key>supportAmpPush</key>
- <true/>
- <key>ampPushHost</key>
- <string>localhost</string>
- <key>ampPushPort</key>
- <integer>62311</integer>
- </dict>
-
- <!-- The profiles define certain types of user behavior on top of the
- client software being simulated. -->
- <key>profiles</key>
- <array>
-
- <!-- First an event-creating profile, which will periodically create
- new events at a random time on a random calendar. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Eventer</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define the interval (in seconds) at which this profile will use
- its client to create a new event. -->
- <key>interval</key>
- <integer>60</integer>
-
- <!-- Define how start times (DTSTART) for the randomly generated events
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>eventStartDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
-
- <!-- Define how recurrences are created. -->
- <key>recurrenceDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. We have a fixed set of
- RRULEs defined for this distribution and pick each based on a
- weight. -->
- <key>type</key>
- <string>contrib.performance.stats.RecurrenceDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- False to disable RRULEs -->
- <key>allowRecurrence</key>
- <true/>
-
- <!-- These are the weights for the specific set of RRULEs. -->
- <key>weights</key>
- <dict>
- <!-- Half of all events will be non-recurring -->
- <key>none</key>
- <integer>50</integer>
-
- <!-- Daily and weekly are pretty common -->
- <key>daily</key>
- <integer>10</integer>
- <key>weekly</key>
- <integer>20</integer>
-
- <!-- Monthly, yearly, daily & weekly limit not so common -->
- <key>monthly</key>
- <integer>2</integer>
- <key>yearly</key>
- <integer>1</integer>
- <key>dailylimit</key>
- <integer>2</integer>
- <key>weeklylimit</key>
- <integer>5</integer>
-
- <!-- Work days pretty common -->
- <key>workdays</key>
- <integer>10</integer>
- </dict>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile invites some number of new attendees to new events. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.RealisticInviter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define the frequency at which new invitations will be sent out. -->
- <key>sendInvitationDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.NormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mu gives the mean of the normal distribution (in seconds). -->
- <key>mu</key>
- <integer>60</integer>
-
- <!-- and sigma gives its standard deviation. -->
- <key>sigma</key>
- <integer>5</integer>
- </dict>
- </dict>
-
- <!-- Define the distribution of who will be invited to an event.
-
- When inviteeClumping is turned on each invitee is based on a sample of
- users "close to" the organizer based on account index. If the clumping
- is too "tight" for the requested number of attendees, then invites for
- those larger numbers will simply fail (the sim will report that situation).
-
- When inviteeClumping is off invitees will be sampled across an entire
- range of account indexes. In this case the distribution ought to be a
- UniformIntegerDistribution with min=0 and max set to the number of accounts.
- -->
- <key>inviteeDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.UniformIntegerDistribution</string>
- <key>params</key>
- <dict>
- <!-- The minimum value (inclusive) of the uniform distribution. -->
- <key>min</key>
- <integer>0</integer>
- <!-- The maximum value (exclusive) of the uniform distribution. -->
- <key>max</key>
- <integer>99</integer>
- </dict>
- </dict>
-
- <key>inviteeClumping</key>
- <true/>
-
- <!-- Define the distribution of how many attendees will be invited to an event.
-
- LogNormal is the best fit to observed data.
-
-
- For LogNormal "mode" is the peak, "mean" is the mean value. For invites,
- mode should typically be 1, and mean whatever matches the user behavior.
- Our typical mean is 6.
- -->
- <key>inviteeCountDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>1</integer>
- <!-- mean - average-->
- <key>median</key>
- <integer>6</integer>
- <!-- maximum -->
- <key>maximum</key>
- <real>60</real>
- </dict>
- </dict>
-
- <!-- Define how start times (DTSTART) for the randomly generated events
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>eventStartDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
-
- <!-- Define how recurrences are created. -->
- <key>recurrenceDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. We have a fixed set of
- RRULEs defined for this distribution and pick each based on a
- weight. -->
- <key>type</key>
- <string>contrib.performance.stats.RecurrenceDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- False to disable RRULEs -->
- <key>allowRecurrence</key>
- <true/>
-
- <!-- These are the weights for the specific set of RRULEs. -->
- <key>weights</key>
- <dict>
- <!-- Half of all events will be non-recurring -->
- <key>none</key>
- <integer>50</integer>
-
- <!-- Daily and weekly are pretty common -->
- <key>daily</key>
- <integer>10</integer>
- <key>weekly</key>
- <integer>20</integer>
-
- <!-- Monthly, yearly, daily & weekly limit not so common -->
- <key>monthly</key>
- <integer>2</integer>
- <key>yearly</key>
- <integer>1</integer>
- <key>dailylimit</key>
- <integer>2</integer>
- <key>weeklylimit</key>
- <integer>5</integer>
-
- <!-- Work days pretty common -->
- <key>workdays</key>
- <integer>10</integer>
- </dict>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile accepts invitations to events, handles cancels, and
- handles replies received. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Accepter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define how long to wait after seeing a new invitation before
- accepting it.
-
- For LogNormal "mode" is the peak, "median" is the 50% cummulative value
- (i.e., half of the user have accepted by that time).
- -->
- <key>acceptDelayDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>300</integer>
- <!-- median - 50% done-->
- <key>median</key>
- <integer>1800</integer>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- A task-creating profile, which will periodically create
- new tasks at a random time on a random calendar. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Tasker</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define the interval (in seconds) at which this profile will use
- its client to create a new task. -->
- <key>interval</key>
- <integer>300</integer>
-
- <!-- Define how due times (DUE) for the randomly generated tasks
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>taskDueDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
- </dict>
- </dict>
-
- </array>
-
- <!-- Determine the frequency at which this client configuration will
- appear in the clients which are created by the load tester. -->
- <key>weight</key>
- <integer>1</integer>
- </dict>
- </array>
-
</del><span class="cx"> <!-- Define some log observers to report on the load test. -->
</span><span class="cx"> <key>observers</key>
</span><span class="cx"> <array>
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestpopulationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/population.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/population.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/population.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -396,6 +396,7 @@
</span><span class="cx"> self._failed_clients = []
</span><span class="cx"> self._failed_sim = collections.defaultdict(int)
</span><span class="cx"> self._startTime = datetime.now()
</span><ins>+ self._expired_data = None
</ins><span class="cx">
</span><span class="cx"> # Load parameters from config
</span><span class="cx"> if "thresholdsPath" in params:
</span><span class="lines">@@ -423,6 +424,13 @@
</span><span class="cx"> self._fail_cut_off = params["failCutoff"]
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def observe(self, event):
+ if event.get('type') == 'sim-expired':
+ self.simExpired(event)
+ else:
+ super(ReportStatistics, self).observe(event)
+
+
</ins><span class="cx"> def countUsers(self):
</span><span class="cx"> return len(self._users)
</span><span class="cx">
</span><span class="lines">@@ -454,6 +462,10 @@
</span><span class="cx"> self._failed_sim[event['reason']] += 1
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def simExpired(self, event):
+ self._expired_data = event['reason']
+
+
</ins><span class="cx"> def printMiscellaneous(self, output, items):
</span><span class="cx"> maxColumnWidth = str(len(max(items.iterkeys(), key=len)))
</span><span class="cx"> fmt = "%" + maxColumnWidth + "s : %-s\n"
</span><span class="lines">@@ -480,7 +492,7 @@
</span><span class="cx"> if result is not None:
</span><span class="cx"> differences.append(result)
</span><span class="cx">
</span><del>- return mean(differences) if differences else "None"
</del><ins>+ return ("%-8.4f" % mean(differences)) if differences else "None"
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def qos_value(self, method, value):
</span><span class="lines">@@ -518,7 +530,7 @@
</span><span class="cx"> 'Start time': self._startTime.strftime('%m/%d %H:%M:%S'),
</span><span class="cx"> 'Run time': "%02d:%02d:%02d" % (runHours, runMinutes, runSeconds),
</span><span class="cx"> 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,),
</span><del>- 'QoS': "%-8.4f" % (self.qos(),),
</del><ins>+ 'QoS': self.qos(),
</ins><span class="cx"> }
</span><span class="cx"> if self.countClientFailures() > 0:
</span><span class="cx"> items['Failed clients'] = self.countClientFailures()
</span><span class="lines">@@ -527,8 +539,22 @@
</span><span class="cx"> if self.countSimFailures() > 0:
</span><span class="cx"> for reason, count in self._failed_sim.items():
</span><span class="cx"> items['Failed operation'] = "%s : %d times" % (reason, count,)
</span><ins>+ output.write("* Client\n")
</ins><span class="cx"> self.printMiscellaneous(output, items)
</span><span class="cx"> output.write("\n")
</span><ins>+
+ if self._expired_data is not None:
+ items = {
+ "Req/sec" : "%.1f" % (self._expired_data[0],),
+ "Response": "%.1f (ms)" % (self._expired_data[1],),
+ "Slots": "%.2f" % (self._expired_data[2],),
+ "CPU": "%.1f%%" % (self._expired_data[3],),
+ }
+ output.write("* Server (Last 5 minutes)\n")
+ self.printMiscellaneous(output, items)
+ output.write("\n")
+ output.write("* Details\n")
+
</ins><span class="cx"> self.printHeader(output, [
</span><span class="cx"> (label, width)
</span><span class="cx"> for (label, width, _ignore_fmt)
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtestsimpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/sim.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/sim.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/sim.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -23,11 +23,15 @@
</span><span class="cx"> from plistlib import readPlist
</span><span class="cx"> from random import Random
</span><span class="cx"> from sys import argv, stdout
</span><ins>+from urlparse import urlsplit
</ins><span class="cx"> from xml.parsers.expat import ExpatError
</span><ins>+import json
+import shutil
+import socket
</ins><span class="cx">
</span><span class="cx"> from twisted.python import context
</span><span class="cx"> from twisted.python.filepath import FilePath
</span><del>-from twisted.python.log import startLogging, addObserver, removeObserver
</del><ins>+from twisted.python.log import startLogging, addObserver, removeObserver, msg
</ins><span class="cx"> from twisted.python.usage import UsageError, Options
</span><span class="cx"> from twisted.python.reflect import namedAny
</span><span class="cx">
</span><span class="lines">@@ -56,6 +60,11 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+def safeDivision(value, total, factor=1):
+ return value * factor / total if total else 0
+
+
+
</ins><span class="cx"> def generateRecords(count, uidPattern="user%d", passwordPattern="user%d",
</span><span class="cx"> namePattern="User %d", emailPattern="user%d@example.com"):
</span><span class="cx"> for i in xrange(count):
</span><span class="lines">@@ -121,6 +130,7 @@
</span><span class="cx"> """
</span><span class="cx"> config = None
</span><span class="cx"> _defaultConfig = FilePath(__file__).sibling("config.plist")
</span><ins>+ _defaultClients = FilePath(__file__).sibling("clients.plist")
</ins><span class="cx">
</span><span class="cx"> optParameters = [
</span><span class="cx"> ("runtime", "t", None,
</span><span class="lines">@@ -129,6 +139,9 @@
</span><span class="cx"> ("config", None, _defaultConfig,
</span><span class="cx"> "Configuration plist file name from which to read simulation parameters.",
</span><span class="cx"> FilePath),
</span><ins>+ ("clients", None, _defaultClients,
+ "Configuration plist file name from which to read client parameters.",
+ FilePath),
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -181,7 +194,23 @@
</span><span class="cx"> finally:
</span><span class="cx"> configFile.close()
</span><span class="cx">
</span><ins>+ try:
+ clientFile = self['clients'].open()
+ except IOError, e:
+ raise UsageError("--clients %s: %s" % (
+ self['clients'].path, e.strerror))
+ try:
+ try:
+ client_config = readPlist(clientFile)
+ self.config["clients"] = client_config["clients"]
+ if "arrivalInterval" in client_config:
+ self.config["arrival"]["params"]["interval"] = client_config["arrivalInterval"]
+ except ExpatError, e:
+ raise UsageError("--clients %s: %s" % (self['clients'].path, e))
+ finally:
+ clientFile.close()
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> Arrival = namedtuple('Arrival', 'factory parameters')
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -200,7 +229,7 @@
</span><span class="cx"> user information about the accounts on the server being put
</span><span class="cx"> under load.
</span><span class="cx"> """
</span><del>- def __init__(self, server, principalPathTemplate, webadminPort, serializationPath, arrival, parameters, observers=None,
</del><ins>+ def __init__(self, server, principalPathTemplate, webadminPort, serverStats, serializationPath, arrival, parameters, observers=None,
</ins><span class="cx"> records=None, reactor=None, runtime=None, workers=None,
</span><span class="cx"> configTemplate=None, workerID=None, workerCount=1):
</span><span class="cx"> if reactor is None:
</span><span class="lines">@@ -208,6 +237,7 @@
</span><span class="cx"> self.server = server
</span><span class="cx"> self.principalPathTemplate = principalPathTemplate
</span><span class="cx"> self.webadminPort = webadminPort
</span><ins>+ self.serverStats = serverStats
</ins><span class="cx"> self.serializationPath = serializationPath
</span><span class="cx"> self.arrival = arrival
</span><span class="cx"> self.parameters = parameters
</span><span class="lines">@@ -260,15 +290,17 @@
</span><span class="cx"> principalPathTemplate = config['principalPathTemplate']
</span><span class="cx">
</span><span class="cx"> if 'clientDataSerialization' in config:
</span><del>- if config['clientDataSerialization']['Enabled']:
- serializationPath = config['clientDataSerialization']['Path']
- if not isdir(serializationPath):
- try:
- mkdir(serializationPath)
- except OSError:
- print("Unable to create client data serialization directory: %s" % (serializationPath))
- print("Please consult the clientDataSerialization stanza of contrib/performance/loadtest/config.plist")
- raise
</del><ins>+ serializationPath = config['clientDataSerialization']['Path']
+ if not config['clientDataSerialization']['UseOldData']:
+ shutil.rmtree(serializationPath)
+ serializationPath = config['clientDataSerialization']['Path']
+ if not isdir(serializationPath):
+ try:
+ mkdir(serializationPath)
+ except OSError:
+ print("Unable to create client data serialization directory: %s" % (serializationPath))
+ print("Please consult the clientDataSerialization stanza of contrib/performance/loadtest/config.plist")
+ raise
</ins><span class="cx">
</span><span class="cx"> if 'arrival' in config:
</span><span class="cx"> arrival = Arrival(
</span><span class="lines">@@ -310,6 +342,11 @@
</span><span class="cx"> if config['webadmin']['enabled']:
</span><span class="cx"> webadminPort = config['webadmin']['HTTPPort']
</span><span class="cx">
</span><ins>+ serverStats = None
+ if 'serverStats' in config:
+ if config['serverStats']['enabled']:
+ serverStats = config['serverStats']
+
</ins><span class="cx"> observers = []
</span><span class="cx"> if 'observers' in config:
</span><span class="cx"> for observer in config['observers']:
</span><span class="lines">@@ -324,11 +361,23 @@
</span><span class="cx"> records.extend(namedAny(loader)(**params))
</span><span class="cx"> output.write("Loaded {0} accounts.\n".format(len(records)))
</span><span class="cx">
</span><del>- return cls(server, principalPathTemplate, webadminPort, serializationPath,
- arrival, parameters, observers=observers,
- records=records, runtime=runtime, reactor=reactor,
- workers=workers, configTemplate=configTemplate,
- workerID=workerID, workerCount=workerCount)
</del><ins>+ return cls(
+ server,
+ principalPathTemplate,
+ webadminPort,
+ serverStats,
+ serializationPath,
+ arrival,
+ parameters,
+ observers=observers,
+ records=records,
+ runtime=runtime,
+ reactor=reactor,
+ workers=workers,
+ configTemplate=configTemplate,
+ workerID=workerID,
+ workerCount=workerCount,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><span class="lines">@@ -409,7 +458,7 @@
</span><span class="cx"> def run(self, output=stdout):
</span><span class="cx"> self.attachServices(output)
</span><span class="cx"> if self.runtime is not None:
</span><del>- self.reactor.callLater(self.runtime, self.reactor.stop)
</del><ins>+ self.reactor.callLater(self.runtime, self.stopAndReport)
</ins><span class="cx"> if self.webadminPort:
</span><span class="cx"> self.reactor.listenTCP(self.webadminPort, server.Site(LoadSimAdminResource(self)))
</span><span class="cx"> self.reactor.run()
</span><span class="lines">@@ -417,16 +466,65 @@
</span><span class="cx">
</span><span class="cx"> def stop(self):
</span><span class="cx"> if self.ms.running:
</span><ins>+ self.updateStats()
</ins><span class="cx"> self.ms.stopService()
</span><del>- self.reactor.callLater(5, self.reactor.stop)
</del><ins>+ self.reactor.callLater(5, self.stopAndReport)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def shutdown(self):
</span><span class="cx"> if self.ms.running:
</span><ins>+ self.updateStats()
</ins><span class="cx"> return self.ms.stopService()
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def updateStats(self):
+ """
+ Capture server stats and stop.
+ """
</ins><span class="cx">
</span><ins>+ if self.serverStats is not None:
+ _ignore_scheme, hostname, _ignore_path, _ignore_query, _ignore_fragment = urlsplit(self.server)
+ data = self.readStatsSock((hostname.split(":")[0], self.serverStats["Port"],), True)
+ if "Failed" not in data:
+ data = data["5 Minutes"]
+ result = (
+ safeDivision(float(data["requests"]), 5 * 60),
+ safeDivision(data["t"], data["requests"]),
+ safeDivision(float(data["slots"]), data["requests"]),
+ safeDivision(data["cpu"], data["requests"]),
+ )
+ msg(type="sim-expired", reason=result)
+
+
+ def stopAndReport(self):
+ """
+ Runtime has expired - capture server stats and stop.
+ """
+
+ self.updateStats()
+ self.reactor.stop()
+
+
+ def readStatsSock(self, sockname, useTCP):
+ try:
+ s = socket.socket(socket.AF_INET if useTCP else socket.AF_UNIX, socket.SOCK_STREAM)
+ s.connect(sockname)
+ data = ""
+ while True:
+ d = s.recv(1024)
+ if d:
+ data += d
+ else:
+ break
+ s.close()
+ data = json.loads(data)
+ except socket.error:
+ data = {"Failed": "Unable to read statistics from server: %s" % (sockname,)}
+ data["Server"] = sockname
+ return data
+
+
+
</ins><span class="cx"> def attachService(reactor, loadsim, service):
</span><span class="cx"> """
</span><span class="cx"> Attach a given L{IService} provider to the given L{IReactorCore}; cause it
</span><span class="lines">@@ -557,7 +655,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def errReceived(self, error):
</span><del>- from twisted.python.log import msg
</del><span class="cx"> msg("stderr received from " + str(self.transport.pid))
</span><span class="cx"> msg(" " + repr(error))
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformanceloadtesttest_simpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/test_sim.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/test_sim.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/loadtest/test_sim.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -253,7 +253,7 @@
</span><span class="cx"> exc = self.assertRaises(
</span><span class="cx"> SystemExit, StubSimulator.main, ['--config', config.path])
</span><span class="cx"> self.assertEquals(
</span><del>- exc.args, (StubSimulator(None, None, None, None, None, None).run(),))
</del><ins>+ exc.args, (StubSimulator(None, None, None, None, None, None, None).run(),))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def test_createSimulator(self):
</span><span class="lines">@@ -264,7 +264,7 @@
</span><span class="cx"> """
</span><span class="cx"> server = 'http://127.0.0.7:1243/'
</span><span class="cx"> reactor = object()
</span><del>- sim = LoadSimulator(server, None, None, None, None, None, reactor=reactor)
</del><ins>+ sim = LoadSimulator(server, None, None, None, None, None, None, reactor=reactor)
</ins><span class="cx"> calsim = sim.createSimulator()
</span><span class="cx"> self.assertIsInstance(calsim, CalendarClientSimulator)
</span><span class="cx"> self.assertIsInstance(calsim.reactor, LagTrackingReactor)
</span><span class="lines">@@ -447,7 +447,7 @@
</span><span class="cx">
</span><span class="cx"> reactor = object()
</span><span class="cx"> sim = LoadSimulator(
</span><del>- None, None, None, None, Arrival(FakeArrival, {'x': 3, 'y': 2}), None, reactor=reactor)
</del><ins>+ None, None, None, None, None, Arrival(FakeArrival, {'x': 3, 'y': 2}), None, reactor=reactor)
</ins><span class="cx"> arrival = sim.createArrivalPolicy()
</span><span class="cx"> self.assertIsInstance(arrival, FakeArrival)
</span><span class="cx"> self.assertIdentical(arrival.reactor, sim.reactor)
</span><span class="lines">@@ -478,7 +478,9 @@
</span><span class="cx"> "weight": 3,
</span><span class="cx"> }]}))
</span><span class="cx">
</span><del>- sim = LoadSimulator.fromCommandLine(['--config', config.path])
</del><ins>+ sim = LoadSimulator.fromCommandLine(
+ ['--config', config.path, '--clients', config.path]
+ )
</ins><span class="cx"> expectedParameters = PopulationParameters()
</span><span class="cx"> expectedParameters.addClient(
</span><span class="cx"> 3, ClientType(OS_X_10_6, {"foo": "bar"}, [ProfileType(Eventer, {
</span><span class="lines">@@ -495,7 +497,9 @@
</span><span class="cx"> """
</span><span class="cx"> config = FilePath(self.mktemp())
</span><span class="cx"> config.setContent(writePlistToString({"clients": []}))
</span><del>- sim = LoadSimulator.fromCommandLine(['--config', config.path])
</del><ins>+ sim = LoadSimulator.fromCommandLine(
+ ['--config', config.path, '--clients', config.path]
+ )
</ins><span class="cx"> expectedParameters = PopulationParameters()
</span><span class="cx"> expectedParameters.addClient(
</span><span class="cx"> 1, ClientType(OS_X_10_6, {}, [Eventer, Inviter, Accepter]))
</span><span class="lines">@@ -528,6 +532,7 @@
</span><span class="cx"> "/principals/users/%s/",
</span><span class="cx"> None,
</span><span class="cx"> None,
</span><ins>+ None,
</ins><span class="cx"> Arrival(lambda reactor: NullArrival(), {}),
</span><span class="cx"> None, observers, reactor=Reactor())
</span><span class="cx"> io = StringIO()
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformancesqlusagerequestshttpTestspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/requests/httpTests.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/requests/httpTests.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/requests/httpTests.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -89,12 +89,21 @@
</span><span class="cx"> pos = line.find(": ")
</span><span class="cx"> return float(line[pos + 2:])
</span><span class="cx">
</span><ins>+ # Need to skip over stats that are unlabeled
</ins><span class="cx"> data = open(self.logFilePath).read()
</span><span class="cx"> lines = data.splitlines()
</span><del>- count = extractInt(lines[4])
- rows = extractInt(lines[5])
- timing = extractFloat(lines[6])
- self.result = HTTPTestBase.SQLResults(count, rows, timing)
</del><ins>+ offset = 0
+ while True:
+ if lines[offset] == "*** SQL Stats ***":
+ if lines[offset + 2].split()[1] != "unlabeled":
+ count = extractInt(lines[offset + 4])
+ rows = extractInt(lines[offset + 5])
+ timing = extractFloat(lines[offset + 6])
+ self.result = HTTPTestBase.SQLResults(count, rows, timing)
+ break
+ offset += 1
+ else:
+ self.result = HTTPTestBase.SQLResults(-1, -1, 0.0)
</ins><span class="cx">
</span><span class="cx"> with open("%s-%d-%s" % (self.logFilePath, event_count, self.label), "w") as f:
</span><span class="cx"> f.write(data)
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribperformancesqlusagesqlusagepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/sqlusage.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/sqlusage.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/performance/sqlusage/sqlusage.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -127,11 +127,17 @@
</span><span class="cx"> ]
</span><span class="cx"> self.requestLabels = [request.label for request in requests]
</span><span class="cx">
</span><del>- # Warm-up server by doing calendar home and calendar propfinds
- props = (davxml.resourcetype,)
- for session in sessions:
- session.getPropertiesOnHierarchy(URL(path=session.homeHref), props)
- session.getPropertiesOnHierarchy(URL(path=session.calendarHref), props)
</del><ins>+ def _warmUp():
+ # Warm-up server by doing calendar home and child collection propfinds.
+ # Do this twice because the very first time might provision DB objects and
+ # blow any DB cache - the second time will warm the DB cache.
+ props = (davxml.resourcetype,)
+ for _ignore in range(2):
+ for session in sessions:
+ session.getPropertiesOnHierarchy(URL(path=session.homeHref), props)
+ session.getPropertiesOnHierarchy(URL(path=session.calendarHref), props)
+ session.getPropertiesOnHierarchy(URL(path=session.inboxHref), props)
+ session.getPropertiesOnHierarchy(URL(path=session.notificationHref), props)
</ins><span class="cx">
</span><span class="cx"> # Now loop over sets of events
</span><span class="cx"> for count in event_counts:
</span><span class="lines">@@ -140,6 +146,7 @@
</span><span class="cx"> result = {}
</span><span class="cx"> for request in requests:
</span><span class="cx"> print(" Test = %s" % (request.label,))
</span><ins>+ _warmUp()
</ins><span class="cx"> result[request.label] = request.execute(count)
</span><span class="cx"> self.results[count] = result
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribtoolsfix_calendar"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/fix_calendar (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/fix_calendar 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/fix_calendar 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -28,9 +28,9 @@
</span><span class="cx"> def usage():
</span><span class="cx"> print """Usage: xattr_fix CALENDARS
</span><span class="cx"> Options:
</span><del>-
</del><ins>+
</ins><span class="cx"> CALENDARS - a list of directories that are to be treated as calendars
</span><del>-
</del><ins>+
</ins><span class="cx"> Description:
</span><span class="cx"> This utility will add xattrs to the specified directories and their contents
</span><span class="cx"> to make them appear to be calendars and calendar resources when used with
</span><span class="lines">@@ -40,8 +40,10 @@
</span><span class="cx"> root without properly preserving the xattrs.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> def fixCalendar(path):
</span><del>-
</del><ins>+
</ins><span class="cx"> # First fix the resourcetype & getctag on the calendar
</span><span class="cx"> x = xattr.xattr(path)
</span><span class="cx"> x["WebDAV:{DAV:}resourcetype"] = """<?xml version='1.0' encoding='UTF-8'?>
</span><span class="lines">@@ -60,7 +62,7 @@
</span><span class="cx"> if not child.endswith(".ics"):
</span><span class="cx"> continue
</span><span class="cx"> fullpath = os.path.join(path, child)
</span><del>-
</del><ins>+
</ins><span class="cx"> # getcontenttype
</span><span class="cx"> x = xattr.xattr(fullpath)
</span><span class="cx"> x["WebDAV:{DAV:}getcontenttype"] = """<?xml version='1.0' encoding='UTF-8'?>
</span><span class="lines">@@ -94,7 +96,7 @@
</span><span class="cx"> if not os.path.exists(arg):
</span><span class="cx"> print "Path does not exist: '%s'. Ignoring." % (arg,)
</span><span class="cx"> continue
</span><del>-
</del><ins>+
</ins><span class="cx"> if os.path.basename(arg) in ("inbox", "outbox", "dropbox",):
</span><span class="cx"> print "Cannot be used on inbox, outbox or dropbox."
</span><span class="cx"> continue
</span><span class="lines">@@ -103,4 +105,3 @@
</span><span class="cx">
</span><span class="cx"> except Exception, e:
</span><span class="cx"> sys.exit(str(e))
</span><del>-
</del><span class="cx">\ No newline at end of file
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixescontribtoolsprotocolanalysispy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/protocolanalysis.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/protocolanalysis.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/contrib/tools/protocolanalysis.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -293,6 +293,12 @@
</span><span class="cx"> self.userCounts = collections.defaultdict(int)
</span><span class="cx"> self.userResponseTimes = collections.defaultdict(float)
</span><span class="cx">
</span><ins>+ self.newEvents = 0
+ self.newInvites = 0
+ self.updateEvents = 0
+ self.updateInvites = 0
+ self.attendeeInvites = 0
+
</ins><span class="cx"> self.otherUserCalendarRequests = {}
</span><span class="cx">
</span><span class="cx"> self.currentLine = None
</span><span class="lines">@@ -416,6 +422,19 @@
</span><span class="cx"> self.hourlyByStatus[" TOTAL"][timeBucketIndex] += 1
</span><span class="cx"> self.hourlyByStatus[self.currentLine.status][timeBucketIndex] += 1
</span><span class="cx">
</span><ins>+ if self.currentLine.status == 201:
+ if adjustedMethod == METHOD_PUT_ICS:
+ self.newEvents += 1
+ elif adjustedMethod == METHOD_PUT_ORGANIZER:
+ self.newInvites += 1
+ elif isOK:
+ if adjustedMethod == METHOD_PUT_ICS:
+ self.updateEvents += 1
+ elif adjustedMethod == METHOD_PUT_ORGANIZER:
+ self.updateInvites += 1
+ elif adjustedMethod == METHOD_PUT_ATTENDEE:
+ self.attendeeInvites += 1
+
</ins><span class="cx"> # Cache analysis
</span><span class="cx"> if adjustedMethod == METHOD_PROPFIND_CALENDAR and self.currentLine.status == 207:
</span><span class="cx"> responses = int(self.currentLine.extended.get("responses", 0))
</span><span class="lines">@@ -1029,7 +1048,10 @@
</span><span class="cx"> #print("User Response times")
</span><span class="cx"> #self.printUserResponseTimes(doTabs)
</span><span class="cx">
</span><ins>+ print("Sim values")
+ self.printSimStats(doTabs)
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def printInfo(self, doTabs):
</span><span class="cx">
</span><span class="cx"> table = tables.Table()
</span><span class="lines">@@ -1083,6 +1105,7 @@
</span><span class="cx"> totalRequests = 0
</span><span class="cx"> totalDepth = 0
</span><span class="cx"> totalTime = 0.0
</span><ins>+ self.timeCounts = 0
</ins><span class="cx"> for ctr in xrange(self.timeBucketCount):
</span><span class="cx"> hour = self.getHourFromIndex(ctr)
</span><span class="cx"> if hour is None:
</span><span class="lines">@@ -1101,12 +1124,13 @@
</span><span class="cx"> totalRequests += countRequests
</span><span class="cx"> totalDepth += countDepth
</span><span class="cx"> totalTime += countTime
</span><ins>+ self.timeCounts += 1
</ins><span class="cx">
</span><span class="cx"> table.addFooter(
</span><span class="cx"> (
</span><span class="cx"> "Total:",
</span><span class="cx"> totalRequests,
</span><del>- (1.0 * totalRequests) / self.timeBucketCount / self.resolutionMinutes / 60,
</del><ins>+ safePercent(totalRequests, self.timeCounts * self.resolutionMinutes * 60, 1.0),
</ins><span class="cx"> safePercent(totalTime, totalRequests, 1.0),
</span><span class="cx"> safePercent(float(totalDepth), totalRequests, 1),
</span><span class="cx"> ),
</span><span class="lines">@@ -1545,7 +1569,38 @@
</span><span class="cx"> print("")
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def printSimStats(self, doTabs):
+ users = len(self.userCounts.keys())
+ hours = self.timeCounts / self.resolutionMinutes / 60
+ table = tables.Table()
+ table.setDefaultColumnFormats((
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ ))
+ table.addHeader(("Item", "Value", "Items, per User, per Day", "Interval (sec), per item, per user"))
+ table.addRow(("Unique Users", users, "", ""))
</ins><span class="cx">
</span><ins>+ def _addRow(title, item):
+ table.addRow((title, item, "%.1f" % (safePercent(24 * item, hours * users, 1.0),), "%.1f" % (safePercent(hours * 60 * 60 * users, item, 1.0),),))
+
+ _addRow("New Events", self.newEvents)
+ _addRow("New Invites", self.newInvites)
+ _addRow("Updated Events", self.updateEvents)
+ _addRow("Updated Invites", self.updateInvites)
+ _addRow("Attendee Invites", self.attendeeInvites)
+ table.addRow((
+ "Recipients",
+ "%.1f" % (safePercent(sum(self.averagedHourlyByRecipientCount["iTIP Average"]), self.timeCounts, 1.0),),
+ "",
+ "",
+ ))
+ table.printTabDelimitedData() if doTabs else table.printTable()
+ print("")
+
+
+
</ins><span class="cx"> class TablePrinter(object):
</span><span class="cx">
</span><span class="cx"> @classmethod
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixessupportversionpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/support/version.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/support/version.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/support/version.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -26,7 +26,7 @@
</span><span class="cx"> # Compute the version number.
</span><span class="cx"> #
</span><span class="cx">
</span><del>- base_version = "5.1"
</del><ins>+ base_version = "5.2"
</ins><span class="cx">
</span><span class="cx"> branches = tuple(
</span><span class="cx"> branch.format(version=base_version)
</span><span class="lines">@@ -36,7 +36,7 @@
</span><span class="cx"> "trunk",
</span><span class="cx"> )
</span><span class="cx"> )
</span><del>-
</del><ins>+
</ins><span class="cx"> source_root = dirname(dirname(__file__))
</span><span class="cx">
</span><span class="cx"> for branch in branches:
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextenterprisedalsyntaxpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/syntax.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/syntax.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/syntax.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -1686,7 +1686,46 @@
</span><span class="cx"> SQLFragment(' in %s mode' % (self.mode,)))
</span><span class="cx">
</span><span class="cx">
</span><ins>+class DatabaseLock(_LockingStatement):
+ """
+ An SQL exclusive session level advisory lock
+ """
</ins><span class="cx">
</span><ins>+ def _toSQL(self, queryGenerator):
+ assert(queryGenerator.dialect == POSTGRES_DIALECT)
+ return SQLFragment('select pg_advisory_lock(1)')
+
+
+ def on(self, txn, *a, **kw):
+ """
+ Override on() to only execute on Postgres
+ """
+ if txn.dialect == POSTGRES_DIALECT:
+ return super(DatabaseLock, self).on(txn, *a, **kw)
+
+ return succeed(None)
+
+
+class DatabaseUnlock(_LockingStatement):
+ """
+ An SQL exclusive session level advisory lock
+ """
+
+ def _toSQL(self, queryGenerator):
+ assert(queryGenerator.dialect == POSTGRES_DIALECT)
+ return SQLFragment('select pg_advisory_unlock(1)')
+
+
+ def on(self, txn, *a, **kw):
+ """
+ Override on() to only execute on Postgres
+ """
+ if txn.dialect == POSTGRES_DIALECT:
+ return super(DatabaseUnlock, self).on(txn, *a, **kw)
+
+ return succeed(None)
+
+
</ins><span class="cx"> class Savepoint(_LockingStatement):
</span><span class="cx"> """
</span><span class="cx"> An SQL 'savepoint' statement.
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextenterprisedaltesttest_sqlsyntaxpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/test/test_sqlsyntax.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/test/test_sqlsyntax.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/enterprise/dal/test/test_sqlsyntax.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -25,7 +25,8 @@
</span><span class="cx"> TableMismatch, Parameter, Max, Len, NotEnoughValues,
</span><span class="cx"> Savepoint, RollbackToSavepoint, ReleaseSavepoint, SavepointAction,
</span><span class="cx"> Union, Intersect, Except, SetExpression, DALError,
</span><del>- ResultAliasSyntax, Count, QueryGenerator, ALL_COLUMNS)
</del><ins>+ ResultAliasSyntax, Count, QueryGenerator, ALL_COLUMNS,
+ DatabaseLock, DatabaseUnlock)
</ins><span class="cx"> from twext.enterprise.dal.syntax import FixedPlaceholder, NumericPlaceholder
</span><span class="cx"> from twext.enterprise.dal.syntax import Function
</span><span class="cx"> from twext.enterprise.dal.syntax import SchemaSyntax
</span><span class="lines">@@ -1314,6 +1315,22 @@
</span><span class="cx"> SQLFragment("lock table FOO in exclusive mode"))
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def test_databaseLock(self):
+ """
+ L{DatabaseLock} generates a ('pg_advisory_lock') statement
+ """
+ self.assertEquals(DatabaseLock().toSQL(),
+ SQLFragment("select pg_advisory_lock(1)"))
+
+
+ def test_databaseUnlock(self):
+ """
+ L{DatabaseUnlock} generates a ('pg_advisory_unlock') statement
+ """
+ self.assertEquals(DatabaseUnlock().toSQL(),
+ SQLFragment("select pg_advisory_unlock(1)"))
+
+
</ins><span class="cx"> def test_savepoint(self):
</span><span class="cx"> """
</span><span class="cx"> L{Savepoint} generates a ('savepoint') statement.
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextpatchespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/patches.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/patches.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/patches.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -26,6 +26,8 @@
</span><span class="cx"> from twisted.python.versions import Version
</span><span class="cx"> from twisted.python.modules import getModule
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> def _hasIPv6ClientSupport():
</span><span class="cx"> """
</span><span class="cx"> Does the loaded version of Twisted have IPv6 client support?
</span><span class="lines">@@ -34,8 +36,9 @@
</span><span class="cx"> if version > lastVersionWithoutIPv6Clients:
</span><span class="cx"> return True
</span><span class="cx"> elif version == lastVersionWithoutIPv6Clients:
</span><del>- # It could be a snapshot of trunk or a branch with this bug fixed. Don't
- # load the module, though, as that would be a bunch of unnecessary work.
</del><ins>+ # It could be a snapshot of trunk or a branch with this bug fixed.
+ # Don't load the module, though, as that would be a bunch of
+ # unnecessary work.
</ins><span class="cx"> return "_resolveIPv6" in (getModule("twisted.internet.tcp")
</span><span class="cx"> .filePath.getContent())
</span><span class="cx"> else:
</span><span class="lines">@@ -45,8 +48,8 @@
</span><span class="cx">
</span><span class="cx"> def _addBackports():
</span><span class="cx"> """
</span><del>- We currently require 2 backported bugfixes from a future release of Twisted,
- for IPv6 support:
</del><ins>+ We currently require 2 backported bugfixes from a future release of
+ Twisted, for IPv6 support:
</ins><span class="cx">
</span><span class="cx"> - U{IPv6 client support <http://tm.tl/5085>}
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhoaggregatepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/aggregate.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/aggregate.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/aggregate.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -45,13 +45,16 @@
</span><span class="cx">
</span><span class="cx"> for service in services:
</span><span class="cx"> if not IDirectoryService.implementedBy(service.__class__):
</span><del>- raise ValueError("Not a directory service: %s" % (service,))
</del><ins>+ raise ValueError(
+ "Not a directory service: {0}".format(service)
+ )
</ins><span class="cx">
</span><span class="cx"> for recordType in service.recordTypes():
</span><span class="cx"> if recordType in recordTypes:
</span><span class="cx"> raise DirectoryConfigurationError(
</span><del>- "Aggregated services may not vend the same record type: %s"
- % (recordType,)
</del><ins>+ "Aggregated services may not vend "
+ "the same record type: {0}"
+ .format(recordType)
</ins><span class="cx"> )
</span><span class="cx"> recordTypes.add(recordType)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhodirectorypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/directory.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/directory.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/directory.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -47,7 +47,7 @@
</span><span class="cx"> fieldName = FieldName
</span><span class="cx">
</span><span class="cx"> normalizedFields = {
</span><del>- FieldName.guid: lambda g: UUID(g).hex,
</del><ins>+ FieldName.guid: lambda g: UUID(g).hex,
</ins><span class="cx"> FieldName.emailAddresses: lambda e: e.lower(),
</span><span class="cx"> }
</span><span class="cx">
</span><span class="lines">@@ -57,9 +57,9 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def __repr__(self):
</span><del>- return "<%s %r>" % (
- self.__class__.__name__,
- self.realmName,
</del><ins>+ return (
+ "<{self.__class__.__name__} {self.realmName!r}>"
+ .format(self=self)
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -76,7 +76,9 @@
</span><span class="cx"> the whole directory should be searched.
</span><span class="cx"> @type records: L{set} or L{frozenset}
</span><span class="cx"> """
</span><del>- return fail(QueryNotSupportedError("Unknown expression: %s" % (expression,)))
</del><ins>+ return fail(QueryNotSupportedError(
+ "Unknown expression: {0}".format(expression)
+ ))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -109,7 +111,9 @@
</span><span class="cx"> elif operand == Operand.OR:
</span><span class="cx"> results |= recordsMatchingExpression
</span><span class="cx"> else:
</span><del>- raise QueryNotSupportedError("Unknown operand: %s" % (operand,))
</del><ins>+ raise QueryNotSupportedError(
+ "Unknown operand: {0}".format(operand)
+ )
</ins><span class="cx">
</span><span class="cx"> returnValue(results)
</span><span class="cx">
</span><span class="lines">@@ -120,12 +124,16 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def recordWithUID(self, uid):
</span><del>- returnValue(uniqueResult((yield self.recordsWithFieldValue(FieldName.uid, uid))))
-
</del><ins>+ returnValue(uniqueResult(
+ (yield self.recordsWithFieldValue(FieldName.uid, uid))
+ ))
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def recordWithGUID(self, guid):
</span><del>- returnValue(uniqueResult((yield self.recordsWithFieldValue(FieldName.guid, guid))))
</del><ins>+ returnValue(uniqueResult(
+ (yield self.recordsWithFieldValue(FieldName.guid, guid))
+ ))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def recordsWithRecordType(self, recordType):
</span><span class="lines">@@ -136,12 +144,15 @@
</span><span class="cx"> def recordWithShortName(self, recordType, shortName):
</span><span class="cx"> returnValue(uniqueResult((yield self.recordsFromQuery((
</span><span class="cx"> MatchExpression(FieldName.recordType, recordType),
</span><del>- MatchExpression(FieldName.shortNames, shortName ),
</del><ins>+ MatchExpression(FieldName.shortNames, shortName),
</ins><span class="cx"> )))))
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def recordsWithEmailAddress(self, emailAddress):
</span><del>- return self.recordsWithFieldValue(FieldName.emailAddresses, emailAddress)
</del><ins>+ return self.recordsWithFieldValue(
+ FieldName.emailAddresses,
+ emailAddress,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def updateRecords(self, records, create=False):
</span><span class="lines">@@ -168,21 +179,31 @@
</span><span class="cx"> def __init__(self, service, fields):
</span><span class="cx"> for fieldName in self.requiredFields:
</span><span class="cx"> if fieldName not in fields or not fields[fieldName]:
</span><del>- raise ValueError("%s field is required." % (fieldName,))
</del><ins>+ raise ValueError("{0} field is required.".format(fieldName))
</ins><span class="cx">
</span><span class="cx"> if FieldName.isMultiValue(fieldName):
</span><span class="cx"> values = fields[fieldName]
</span><span class="cx"> if len(values) == 0:
</span><del>- raise ValueError("%s field must have at least one value." % (fieldName,))
</del><ins>+ raise ValueError(
+ "{0} field must have at least one value."
+ .format(fieldName)
+ )
</ins><span class="cx"> for value in values:
</span><span class="cx"> if not value:
</span><del>- raise ValueError("%s field must not be empty." % (fieldName,))
</del><ins>+ raise ValueError(
+ "{0} field must not be empty.".format(fieldName)
+ )
</ins><span class="cx">
</span><del>- if fields[FieldName.recordType] not in service.recordType.iterconstants():
- raise ValueError("Record type must be one of %r, not %r." % (
- tuple(service.recordType.iterconstants()),
- fields[FieldName.recordType]
- ))
</del><ins>+ if (
+ fields[FieldName.recordType] not in
+ service.recordType.iterconstants()
+ ):
+ raise ValueError(
+ "Record type must be one of {0!r}, not {1!r}.".format(
+ tuple(service.recordType.iterconstants()),
+ fields[FieldName.recordType],
+ )
+ )
</ins><span class="cx">
</span><span class="cx"> # Normalize fields
</span><span class="cx"> normalizedFields = {}
</span><span class="lines">@@ -197,16 +218,18 @@
</span><span class="cx"> normalizedFields[name] = tuple((normalize(v) for v in value))
</span><span class="cx"> else:
</span><span class="cx"> normalizedFields[name] = normalize(value)
</span><del>-
</del><ins>+
</ins><span class="cx"> self.service = service
</span><span class="cx"> self.fields = normalizedFields
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def __repr__(self):
</span><del>- return "<%s (%s)%s>" % (
- self.__class__.__name__,
- describe(self.recordType),
- self.shortNames[0],
</del><ins>+ return (
+ "<{self.__class__.__name__} ({recordType}){shortName}>".format(
+ self=self,
+ recordType=describe(self.recordType),
+ shortName=self.shortNames[0],
+ )
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -262,9 +285,9 @@
</span><span class="cx">
</span><span class="cx"> def members(self):
</span><span class="cx"> if self.recordType == RecordType.group:
</span><del>- raise NotImplementedError()
</del><ins>+ raise NotImplementedError("Subclasses must implement members()")
</ins><span class="cx"> return succeed(())
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def groups(self):
</span><del>- raise NotImplementedError()
</del><ins>+ raise NotImplementedError("Subclasses must implement groups()")
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhoexpressionpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/expression.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/expression.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/expression.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -72,7 +72,11 @@
</span><span class="cx"> @ivar flags: L{NamedConstant} specifying additional options
</span><span class="cx"> """
</span><span class="cx">
</span><del>- def __init__(self, fieldName, fieldValue, matchType=MatchType.equals, flags=None):
</del><ins>+ def __init__(
+ self,
+ fieldName, fieldValue,
+ matchType=MatchType.equals, flags=None
+ ):
</ins><span class="cx"> self.fieldName = fieldName
</span><span class="cx"> self.fieldValue = fieldValue
</span><span class="cx"> self.matchType = matchType
</span><span class="lines">@@ -85,12 +89,16 @@
</span><span class="cx"> if self.flags is None:
</span><span class="cx"> flags = ""
</span><span class="cx"> else:
</span><del>- flags = " (%s)" % (describe(self.flags),)
</del><ins>+ flags = " ({0})".format(describe(self.flags))
</ins><span class="cx">
</span><del>- return "<%s: %r %s %r%s>" % (
- self.__class__.__name__,
- describe(self.fieldName),
- describe(self.matchType),
- describe(self.fieldValue),
- flags
</del><ins>+ return (
+ "<{self.__class__.__name__}: {fieldName!r} "
+ "{matchType} {fieldValue!r}{flags}>"
+ .format(
+ self=self,
+ fieldName=describe(self.fieldName),
+ matchType=describe(self.matchType),
+ fieldValue=describe(self.fieldValue),
+ flags=flags,
+ )
</ins><span class="cx"> )
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhoidirectorypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/idirectory.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/idirectory.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/idirectory.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -51,16 +51,22 @@
</span><span class="cx"> Directory service generic error.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class DirectoryConfigurationError(DirectoryServiceError):
</span><span class="cx"> """
</span><span class="cx"> Directory configurtion error.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class DirectoryAvailabilityError(DirectoryServiceError):
</span><span class="cx"> """
</span><span class="cx"> Directory not available.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class UnknownRecordTypeError(DirectoryServiceError):
</span><span class="cx"> """
</span><span class="cx"> Unknown record type.
</span><span class="lines">@@ -69,16 +75,22 @@
</span><span class="cx"> DirectoryServiceError.__init__(self, token)
</span><span class="cx"> self.token = token
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class QueryNotSupportedError(DirectoryServiceError):
</span><span class="cx"> """
</span><span class="cx"> Query not supported.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class NoSuchRecordError(DirectoryServiceError):
</span><span class="cx"> """
</span><span class="cx"> Record does not exist.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class NotAllowedError(DirectoryServiceError):
</span><span class="cx"> """
</span><span class="cx"> Apparently, you can't do that.
</span><span class="lines">@@ -123,6 +135,7 @@
</span><span class="cx"> fullNames.multiValue = True
</span><span class="cx"> emailAddresses.multiValue = True
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @staticmethod
</span><span class="cx"> def isMultiValue(name):
</span><span class="cx"> return getattr(name, "multiValue", False)
</span><span class="lines">@@ -157,14 +170,18 @@
</span><span class="cx"> A directory service may allow support the editing, removal and
</span><span class="cx"> addition of records.
</span><span class="cx"> """
</span><del>- realmName = Attribute("The name of the authentication realm this service represents.")
</del><ins>+ realmName = Attribute(
+ "The name of the authentication realm this service represents."
+ )
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordTypes():
</span><span class="cx"> """
</span><span class="cx"> @return: an iterable of L{NamedConstant}s denoting the record
</span><span class="cx"> types that are kept in this directory.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordsFromExpression(self, expression):
</span><span class="cx"> """
</span><span class="cx"> Find records matching an expression.
</span><span class="lines">@@ -175,6 +192,7 @@
</span><span class="cx"> supported by this directory service.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordsFromQuery(expressions, operand=Operand.AND):
</span><span class="cx"> """
</span><span class="cx"> Find records by composing a query consisting of an iterable of
</span><span class="lines">@@ -188,6 +206,7 @@
</span><span class="cx"> supported by this directory service.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordsWithFieldValue(fieldName, value):
</span><span class="cx"> """
</span><span class="cx"> Find records that have the given field name with the given
</span><span class="lines">@@ -199,6 +218,7 @@
</span><span class="cx"> @return: a deferred iterable of L{IDirectoryRecord}s.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordWithUID(uid):
</span><span class="cx"> """
</span><span class="cx"> Find the record that has the given UID.
</span><span class="lines">@@ -207,7 +227,8 @@
</span><span class="cx"> @return: a deferred iterable of L{IDirectoryRecord}s, or
</span><span class="cx"> C{None} if there is no such record.
</span><span class="cx"> """
</span><del>-
</del><ins>+
+
</ins><span class="cx"> def recordWithGUID(guid):
</span><span class="cx"> """
</span><span class="cx"> Find the record that has the given GUID.
</span><span class="lines">@@ -217,6 +238,7 @@
</span><span class="cx"> C{None} if there is no such record.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordsWithRecordType(recordType):
</span><span class="cx"> """
</span><span class="cx"> Find the records that have the given record type.
</span><span class="lines">@@ -225,6 +247,7 @@
</span><span class="cx"> @return: a deferred iterable of L{IDirectoryRecord}s.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordWithShortName(recordType, shortName):
</span><span class="cx"> """
</span><span class="cx"> Find the record that has the given record type and short name.
</span><span class="lines">@@ -236,6 +259,7 @@
</span><span class="cx"> C{None} if there is no such record.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def recordsWithEmailAddress(emailAddress):
</span><span class="cx"> """
</span><span class="cx"> Find the records that have the given email address.
</span><span class="lines">@@ -245,6 +269,7 @@
</span><span class="cx"> C{None} if there is no such record.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def updateRecords(records, create=False):
</span><span class="cx"> """
</span><span class="cx"> Updates existing directory records.
</span><span class="lines">@@ -254,6 +279,7 @@
</span><span class="cx"> @type create: boolean
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def removeRecords(uids):
</span><span class="cx"> """
</span><span class="cx"> Removes the records with the given UIDs.
</span><span class="lines">@@ -294,6 +320,7 @@
</span><span class="cx"> service = Attribute("The L{IDirectoryService} this record exists in.")
</span><span class="cx"> fields = Attribute("A mapping with L{NamedConstant} keys.")
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def members():
</span><span class="cx"> """
</span><span class="cx"> Find the records that are members of this group. Only direct
</span><span class="lines">@@ -302,6 +329,7 @@
</span><span class="cx"> direct members of this group.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def groups():
</span><span class="cx"> """
</span><span class="cx"> Find the group records that this record is a member of. Only
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhoindexpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/index.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/index.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/index.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -29,7 +29,8 @@
</span><span class="cx"> from twisted.python.constants import Names, NamedConstant
</span><span class="cx"> from twisted.internet.defer import succeed, inlineCallbacks, returnValue
</span><span class="cx">
</span><del>-from twext.who.util import ConstantsContainer, describe, uniqueResult, iterFlags
</del><ins>+from twext.who.util import ConstantsContainer
+from twext.who.util import describe, uniqueResult, iterFlags
</ins><span class="cx"> from twext.who.idirectory import FieldName as BaseFieldName
</span><span class="cx"> from twext.who.expression import MatchExpression, MatchType, MatchFlags
</span><span class="cx"> from twext.who.directory import DirectoryService as BaseDirectoryService
</span><span class="lines">@@ -57,7 +58,10 @@
</span><span class="cx"> XML directory service.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- fieldName = ConstantsContainer(chain(BaseDirectoryService.fieldName.iterconstants(), FieldName.iterconstants()))
</del><ins>+ fieldName = ConstantsContainer(chain(
+ BaseDirectoryService.fieldName.iterconstants(),
+ FieldName.iterconstants()
+ ))
</ins><span class="cx">
</span><span class="cx"> indexedFields = (
</span><span class="cx"> BaseFieldName.recordType,
</span><span class="lines">@@ -90,7 +94,7 @@
</span><span class="cx"> """
</span><span class="cx"> Load records.
</span><span class="cx"> """
</span><del>- raise NotImplementedError("Subclasses should implement loadRecords().")
</del><ins>+ raise NotImplementedError("Subclasses must implement loadRecords().")
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def flush(self):
</span><span class="lines">@@ -112,7 +116,9 @@
</span><span class="cx"> elif flag == MatchFlags.caseInsensitive:
</span><span class="cx"> normalize = lambda x: x.lower()
</span><span class="cx"> else:
</span><del>- raise NotImplementedError("Unknown query flag: %s" % (describe(flag),))
</del><ins>+ raise NotImplementedError(
+ "Unknown query flag: {0}".format(describe(flag))
+ )
</ins><span class="cx">
</span><span class="cx"> return predicate, normalize
</span><span class="cx">
</span><span class="lines">@@ -131,16 +137,27 @@
</span><span class="cx"> matchType = expression.matchType
</span><span class="cx">
</span><span class="cx"> if matchType == MatchType.startsWith:
</span><del>- indexKeys = (key for key in fieldIndex if predicate(normalize(key).startswith(matchValue)))
</del><ins>+ indexKeys = (
+ key for key in fieldIndex
+ if predicate(normalize(key).startswith(matchValue))
+ )
</ins><span class="cx"> elif matchType == MatchType.contains:
</span><del>- indexKeys = (key for key in fieldIndex if predicate(matchValue in normalize(key)))
</del><ins>+ indexKeys = (
+ key for key in fieldIndex
+ if predicate(matchValue in normalize(key))
+ )
</ins><span class="cx"> elif matchType == MatchType.equals:
</span><span class="cx"> if predicate(True):
</span><span class="cx"> indexKeys = (matchValue,)
</span><span class="cx"> else:
</span><del>- indexKeys = (key for key in fieldIndex if normalize(key) != matchValue)
</del><ins>+ indexKeys = (
+ key for key in fieldIndex
+ if normalize(key) != matchValue
+ )
</ins><span class="cx"> else:
</span><del>- raise NotImplementedError("Unknown match type: %s" % (describe(matchType),))
</del><ins>+ raise NotImplementedError(
+ "Unknown match type: {0}".format(describe(matchType))
+ )
</ins><span class="cx">
</span><span class="cx"> matchingRecords = set()
</span><span class="cx"> for key in indexKeys:
</span><span class="lines">@@ -165,18 +182,25 @@
</span><span class="cx"> matchType = expression.matchType
</span><span class="cx">
</span><span class="cx"> if matchType == MatchType.startsWith:
</span><del>- match = lambda fieldValue: predicate(fieldValue.startswith(matchValue))
</del><ins>+ match = lambda fieldValue: predicate(
+ fieldValue.startswith(matchValue)
+ )
</ins><span class="cx"> elif matchType == MatchType.contains:
</span><span class="cx"> match = lambda fieldValue: predicate(matchValue in fieldValue)
</span><span class="cx"> elif matchType == MatchType.equals:
</span><span class="cx"> match = lambda fieldValue: predicate(fieldValue == matchValue)
</span><span class="cx"> else:
</span><del>- raise NotImplementedError("Unknown match type: %s" % (describe(matchType),))
</del><ins>+ raise NotImplementedError(
+ "Unknown match type: {0}".format(describe(matchType))
+ )
</ins><span class="cx">
</span><span class="cx"> result = set()
</span><span class="cx">
</span><span class="cx"> if records is None:
</span><del>- records = (uniqueResult(values) for values in self.index[self.fieldName.uid].itervalues())
</del><ins>+ records = (
+ uniqueResult(values) for values
+ in self.index[self.fieldName.uid].itervalues()
+ )
</ins><span class="cx">
</span><span class="cx"> for record in records:
</span><span class="cx"> fieldValues = record.fields.get(expression.fieldName, None)
</span><span class="lines">@@ -194,11 +218,17 @@
</span><span class="cx"> def recordsFromExpression(self, expression, records=None):
</span><span class="cx"> if isinstance(expression, MatchExpression):
</span><span class="cx"> if expression.fieldName in self.indexedFields:
</span><del>- return self.indexedRecordsFromMatchExpression(expression, records=records)
</del><ins>+ return self.indexedRecordsFromMatchExpression(
+ expression, records=records
+ )
</ins><span class="cx"> else:
</span><del>- return self.unIndexedRecordsFromMatchExpression(expression, records=records)
</del><ins>+ return self.unIndexedRecordsFromMatchExpression(
+ expression, records=records
+ )
</ins><span class="cx"> else:
</span><del>- return BaseDirectoryService.recordsFromExpression(self, expression, records=records)
</del><ins>+ return BaseDirectoryService.recordsFromExpression(
+ self, expression, records=records
+ )
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -206,6 +236,7 @@
</span><span class="cx"> """
</span><span class="cx"> XML directory record
</span><span class="cx"> """
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def members(self):
</span><span class="cx"> members = set()
</span><span class="lines">@@ -215,4 +246,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def groups(self):
</span><del>- return self.service.recordsWithFieldValue(FieldName.memberUIDs, self.uid)
</del><ins>+ return self.service.recordsWithFieldValue(
+ FieldName.memberUIDs, self.uid
+ )
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhoutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/util.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/util.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/util.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -40,7 +40,7 @@
</span><span class="cx"> myConstants = {}
</span><span class="cx"> for constant in constants:
</span><span class="cx"> if constant.name in myConstants:
</span><del>- raise ValueError("Name conflict: %r" % (constant.name,))
</del><ins>+ raise ValueError("Name conflict: {0}".format(constant.name))
</ins><span class="cx"> myConstants[constant.name] = constant
</span><span class="cx">
</span><span class="cx"> self._constants = myConstants
</span><span class="lines">@@ -67,7 +67,9 @@
</span><span class="cx"> if result is None:
</span><span class="cx"> result = value
</span><span class="cx"> else:
</span><del>- raise DirectoryServiceError("Multiple values found where one expected.")
</del><ins>+ raise DirectoryServiceError(
+ "Multiple values found where one expected."
+ )
</ins><span class="cx"> return result
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwextwhoxmlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/xml.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/xml.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twext/who/xml.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -144,9 +144,11 @@
</span><span class="cx"> else:
</span><span class="cx"> realmName = repr(realmName)
</span><span class="cx">
</span><del>- return "<%s %s>" % (
- self.__class__.__name__,
- realmName,
</del><ins>+ return (
+ "<{self.__class__.__name__} {realmName}>".format(
+ self=self,
+ realmName=realmName,
+ )
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -201,7 +203,10 @@
</span><span class="cx"> #
</span><span class="cx"> if stat:
</span><span class="cx"> self.filePath.restat()
</span><del>- cacheTag = (self.filePath.getModificationTime(), self.filePath.getsize())
</del><ins>+ cacheTag = (
+ self.filePath.getModificationTime(),
+ self.filePath.getsize()
+ )
</ins><span class="cx"> if cacheTag == self._cacheTag:
</span><span class="cx"> return
</span><span class="cx"> else:
</span><span class="lines">@@ -225,9 +230,13 @@
</span><span class="cx"> #
</span><span class="cx"> directoryNode = etree.getroot()
</span><span class="cx"> if directoryNode.tag != self.element.directory.value:
</span><del>- raise ParseError("Incorrect root element: %s" % (directoryNode.tag,))
</del><ins>+ raise ParseError(
+ "Incorrect root element: {0}".format(directoryNode.tag)
+ )
</ins><span class="cx">
</span><del>- realmName = directoryNode.get(self.attribute.realm.value, "").encode("utf-8")
</del><ins>+ realmName = directoryNode.get(
+ self.attribute.realm.value, ""
+ ).encode("utf-8")
</ins><span class="cx">
</span><span class="cx"> if not realmName:
</span><span class="cx"> raise ParseError("No realm name.")
</span><span class="lines">@@ -239,7 +248,9 @@
</span><span class="cx">
</span><span class="cx"> for recordNode in directoryNode:
</span><span class="cx"> try:
</span><del>- records.add(self.parseRecordNode(recordNode, unknownFieldElements))
</del><ins>+ records.add(
+ self.parseRecordNode(recordNode, unknownFieldElements)
+ )
</ins><span class="cx"> except UnknownRecordTypeError as e:
</span><span class="cx"> unknownRecordTypes.add(e.token)
</span><span class="cx">
</span><span class="lines">@@ -277,10 +288,14 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def parseRecordNode(self, recordNode, unknownFieldElements=None):
</span><del>- recordTypeAttribute = recordNode.get(self.attribute.recordType.value, "").encode("utf-8")
</del><ins>+ recordTypeAttribute = recordNode.get(
+ self.attribute.recordType.value, ""
+ ).encode("utf-8")
</ins><span class="cx"> if recordTypeAttribute:
</span><span class="cx"> try:
</span><del>- recordType = self.value.lookupByValue(recordTypeAttribute).recordType
</del><ins>+ recordType = (
+ self.value.lookupByValue(recordTypeAttribute).recordType
+ )
</ins><span class="cx"> except (ValueError, AttributeError):
</span><span class="cx"> raise UnknownRecordTypeError(recordTypeAttribute)
</span><span class="cx"> else:
</span><span class="lines">@@ -357,9 +372,14 @@
</span><span class="cx"> for (name, value) in record.fields.items():
</span><span class="cx"> if name == self.fieldName.recordType:
</span><span class="cx"> if value in recordTypes:
</span><del>- recordNode.set(self.attribute.recordType.value, recordTypes[value])
</del><ins>+ recordNode.set(
+ self.attribute.recordType.value,
+ recordTypes[value]
+ )
</ins><span class="cx"> else:
</span><del>- raise AssertionError("Unknown record type: %r" % (value,))
</del><ins>+ raise AssertionError(
+ "Unknown record type: {0}".format(value)
+ )
</ins><span class="cx">
</span><span class="cx"> else:
</span><span class="cx"> if name in fieldNames:
</span><span class="lines">@@ -376,7 +396,9 @@
</span><span class="cx"> recordNode.append(subNode)
</span><span class="cx">
</span><span class="cx"> else:
</span><del>- raise AssertionError("Unknown field name: %r" % (name,))
</del><ins>+ raise AssertionError(
+ "Unknown field name: {0!r}".format(name)
+ )
</ins><span class="cx">
</span><span class="cx"> # Walk through the record nodes in the XML tree and apply
</span><span class="cx"> # updates.
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavdirectorydirectorypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/directory.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/directory.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/directory.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -533,10 +533,11 @@
</span><span class="cx"> )
</span><span class="cx"> for record in resources:
</span><span class="cx"> guid = record.guid
</span><del>- assignments.append(("%s#calendar-proxy-write" % (guid,),
- record.externalProxies()))
- assignments.append(("%s#calendar-proxy-read" % (guid,),
- record.externalReadOnlyProxies()))
</del><ins>+ if record.enabledForCalendaring:
+ assignments.append(("%s#calendar-proxy-write" % (guid,),
+ record.externalProxies()))
+ assignments.append(("%s#calendar-proxy-read" % (guid,),
+ record.externalReadOnlyProxies()))
</ins><span class="cx">
</span><span class="cx"> return assignments
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavdirectoryldapdirectorypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/ldapdirectory.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/ldapdirectory.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/ldapdirectory.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -391,6 +391,12 @@
</span><span class="cx">
</span><span class="cx"> # Build filter
</span><span class="cx"> filterstr = "(|(%s=*)(%s=*))" % (readAttr, writeAttr)
</span><ins>+ # ...taking into account only calendar-enabled records
+ enabledAttr = self.rdnSchema["locations"]["calendarEnabledAttr"]
+ enabledValue = self.rdnSchema["locations"]["calendarEnabledValue"]
+ if enabledAttr and enabledValue:
+ filterstr = "(&(%s=%s)%s)" % (enabledAttr, enabledValue, filterstr)
+
</ins><span class="cx"> attrlist = [guidAttr, readAttr, writeAttr]
</span><span class="cx">
</span><span class="cx"> # Query the LDAP server
</span><span class="lines">@@ -1046,7 +1052,7 @@
</span><span class="cx">
</span><span class="cx"> try:
</span><span class="cx"> record = self._ldapResultToRecord(dn, attrs, recordType)
</span><del>- self.log.debug("Got LDAP record %s" % (record,))
</del><ins>+ self.log.debug("Got LDAP record {rec}", rec=record)
</ins><span class="cx">
</span><span class="cx"> if not unrestricted:
</span><span class="cx"> self.log.debug("%s is not enabled because it's not a member of group: %s" % (dn, self.restrictToGroup))
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavdirectorytesttest_directorypy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/test/test_directory.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/test/test_directory.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/directory/test/test_directory.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -540,7 +540,167 @@
</span><span class="cx"> groups,
</span><span class="cx"> )
</span><span class="cx">
</span><ins>+ #
+ # Now remove all external assignments, and those should take effect.
+ #
+ def fakeExternalProxiesEmpty():
+ return []
</ins><span class="cx">
</span><ins>+ updater = GroupMembershipCacheUpdater(
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30, 30,
+ cache=cache, useExternalProxies=True,
+ externalProxiesSource=fakeExternalProxiesEmpty)
+
+ yield updater.updateCache()
+
+ delegates = (
+
+ # record name
+ # read-write delegators
+ # read-only delegators
+ # groups delegate is in (restricted to only those groups
+ # participating in delegation)
+
+ # Note: "transporter" is now gone for everyone
+
+ ("wsanchez",
+ set(["mercury", "apollo", "orion", "gemini"]),
+ set(["non_calendar_proxy"]),
+ set(['left_coast',
+ 'both_coasts',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ 'gemini#calendar-proxy-write',
+ ]),
+ ),
+ ("cdaboo",
+ set(["apollo", "orion", "non_calendar_proxy"]),
+ set(["non_calendar_proxy"]),
+ set(['both_coasts',
+ 'non_calendar_group',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ ]),
+ ),
+ ("lecroy",
+ set(["apollo", "mercury", "non_calendar_proxy"]),
+ set(),
+ set(['both_coasts',
+ 'left_coast',
+ 'non_calendar_group',
+ ]),
+ ),
+ )
+
+ for name, write, read, groups in delegates:
+ delegate = self._getPrincipalByShortName(DirectoryService.recordType_users, name)
+
+ proxyFor = (yield delegate.proxyFor(True))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ write,
+ )
+ proxyFor = (yield delegate.proxyFor(False))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ read,
+ )
+ groupsIn = (yield delegate.groupMemberships())
+ uids = set()
+ for group in groupsIn:
+ try:
+ uid = group.uid # a sub-principal
+ except AttributeError:
+ uid = group.record.guid # a regular group
+ uids.add(uid)
+ self.assertEquals(
+ set(uids),
+ groups,
+ )
+
+ #
+ # Now add back an external assignments, and those should take effect.
+ #
+ def fakeExternalProxiesAdded():
+ return [
+ (
+ "transporter#calendar-proxy-write",
+ set(["8B4288F6-CC82-491D-8EF9-642EF4F3E7D0"])
+ ),
+ ]
+
+ updater = GroupMembershipCacheUpdater(
+ calendaruserproxy.ProxyDBService, self.directoryService, 30, 30, 30,
+ cache=cache, useExternalProxies=True,
+ externalProxiesSource=fakeExternalProxiesAdded)
+
+ yield updater.updateCache()
+
+ delegates = (
+
+ # record name
+ # read-write delegators
+ # read-only delegators
+ # groups delegate is in (restricted to only those groups
+ # participating in delegation)
+
+ ("wsanchez",
+ set(["mercury", "apollo", "orion", "gemini"]),
+ set(["non_calendar_proxy"]),
+ set(['left_coast',
+ 'both_coasts',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ 'gemini#calendar-proxy-write',
+ ]),
+ ),
+ ("cdaboo",
+ set(["apollo", "orion", "non_calendar_proxy"]),
+ set(["non_calendar_proxy"]),
+ set(['both_coasts',
+ 'non_calendar_group',
+ 'recursive1_coasts',
+ 'recursive2_coasts',
+ ]),
+ ),
+ ("lecroy",
+ set(["apollo", "mercury", "non_calendar_proxy", "transporter"]),
+ set(),
+ set(['both_coasts',
+ 'left_coast',
+ 'non_calendar_group',
+ 'transporter#calendar-proxy-write',
+ ]),
+ ),
+ )
+
+ for name, write, read, groups in delegates:
+ delegate = self._getPrincipalByShortName(DirectoryService.recordType_users, name)
+
+ proxyFor = (yield delegate.proxyFor(True))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ write,
+ )
+ proxyFor = (yield delegate.proxyFor(False))
+ self.assertEquals(
+ set([p.record.guid for p in proxyFor]),
+ read,
+ )
+ groupsIn = (yield delegate.groupMemberships())
+ uids = set()
+ for group in groupsIn:
+ try:
+ uid = group.uid # a sub-principal
+ except AttributeError:
+ uid = group.record.guid # a regular group
+ uids.add(uid)
+ self.assertEquals(
+ set(uids),
+ groups,
+ )
+
+
</ins><span class="cx"> def test_diffAssignments(self):
</span><span class="cx"> """
</span><span class="cx"> Ensure external proxy assignment diffing works
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/resource.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/resource.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/resource.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -333,6 +333,12 @@
</span><span class="cx"> else:
</span><span class="cx"> yield transaction.commit()
</span><span class="cx">
</span><ins>+ # Log extended item
+ if transaction.logItems:
+ if not hasattr(request, "extendedLogItems"):
+ request.extendedLogItems = {}
+ request.extendedLogItems.update(transaction.logItems)
+
</ins><span class="cx"> # May need to reset the last-modified header in the response as txn.commit() can change it due to pre-commit hooks
</span><span class="cx"> if response.headers.hasHeader("last-modified"):
</span><span class="cx"> response.headers.setHeader("last-modified", self.lastModified())
</span><span class="lines">@@ -2551,15 +2557,6 @@
</span><span class="cx"> return self._newStoreHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object._newStoreObject, mode)
</span><span class="cx">
</span><span class="cx">
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
- """
- Return all child object resources with the specified UID.
-
- Pass through direct to store.
- """
- return self._newStoreHome.getCalendarResourcesForUID(uid, allow_shared)
-
-
</del><span class="cx"> def defaultAccessControlList(self):
</span><span class="cx"> myPrincipal = self.principalForRecord()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavscheduling_storecaldavresourcepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/scheduling_store/caldav/resource.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/scheduling_store/caldav/resource.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/scheduling_store/caldav/resource.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -422,8 +422,12 @@
</span><span class="cx"> authz = (yield request.locateResource(principalURL))
</span><span class="cx"> self._associatedTransaction._authz_uid = authz.record.guid
</span><span class="cx">
</span><ins>+ # Log extended item
+ if not hasattr(request, "extendedLogItems"):
+ request.extendedLogItems = {}
+
</ins><span class="cx"> # This is a local CALDAV scheduling operation.
</span><del>- scheduler = CalDAVScheduler(self._associatedTransaction, self.parent._newStoreHome.uid())
</del><ins>+ scheduler = CalDAVScheduler(self._associatedTransaction, self.parent._newStoreHome.uid(), logItems=request.extendedLogItems)
</ins><span class="cx">
</span><span class="cx"> # Do the POST processing treating
</span><span class="cx"> result = (yield scheduler.doSchedulingViaPOST(originator, recipients, calendar))
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestwistedcaldavstdconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/stdconfig.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/stdconfig.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/twistedcaldav/stdconfig.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -307,9 +307,15 @@
</span><span class="cx"> "FailIfUpgradeNeeded" : True, # Set to True to prevent the server or utility tools
</span><span class="cx"> # tools from running if the database needs a schema
</span><span class="cx"> # upgrade.
</span><del>- "StopAfterUpgradeTriggerFile" : "stop_after_upgrade", # if this file exists
- # in ConfigRoot, stop the service after finishing upgrade phase
</del><ins>+ "StopAfterUpgradeTriggerFile" : "stop_after_upgrade", # if this file exists in ConfigRoot, stop
+ # the service after finishing upgrade phase
</ins><span class="cx">
</span><ins>+ "UpgradeHomePrefix" : "", # When upgrading, only upgrade homes where the owner UID starts with
+ # with the specified prefix. The upgrade will only be partial and only
+ # apply to upgrade pieces that affect entire homes. The upgrade will
+ # need to be run again without this prefix set to complete the overall
+ # upgrade.
+
</ins><span class="cx"> #
</span><span class="cx"> # Types of service provided
</span><span class="cx"> #
</span><span class="lines">@@ -564,8 +570,8 @@
</span><span class="cx"> }
</span><span class="cx"> },
</span><span class="cx">
</span><del>- "EnableTimezonesByReference" : False, # Strip out VTIMEZONES that are known
- "UsePackageTimezones" : False, # Use timezone data from twistedcaldav.zoneinfo - don't copy to Data directory
</del><ins>+ "EnableTimezonesByReference" : True, # Strip out VTIMEZONES that are known
+ "UsePackageTimezones" : False, # Use timezone data from twistedcaldav.zoneinfo - don't copy to Data directory
</ins><span class="cx">
</span><span class="cx"> "EnableBatchUpload" : True, # POST batch uploads
</span><span class="cx"> "MaxResourcesBatchUpload" : 100, # Maximum number of resources in a batch POST
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavbasedatastoresubpostgrespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/subpostgres.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/subpostgres.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/subpostgres.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -454,6 +454,10 @@
</span><span class="cx"> self.deactivateDelayedShutdown()
</span><span class="cx">
</span><span class="cx"> def gotReady(result):
</span><ins>+ """
+ We started postgres; we're responsible for stopping it later.
+ Call pgCtl status to get the pid.
+ """
</ins><span class="cx"> log.warn("{cmd} exited", cmd=pgCtl)
</span><span class="cx"> self.shouldStopDatabase = True
</span><span class="cx"> d = Deferred()
</span><span class="lines">@@ -465,13 +469,32 @@
</span><span class="cx"> )
</span><span class="cx"> return d.addCallback(gotStatus)
</span><span class="cx">
</span><del>- def reportit(f):
- log.failure("starting postgres", f)
</del><ins>+ def couldNotStart(f):
+ """
+ There was an error trying to start postgres. Try to connect
+ because it might already be running. In this case, we won't
+ be the one to stop it.
+ """
+ d = Deferred()
+ statusMonitor = CapturingProcessProtocol(d, None)
+ self.reactor.spawnProcess(
+ statusMonitor, pgCtl, [pgCtl, "status"],
+ env=self.env, path=self.workingDir.path,
+ uid=self.uid, gid=self.gid,
+ )
+ return d.addCallback(gotStatus).addErrback(giveUp)
+
+ def giveUp(f):
+ """
+ We can't start postgres or connect to a running instance. Shut
+ down.
+ """
+ log.failure("Can't start or connect to postgres", f)
</ins><span class="cx"> self.deactivateDelayedShutdown()
</span><span class="cx"> self.reactor.stop()
</span><del>-
</del><ins>+
</ins><span class="cx"> self.monitor.completionDeferred.addCallback(
</span><del>- gotReady).addErrback(reportit)
</del><ins>+ gotReady).addErrback(couldNotStart)
</ins><span class="cx">
</span><span class="cx"> shouldStopDatabase = False
</span><span class="cx">
</span><span class="lines">@@ -549,6 +572,7 @@
</span><span class="cx"> # d.addCallback(maybeStopSubprocess)
</span><span class="cx"> # return d
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def hardStop(self):
</span><span class="cx"> """
</span><span class="cx"> Stop postgres quickly by sending it SIGQUIT
</span><span class="lines">@@ -556,5 +580,5 @@
</span><span class="cx"> if self._postgresPid is not None:
</span><span class="cx"> try:
</span><span class="cx"> os.kill(self._postgresPid, signal.SIGQUIT)
</span><del>- except OSError:
</del><ins>+ except OSError:
</ins><span class="cx"> pass
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavbasedatastoretesttest_subpostgrespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/test/test_subpostgres.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/test/test_subpostgres.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/test/test_subpostgres.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -189,5 +189,3 @@
</span><span class="cx"> cursor.execute("select * from import_test_table")
</span><span class="cx"> values = cursor.fetchall()
</span><span class="cx"> self.assertEquals(values, [["value1"], ["value2"]])
</span><del>-
-
</del></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavbasedatastoreutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/util.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/util.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/base/datastore/util.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -92,6 +92,12 @@
</span><span class="cx"> return "objectWithName:%s:%s" % (homeResourceID, name)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ # Home child objects by id
+
+ def keyForObjectWithResourceID(self, homeResourceID, resourceID):
+ return "objectWithName:%s:%s" % (homeResourceID, resourceID)
+
+
</ins><span class="cx"> # Home metadata (Created/Modified)
</span><span class="cx">
</span><span class="cx"> def keyForHomeMetaData(self, homeResourceID):
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastorefilepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/file.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/file.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/file.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -128,7 +128,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def hasCalendarResourceUIDSomewhereElse(self, uid, ok_object, type):
</span><span class="cx">
</span><del>- objectResources = (yield self.objectResourcesWithUID(uid, ("inbox",)))
</del><ins>+ objectResources = (yield self.getCalendarResourcesForUID(uid))
</ins><span class="cx"> for objectResource in objectResources:
</span><span class="cx"> if ok_object and objectResource._path == ok_object._path:
</span><span class="cx"> continue
</span><span class="lines">@@ -140,14 +140,9 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
</del><ins>+ def getCalendarResourcesForUID(self, uid):
</ins><span class="cx">
</span><del>- results = []
- objectResources = (yield self.objectResourcesWithUID(uid, ("inbox",)))
- for objectResource in objectResources:
- if allow_shared or objectResource._parentCollection.owned():
- results.append(objectResource)
-
</del><ins>+ results = (yield self.objectResourcesWithUID(uid, ("inbox",)))
</ins><span class="cx"> returnValue(results)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/schedule.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/schedule.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/schedule.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -112,8 +112,8 @@
</span><span class="cx"> return self._calendarHome.hasCalendarResourceUIDSomewhereElse(uid, ok_object, type)
</span><span class="cx">
</span><span class="cx">
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
- return self._calendarHome.getCalendarResourcesForUID(uid, allow_shared)
</del><ins>+ def getCalendarResourcesForUID(self, uid):
+ return self._calendarHome.getCalendarResourcesForUID(uid)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingimipinboundpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/inbound.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/inbound.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/inbound.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -143,6 +143,7 @@
</span><span class="cx"> yield scheduleNextMailPoll(self.store, seconds)
</span><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def shouldDeleteAllMail(serverHostName, inboundServer, username):
</span><span class="cx"> """
</span><span class="cx"> Given the hostname of the calendar server, the hostname of the pop/imap
</span><span class="lines">@@ -165,6 +166,7 @@
</span><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def scheduleNextMailPoll(store, seconds):
</span><span class="cx"> txn = store.newTransaction()
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingimiptesttest_inboundpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/test/test_inbound.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/test/test_inbound.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/imip/test/test_inbound.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -436,13 +436,13 @@
</span><span class="cx"> self.assertEquals(self.flagDeletedResult, "xyzzy")
</span><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class StubFactory(object):
</span><span class="cx">
</span><span class="cx"> def __init__(self, actionTaken, deleteAllMail):
</span><span class="cx"> self.actionTaken = actionTaken
</span><span class="cx"> self.deleteAllMail = deleteAllMail
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> def handleMessage(self, messageData):
</span><span class="cx"> return succeed(self.actionTaken)
</span><del>-
-
</del></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingimplicitpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/implicit.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/implicit.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/implicit.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -56,10 +56,10 @@
</span><span class="cx"> STATUS_ORPHANED_CANCELLED_EVENT = 1
</span><span class="cx"> STATUS_ORPHANED_EVENT = 2
</span><span class="cx">
</span><del>- def __init__(self):
</del><ins>+ def __init__(self, logItems=None):
</ins><span class="cx">
</span><span class="cx"> self.return_status = ImplicitScheduler.STATUS_OK
</span><del>- self.logItems = {}
</del><ins>+ self.logItems = logItems
</ins><span class="cx"> self.allowed_to_schedule = True
</span><span class="cx"> self.suppress_refresh = False
</span><span class="cx">
</span><span class="lines">@@ -383,7 +383,7 @@
</span><span class="cx"> if self.txn.doing_attendee_refresh == 0:
</span><span class="cx"> delattr(self.txn, "doing_attendee_refresh")
</span><span class="cx">
</span><del>- if refreshCount:
</del><ins>+ if refreshCount and self.logItems is not None:
</ins><span class="cx"> self.logItems["itip.refreshes"] = refreshCount
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -925,7 +925,8 @@
</span><span class="cx"> if self.action in ("create", "modify",):
</span><span class="cx"> total += (yield self.processRequests())
</span><span class="cx">
</span><del>- self.logItems["itip.requests"] = total
</del><ins>+ if self.logItems is not None:
+ self.logItems["itip.requests"] = total
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1304,7 +1305,8 @@
</span><span class="cx"> # First make sure we are allowed to schedule
</span><span class="cx"> self.testSchedulingAllowed()
</span><span class="cx">
</span><del>- self.logItems["itip.reply"] = "reply"
</del><ins>+ if self.logItems is not None:
+ self.logItems["itip.reply"] = "reply"
</ins><span class="cx">
</span><span class="cx"> itipmsg = iTipGenerator.generateAttendeeReply(self.calendar, self.attendee, changedRids=changedRids)
</span><span class="cx">
</span><span class="lines">@@ -1317,7 +1319,8 @@
</span><span class="cx"> # First make sure we are allowed to schedule
</span><span class="cx"> self.testSchedulingAllowed()
</span><span class="cx">
</span><del>- self.logItems["itip.reply"] = "cancel"
</del><ins>+ if self.logItems is not None:
+ self.logItems["itip.reply"] = "cancel"
</ins><span class="cx">
</span><span class="cx"> itipmsg = iTipGenerator.generateAttendeeReply(self.calendar, self.attendee, force_decline=True)
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoreschedulingutilspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/utils.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/utils.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/scheduling/utils.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -21,7 +21,7 @@
</span><span class="cx"> log = Logger()
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>-def getCalendarObjectForRecord(txn, record, uid, allow_shared=False):
</del><ins>+def getCalendarObjectForRecord(txn, record, uid):
</ins><span class="cx"> """
</span><span class="cx"> Get a copy of the event for a calendar user identified by a directory record.
</span><span class="cx">
</span><span class="lines">@@ -34,7 +34,7 @@
</span><span class="cx"> calendar_home = yield txn.calendarHomeWithUID(record.uid)
</span><span class="cx">
</span><span class="cx"> # Get matching newstore objects
</span><del>- objectResources = (yield calendar_home.getCalendarResourcesForUID(uid, allow_shared))
</del><ins>+ objectResources = (yield calendar_home.getCalendarResourcesForUID(uid))
</ins><span class="cx">
</span><span class="cx"> if len(objectResources) > 1:
</span><span class="cx"> # Delete all but the first one
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/sql.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/sql.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/sql.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -527,9 +527,7 @@
</span><span class="cx"> # refer to calendar *object* UIDs, since calendar *resources* are an
</span><span class="cx"> # HTTP protocol layer thing, not a data store thing. (See also
</span><span class="cx"> # objectResourcesWithUID.)
</span><del>- objectResources = (
- yield self.objectResourcesWithUID(uid, ["inbox"], False)
- )
</del><ins>+ objectResources = (yield self.getCalendarResourcesForUID(uid))
</ins><span class="cx"> for objectResource in objectResources:
</span><span class="cx"> if ok_object and objectResource._resourceID == ok_object._resourceID:
</span><span class="cx"> continue
</span><span class="lines">@@ -541,15 +539,22 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def getCalendarResourcesForUID(self, uid, allow_shared=False):
</del><ins>+ def getCalendarResourcesForUID(self, uid):
+ """
+ Find all calendar object resources in the calendar home that are not in the "inbox" collection
+ and not in shared collections.
+ Cache the result of this query as it can happen multiple times during scheduling under slightly
+ different circumstances.
</ins><span class="cx">
</span><del>- results = []
- objectResources = (yield self.objectResourcesWithUID(uid, ["inbox"]))
- for objectResource in objectResources:
- if allow_shared or objectResource._parentCollection.owned():
- results.append(objectResource)
</del><ins>+ @param uid: the UID of the calendar object resources to find
+ @type uid: C{str}
+ """
</ins><span class="cx">
</span><del>- returnValue(results)
</del><ins>+ if not hasattr(self, "_cachedCalendarResourcesForUID"):
+ self._cachedCalendarResourcesForUID = {}
+ if uid not in self._cachedCalendarResourcesForUID:
+ self._cachedCalendarResourcesForUID[uid] = (yield self.objectResourcesWithUID(uid, ["inbox"], allowShared=False))
+ returnValue(self._cachedCalendarResourcesForUID[uid])
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1576,10 +1581,6 @@
</span><span class="cx"> if calsize > config.MaxResourceSize:
</span><span class="cx"> raise ObjectResourceTooBigError()
</span><span class="cx">
</span><del>- # Possible timezone stripping
- if config.EnableTimezonesByReference:
- component.stripKnownTimezones()
-
</del><span class="cx"> # Do validation on external requests
</span><span class="cx"> if internal_state == ComponentUpdateState.NORMAL:
</span><span class="cx">
</span><span class="lines">@@ -1597,6 +1598,10 @@
</span><span class="cx"> # calendar data
</span><span class="cx"> component.normalizeCalendarUserAddresses(normalizationLookup, self.directoryService().recordWithCalendarUserAddress)
</span><span class="cx">
</span><ins>+ # Possible timezone stripping
+ if config.EnableTimezonesByReference:
+ component.stripKnownTimezones()
+
</ins><span class="cx"> # Check location/resource organizer requirement
</span><span class="cx"> self.validLocationResourceOrganizer(component, inserting, internal_state)
</span><span class="cx">
</span><span class="lines">@@ -1953,7 +1958,7 @@
</span><span class="cx"> user_uuid = self._parentCollection.viewerHome().uid()
</span><span class="cx"> component = PerUserDataFilter(user_uuid).filter(component.duplicate())
</span><span class="cx">
</span><del>- scheduler = ImplicitScheduler()
</del><ins>+ scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx">
</span><span class="cx"> # PUT
</span><span class="cx"> do_implicit_action, is_scheduling_resource = (yield scheduler.testImplicitSchedulingPUT(
</span><span class="lines">@@ -2610,7 +2615,7 @@
</span><span class="cx"> if not isinbox and internal_state == ComponentRemoveState.NORMAL:
</span><span class="cx"> # Get data we need for implicit scheduling
</span><span class="cx"> calendar = (yield self.componentForUser())
</span><del>- scheduler = ImplicitScheduler()
</del><ins>+ scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx"> do_implicit_action, _ignore = (yield scheduler.testImplicitSchedulingDELETE(
</span><span class="cx"> self.calendar(),
</span><span class="cx"> self,
</span><span class="lines">@@ -2929,7 +2934,7 @@
</span><span class="cx">
</span><span class="cx"> # Only allow organizers to manipulate managed attachments for now
</span><span class="cx"> calendar = (yield self.componentForUser())
</span><del>- scheduler = ImplicitScheduler()
</del><ins>+ scheduler = ImplicitScheduler(logItems=self._txn.logItems)
</ins><span class="cx"> is_attendee = (yield scheduler.testAttendeeEvent(self.calendar(), self, calendar,))
</span><span class="cx"> if is_attendee:
</span><span class="cx"> raise InvalidAttachmentOperation("Attendees are not allowed to manipulate managed attachments")
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoretestcommonpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/common.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/common.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/common.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -84,73 +84,75 @@
</span><span class="cx">
</span><span class="cx"> OTHER_HOME_UID = "home_splits"
</span><span class="cx">
</span><del>-test_event_text = (
- "BEGIN:VCALENDAR\r\n"
- "VERSION:2.0\r\n"
- "PRODID:-//Apple Inc.//iCal 4.0.1//EN\r\n"
- "CALSCALE:GREGORIAN\r\n"
- "BEGIN:VTIMEZONE\r\n"
- "TZID:US/Pacific\r\n"
- "BEGIN:DAYLIGHT\r\n"
- "TZOFFSETFROM:-0800\r\n"
- "RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU\r\n"
- "DTSTART:20070311T020000\r\n"
- "TZNAME:PDT\r\n"
- "TZOFFSETTO:-0700\r\n"
- "END:DAYLIGHT\r\n"
- "BEGIN:STANDARD\r\n"
- "TZOFFSETFROM:-0700\r\n"
- "RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU\r\n"
- "DTSTART:20071104T020000\r\n"
- "TZNAME:PST\r\n"
- "TZOFFSETTO:-0800\r\n"
- "END:STANDARD\r\n"
- "END:VTIMEZONE\r\n"
- "BEGIN:VEVENT\r\n"
- "CREATED:20100203T013849Z\r\n"
- "UID:uid-test\r\n"
- "DTEND;TZID=US/Pacific:20100207T173000\r\n"
- "TRANSP:OPAQUE\r\n"
- "SUMMARY:New Event\r\n"
- "DTSTART;TZID=US/Pacific:20100207T170000\r\n"
- "DTSTAMP:20100203T013909Z\r\n"
- "SEQUENCE:3\r\n"
- "X-APPLE-DROPBOX:/calendars/users/wsanchez/dropbox/uid-test.dropbox\r\n"
- "BEGIN:VALARM\r\n"
- "X-WR-ALARMUID:1377CCC7-F85C-4610-8583-9513D4B364E1\r\n"
- "TRIGGER:-PT20M\r\n"
- "ATTACH:Basso\r\n"
- "ACTION:AUDIO\r\n"
- "END:VALARM\r\n"
- "END:VEVENT\r\n"
- "END:VCALENDAR\r\n"
-)
</del><ins>+test_event_text = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VTIMEZONE
+TZID:US/Pacific
+BEGIN:DAYLIGHT
+TZOFFSETFROM:-0800
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+DTSTART:20070311T020000
+TZNAME:PDT
+TZOFFSETTO:-0700
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:-0700
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+DTSTART:20071104T020000
+TZNAME:PST
+TZOFFSETTO:-0800
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20100203T013849Z
+UID:uid-test
+DTEND;TZID=US/Pacific:20100207T173000
+TRANSP:OPAQUE
+SUMMARY:New Event
+DTSTART;TZID=US/Pacific:20100207T170000
+DTSTAMP:20100203T013909Z
+SEQUENCE:3
+X-APPLE-DROPBOX:/calendars/users/wsanchez/dropbox/uid-test.dropbox
+BEGIN:VALARM
+X-WR-ALARMUID:1377CCC7-F85C-4610-8583-9513D4B364E1
+TRIGGER:-PT20M
+ATTACH:Basso
+ACTION:AUDIO
+END:VALARM
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-test_event_notCalDAV_text = (
- "BEGIN:VCALENDAR\r\n"
- "VERSION:2.0\r\n"
- "PRODID:-//Apple Inc.//iCal 4.0.1//EN\r\n"
- "CALSCALE:GREGORIAN\r\n"
- "BEGIN:VEVENT\r\n"
- "CREATED:20100203T013849Z\r\n"
- "UID:test\r\n"
- "DTEND;TZID=US/Pacific:20100207T173000\r\n" # TZID without VTIMEZONE
- "TRANSP:OPAQUE\r\n"
- "SUMMARY:New Event\r\n"
- "DTSTART;TZID=US/Pacific:20100207T170000\r\n"
- "DTSTAMP:20100203T013909Z\r\n"
- "SEQUENCE:3\r\n"
- "BEGIN:VALARM\r\n"
- "X-WR-ALARMUID:1377CCC7-F85C-4610-8583-9513D4B364E1\r\n"
- "TRIGGER:-PT20M\r\n"
- "ATTACH:Basso\r\n"
- "ACTION:AUDIO\r\n"
- "END:VALARM\r\n"
- "END:VEVENT\r\n"
- "END:VCALENDAR\r\n"
-)
</del><ins>+test_event_notCalDAV_text = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Apple Inc.//iCal 4.0.1//EN
+CALSCALE:GREGORIAN
+BEGIN:VEVENT
+CREATED:20100203T013849Z
+UID:test-bad1
+DTEND:20100207T173000Z
+TRANSP:OPAQUE
+SUMMARY:New Event
+DTSTART:20100207T170000Z
+DTSTAMP:20100203T013909Z
+SEQUENCE:3
+END:VEVENT
+BEGIN:VEVENT
+CREATED:20100203T013849Z
+UID:test-bad2
+DTEND:20100207T173000Z
+TRANSP:OPAQUE
+SUMMARY:New Event
+DTSTART:20100207T170000Z
+DTSTAMP:20100203T013909Z
+SEQUENCE:3
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -450,9 +452,7 @@
</span><span class="cx"> yield notifications.writeNotificationObject("abc", inviteNotification,
</span><span class="cx"> inviteNotification.toxml())
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notification fired after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -460,6 +460,7 @@
</span><span class="cx"> "/CalDAV/example.com/home1/notification/",
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx"> notifications = yield self.transactionUnderTest().notificationsWithUID(
</span><span class="cx"> "home1"
</span><span class="lines">@@ -469,9 +470,7 @@
</span><span class="cx"> abc = yield notifications.notificationObjectWithUID("abc")
</span><span class="cx"> self.assertEquals(abc, None)
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notification fired after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -479,6 +478,7 @@
</span><span class="cx"> "/CalDAV/example.com/home1/notification/",
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -697,11 +697,10 @@
</span><span class="cx"> self.assertNotIdentical((yield home.calendarWithName(name)), None)
</span><span class="cx"> calendarProperties = (yield home.calendarWithName(name)).properties()
</span><span class="cx"> self.assertEqual(len(calendarProperties), 0)
</span><ins>+ # notify is called prior to commit
+ self.assertTrue("/CalDAV/example.com/home1/" in self.notifierFactory.history)
</ins><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Make sure notification fired after commit
- self.assertTrue("/CalDAV/example.com/home1/" in self.notifierFactory.history)
-
</del><span class="cx"> # Make sure it's available in a new transaction; i.e. test the commit.
</span><span class="cx"> home = yield self.homeUnderTest()
</span><span class="cx"> self.assertNotIdentical((yield home.calendarWithName(name)), None)
</span><span class="lines">@@ -915,8 +914,7 @@
</span><span class="cx"> None
</span><span class="cx"> )
</span><span class="cx">
</span><del>- # Make sure notifications are fired after commit
- yield self.commit()
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -924,6 +922,7 @@
</span><span class="cx"> "/CalDAV/example.com/home1/calendar_1/",
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1471,9 +1470,7 @@
</span><span class="cx"> self.assertEquals((yield calendarObject.componentForUser()), component)
</span><span class="cx"> self.assertEquals((yield calendarObject.getMetadata()), metadata)
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notifications fire after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -1481,6 +1478,7 @@
</span><span class="cx"> "/CalDAV/example.com/home1/calendar_1/",
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -1591,9 +1589,7 @@
</span><span class="cx"> calendarObject = yield calendar1.calendarObjectWithName("1.ics")
</span><span class="cx"> self.assertEquals((yield calendarObject.componentForUser()), component)
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notification fired after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -1601,6 +1597,7 @@
</span><span class="cx"> "/CalDAV/example.com/home1/calendar_1/",
</span><span class="cx"> ])
</span><span class="cx"> )
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def checkPropertiesMethod(self, thunk):
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcaldavdatastoretesttest_utilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/test_util.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/test_util.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/caldav/datastore/test/test_util.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -525,16 +525,25 @@
</span><span class="cx"> "different-name": self.sampleEvent("other-uid", "tgt other"),
</span><span class="cx"> },
</span><span class="cx"> )
</span><ins>+
</ins><span class="cx"> txn = self.transactionUnderTest()
</span><del>- c1 = yield txn.calendarHomeWithUID("conflict1")
</del><span class="cx"> c2 = yield txn.calendarHomeWithUID("conflict2")
</span><span class="cx"> otherCal = yield c2.createCalendarWithName("othercal")
</span><del>- otherCal.createCalendarObjectWithName(
</del><ins>+ yield otherCal.createCalendarObjectWithName(
</ins><span class="cx"> "some-name", Component.fromString(
</span><span class="cx"> self.sampleEvent("oc", "target calendar")[0]
</span><span class="cx"> )
</span><span class="cx"> )
</span><ins>+ yield self.commit()
+
+ txn = self.transactionUnderTest()
+ c1 = yield txn.calendarHomeWithUID("conflict1")
+ c2 = yield txn.calendarHomeWithUID("conflict2")
</ins><span class="cx"> yield migrateHome(c1, c2, merge=True)
</span><ins>+ yield self.commit()
+
+ txn = self.transactionUnderTest()
+ c2 = yield txn.calendarHomeWithUID("conflict2")
</ins><span class="cx"> targetCal = yield c2.calendarWithName("conflicted")
</span><span class="cx"> yield self.checkSummary("same-name", "target", targetCal)
</span><span class="cx"> yield self.checkSummary("different-name", "tgt other", targetCal)
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcarddavdatastoretestcommonpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/carddav/datastore/test/common.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/carddav/datastore/test/common.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/carddav/datastore/test/common.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -371,11 +371,10 @@
</span><span class="cx"> #self.assertIdentical((yield home.addressbookWithName(name)), None)
</span><span class="cx"> yield home.removeAddressBookWithName(name)
</span><span class="cx"> self.assertNotIdentical((yield home.addressbookWithName(name)), None)
</span><ins>+ # notify is called prior to commit
+ self.assertTrue("/CardDAV/example.com/home1/" in self.notifierFactory.history)
</ins><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Make sure notification fired after commit
- self.assertTrue("/CardDAV/example.com/home1/" in self.notifierFactory.history)
-
</del><span class="cx"> # Make sure it's available in a new transaction; i.e. test the commit.
</span><span class="cx"> home = yield self.homeUnderTest()
</span><span class="cx"> self.assertNotIdentical((yield home.addressbookWithName(name)), None)
</span><span class="lines">@@ -396,9 +395,7 @@
</span><span class="cx"> ab = yield home.addressbookWithName(name)
</span><span class="cx"> self.assertEquals((yield ab.listAddressBookObjects()), [])
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notification fired after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -407,7 +404,9 @@
</span><span class="cx"> ])
</span><span class="cx"> )
</span><span class="cx">
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def test_removeAddressBookWithName_absent(self):
</span><span class="cx"> """
</span><span class="lines">@@ -530,8 +529,6 @@
</span><span class="cx"> (yield addressbook.addressbookObjectWithName(name)), None
</span><span class="cx"> )
</span><span class="cx">
</span><del>- # Make sure notifications are fired after commit
- yield self.commit()
</del><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -692,9 +689,7 @@
</span><span class="cx"> addressbookObject = yield addressbook1.addressbookObjectWithName(name)
</span><span class="cx"> self.assertEquals((yield addressbookObject.component()), component)
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notifications fire after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -703,7 +698,9 @@
</span><span class="cx"> ])
</span><span class="cx"> )
</span><span class="cx">
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def test_createAddressBookObjectWithName_exists(self):
</span><span class="cx"> """
</span><span class="lines">@@ -808,9 +805,7 @@
</span><span class="cx"> addressbookObject = yield addressbook1.addressbookObjectWithName("1.vcf")
</span><span class="cx"> self.assertEquals((yield addressbookObject.component()), component)
</span><span class="cx">
</span><del>- yield self.commit()
-
- # Make sure notification fired after commit
</del><ins>+ # notify is called prior to commit
</ins><span class="cx"> self.assertEquals(
</span><span class="cx"> set(self.notifierFactory.history),
</span><span class="cx"> set([
</span><span class="lines">@@ -819,7 +814,9 @@
</span><span class="cx"> ])
</span><span class="cx"> )
</span><span class="cx">
</span><ins>+ yield self.commit()
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def checkPropertiesMethod(self, thunk):
</span><span class="cx"> """
</span><span class="cx"> Verify that the given object has a properties method that returns an
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastorefilepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/file.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/file.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/file.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -926,6 +926,7 @@
</span><span class="cx"> return (self._notifierPrefix, self.uid(),)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def notifyChanged(self):
</span><span class="cx"> """
</span><span class="cx"> Trigger a notification of a change
</span><span class="lines">@@ -933,8 +934,14 @@
</span><span class="cx">
</span><span class="cx"> # Only send one set of change notifications per transaction
</span><span class="cx"> if self._notifiers and not self._transaction.isNotifiedAlready(self):
</span><del>- for notifier in self._notifiers.values():
</del><ins>+ # cache notifiers run in post commit
+ notifier = self._notifiers.get("cache", None)
+ if notifier:
</ins><span class="cx"> self._transaction.postCommit(notifier.notify)
</span><ins>+ # push notifiers add their work items immediately
+ notifier = self._notifiers.get("push", None)
+ if notifier:
+ yield notifier.notify(self._transaction)
</ins><span class="cx"> self._transaction.notificationAddedForObject(self)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -1272,6 +1279,7 @@
</span><span class="cx"> return self.ownerHome().notifierID()
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @inlineCallbacks
</ins><span class="cx"> def notifyChanged(self):
</span><span class="cx"> """
</span><span class="cx"> Trigger a notification of a change
</span><span class="lines">@@ -1279,8 +1287,14 @@
</span><span class="cx">
</span><span class="cx"> # Only send one set of change notifications per transaction
</span><span class="cx"> if self._notifiers and not self._transaction.isNotifiedAlready(self):
</span><del>- for notifier in self._notifiers.values():
</del><ins>+ # cache notifiers run in post commit
+ notifier = self._notifiers.get("cache", None)
+ if notifier:
</ins><span class="cx"> self._transaction.postCommit(notifier.notify)
</span><ins>+ # push notifiers add their work items immediately
+ notifier = self._notifiers.get("push", None)
+ if notifier:
+ yield notifier.notify(self._transaction)
</ins><span class="cx"> self._transaction.notificationAddedForObject(self)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -29,9 +29,10 @@
</span><span class="cx">
</span><span class="cx"> from pycalendar.datetime import PyCalendarDateTime
</span><span class="cx">
</span><del>-from twext.enterprise.dal.syntax import \
- Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction, \
- Select, Update, ColumnSyntax, TableSyntax, Upper, Count, ALL_COLUMNS, Sum
</del><ins>+from twext.enterprise.dal.syntax import (
+ Delete, utcNowSQL, Union, Insert, Len, Max, Parameter, SavepointAction,
+ Select, Update, ColumnSyntax, TableSyntax, Upper, Count, ALL_COLUMNS, Sum,
+ DatabaseLock, DatabaseUnlock)
</ins><span class="cx"> from twext.enterprise.ienterprise import AlreadyFinishedError
</span><span class="cx"> from twext.enterprise.queue import LocalQueuer
</span><span class="cx"> from twext.enterprise.util import parseSQLTimestamp
</span><span class="lines">@@ -315,6 +316,7 @@
</span><span class="cx"> self.label = label
</span><span class="cx"> self.logFileName = logFileName
</span><span class="cx"> self.statements = []
</span><ins>+ self.startTime = time.time()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def startStatement(self, sql, args):
</span><span class="lines">@@ -330,7 +332,7 @@
</span><span class="cx"> """
</span><span class="cx"> args = ["%s" % (arg,) for arg in args]
</span><span class="cx"> args = [((arg[:10] + "...") if len(arg) > 40 else arg) for arg in args]
</span><del>- self.statements.append(["%s %s" % (sql, args,), 0, 0])
</del><ins>+ self.statements.append(["%s %s" % (sql, args,), 0, 0, 0])
</ins><span class="cx"> return len(self.statements) - 1, time.time()
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -344,8 +346,10 @@
</span><span class="cx"> @type rows: C{int}
</span><span class="cx"> """
</span><span class="cx"> index, tstamp = context
</span><ins>+ t = time.time()
</ins><span class="cx"> self.statements[index][1] = len(rows) if rows else 0
</span><del>- self.statements[index][2] = time.time() - tstamp
</del><ins>+ self.statements[index][2] = t - tstamp
+ self.statements[index][3] = t
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def printReport(self):
</span><span class="lines">@@ -353,19 +357,28 @@
</span><span class="cx"> Print a report of all the SQL statements executed to date.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ total_statements = len(self.statements)
+ total_rows = sum([statement[1] for statement in self.statements])
+ total_time = sum([statement[2] for statement in self.statements]) * 1000.0
+
</ins><span class="cx"> toFile = StringIO()
</span><span class="cx"> toFile.write("*** SQL Stats ***\n")
</span><span class="cx"> toFile.write("\n")
</span><span class="cx"> toFile.write("Label: %s\n" % (self.label,))
</span><span class="cx"> toFile.write("Unique statements: %d\n" % (len(set([statement[0] for statement in self.statements]),),))
</span><del>- toFile.write("Total statements: %d\n" % (len(self.statements),))
- toFile.write("Total rows: %d\n" % (sum([statement[1] for statement in self.statements]),))
- toFile.write("Total time (ms): %.3f\n" % (sum([statement[2] for statement in self.statements]) * 1000.0,))
- for sql, rows, t in self.statements:
</del><ins>+ toFile.write("Total statements: %d\n" % (total_statements,))
+ toFile.write("Total rows: %d\n" % (total_rows,))
+ toFile.write("Total time (ms): %.3f\n" % (total_time,))
+ t_last_end = self.startTime
+ for sql, rows, t_taken, t_end in self.statements:
</ins><span class="cx"> toFile.write("\n")
</span><span class="cx"> toFile.write("SQL: %s\n" % (sql,))
</span><span class="cx"> toFile.write("Rows: %s\n" % (rows,))
</span><del>- toFile.write("Time (ms): %.3f\n" % (t * 1000.0,))
</del><ins>+ toFile.write("Time (ms): %.3f\n" % (t_taken * 1000.0,))
+ toFile.write("Idle (ms): %.3f\n" % ((t_end - t_taken - t_last_end) * 1000.0,))
+ toFile.write("Elapsed (ms): %.3f\n" % ((t_end - self.startTime) * 1000.0,))
+ t_last_end = t_end
+ toFile.write("Commit (ms): %.3f\n" % ((time.time() - t_last_end) * 1000.0,))
</ins><span class="cx"> toFile.write("***\n\n")
</span><span class="cx">
</span><span class="cx"> if self.logFileName:
</span><span class="lines">@@ -373,8 +386,10 @@
</span><span class="cx"> else:
</span><span class="cx"> log.error(toFile.getvalue())
</span><span class="cx">
</span><ins>+ return (total_statements, total_rows, total_time,)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class CommonStoreTransactionMonitor(object):
</span><span class="cx"> """
</span><span class="cx"> Object that monitors the state of a transaction over time and logs or times out
</span><span class="lines">@@ -484,7 +499,9 @@
</span><span class="cx"> self.iudCount = 0
</span><span class="cx"> self.currentStatement = None
</span><span class="cx">
</span><ins>+ self.logItems = {}
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> def enqueue(self, workItem, **kw):
</span><span class="cx"> """
</span><span class="cx"> Enqueue a L{twext.enterprise.queue.WorkItem} for later execution.
</span><span class="lines">@@ -1033,7 +1050,7 @@
</span><span class="cx">
</span><span class="cx"> # Do stats logging as a postCommit because there might be some pending preCommit SQL we want to log
</span><span class="cx"> if self._stats:
</span><del>- self.postCommit(self._stats.printReport)
</del><ins>+ self.postCommit(self.statsReport)
</ins><span class="cx"> return self._sqlTxn.commit()
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -1044,6 +1061,16 @@
</span><span class="cx"> return self._sqlTxn.abort()
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def statsReport(self):
+ """
+ Print the stats report and record log items
+ """
+ sql_statements, sql_rows, sql_time = self._stats.printReport()
+ self.logItems["sql-s"] = str(sql_statements)
+ self.logItems["sql-r"] = str(sql_rows)
+ self.logItems["sql-t"] = "%.1f" % (sql_time,)
+
+
</ins><span class="cx"> def _oldEventsBase(self, limit):
</span><span class="cx"> ch = schema.CALENDAR_HOME
</span><span class="cx"> co = schema.CALENDAR_OBJECT
</span><span class="lines">@@ -1376,11 +1403,11 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def acquireUpgradeLock(self):
</span><del>- return self.execSQL("select pg_advisory_lock(1)")
</del><ins>+ return DatabaseLock().on(self)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def releaseUpgradeLock(self):
</span><del>- return self.execSQL("select pg_advisory_unlock(1)")
</del><ins>+ return DatabaseUnlock().on(self)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -1418,6 +1445,7 @@
</span><span class="cx"> self._txn = transaction
</span><span class="cx"> self._ownerUID = ownerUID
</span><span class="cx"> self._resourceID = None
</span><ins>+ self._dataVersion = None
</ins><span class="cx"> self._childrenLoaded = False
</span><span class="cx"> self._children = {}
</span><span class="cx"> self._notifiers = None
</span><span class="lines">@@ -1663,6 +1691,23 @@
</span><span class="cx"> yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</span><span class="cx">
</span><span class="cx">
</span><ins>+ @classproperty
+ def _dataVersionQuery(cls): #@NoSelf
+ ch = cls._homeSchema
+ return Select(
+ [ch.DATAVERSION], From=ch,
+ Where=ch.RESOURCE_ID == Parameter("resourceID")
+ )
+
+
+ @inlineCallbacks
+ def dataVersion(self):
+ if self._dataVersion is None:
+ self._dataVersion = (yield self._dataVersionQuery.on(
+ self._txn, resourceID=self._resourceID))[0][0]
+ returnValue(self._dataVersion)
+
+
</ins><span class="cx"> def name(self):
</span><span class="cx"> """
</span><span class="cx"> Implement L{IDataStoreObject.name} to return the uid.
</span><span class="lines">@@ -2164,6 +2209,7 @@
</span><span class="cx"> the resource has changed. We ensure we only do this once per object
</span><span class="cx"> per transaction.
</span><span class="cx"> """
</span><ins>+
</ins><span class="cx"> if self._txn.isNotifiedAlready(self):
</span><span class="cx"> returnValue(None)
</span><span class="cx"> self._txn.notificationAddedForObject(self)
</span><span class="lines">@@ -2174,8 +2220,14 @@
</span><span class="cx">
</span><span class="cx"> # Send notifications
</span><span class="cx"> if self._notifiers:
</span><del>- for notifier in self._notifiers.values():
</del><ins>+ # cache notifiers run in post commit
+ notifier = self._notifiers.get("cache", None)
+ if notifier:
</ins><span class="cx"> self._txn.postCommit(notifier.notify)
</span><ins>+ # push notifiers add their work items immediately
+ notifier = self._notifiers.get("push", None)
+ if notifier:
+ yield notifier.notify(self._txn)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classproperty
</span><span class="lines">@@ -2289,16 +2341,20 @@
</span><span class="cx"> raise NotImplementedError()
</span><span class="cx">
</span><span class="cx">
</span><del>- @classproperty
- def _objectNamesSinceRevisionQuery(cls): #@NoSelf
</del><ins>+ @classmethod
+ def _objectNamesSinceRevisionQuery(cls, deleted=True): #@NoSelf
</ins><span class="cx"> """
</span><span class="cx"> DAL query for (resource, deleted-flag)
</span><span class="cx"> """
</span><span class="cx"> rev = cls._revisionsSchema
</span><del>- return Select([rev.RESOURCE_NAME, rev.DELETED],
- From=rev,
- Where=(rev.REVISION > Parameter("revision")).And(
- rev.RESOURCE_ID == Parameter("resourceID")))
</del><ins>+ where = (rev.REVISION > Parameter("revision")).And(rev.RESOURCE_ID == Parameter("resourceID"))
+ if not deleted:
+ where = where.And(rev.DELETED == False)
+ return Select(
+ [rev.RESOURCE_NAME, rev.DELETED],
+ From=rev,
+ Where=where,
+ )
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def resourceNamesSinceToken(self, token):
</span><span class="lines">@@ -2323,10 +2379,10 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> results = [
</span><del>- (name if name else "", deleted)
- for name, deleted in
- (yield self._objectNamesSinceRevisionQuery.on(
- self._txn, revision=revision, resourceID=self._resourceID))
</del><ins>+ (name if name else "", deleted) for name, deleted in
+ (yield self._objectNamesSinceRevisionQuery(deleted=(revision != 0)).on(
+ self._txn, revision=revision, resourceID=self._resourceID)
+ )
</ins><span class="cx"> ]
</span><span class="cx"> results.sort(key=lambda x: x[1])
</span><span class="cx">
</span><span class="lines">@@ -2989,7 +3045,9 @@
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(shareeView._home._resourceID, shareeView._name)
</span><del>- queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+ cacheKey = queryCacher.keyForObjectWithResourceID(shareeView._home._resourceID, shareeView._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</ins><span class="cx">
</span><span class="cx"> shareeView._name = sharedname[0][0]
</span><span class="cx">
</span><span class="lines">@@ -3047,7 +3105,9 @@
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(shareeHome._resourceID, shareeChild._name)
</span><del>- queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
+ cacheKey = queryCacher.keyForObjectWithResourceID(shareeHome._resourceID, shareeChild._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</ins><span class="cx"> else:
</span><span class="cx"> deletedBindName = None
</span><span class="cx">
</span><span class="lines">@@ -3313,10 +3373,9 @@
</span><span class="cx"> def invalidateQueryCache(self):
</span><span class="cx"> queryCacher = self._txn._queryCacher
</span><span class="cx"> if queryCacher is not None:
</span><del>- cacheKey = queryCacher.keyForHomeChildMetaData(self._resourceID)
- yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
- cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name)
- yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
</del><ins>+ yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForHomeChildMetaData(self._resourceID))
+ yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForObjectWithName(self._home._resourceID, self._name))
+ yield queryCacher.invalidateAfterCommit(self._txn, queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -3493,6 +3552,7 @@
</span><span class="cx"> if rows and queryCacher:
</span><span class="cx"> # Cache the result
</span><span class="cx"> queryCacher.setAfterCommit(home._txn, cacheKey, rows)
</span><ins>+ queryCacher.setAfterCommit(home._txn, queryCacher.keyForObjectWithResourceID(home._resourceID, rows[0][2]), rows)
</ins><span class="cx">
</span><span class="cx"> if not rows:
</span><span class="cx"> returnValue(None)
</span><span class="lines">@@ -3533,8 +3593,24 @@
</span><span class="cx"> @return: an L{CommonHomeChild} or C{None} if no such child
</span><span class="cx"> exists.
</span><span class="cx"> """
</span><del>- rows = yield cls._bindForResourceIDAndHomeID.on(
- home._txn, resourceID=resourceID, homeID=home._resourceID)
</del><ins>+
+ rows = None
+ queryCacher = home._txn._queryCacher
+
+ if queryCacher:
+ # Retrieve data from cache
+ cacheKey = queryCacher.keyForObjectWithResourceID(home._resourceID, resourceID)
+ rows = yield queryCacher.get(cacheKey)
+
+ if rows is None:
+ # No cached copy
+ rows = yield cls._bindForResourceIDAndHomeID.on(home._txn, resourceID=resourceID, homeID=home._resourceID)
+
+ if rows and queryCacher:
+ # Cache the result (under both the ID and name values)
+ queryCacher.setAfterCommit(home._txn, cacheKey, rows)
+ queryCacher.setAfterCommit(home._txn, queryCacher.keyForObjectWithName(home._resourceID, rows[0][3]), rows)
+
</ins><span class="cx"> if not rows:
</span><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="lines">@@ -3715,6 +3791,8 @@
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, oldName)
</span><span class="cx"> yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</span><ins>+ cacheKey = queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</ins><span class="cx">
</span><span class="cx"> yield self._renameQuery.on(self._txn, name=name,
</span><span class="cx"> resourceID=self._resourceID,
</span><span class="lines">@@ -3748,6 +3826,8 @@
</span><span class="cx"> if queryCacher:
</span><span class="cx"> cacheKey = queryCacher.keyForObjectWithName(self._home._resourceID, self._name)
</span><span class="cx"> yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</span><ins>+ cacheKey = queryCacher.keyForObjectWithResourceID(self._home._resourceID, self._resourceID)
+ yield queryCacher.invalidateAfterCommit(self._home._txn, cacheKey)
</ins><span class="cx">
</span><span class="cx"> yield self._deletedSyncToken()
</span><span class="cx"> yield self._deleteQuery.on(self._txn, NoSuchHomeChildError,
</span><span class="lines">@@ -4299,8 +4379,14 @@
</span><span class="cx">
</span><span class="cx"> # Send notifications
</span><span class="cx"> if self._notifiers:
</span><del>- for notifier in self._notifiers.values():
</del><ins>+ # cache notifiers run in post commit
+ notifier = self._notifiers.get("cache", None)
+ if notifier:
</ins><span class="cx"> self._txn.postCommit(notifier.notify)
</span><ins>+ # push notifiers add their work items immediately
+ notifier = self._notifiers.get("push", None)
+ if notifier:
+ yield notifier.notify(self._txn)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classproperty
</span><span class="lines">@@ -4523,7 +4609,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def create(cls, parent, name, component, options=None):
</span><span class="cx">
</span><del>- child = (yield cls.objectWithName(parent, name, None))
</del><ins>+ child = (yield parent.objectResourceWithName(name))
</ins><span class="cx"> if child:
</span><span class="cx"> raise ObjectResourceNameAlreadyExistsError(name)
</span><span class="cx">
</span><span class="lines">@@ -5120,15 +5206,21 @@
</span><span class="cx"> the resource has changed. We ensure we only do this once per object
</span><span class="cx"> per transaction.
</span><span class="cx"> """
</span><del>- yield
</del><span class="cx"> if self._txn.isNotifiedAlready(self):
</span><span class="cx"> returnValue(None)
</span><span class="cx"> self._txn.notificationAddedForObject(self)
</span><span class="cx">
</span><span class="cx"> # Send notifications
</span><span class="cx"> if self._notifiers:
</span><del>- for notifier in self._notifiers.values():
</del><ins>+ # cache notifiers run in post commit
+ notifier = self._notifiers.get("cache", None)
+ if notifier:
</ins><span class="cx"> self._txn.postCommit(notifier.notify)
</span><ins>+ # push notifiers add their work items immediately
+ notifier = self._notifiers.get("push", None)
+ if notifier:
+ yield notifier.notify(self._txn)
+
</ins><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_19_to_20sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_19_to_20.sql (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_19_to_20.sql 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_19_to_20.sql 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -113,8 +113,12 @@
</span><span class="cx"> -- Alter ADDRESSBOOK_HOME --
</span><span class="cx"> -----------------------------
</span><span class="cx">
</span><ins>+-- This is tricky as we have to create a new not null column and populate it, but we can't do
+-- not null immediately without a default - which we do not want. So we create the column without not null,
+-- do the updates, then add the constraint.
+
</ins><span class="cx"> alter table ADDRESSBOOK_HOME
</span><del>- add ("ADDRESSBOOK_PROPERTY_STORE_ID" integer not null);
</del><ins>+ add ("ADDRESSBOOK_PROPERTY_STORE_ID" integer);
</ins><span class="cx">
</span><span class="cx"> update ADDRESSBOOK_HOME
</span><span class="cx"> set ADDRESSBOOK_PROPERTY_STORE_ID = (
</span><span class="lines">@@ -133,8 +137,11 @@
</span><span class="cx"> ADDRESSBOOK_BIND.BIND_MODE = 0 and -- CALENDAR_BIND_MODE 'own'
</span><span class="cx"> ADDRESSBOOK_BIND.ADDRESSBOOK_RESOURCE_NAME = 'addressbook'
</span><span class="cx"> );
</span><del>-
</del><span class="cx">
</span><ins>+alter table ADDRESSBOOK_HOME
+ modify ("ADDRESSBOOK_PROPERTY_STORE_ID" not null);
+
+
</ins><span class="cx"> --------------------------------
</span><span class="cx"> -- change ADDRESSBOOK_OBJECT --
</span><span class="cx"> --------------------------------
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_13_to_14sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_13_to_14.sql (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_13_to_14.sql 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_13_to_14.sql 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -26,6 +26,11 @@
</span><span class="cx"> drop column SEEN_BY_OWNER;
</span><span class="cx"> alter table CALENDAR_BIND
</span><span class="cx"> drop column SEEN_BY_SHAREE;
</span><ins>+
+-- Don't allow nulls in the column we are about to constrain
+update CALENDAR_BIND
+ set CALENDAR_RESOURCE_NAME = 'Shared_' || CALENDAR_RESOURCE_ID || '_' || CALENDAR_HOME_RESOURCE_ID
+ where CALENDAR_RESOURCE_NAME is null;
</ins><span class="cx"> alter table CALENDAR_BIND
</span><span class="cx"> alter column CALENDAR_RESOURCE_NAME
</span><span class="cx"> set not null;
</span><span class="lines">@@ -34,6 +39,11 @@
</span><span class="cx"> drop column SEEN_BY_OWNER;
</span><span class="cx"> alter table ADDRESSBOOK_BIND
</span><span class="cx"> drop column SEEN_BY_SHAREE;
</span><ins>+
+-- Don't allow nulls in the column we are about to constrain
+update ADDRESSBOOK_BIND
+ set ADDRESSBOOK_RESOURCE_NAME = 'Shared_' || ADDRESSBOOK_RESOURCE_ID || '_' || ADDRESSBOOK_HOME_RESOURCE_ID
+ where ADDRESSBOOK_RESOURCE_NAME is null;
</ins><span class="cx"> alter table ADDRESSBOOK_BIND
</span><span class="cx"> alter column ADDRESSBOOK_RESOURCE_NAME
</span><span class="cx"> set not null;
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoretestutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/test/util.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/test/util.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/test/util.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -726,7 +726,7 @@
</span><span class="cx"> return "/%s/%s/%s/" % (prefix, self.hostname, id)
</span><span class="cx">
</span><span class="cx">
</span><del>- def send(self, prefix, id):
</del><ins>+ def send(self, prefix, id, txn):
</ins><span class="cx"> self.history.append(self.pushKeyForId(prefix, id))
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqltesttest_upgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -25,8 +25,8 @@
</span><span class="cx"> from twisted.trial.unittest import TestCase
</span><span class="cx"> from txdav.common.datastore.sql_dump import dumpSchema
</span><span class="cx"> from txdav.common.datastore.test.util import theStoreBuilder, StubNotifierFactory
</span><del>-from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaStep, \
- UpgradeDatabaseAddressBookDataStep, UpgradeDatabaseCalendarDataStep
</del><ins>+from txdav.common.datastore.upgrade.sql.upgrade import (
+ UpgradeDatabaseSchemaStep, UpgradeDatabaseAddressBookDataStep, UpgradeDatabaseCalendarDataStep, NotAllowedToUpgrade)
</ins><span class="cx"> import re
</span><span class="cx">
</span><span class="cx"> class SchemaUpgradeTests(TestCase):
</span><span class="lines">@@ -215,12 +215,12 @@
</span><span class="cx"> old_version = yield _loadVersion()
</span><span class="cx"> try:
</span><span class="cx"> yield upgrader.databaseUpgrade()
</span><del>- except RuntimeError:
</del><ins>+ except NotAllowedToUpgrade:
</ins><span class="cx"> pass
</span><span class="cx"> except Exception:
</span><del>- self.fail("RuntimeError not raised")
</del><ins>+ self.fail("NotAllowedToUpgrade not raised")
</ins><span class="cx"> else:
</span><del>- self.fail("RuntimeError not raised")
</del><ins>+ self.fail("NotAllowedToUpgrade not raised")
</ins><span class="cx"> new_version = yield _loadVersion()
</span><span class="cx"> yield _unloadOldSchema()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrade.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrade.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrade.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -74,11 +74,15 @@
</span><span class="cx"> yield sqlTxn.commit()
</span><span class="cx">
</span><span class="cx">
</span><del>- def stepWithFailure(self, failure):
- return self.stepWithResult(None)
</del><span class="cx">
</span><ins>+class NotAllowedToUpgrade(Exception):
+ """
+ Exception indicating an upgrade is needed but we're not configured to
+ perform it.
+ """
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> class UpgradeDatabaseCoreStep(object):
</span><span class="cx"> """
</span><span class="cx"> Base class for either schema or data upgrades on the database.
</span><span class="lines">@@ -136,8 +140,7 @@
</span><span class="cx"> self.log.error(msg)
</span><span class="cx"> raise RuntimeError(msg)
</span><span class="cx"> elif self.failIfUpgradeNeeded:
</span><del>- # TODO: change this exception to be upgrade-specific
- raise RuntimeError("Database upgrade is needed but not allowed.")
</del><ins>+ raise NotAllowedToUpgrade()
</ins><span class="cx"> else:
</span><span class="cx"> self.sqlStore.setUpgrading(True)
</span><span class="cx"> yield self.upgradeVersion(actual_version, required_version, dialect)
</span></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradesaddressbook_upgrade_from_1_to_2py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/addressbook_upgrade_from_1_to_2.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/addressbook_upgrade_from_1_to_2.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/addressbook_upgrade_from_1_to_2.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -22,7 +22,8 @@
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="cx"> from txdav.common.datastore.sql_tables import _ABO_KIND_GROUP, schema
</span><span class="cx"> from txdav.common.datastore.upgrade.sql.upgrades.util import updateAddressBookDataVersion, \
</span><del>- doToEachHomeNotAtVersion, removeProperty, cleanPropertyStore
</del><ins>+ doToEachHomeNotAtVersion, removeProperty, cleanPropertyStore, \
+ logUpgradeStatus
</ins><span class="cx"> from txdav.xml import element
</span><span class="cx">
</span><span class="cx"> """
</span><span class="lines">@@ -73,14 +74,20 @@
</span><span class="cx"> #update rest
</span><span class="cx"> yield abObject.setComponent(component)
</span><span class="cx">
</span><ins>+ logUpgradeStatus("Starting Addressbook Populate Members")
+
</ins><span class="cx"> # Do this to each calendar home not already at version 2
</span><del>- yield doToEachHomeNotAtVersion(sqlStore, schema.ADDRESSBOOK_HOME, UPGRADE_TO_VERSION, doIt)
</del><ins>+ yield doToEachHomeNotAtVersion(sqlStore, schema.ADDRESSBOOK_HOME, UPGRADE_TO_VERSION, doIt, "Populate Members")
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def removeResourceType(sqlStore):
</span><ins>+ logUpgradeStatus("Starting Addressbook Remove Resource Type")
+
</ins><span class="cx"> sqlTxn = sqlStore.newTransaction()
</span><span class="cx"> yield removeProperty(sqlTxn, PropertyName.fromElement(element.ResourceType))
</span><span class="cx"> yield sqlTxn.commit()
</span><span class="cx"> yield cleanPropertyStore()
</span><ins>+
+ logUpgradeStatus("End Addressbook Remove Resource Type")
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_1_to_2py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_1_to_2.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -16,12 +16,16 @@
</span><span class="cx"> ##
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.syntax import Update
</span><del>-from txdav.xml.parser import WebDAVDocument
</del><ins>+
</ins><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><ins>+
</ins><span class="cx"> from twistedcaldav import caldavxml
</span><ins>+
</ins><span class="cx"> from txdav.common.datastore.sql_tables import schema
</span><span class="cx"> from txdav.common.datastore.upgrade.sql.upgrades.util import rowsForProperty,\
</span><del>- removeProperty, updateCalendarDataVersion, doToEachHomeNotAtVersion
</del><ins>+ removeProperty, updateCalendarDataVersion, doToEachHomeNotAtVersion, \
+ logUpgradeStatus, logUpgradeError
+from txdav.xml.parser import WebDAVDocument
</ins><span class="cx">
</span><span class="cx"> """
</span><span class="cx"> Calendar data upgrade from database version 1 to 2
</span><span class="lines">@@ -50,9 +54,14 @@
</span><span class="cx"> extracting the new format value from the XML property.
</span><span class="cx"> """
</span><span class="cx">
</span><ins>+ logUpgradeStatus("Starting Move supported-component-set")
+
</ins><span class="cx"> sqlTxn = sqlStore.newTransaction()
</span><span class="cx"> try:
</span><ins>+ calendar_rid = None
</ins><span class="cx"> rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
</span><ins>+ total = len(rows)
+ count = 0
</ins><span class="cx"> for calendar_rid, value in rows:
</span><span class="cx"> prop = WebDAVDocument.fromString(value).root_element
</span><span class="cx"> supported_components = ",".join(sorted([comp.attributes["name"].upper() for comp in prop.children]))
</span><span class="lines">@@ -63,11 +72,19 @@
</span><span class="cx"> },
</span><span class="cx"> Where=(meta.RESOURCE_ID == calendar_rid)
</span><span class="cx"> ).on(sqlTxn)
</span><ins>+ count += 1
+ logUpgradeStatus("Move supported-component-set", count, total)
</ins><span class="cx">
</span><span class="cx"> yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)
</span><span class="cx"> yield sqlTxn.commit()
</span><ins>+
+ logUpgradeStatus("End Move supported-component-set")
</ins><span class="cx"> except RuntimeError:
</span><span class="cx"> yield sqlTxn.abort()
</span><ins>+ logUpgradeError(
+ "Move supported-component-set",
+ "Last calendar: {}".format(calendar_rid)
+ )
</ins><span class="cx"> raise
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -86,5 +103,7 @@
</span><span class="cx"> home = yield txn.calendarHomeWithResourceID(homeResourceID)
</span><span class="cx"> yield home.splitCalendars()
</span><span class="cx">
</span><ins>+ logUpgradeStatus("Starting Split Calendars")
+
</ins><span class="cx"> # Do this to each calendar home not already at version 2
</span><del>- yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, doIt)
</del><ins>+ yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, doIt, "Split Calendars")
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_3_to_4py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_3_to_4.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -15,19 +15,17 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><del>-from twext.enterprise.dal.syntax import Select, Delete, Parameter
-
</del><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><span class="cx">
</span><span class="cx"> from twistedcaldav import caldavxml, customxml
</span><span class="cx">
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><del>-from txdav.common.datastore.sql_tables import schema, _BIND_MODE_OWN
-from txdav.common.datastore.upgrade.sql.upgrades.util import rowsForProperty, updateCalendarDataVersion, \
- updateAllCalendarHomeDataVersions, removeProperty, cleanPropertyStore
-from txdav.xml.parser import WebDAVDocument
</del><ins>+from txdav.caldav.icalendarstore import InvalidDefaultCalendar
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.upgrade.sql.upgrades.util import updateCalendarDataVersion, \
+ removeProperty, cleanPropertyStore, logUpgradeStatus, doToEachHomeNotAtVersion
</ins><span class="cx"> from txdav.xml import element
</span><del>-from twisted.python.failure import Failure
</del><ins>+from twistedcaldav.config import config
</ins><span class="cx">
</span><span class="cx"> """
</span><span class="cx"> Data upgrade from database version 3 to 4
</span><span class="lines">@@ -41,165 +39,111 @@
</span><span class="cx"> """
</span><span class="cx"> Do the required upgrade steps.
</span><span class="cx"> """
</span><del>- yield moveDefaultCalendarProperties(sqlStore)
- yield moveCalendarTranspProperties(sqlStore)
- yield moveDefaultAlarmProperties(sqlStore)
- yield removeResourceType(sqlStore)
</del><ins>+ yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix)
</ins><span class="cx">
</span><del>- # Always bump the DB value
- yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
- yield updateAllCalendarHomeDataVersions(sqlStore, UPGRADE_TO_VERSION)
</del><ins>+ # Don't do remaining upgrade if we are only process a subset of the homes
+ if not config.UpgradeHomePrefix:
+ yield removeResourceType(sqlStore)
</ins><span class="cx">
</span><ins>+ # Always bump the DB value
+ yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><del>-def moveDefaultCalendarProperties(sqlStore):
</del><ins>+def updateCalendarHomes(sqlStore, prefix=None):
</ins><span class="cx"> """
</span><del>- Need to move all the CalDAV:default-calendar and CS:default-tasks properties in the
- RESOURCE_PROPERTY table to the new CALENDAR_HOME_METADATA table columns, extracting
- the new value from the XML property.
</del><ins>+ For each calendar home, update the associated properties on the home or its owned calendars.
</ins><span class="cx"> """
</span><span class="cx">
</span><del>- meta = schema.CALENDAR_HOME_METADATA
- yield _processDefaultCalendarProperty(sqlStore, caldavxml.ScheduleDefaultCalendarURL, meta.DEFAULT_EVENTS)
- yield _processDefaultCalendarProperty(sqlStore, customxml.ScheduleDefaultTasksURL, meta.DEFAULT_TASKS)
</del><ins>+ yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>-def _processDefaultCalendarProperty(sqlStore, propname, colname):
</del><ins>+def updateCalendarHome(txn, homeResourceID):
</ins><span class="cx"> """
</span><del>- Move the specified property value to the matching CALENDAR_HOME_METADATA table column.
-
- Since the number of calendar homes may well be large, we need to do this in batches.
</del><ins>+ For this calendar home, update the associated properties on the home or its owned calendars.
</ins><span class="cx"> """
</span><span class="cx">
</span><del>- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
</del><ins>+ home = yield txn.calendarHomeWithResourceID(homeResourceID)
+ yield moveDefaultCalendarProperties(home)
+ yield moveCalendarTranspProperties(home)
+ yield moveDefaultAlarmProperties(home)
+ yield cleanPropertyStore()
</ins><span class="cx">
</span><del>- try:
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
- delete_ids = []
- for inbox_rid, value in rows:
- delete_ids.append(inbox_rid)
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == inbox_rid,
- ).on(sqlTxn)
- if len(ids) > 0:
</del><span class="cx">
</span><del>- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(ids[0][0]))
- if calendarHome is not None:
</del><span class="cx">
</span><del>- prop = WebDAVDocument.fromString(value).root_element
- defaultCalendar = str(prop.children[0])
- parts = defaultCalendar.split("/")
- if len(parts) == 5:
</del><ins>+@inlineCallbacks
+def moveDefaultCalendarProperties(home):
+ """
+ Need to move any the CalDAV:default-calendar and CS:default-tasks properties in the
+ RESOURCE_PROPERTY table to the new CALENDAR_HOME_METADATA table columns, extracting
+ the new value from the XML property.
+ """
</ins><span class="cx">
</span><del>- calendarName = parts[-1]
- calendarHomeUID = parts[-2]
- expectedHome = (yield sqlTxn.calendarHomeWithUID(calendarHomeUID))
- if expectedHome is not None and expectedHome.id() == calendarHome.id():
</del><ins>+ yield _processDefaultCalendarProperty(home, caldavxml.ScheduleDefaultCalendarURL)
+ yield _processDefaultCalendarProperty(home, customxml.ScheduleDefaultTasksURL)
</ins><span class="cx">
</span><del>- calendar = (yield calendarHome.calendarWithName(calendarName))
- if calendar is not None:
- yield calendarHome.setDefaultCalendar(
- calendar, tasks=(propname == customxml.ScheduleDefaultTasksURL)
- )
</del><span class="cx">
</span><del>- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(propname).toString()),
- ).on(sqlTxn, ids=delete_ids)
</del><span class="cx">
</span><del>- yield sqlTxn.commit()
</del><ins>+@inlineCallbacks
+def _processDefaultCalendarProperty(home, propname):
+ """
+ Move the specified property value to the matching CALENDAR_HOME_METADATA table column.
+ """
</ins><span class="cx">
</span><del>- yield cleanPropertyStore()
</del><ins>+ inbox = (yield home.calendarWithName("inbox"))
+ prop = inbox.properties().get(PropertyName.fromElement(propname))
+ if prop is not None:
+ defaultCalendar = str(prop.children[0])
+ parts = defaultCalendar.split("/")
+ if len(parts) == 5:
</ins><span class="cx">
</span><del>- except RuntimeError:
- f = Failure()
- yield sqlTxn.abort()
- f.raiseException()
</del><ins>+ calendarName = parts[-1]
+ calendarHomeUID = parts[-2]
+ if calendarHomeUID == home.uid():
</ins><span class="cx">
</span><ins>+ calendar = (yield home.calendarWithName(calendarName))
+ if calendar is not None:
+ try:
+ yield home.setDefaultCalendar(
+ calendar, tasks=(propname == customxml.ScheduleDefaultTasksURL)
+ )
+ except InvalidDefaultCalendar:
+ # Ignore these - the server will recover
+ pass
</ins><span class="cx">
</span><ins>+ del inbox.properties()[PropertyName.fromElement(propname)]
</ins><span class="cx">
</span><ins>+
+
</ins><span class="cx"> @inlineCallbacks
</span><del>-def moveCalendarTranspProperties(sqlStore):
</del><ins>+def moveCalendarTranspProperties(home):
</ins><span class="cx"> """
</span><span class="cx"> Need to move all the CalDAV:schedule-calendar-transp properties in the
</span><span class="cx"> RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
</span><span class="cx"> the new value from the XML property.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
</del><ins>+ # Iterate over each calendar (both owned and shared)
+ calendars = (yield home.loadChildren())
+ for calendar in calendars:
+ if calendar.isInbox():
+ continue
+ prop = calendar.properties().get(PropertyName.fromElement(caldavxml.ScheduleCalendarTransp))
+ if prop is not None:
+ yield calendar.setUsedForFreeBusy(prop == caldavxml.ScheduleCalendarTransp(caldavxml.Opaque()))
+ del calendar.properties()[PropertyName.fromElement(caldavxml.ScheduleCalendarTransp)]
+ inbox = (yield home.calendarWithName("inbox"))
+ prop = inbox.properties().get(PropertyName.fromElement(caldavxml.CalendarFreeBusySet))
+ if prop is not None:
+ del inbox.properties()[PropertyName.fromElement(caldavxml.CalendarFreeBusySet)]
</ins><span class="cx">
</span><del>- try:
- calendars_for_id = {}
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, caldavxml.ScheduleCalendarTransp, with_uid=True, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
- delete_ids = []
- for calendar_rid, value, viewer in rows:
- delete_ids.append(calendar_rid)
- if calendar_rid not in calendars_for_id:
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, cb.BIND_MODE, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == calendar_rid,
- ).on(sqlTxn)
- calendars_for_id[calendar_rid] = ids
</del><span class="cx">
</span><del>- if viewer:
- calendarHome = (yield sqlTxn.calendarHomeWithUID(viewer))
- else:
- calendarHome = None
- for row in calendars_for_id[calendar_rid]:
- home_id, bind_mode = row
- if bind_mode == _BIND_MODE_OWN:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(home_id))
- break
</del><span class="cx">
</span><del>- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- calendar = (yield calendarHome.childWithID(calendar_rid))
- if calendar is not None:
- yield calendar.setUsedForFreeBusy(prop == caldavxml.ScheduleCalendarTransp(caldavxml.Opaque()))
-
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(caldavxml.ScheduleCalendarTransp).toString()),
- ).on(sqlTxn, ids=delete_ids)
-
- yield sqlTxn.commit()
-
- sqlTxn = sqlStore.newTransaction()
- yield removeProperty(sqlTxn, PropertyName.fromElement(caldavxml.CalendarFreeBusySet))
- yield sqlTxn.commit()
- yield cleanPropertyStore()
-
- except RuntimeError:
- f = Failure()
- yield sqlTxn.abort()
- f.raiseException()
-
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>-def moveDefaultAlarmProperties(sqlStore):
</del><ins>+def moveDefaultAlarmProperties(home):
</ins><span class="cx"> """
</span><span class="cx"> Need to move all the CalDAV:default-calendar and CS:default-tasks properties in the
</span><span class="cx"> RESOURCE_PROPERTY table to the new CALENDAR_HOME_METADATA table columns, extracting
</span><span class="lines">@@ -207,25 +151,25 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> yield _processDefaultAlarmProperty(
</span><del>- sqlStore,
</del><ins>+ home,
</ins><span class="cx"> caldavxml.DefaultAlarmVEventDateTime,
</span><span class="cx"> True,
</span><span class="cx"> True,
</span><span class="cx"> )
</span><span class="cx"> yield _processDefaultAlarmProperty(
</span><del>- sqlStore,
</del><ins>+ home,
</ins><span class="cx"> caldavxml.DefaultAlarmVEventDate,
</span><span class="cx"> True,
</span><span class="cx"> False,
</span><span class="cx"> )
</span><span class="cx"> yield _processDefaultAlarmProperty(
</span><del>- sqlStore,
</del><ins>+ home,
</ins><span class="cx"> caldavxml.DefaultAlarmVToDoDateTime,
</span><span class="cx"> False,
</span><span class="cx"> True,
</span><span class="cx"> )
</span><span class="cx"> yield _processDefaultAlarmProperty(
</span><del>- sqlStore,
</del><ins>+ home,
</ins><span class="cx"> caldavxml.DefaultAlarmVToDoDate,
</span><span class="cx"> False,
</span><span class="cx"> False,
</span><span class="lines">@@ -234,90 +178,40 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>-def _processDefaultAlarmProperty(sqlStore, propname, vevent, timed):
</del><ins>+def _processDefaultAlarmProperty(home, propname, vevent, timed):
</ins><span class="cx"> """
</span><span class="cx"> Move the specified property value to the matching CALENDAR_HOME_METADATA or CALENDAR_BIND table column.
</span><span class="cx">
</span><span class="cx"> Since the number of properties may well be large, we need to do this in batches.
</span><span class="cx"> """
</span><span class="cx">
</span><del>- hm = schema.CALENDAR_HOME_METADATA
- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
</del><ins>+ # Check the home first
+ prop = home.properties().get(PropertyName.fromElement(propname))
+ if prop is not None:
+ alarm = str(prop.children[0]) if prop.children else None
+ yield home.setDefaultAlarm(alarm, vevent, timed)
+ del home.properties()[PropertyName.fromElement(propname)]
</ins><span class="cx">
</span><del>- try:
- calendars_for_id = {}
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, propname, with_uid=True, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
- delete_ids = []
- for rid, value, viewer in rows:
- delete_ids.append(rid)
</del><ins>+ # Now each child
+ calendars = (yield home.loadChildren())
+ for calendar in calendars:
+ if calendar.isInbox():
+ continue
+ prop = calendar.properties().get(PropertyName.fromElement(propname))
+ if prop is not None:
+ alarm = str(prop.children[0]) if prop.children else None
+ yield calendar.setDefaultAlarm(alarm, vevent, timed)
+ del calendar.properties()[PropertyName.fromElement(propname)]
</ins><span class="cx">
</span><del>- prop = WebDAVDocument.fromString(value).root_element
- alarm = str(prop.children[0]) if prop.children else None
</del><span class="cx">
</span><del>- # First check if the rid is a home - this is the most common case
- ids = yield Select(
- [hm.RESOURCE_ID, ],
- From=hm,
- Where=hm.RESOURCE_ID == rid,
- ).on(sqlTxn)
</del><span class="cx">
</span><del>- if len(ids) > 0:
- # Home object
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(ids[0][0]))
- if calendarHome is not None:
- yield calendarHome.setDefaultAlarm(alarm, vevent, timed)
- else:
- # rid is a calendar - we need to find the per-user calendar for the resource viewer
- if rid not in calendars_for_id:
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, cb.BIND_MODE, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == rid,
- ).on(sqlTxn)
- calendars_for_id[rid] = ids
-
- if viewer:
- calendarHome = (yield sqlTxn.calendarHomeWithUID(viewer))
- else:
- calendarHome = None
- for row in calendars_for_id[rid]:
- home_id, bind_mode = row
- if bind_mode == _BIND_MODE_OWN:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(home_id))
- break
-
- if calendarHome is not None:
- calendar = yield calendarHome.childWithID(rid)
- if calendar is not None:
- yield calendar.setDefaultAlarm(alarm, vevent, timed)
-
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(propname).toString()),
- ).on(sqlTxn, ids=delete_ids)
-
- yield sqlTxn.commit()
-
- yield cleanPropertyStore()
-
- except RuntimeError:
- f = Failure()
- yield sqlTxn.abort()
- f.raiseException()
-
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def removeResourceType(sqlStore):
</span><ins>+ logUpgradeStatus("Starting Calendar Remove Resource Type")
+
</ins><span class="cx"> sqlTxn = sqlStore.newTransaction()
</span><span class="cx"> yield removeProperty(sqlTxn, PropertyName.fromElement(element.ResourceType))
</span><span class="cx"> yield sqlTxn.commit()
</span><span class="cx"> yield cleanPropertyStore()
</span><ins>+
+ logUpgradeStatus("End Calendar Remove Resource Type")
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradescalendar_upgrade_from_4_to_5py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -15,21 +15,18 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><del>-from twext.enterprise.dal.syntax import Select, Delete, Parameter
</del><ins>+from twext.web2.dav.resource import TwistedQuotaUsedProperty, TwistedGETContentMD5
</ins><span class="cx">
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><del>-from twisted.python.failure import Failure
</del><span class="cx">
</span><span class="cx"> from twistedcaldav import caldavxml, customxml
</span><ins>+from twistedcaldav.config import config
</ins><span class="cx">
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><del>-from txdav.common.datastore.sql_tables import schema, _BIND_MODE_OWN
-from txdav.common.datastore.upgrade.sql.upgrades.util import rowsForProperty, updateCalendarDataVersion, \
- updateAllCalendarHomeDataVersions, removeProperty, cleanPropertyStore
</del><ins>+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.upgrade.sql.upgrades.util import updateCalendarDataVersion, \
+ removeProperty, cleanPropertyStore, logUpgradeStatus, doToEachHomeNotAtVersion
</ins><span class="cx"> from txdav.xml import element
</span><del>-from txdav.xml.parser import WebDAVDocument
-from twext.web2.dav.resource import TwistedQuotaUsedProperty, \
- TwistedGETContentMD5
</del><span class="cx">
</span><span class="cx"> """
</span><span class="cx"> Data upgrade from database version 4 to 5
</span><span class="lines">@@ -43,136 +40,75 @@
</span><span class="cx"> """
</span><span class="cx"> Do the required upgrade steps.
</span><span class="cx"> """
</span><del>- yield moveCalendarTimezoneProperties(sqlStore)
- yield moveCalendarAvailabilityProperties(sqlStore)
- yield removeOtherProperties(sqlStore)
</del><ins>+ yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix)
</ins><span class="cx">
</span><del>- # Always bump the DB value
- yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
- yield updateAllCalendarHomeDataVersions(sqlStore, UPGRADE_TO_VERSION)
</del><ins>+ # Don't do remaining upgrade if we are only process a subset of the homes
+ if not config.UpgradeHomePrefix:
+ yield removeOtherProperties(sqlStore)
</ins><span class="cx">
</span><ins>+ # Always bump the DB value
+ yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><del>-def moveCalendarTimezoneProperties(sqlStore):
</del><ins>+def updateCalendarHomes(sqlStore, prefix=None):
</ins><span class="cx"> """
</span><del>- Need to move all the CalDAV:calendar-timezone properties in the
- RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
- the new value from the XML property.
</del><ins>+ For each calendar home, update the associated properties on the home or its owned calendars.
</ins><span class="cx"> """
</span><span class="cx">
</span><del>- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
</del><ins>+ yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix)
</ins><span class="cx">
</span><del>- try:
- calendars_for_id = {}
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, caldavxml.CalendarTimeZone, with_uid=True, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
- delete_ids = []
- for calendar_rid, value, viewer in rows:
- delete_ids.append(calendar_rid)
- if calendar_rid not in calendars_for_id:
- ids = yield Select(
- [cb.CALENDAR_HOME_RESOURCE_ID, cb.BIND_MODE, ],
- From=cb,
- Where=cb.CALENDAR_RESOURCE_ID == calendar_rid,
- ).on(sqlTxn)
- calendars_for_id[calendar_rid] = ids
</del><span class="cx">
</span><del>- if viewer:
- calendarHome = (yield sqlTxn.calendarHomeWithUID(viewer))
- else:
- calendarHome = None
- for row in calendars_for_id[calendar_rid]:
- home_id, bind_mode = row
- if bind_mode == _BIND_MODE_OWN:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(home_id))
- break
</del><span class="cx">
</span><del>- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- calendar = (yield calendarHome.childWithID(calendar_rid))
- if calendar is not None:
- yield calendar.setTimezone(prop.calendar())
</del><ins>+@inlineCallbacks
+def updateCalendarHome(txn, homeResourceID):
+ """
+ For this calendar home, update the associated properties on the home or its owned calendars.
+ """
</ins><span class="cx">
</span><del>- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(delete_ids)))).And
- (rp.NAME == PropertyName.fromElement(caldavxml.CalendarTimeZone).toString()),
- ).on(sqlTxn, ids=delete_ids)
</del><ins>+ home = yield txn.calendarHomeWithResourceID(homeResourceID)
+ yield moveCalendarTimezoneProperties(home)
+ yield moveCalendarAvailabilityProperties(home)
+ yield cleanPropertyStore()
</ins><span class="cx">
</span><del>- yield sqlTxn.commit()
</del><span class="cx">
</span><del>- yield cleanPropertyStore()
</del><span class="cx">
</span><del>- except RuntimeError:
- f = Failure()
- yield sqlTxn.abort()
- f.raiseException()
</del><ins>+@inlineCallbacks
+def moveCalendarTimezoneProperties(home):
+ """
+ Need to move all the CalDAV:calendar-timezone properties in the
+ RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
+ the new value from the XML property.
+ """
</ins><span class="cx">
</span><ins>+ # Iterate over each calendar (both owned and shared)
+ calendars = (yield home.loadChildren())
+ for calendar in calendars:
+ if calendar.isInbox():
+ continue
+ prop = calendar.properties().get(PropertyName.fromElement(caldavxml.CalendarTimeZone))
+ if prop is not None:
+ yield calendar.setTimezone(prop.calendar())
+ del calendar.properties()[PropertyName.fromElement(caldavxml.CalendarTimeZone)]
</ins><span class="cx">
</span><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><del>-def moveCalendarAvailabilityProperties(sqlStore):
</del><ins>+def moveCalendarAvailabilityProperties(home):
</ins><span class="cx"> """
</span><span class="cx"> Need to move all the CS:calendar-availability properties in the
</span><span class="cx"> RESOURCE_PROPERTY table to the new CALENDAR_BIND table columns, extracting
</span><span class="cx"> the new value from the XML property.
</span><span class="cx"> """
</span><ins>+ inbox = (yield home.calendarWithName("inbox"))
+ prop = inbox.properties().get(PropertyName.fromElement(customxml.CalendarAvailability))
+ if prop is not None:
+ yield home.setAvailability(prop.calendar())
+ del inbox.properties()[customxml.CalendarAvailability]
</ins><span class="cx">
</span><del>- cb = schema.CALENDAR_BIND
- rp = schema.RESOURCE_PROPERTY
</del><span class="cx">
</span><del>- try:
- while True:
- sqlTxn = sqlStore.newTransaction()
- rows = (yield rowsForProperty(sqlTxn, customxml.CalendarAvailability, batch=BATCH_SIZE))
- if len(rows) == 0:
- yield sqlTxn.commit()
- break
</del><span class="cx">
</span><del>- # Map each calendar to a home id using a single query for efficiency
- calendar_ids = [row[0] for row in rows]
-
- home_map = yield Select(
- [cb.CALENDAR_RESOURCE_ID, cb.CALENDAR_HOME_RESOURCE_ID, ],
- From=cb,
- Where=(cb.CALENDAR_RESOURCE_ID.In(Parameter("ids", len(calendar_ids)))).And(cb.BIND_MODE == _BIND_MODE_OWN),
- ).on(sqlTxn, ids=calendar_ids)
- calendar_to_home = dict(home_map)
-
- # Move property to each home
- for calendar_rid, value in rows:
- if calendar_rid in calendar_to_home:
- calendarHome = (yield sqlTxn.calendarHomeWithResourceID(calendar_to_home[calendar_rid]))
-
- if calendarHome is not None:
- prop = WebDAVDocument.fromString(value).root_element
- yield calendarHome.setAvailability(prop.calendar())
-
- # Always delete the rows so that batch processing works correctly
- yield Delete(
- From=rp,
- Where=(rp.RESOURCE_ID.In(Parameter("ids", len(calendar_ids)))).And
- (rp.NAME == PropertyName.fromElement(customxml.CalendarAvailability).toString()),
- ).on(sqlTxn, ids=calendar_ids)
-
- yield sqlTxn.commit()
-
- yield cleanPropertyStore()
-
- except RuntimeError:
- f = Failure()
- yield sqlTxn.abort()
- f.raiseException()
-
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def removeOtherProperties(sqlStore):
</span><span class="cx"> """
</span><span class="lines">@@ -190,6 +126,8 @@
</span><span class="cx"> {http://twistedmatrix.com/xml_namespace/dav/}schedule-auto-respond
</span><span class="cx">
</span><span class="cx"> """
</span><ins>+ logUpgradeStatus("Starting Calendar Remove Other Properties")
+
</ins><span class="cx"> sqlTxn = sqlStore.newTransaction()
</span><span class="cx">
</span><span class="cx"> yield removeProperty(sqlTxn, PropertyName.fromElement(element.ACL))
</span><span class="lines">@@ -205,3 +143,5 @@
</span><span class="cx">
</span><span class="cx"> yield sqlTxn.commit()
</span><span class="cx"> yield cleanPropertyStore()
</span><ins>+
+ logUpgradeStatus("End Calendar Remove Other Properties")
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradestesttest_upgrade_from_3_to_4py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_3_to_4.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -13,23 +13,27 @@
</span><span class="cx"> # See the License for the specific language governing permissions and
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><ins>+
+from twext.enterprise.dal.syntax import Update, Insert
+
+from twistedcaldav import caldavxml
</ins><span class="cx"> from twistedcaldav.caldavxml import ScheduleDefaultCalendarURL, \
</span><del>- CalendarFreeBusySet, Opaque, ScheduleCalendarTransp
</del><ins>+ CalendarFreeBusySet, Opaque, ScheduleCalendarTransp, Transparent
+
</ins><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="cx"> from txdav.caldav.datastore.test.util import CommonStoreTests
</span><ins>+from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_3_to_4 import updateCalendarHomes, \
+ doUpgrade
+from txdav.xml import element
</ins><span class="cx"> from txdav.xml.element import HRef
</span><del>-from twext.enterprise.dal.syntax import Update, Insert
-from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_3_to_4 import moveDefaultCalendarProperties, \
- moveCalendarTranspProperties, removeResourceType, moveDefaultAlarmProperties
-from txdav.xml import element
-from twistedcaldav import caldavxml
-from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
</del><ins>+from twistedcaldav.config import config
</ins><span class="cx">
</span><span class="cx"> """
</span><span class="cx"> Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
</span><span class="cx"> """
</span><span class="cx">
</span><del>-from twisted.internet.defer import inlineCallbacks
</del><ins>+from twisted.internet.defer import inlineCallbacks, returnValue
</ins><span class="cx">
</span><span class="cx"> class Upgrade_from_3_to_4(CommonStoreTests):
</span><span class="cx"> """
</span><span class="lines">@@ -37,7 +41,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_defaultCalendarUpgrade(self):
</del><ins>+ def _defaultCalendarUpgrade_setup(self):
</ins><span class="cx">
</span><span class="cx"> # Set dead property on inbox
</span><span class="cx"> for user in ("user01", "user02",):
</span><span class="lines">@@ -52,39 +56,132 @@
</span><span class="cx"> Where=chm.RESOURCE_ID == home._resourceID,
</span><span class="cx"> ).on(self.transactionUnderTest())
</span><span class="cx">
</span><del>- # Force data version to previous
- ch = home._homeSchema
- yield Update(
- {ch.DATAVERSION: 3},
- Where=ch.RESOURCE_ID == home._resourceID,
- ).on(self.transactionUnderTest())
</del><ins>+ # Force data version to previous
+ ch = home._homeSchema
+ yield Update(
+ {ch.DATAVERSION: 3},
+ Where=ch.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
</ins><span class="cx">
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield moveDefaultCalendarProperties(self._sqlCalendarStore)
</del><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _defaultCalendarUpgrade_check(self, changed_users, unchanged_users):
+
</ins><span class="cx"> # Test results
</span><del>- for user in ("user01", "user02",):
</del><ins>+ for user in changed_users:
</ins><span class="cx"> home = (yield self.homeUnderTest(name=user))
</span><ins>+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
</ins><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><span class="cx"> self.assertTrue(home.isDefaultCalendar(calendar))
</span><span class="cx"> inbox = (yield self.calendarUnderTest(name="inbox", home=user))
</span><span class="cx"> self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) not in inbox.properties())
</span><span class="cx">
</span><ins>+ for user in unchanged_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertFalse(home.isDefaultCalendar(calendar))
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) in inbox.properties())
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><del>- def test_calendarTranspUpgrade(self):
</del><ins>+ def test_defaultCalendarUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01", "user02",), ())
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def test_partialDefaultCalendarUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._defaultCalendarUpgrade_check(("user01",), ("user02",))
+
+
+ @inlineCallbacks
+ def _invalidDefaultCalendarUpgrade_setup(self):
+
</ins><span class="cx"> # Set dead property on inbox
</span><span class="cx"> for user in ("user01", "user02",):
</span><span class="cx"> inbox = (yield self.calendarUnderTest(name="inbox", home=user))
</span><ins>+ inbox.properties()[PropertyName.fromElement(ScheduleDefaultCalendarURL)] = ScheduleDefaultCalendarURL(HRef.fromString("/calendars/__uids__/%s/tasks_1" % (user,)))
+
+ # Force current default to null
+ home = (yield self.homeUnderTest(name=user))
+ chm = home._homeMetaDataSchema
+ yield Update(
+ {chm.DEFAULT_EVENTS: None},
+ Where=chm.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
+
+ # Create tasks only calendar
+ tasks = (yield home.createCalendarWithName("tasks_1"))
+ yield tasks.setSupportedComponents("VTODO")
+
+ # Force data version to previous
+ ch = home._homeSchema
+ yield Update(
+ {ch.DATAVERSION: 3},
+ Where=ch.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
+
+ yield self.commit()
+
+
+ @inlineCallbacks
+ def _invalidDefaultCalendarUpgrade_check(self, changed_users, unchanged_users):
+
+ # Test results
+ for user in changed_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ calendar = (yield self.calendarUnderTest(name="tasks_1", home=user))
+ self.assertFalse(home.isDefaultCalendar(calendar))
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) not in inbox.properties())
+
+ for user in unchanged_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ calendar = (yield self.calendarUnderTest(name="tasks_1", home=user))
+ self.assertFalse(home.isDefaultCalendar(calendar))
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(ScheduleDefaultCalendarURL) in inbox.properties())
+
+
+ @inlineCallbacks
+ def test_invalidDefaultCalendarUpgrade(self):
+ yield self._invalidDefaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._invalidDefaultCalendarUpgrade_check(("user01", "user02",), ())
+
+
+ @inlineCallbacks
+ def test_partialInvalidDefaultCalendarUpgrade(self):
+ yield self._invalidDefaultCalendarUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._invalidDefaultCalendarUpgrade_check(("user01",), ("user02",))
+
+
+ @inlineCallbacks
+ def _calendarTranspUpgrade_setup(self):
+
+ # Set dead property on inbox
+ for user in ("user01", "user02",):
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
</ins><span class="cx"> inbox.properties()[PropertyName.fromElement(CalendarFreeBusySet)] = CalendarFreeBusySet(HRef.fromString("/calendars/__uids__/%s/calendar_1" % (user,)))
</span><span class="cx">
</span><span class="cx"> # Force current to transparent
</span><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><span class="cx"> yield calendar.setUsedForFreeBusy(False)
</span><del>- calendar.properties()[PropertyName.fromElement(ScheduleCalendarTransp)] = ScheduleCalendarTransp(Opaque())
</del><ins>+ calendar.properties()[PropertyName.fromElement(ScheduleCalendarTransp)] = ScheduleCalendarTransp(Opaque() if user == "user01" else Transparent())
</ins><span class="cx">
</span><span class="cx"> # Force data version to previous
</span><span class="cx"> home = (yield self.homeUnderTest(name=user))
</span><span class="lines">@@ -118,21 +215,55 @@
</span><span class="cx"> ).on(txn)
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield moveCalendarTranspProperties(self._sqlCalendarStore)
</del><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _calendarTranspUpgrade_check(self, changed_users, unchanged_users):
+
</ins><span class="cx"> # Test results
</span><del>- for user in ("user01", "user02",):
</del><ins>+ for user in changed_users:
</ins><span class="cx"> home = (yield self.homeUnderTest(name=user))
</span><ins>+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
</ins><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><del>- self.assertTrue(calendar.isUsedForFreeBusy())
</del><ins>+ if user == "user01":
+ self.assertTrue(calendar.isUsedForFreeBusy())
+ else:
+ self.assertFalse(calendar.isUsedForFreeBusy())
+ self.assertTrue(PropertyName.fromElement(caldavxml.ScheduleCalendarTransp) not in calendar.properties())
</ins><span class="cx"> inbox = (yield self.calendarUnderTest(name="inbox", home=user))
</span><span class="cx"> self.assertTrue(PropertyName.fromElement(CalendarFreeBusySet) not in inbox.properties())
</span><span class="cx">
</span><ins>+ for user in unchanged_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ if user == "user01":
+ self.assertFalse(calendar.isUsedForFreeBusy())
+ else:
+ self.assertFalse(calendar.isUsedForFreeBusy())
+ self.assertTrue(PropertyName.fromElement(caldavxml.ScheduleCalendarTransp) in calendar.properties())
+ inbox = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertTrue(PropertyName.fromElement(CalendarFreeBusySet) in inbox.properties())
</ins><span class="cx">
</span><ins>+
</ins><span class="cx"> @inlineCallbacks
</span><del>- def test_defaultAlarmUpgrade(self):
</del><ins>+ def test_calendarTranspUpgrade(self):
+ yield self._calendarTranspUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarTranspUpgrade_check(("user01", "user02",), ())
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def test_partialCalendarTranspUpgrade(self):
+ yield self._calendarTranspUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarTranspUpgrade_check(("user01",), ("user02",))
+
+
+ @inlineCallbacks
+ def _defaultAlarmUpgrade_setup(self):
+
</ins><span class="cx"> alarmhome1 = """BEGIN:VALARM
</span><span class="cx"> ACTION:AUDIO
</span><span class="cx"> TRIGGER;RELATED=START:-PT1M
</span><span class="lines">@@ -236,13 +367,28 @@
</span><span class="cx"> shared = yield self.calendarUnderTest(name=shared_name, home="user02")
</span><span class="cx"> for _ignore_vevent, _ignore_timed, alarm, prop in detailsshared:
</span><span class="cx"> shared.properties()[PropertyName.fromElement(prop)] = prop(alarm)
</span><ins>+
+ for user in ("user01", "user02",):
+ # Force data version to previous
+ home = (yield self.homeUnderTest(name=user))
+ ch = home._homeSchema
+ yield Update(
+ {ch.DATAVERSION: 3},
+ Where=ch.RESOURCE_ID == home._resourceID,
+ ).on(self.transactionUnderTest())
+
</ins><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield moveDefaultAlarmProperties(self._sqlCalendarStore)
</del><ins>+ returnValue((detailshome, detailscalendar, detailsshared, shared_name,))
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def _defaultAlarmUpgrade_check(self, changed_users, unchanged_users, detailshome, detailscalendar, detailsshared, shared_name):
+
</ins><span class="cx"> # Check each type of collection
</span><span class="cx"> home = yield self.homeUnderTest(name="user01")
</span><ins>+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
</ins><span class="cx"> for vevent, timed, alarm, prop in detailshome:
</span><span class="cx"> alarm_result = (yield home.getDefaultAlarm(vevent, timed))
</span><span class="cx"> self.assertEquals(alarm_result, alarm)
</span><span class="lines">@@ -252,18 +398,67 @@
</span><span class="cx"> for vevent, timed, alarm, prop in detailscalendar:
</span><span class="cx"> alarm_result = (yield calendar.getDefaultAlarm(vevent, timed))
</span><span class="cx"> self.assertEquals(alarm_result, alarm)
</span><del>- self.assertTrue(PropertyName.fromElement(prop) not in home.properties())
</del><ins>+ self.assertTrue(PropertyName.fromElement(prop) not in calendar.properties())
</ins><span class="cx">
</span><del>- shared = yield self.calendarUnderTest(name=shared_name, home="user02")
- for vevent, timed, alarm, prop in detailsshared:
- alarm_result = (yield shared.getDefaultAlarm(vevent, timed))
- self.assertEquals(alarm_result, alarm)
- self.assertTrue(PropertyName.fromElement(prop) not in home.properties())
</del><ins>+ if "user02" in changed_users:
+ home = (yield self.homeUnderTest(name="user02"))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ shared = yield self.calendarUnderTest(name=shared_name, home="user02")
+ for vevent, timed, alarm, prop in detailsshared:
+ alarm_result = (yield shared.getDefaultAlarm(vevent, timed))
+ self.assertEquals(alarm_result, alarm)
+ self.assertTrue(PropertyName.fromElement(prop) not in shared.properties())
+ else:
+ home = (yield self.homeUnderTest(name="user02"))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 3)
+ shared = yield self.calendarUnderTest(name=shared_name, home="user02")
+ for vevent, timed, alarm, prop in detailsshared:
+ alarm_result = (yield shared.getDefaultAlarm(vevent, timed))
+ self.assertEquals(alarm_result, None)
+ self.assertTrue(PropertyName.fromElement(prop) in shared.properties())
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_resourceTypeUpgrade(self):
</del><ins>+ def test_defaultAlarmUpgrade(self):
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._defaultAlarmUpgrade_check(("user01", "user02",), (), detailshome, detailscalendar, detailsshared, shared_name)
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def test_partialDefaultAlarmUpgrade(self):
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._defaultAlarmUpgrade_check(("user01",), ("user02",), detailshome, detailscalendar, detailsshared, shared_name)
+
+
+ @inlineCallbacks
+ def test_combinedUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01", "user02",), ())
+ yield self._calendarTranspUpgrade_check(("user01", "user02",), ())
+ yield self._defaultAlarmUpgrade_check(("user01", "user02",), (), detailshome, detailscalendar, detailsshared, shared_name)
+
+
+ @inlineCallbacks
+ def test_partialCombinedUpgrade(self):
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._defaultCalendarUpgrade_check(("user01",), ("user02",))
+ yield self._calendarTranspUpgrade_check(("user01",), ("user02",))
+ yield self._defaultAlarmUpgrade_check(("user01",), ("user02",), detailshome, detailscalendar, detailsshared, shared_name)
+
+
+ @inlineCallbacks
+ def _resourceTypeUpgrade_setup(self):
+
</ins><span class="cx"> # Set dead property on calendar
</span><span class="cx"> for user in ("user01", "user02",):
</span><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><span class="lines">@@ -273,12 +468,60 @@
</span><span class="cx"> for user in ("user01", "user02",):
</span><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><span class="cx"> self.assertTrue(PropertyName.fromElement(element.ResourceType) in calendar.properties())
</span><ins>+
+ yield self.transactionUnderTest().updateCalendarserverValue("CALENDAR-DATAVERSION", "3")
+
</ins><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield removeResourceType(self._sqlCalendarStore)
</del><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _resourceTypeUpgrade_check(self, full=True):
+
</ins><span class="cx"> # Test results
</span><del>- for user in ("user01", "user02",):
- calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
- self.assertTrue(PropertyName.fromElement(element.ResourceType) not in calendar.properties())
</del><ins>+ if full:
+ for user in ("user01", "user02",):
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceType) not in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 4)
+ else:
+ for user in ("user01", "user02",):
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceType) in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 3)
+
+
+ @inlineCallbacks
+ def test_resourceTypeUpgrade(self):
+ yield self._resourceTypeUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._resourceTypeUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_fullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "")
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield self._resourceTypeUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01", "user02",), ())
+ yield self._calendarTranspUpgrade_check(("user01", "user02",), ())
+ yield self._defaultAlarmUpgrade_check(("user01", "user02",), (), detailshome, detailscalendar, detailsshared, shared_name)
+ yield self._resourceTypeUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_partialFullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "user01")
+ yield self._defaultCalendarUpgrade_setup()
+ yield self._calendarTranspUpgrade_setup()
+ yield self._resourceTypeUpgrade_setup()
+ detailshome, detailscalendar, detailsshared, shared_name = (yield self._defaultAlarmUpgrade_setup())
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._defaultCalendarUpgrade_check(("user01",), ("user02",))
+ yield self._calendarTranspUpgrade_check(("user01",), ("user02",))
+ yield self._defaultAlarmUpgrade_check(("user01",), ("user02",), detailshome, detailscalendar, detailsshared, shared_name)
+ yield self._resourceTypeUpgrade_check(False)
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradestesttest_upgrade_from_4_to_5py"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/test/test_upgrade_from_4_to_5.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -13,21 +13,24 @@
</span><span class="cx"> # See the License for the specific language governing permissions and
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><del>-from twistedcaldav import caldavxml, customxml
-from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_4_to_5 import moveCalendarTimezoneProperties, \
- removeOtherProperties, moveCalendarAvailabilityProperties
-from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
-from txdav.xml import element
</del><span class="cx">
</span><span class="cx"> """
</span><span class="cx"> Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.dal.syntax import Update, Insert
</span><del>-from twisted.internet.defer import inlineCallbacks
</del><ins>+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from twistedcaldav import caldavxml, customxml
+from twistedcaldav.config import config
</ins><span class="cx"> from twistedcaldav.ical import Component
</span><ins>+
</ins><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="cx"> from txdav.caldav.datastore.test.util import CommonStoreTests
</span><ins>+from txdav.common.datastore.sql_tables import _BIND_MODE_WRITE, schema
+from txdav.common.datastore.upgrade.sql.upgrades.calendar_upgrade_from_4_to_5 import updateCalendarHomes, doUpgrade
+from txdav.xml import element
</ins><span class="cx">
</span><span class="cx"> class Upgrade_from_4_to_5(CommonStoreTests):
</span><span class="cx"> """
</span><span class="lines">@@ -35,7 +38,7 @@
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_calendarTimezoneUpgrade(self):
</del><ins>+ def _calendarTimezoneUpgrade_setup(self):
</ins><span class="cx">
</span><span class="cx"> tz1 = Component.fromString("""BEGIN:VCALENDAR
</span><span class="cx"> VERSION:2.0
</span><span class="lines">@@ -137,19 +140,47 @@
</span><span class="cx"> ).on(txn)
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield moveCalendarTimezoneProperties(self._sqlCalendarStore)
</del><ins>+ returnValue(user_details)
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def _calendarTimezoneUpgrade_check(self, changed_users, unchanged_users, user_details):
+
</ins><span class="cx"> # Test results
</span><span class="cx"> for user, calname, tz in user_details:
</span><del>- calendar = (yield self.calendarUnderTest(name=calname, home=user))
- self.assertEqual(calendar.getTimezone(), tz)
- self.assertTrue(PropertyName.fromElement(caldavxml.CalendarTimeZone) not in calendar.properties())
</del><ins>+ if user in changed_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 5)
+ calendar = (yield self.calendarUnderTest(name=calname, home=user))
+ self.assertEqual(calendar.getTimezone(), tz)
+ self.assertTrue(PropertyName.fromElement(caldavxml.CalendarTimeZone) not in calendar.properties())
+ else:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ calendar = (yield self.calendarUnderTest(name=calname, home=user))
+ self.assertEqual(calendar.getTimezone(), None)
+ self.assertTrue(PropertyName.fromElement(caldavxml.CalendarTimeZone) in calendar.properties())
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_calendarAvailabilityUpgrade(self):
</del><ins>+ def test_calendarTimezoneUpgrade(self):
+ user_details = yield self._calendarTimezoneUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01", "user02", "user03",), (), user_details)
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def test_partialCalendarTimezoneUpgrade(self):
+ user_details = yield self._calendarTimezoneUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarTimezoneUpgrade_check(("user01",), ("user02", "user03",), user_details)
+
+
+ @inlineCallbacks
+ def _calendarAvailabilityUpgrade_setup(self):
+
</ins><span class="cx"> av1 = Component.fromString("""BEGIN:VCALENDAR
</span><span class="cx"> VERSION:2.0
</span><span class="cx"> CALSCALE:GREGORIAN
</span><span class="lines">@@ -220,20 +251,65 @@
</span><span class="cx"> self.assertEqual(PropertyName.fromElement(customxml.CalendarAvailability) in calendar.properties(), av is not None)
</span><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield moveCalendarAvailabilityProperties(self._sqlCalendarStore)
</del><ins>+ returnValue(user_details)
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def _calendarAvailabilityUpgrade_check(self, changed_users, unchanged_users, user_details):
+
</ins><span class="cx"> # Test results
</span><span class="cx"> for user, av in user_details:
</span><del>- home = (yield self.homeUnderTest(name=user))
- calendar = (yield self.calendarUnderTest(name="inbox", home=user))
- self.assertEqual(home.getAvailability(), av)
- self.assertTrue(PropertyName.fromElement(customxml.CalendarAvailability) not in calendar.properties())
</del><ins>+ if user in changed_users:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 5)
+ calendar = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertEqual(home.getAvailability(), av)
+ self.assertTrue(PropertyName.fromElement(customxml.CalendarAvailability) not in calendar.properties())
+ else:
+ home = (yield self.homeUnderTest(name=user))
+ version = (yield home.dataVersion())
+ self.assertEqual(version, 4)
+ calendar = (yield self.calendarUnderTest(name="inbox", home=user))
+ self.assertEqual(home.getAvailability(), None)
+ self.assertTrue(PropertyName.fromElement(customxml.CalendarAvailability) in calendar.properties())
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_removeOtherPropertiesUpgrade(self):
</del><ins>+ def test_calendarAvailabilityUpgrade(self):
+ user_details = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarAvailabilityUpgrade_check(("user01", "user02", "user03",), (), user_details)
</ins><span class="cx">
</span><ins>+
+ @inlineCallbacks
+ def test_partialCalendarAvailabilityUpgrade(self):
+ user_details = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarAvailabilityUpgrade_check(("user01",), ("user02", "user03",), user_details)
+
+
+ @inlineCallbacks
+ def test_combinedUpgrade(self):
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01", "user02", "user03",), (), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01", "user02", "user03",), (), user_details2)
+
+
+ @inlineCallbacks
+ def test_partialCombinedUpgrade(self):
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield updateCalendarHomes(self._sqlCalendarStore, "user01")
+ yield self._calendarTimezoneUpgrade_check(("user01",), ("user02", "user03",), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01",), ("user02", "user03",), user_details2)
+
+
+ @inlineCallbacks
+ def _removeOtherPropertiesUpgrade_setup(self):
+
</ins><span class="cx"> # Set dead property on calendar
</span><span class="cx"> for user in ("user01", "user02",):
</span><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><span class="lines">@@ -243,12 +319,55 @@
</span><span class="cx"> for user in ("user01", "user02",):
</span><span class="cx"> calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
</span><span class="cx"> self.assertTrue(PropertyName.fromElement(element.ResourceID) in calendar.properties())
</span><ins>+
+ yield self.transactionUnderTest().updateCalendarserverValue("CALENDAR-DATAVERSION", "4")
+
</ins><span class="cx"> yield self.commit()
</span><span class="cx">
</span><del>- # Trigger upgrade
- yield removeOtherProperties(self._sqlCalendarStore)
</del><span class="cx">
</span><ins>+ @inlineCallbacks
+ def _removeOtherPropertiesUpgrade_check(self, full=True):
+
</ins><span class="cx"> # Test results
</span><span class="cx"> for user in ("user01", "user02",):
</span><del>- calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
- self.assertTrue(PropertyName.fromElement(element.ResourceID) not in calendar.properties())
</del><ins>+ if full:
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceID) not in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 5)
+ else:
+ calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
+ self.assertTrue(PropertyName.fromElement(element.ResourceID) in calendar.properties())
+ version = yield self.transactionUnderTest().calendarserverValue("CALENDAR-DATAVERSION")
+ self.assertEqual(int(version), 4)
+
+
+ @inlineCallbacks
+ def test_removeOtherPropertiesUpgrade(self):
+ yield self._removeOtherPropertiesUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._removeOtherPropertiesUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_fullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "")
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield self._removeOtherPropertiesUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01", "user02", "user03",), (), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01", "user02", "user03",), (), user_details2)
+ yield self._removeOtherPropertiesUpgrade_check()
+
+
+ @inlineCallbacks
+ def test_partialFullUpgrade(self):
+ self.patch(config, "UpgradeHomePrefix", "user01")
+ user_details1 = yield self._calendarTimezoneUpgrade_setup()
+ user_details2 = yield self._calendarAvailabilityUpgrade_setup()
+ yield self._removeOtherPropertiesUpgrade_setup()
+ yield doUpgrade(self._sqlCalendarStore)
+ yield self._calendarTimezoneUpgrade_check(("user01",), ("user02", "user03",), user_details1)
+ yield self._calendarAvailabilityUpgrade_check(("user01",), ("user02", "user03",), user_details2)
+ yield self._removeOtherPropertiesUpgrade_check(False)
</ins></span></pre></div>
<a id="CalendarServerbranchesusersgayasharedgroupfixestxdavcommondatastoreupgradesqlupgradesutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/util.py (11860 => 11861)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/util.py 2013-10-31 21:40:44 UTC (rev 11860)
+++ CalendarServer/branches/users/gaya/sharedgroupfixes/txdav/common/datastore/upgrade/sql/upgrades/util.py 2013-10-31 22:23:31 UTC (rev 11861)
</span><span class="lines">@@ -14,7 +14,7 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><del>-from twext.enterprise.dal.syntax import Select, Delete, Update
</del><ins>+from twext.enterprise.dal.syntax import Select, Delete, Update, Count
</ins><span class="cx"> from twext.python.log import Logger
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks, returnValue
</span><span class="cx"> from txdav.base.propertystore.base import PropertyName
</span><span class="lines">@@ -44,6 +44,21 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><ins>+def countProperty(txn, propelement):
+ pname = PropertyName.fromElement(propelement)
+
+ rp = schema.RESOURCE_PROPERTY
+ count = (yield Select(
+ [Count(rp.RESOURCE_ID), ],
+ From=rp,
+ Where=rp.NAME == pname.toString(),
+ ).on(txn))[0][0]
+
+ returnValue(count)
+
+
+
+@inlineCallbacks
</ins><span class="cx"> def cleanPropertyStore():
</span><span class="cx"> """
</span><span class="cx"> We have manually manipulated the SQL property store by-passing the underlying implementation's caching
</span><span class="lines">@@ -114,27 +129,43 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>-def doToEachHomeNotAtVersion(store, homeSchema, version, doIt):
</del><ins>+def doToEachHomeNotAtVersion(store, homeSchema, version, doIt, logStr, filterOwnerUID=None):
</ins><span class="cx"> """
</span><span class="cx"> Do something to each home whose version column indicates it is older
</span><del>- than the specified version. Do this in batches as there may be a lot of work to do.
</del><ins>+ than the specified version. Do this in batches as there may be a lot of work to do. Also,
+ allow the GUID to be filtered to support a parallel mode of operation.
</ins><span class="cx"> """
</span><span class="cx">
</span><ins>+ txn = store.newTransaction("updateDataVersion")
+ where = homeSchema.DATAVERSION < version
+ if filterOwnerUID:
+ where = where.And(homeSchema.OWNER_UID.StartsWith(filterOwnerUID))
+ total = (yield Select(
+ [Count(homeSchema.RESOURCE_ID), ],
+ From=homeSchema,
+ Where=where,
+ ).on(txn))[0][0]
+ yield txn.commit()
+ count = 0
+
</ins><span class="cx"> while True:
</span><span class="cx">
</span><ins>+ logUpgradeStatus(logStr, count, total)
+
</ins><span class="cx"> # Get the next home with an old version
</span><span class="cx"> txn = store.newTransaction("updateDataVersion")
</span><span class="cx"> try:
</span><span class="cx"> rows = yield Select(
</span><span class="cx"> [homeSchema.RESOURCE_ID, homeSchema.OWNER_UID, ],
</span><span class="cx"> From=homeSchema,
</span><del>- Where=homeSchema.DATAVERSION < version,
</del><ins>+ Where=where,
</ins><span class="cx"> OrderBy=homeSchema.OWNER_UID,
</span><span class="cx"> Limit=1,
</span><span class="cx"> ).on(txn)
</span><span class="cx">
</span><span class="cx"> if len(rows) == 0:
</span><span class="cx"> yield txn.commit()
</span><ins>+ logUpgradeStatus("End {}".format(logStr), count, total)
</ins><span class="cx"> returnValue(None)
</span><span class="cx">
</span><span class="cx"> # Apply to the home
</span><span class="lines">@@ -149,6 +180,26 @@
</span><span class="cx"> yield txn.commit()
</span><span class="cx"> except RuntimeError, e:
</span><span class="cx"> f = Failure()
</span><del>- log.error("Failed to upgrade %s to %s: %s" % (homeSchema, version, e))
</del><ins>+ logUpgradeError(
+ logStr,
+ "Failed to upgrade {} to {}: {}".format(homeSchema, version, e)
+ )
</ins><span class="cx"> yield txn.abort()
</span><span class="cx"> f.raiseException()
</span><ins>+
+ count += 1
+
+
+
+def logUpgradeStatus(title, count=None, total=None):
+ if total is None:
+ log.info("Database upgrade {title}", title=title)
+ else:
+ divisor = 1000 if total > 1000 else 100
+ if (divmod(count, divisor)[1] == 0) or (count == total):
+ log.info("Database upgrade {title}: {count} of {total}", title=title, count=count, total=total)
+
+
+
+def logUpgradeError(title, details):
+ log.error("Database upgrade {title} failed: {details}", title=title, details=details)
</ins></span></pre>
</div>
</div>
</body>
</html>