<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[15046] CalendarServer/branches/users/sredmond</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/15046">15046</a></dd>
<dt>Author</dt> <dd>sredmond@apple.com</dd>
<dt>Date</dt> <dd>2015-08-17 13:29:08 -0700 (Mon, 17 Aug 2015)</dd>
</dl>
<h3>Log Message</h3>
<pre>Creates private copy of (locally-modified) CalendarServer/trunk for client sim development</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestaccountscsv">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/accounts.csv</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestampsimpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtesticalpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestloggerpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestpopulationpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestprofilespy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestrequestdataOS_X_10_7user_list_principal_property_searchrequest">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsimpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadteststandardconfigseventsonlyplist">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/standard-configs/events-only.plist</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestwebadminpy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/webadmin.py</a></li>
<li><a href="#CalendarServerbranchesuserssredmondclientsimcontribperformancestatspy">CalendarServer/branches/users/sredmond/clientsim/contrib/performance/stats.py</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li>CalendarServer/branches/users/sredmond/clientsim/</li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServerbranchesuserssredmondclientsim"></a>
<div class="propset"><h4>Property changes: CalendarServer/branches/users/sredmond/clientsim</h4>
<pre class="diff"><span>
</span></pre></div>
<a id="svnignore"></a>
<div class="addfile"><h4>Added: svn:ignore</h4></div>
<span class="cx">.settings
</span><span class="cx">.project
</span><span class="cx">.pydevproject
</span><span class="cx">*.egg-info
</span><span class="cx">*.pyc
</span><span class="cx">data
</span><span class="cx">build
</span><span class="cx">subprojects
</span><span class="cx">_run
</span><a id="svnmergeinfo"></a>
<div class="addfile"><h4>Added: svn:mergeinfo</h4></div>
<span class="cx">/CalendarServer/branches/config-separation:4379-4443
</span><span class="cx">/CalendarServer/branches/egg-info-351:4589-4625
</span><span class="cx">/CalendarServer/branches/generic-sqlstore:6167-6191
</span><span class="cx">/CalendarServer/branches/new-store:5594-5934
</span><span class="cx">/CalendarServer/branches/new-store-no-caldavfile:5911-5935
</span><span class="cx">/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
</span><span class="cx">/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
</span><span class="cx">/CalendarServer/branches/release/CalendarServer-5.1-dev:11846
</span><span class="cx">/CalendarServer/branches/release/CalendarServer-5.2-dev:11972,12357-12358,12794,12814
</span><span class="cx">/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
</span><span class="cx">/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
</span><span class="cx">/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
</span><span class="cx">/CalendarServer/branches/users/cdaboo/cross-pod-sharing:12038-12191
</span><span class="cx">/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
</span><span class="cx">/CalendarServer/branches/users/cdaboo/fix-no-ischedule:11607-11871
</span><span class="cx">/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
</span><span class="cx">/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
</span><span class="cx">/CalendarServer/branches/users/cdaboo/json:11622-11912
</span><span class="cx">/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
</span><span class="cx">/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
</span><span class="cx">/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
</span><span class="cx">/CalendarServer/branches/users/cdaboo/performance-tweaks:11824-11836
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pod2pod-migration:14338-14520
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pods:7297-7377
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
</span><span class="cx">/CalendarServer/branches/users/cdaboo/pycard:7227-7237
</span><span class="cx">/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
</span><span class="cx">/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
</span><span class="cx">/CalendarServer/branches/users/cdaboo/reverse-proxy-pods:11875-11900
</span><span class="cx">/CalendarServer/branches/users/cdaboo/scheduling-queue-refresh:11783-12557
</span><span class="cx">/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
</span><span class="cx">/CalendarServer/branches/users/cdaboo/sharing-in-the-store:11935-12016
</span><span class="cx">/CalendarServer/branches/users/cdaboo/store-scheduling:10876-11129
</span><span class="cx">/CalendarServer/branches/users/cdaboo/timezones:7443-7699
</span><span class="cx">/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
</span><span class="cx">/CalendarServer/branches/users/gaya/cleanrevisions:12152-12334
</span><span class="cx">/CalendarServer/branches/users/gaya/groupsharee2:13669-13773
</span><span class="cx">/CalendarServer/branches/users/gaya/sharedgroupfixes:12120-12142
</span><span class="cx">/CalendarServer/branches/users/gaya/sharedgroups-3:11088-11204
</span><span class="cx">/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
</span><span class="cx">/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
</span><span class="cx">/CalendarServer/branches/users/glyph/conn-limit:6574-6577
</span><span class="cx">/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
</span><span class="cx">/CalendarServer/branches/users/glyph/dalify:6932-7023
</span><span class="cx">/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
</span><span class="cx">/CalendarServer/branches/users/glyph/deploybuild:7563-7572
</span><span class="cx">/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
</span><span class="cx">/CalendarServer/branches/users/glyph/disable-quota:7718-7727
</span><span class="cx">/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
</span><span class="cx">/CalendarServer/branches/users/glyph/enforce-max-requests:11640-11643
</span><span class="cx">/CalendarServer/branches/users/glyph/hang-fix:11465-11491
</span><span class="cx">/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
</span><span class="cx">/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
</span><span class="cx">/CalendarServer/branches/users/glyph/launchd-wrapper-bis:11413-11436
</span><span class="cx">/CalendarServer/branches/users/glyph/linux-tests:6893-6900
</span><span class="cx">/CalendarServer/branches/users/glyph/log-cleanups:11691-11731
</span><span class="cx">/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
</span><span class="cx">/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
</span><span class="cx">/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
</span><span class="cx">/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
</span><span class="cx">/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
</span><span class="cx">/CalendarServer/branches/users/glyph/new-export:7444-7485
</span><span class="cx">/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
</span><span class="cx">/CalendarServer/branches/users/glyph/oracle:7106-7155
</span><span class="cx">/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
</span><span class="cx">/CalendarServer/branches/users/glyph/other-html:8062-8091
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
</span><span class="cx">/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
</span><span class="cx">/CalendarServer/branches/users/glyph/q:9560-9688
</span><span class="cx">/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
</span><span class="cx">/CalendarServer/branches/users/glyph/quota:7604-7637
</span><span class="cx">/CalendarServer/branches/users/glyph/sendfdport:5388-5424
</span><span class="cx">/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
</span><span class="cx">/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
</span><span class="cx">/CalendarServer/branches/users/glyph/sharedpool:6490-6550
</span><span class="cx">/CalendarServer/branches/users/glyph/sharing-api:9192-9205
</span><span class="cx">/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
</span><span class="cx">/CalendarServer/branches/users/glyph/sql-store:5929-6073
</span><span class="cx">/CalendarServer/branches/users/glyph/start-service-start-loop:11060-11065
</span><span class="cx">/CalendarServer/branches/users/glyph/subtransactions:7248-7258
</span><span class="cx">/CalendarServer/branches/users/glyph/table-alias:8651-8664
</span><span class="cx">/CalendarServer/branches/users/glyph/uidexport:7673-7676
</span><span class="cx">/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
</span><span class="cx">/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
</span><span class="cx">/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
</span><span class="cx">/CalendarServer/branches/users/glyph/warning-cleanups:11347-11357
</span><span class="cx">/CalendarServer/branches/users/glyph/whenNotProposed:11881-11897
</span><span class="cx">/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
</span><span class="cx">/CalendarServer/branches/users/sagen/applepush:8126-8184
</span><span class="cx">/CalendarServer/branches/users/sagen/inboxitems:7380-7381
</span><span class="cx">/CalendarServer/branches/users/sagen/locations-resources:5032-5051
</span><span class="cx">/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
</span><span class="cx">/CalendarServer/branches/users/sagen/move2who:12819-12860
</span><span class="cx">/CalendarServer/branches/users/sagen/move2who-2:12861-12898
</span><span class="cx">/CalendarServer/branches/users/sagen/move2who-3:12899-12913
</span><span class="cx">/CalendarServer/branches/users/sagen/move2who-4:12914-13157
</span><span class="cx">/CalendarServer/branches/users/sagen/move2who-5:13158-13163
</span><span class="cx">/CalendarServer/branches/users/sagen/newcua:13309-13327
</span><span class="cx">/CalendarServer/branches/users/sagen/newcua-1:13328-13330
</span><span class="cx">/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
</span><span class="cx">/CalendarServer/branches/users/sagen/recordtypes:13648-13656
</span><span class="cx">/CalendarServer/branches/users/sagen/recordtypes-2:13657
</span><span class="cx">/CalendarServer/branches/users/sagen/request-socket:14748-14767
</span><span class="cx">/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
</span><span class="cx">/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
</span><span class="cx">/CalendarServer/branches/users/sagen/resources-2:5084-5093
</span><span class="cx">/CalendarServer/branches/users/sagen/testing:10827-10851,10853-10855
</span><span class="cx">/CalendarServer/branches/users/sagen/trashcan:14185-14269
</span><span class="cx">/CalendarServer/branches/users/sagen/trashcan-2:14270-14324
</span><span class="cx">/CalendarServer/branches/users/sagen/trashcan-3:14325-14450
</span><span class="cx">/CalendarServer/branches/users/sagen/trashcan-4:14451-14471
</span><span class="cx">/CalendarServer/branches/users/sagen/trashcan-5:14471-14555
</span><span class="cx">/CalendarServer/branches/users/wsanchez/psycopg2cffi:14427-14439
</span><span class="cx">/CalendarServer/branches/users/wsanchez/transations:5515-5593
</span><a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestaccountscsv"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/accounts.csv (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/accounts.csv 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/accounts.csv 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -96,4 +96,4 @@
</span><span class="cx"> user96,user96,User 96,user96@example.com,10000000-0000-0000-0000-000000000096
</span><span class="cx"> user97,user97,User 97,user97@example.com,10000000-0000-0000-0000-000000000097
</span><span class="cx"> user98,user98,User 98,user98@example.com,10000000-0000-0000-0000-000000000098
</span><del>-user99,user99,User 99,user99@example.com,10000000-0000-0000-0000-000000000099
</del><ins>+user99,user99,User 99,user99@example.com,10000000-0000-0000-0000-000000000099
</ins><span class="cx">\ No newline at end of file
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestampsimpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/ampsim.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ampsim.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -56,7 +56,8 @@
</span><span class="cx">
</span><span class="cx"> from twext.enterprise.adbapi2 import Pickle
</span><span class="cx">
</span><del>-from contrib.performance.loadtest.sim import _DirectoryRecord, LoadSimulator
</del><ins>+from contrib.performance.loadtest.sim import LoadSimulator
+from contrib.performance.loadtest.records import DirectoryRecord
</ins><span class="cx">
</span><span class="cx"> class Configure(Command):
</span><span class="cx"> """
</span><span class="lines">@@ -79,7 +80,7 @@
</span><span class="cx">
</span><span class="cx"> class Account(Command):
</span><span class="cx"> """
</span><del>- This message represents a L{_DirectoryRecord} loaded by the manager process
</del><ins>+ This message represents a L{DirectoryRecord} loaded by the manager process
</ins><span class="cx"> being relayed to a worker.
</span><span class="cx"> """
</span><span class="cx"> arguments = [
</span><span class="lines">@@ -106,7 +107,7 @@
</span><span class="cx">
</span><span class="cx"> @Account.responder
</span><span class="cx"> def account(self, **kw):
</span><del>- self.records.append(_DirectoryRecord(**kw))
</del><ins>+ self.records.append(DirectoryRecord(**kw))
</ins><span class="cx"> return {}
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtesticalpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/ical.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/ical.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -18,40 +18,28 @@
</span><span class="cx">
</span><span class="cx"> from caldavclientlibrary.protocol.caldav.definitions import caldavxml
</span><span class="cx"> from caldavclientlibrary.protocol.caldav.definitions import csxml
</span><del>-from caldavclientlibrary.protocol.url import URL
</del><span class="cx"> from caldavclientlibrary.protocol.webdav.definitions import davxml
</span><del>-from caldavclientlibrary.protocol.webdav.propfindparser import PropFindParser
</del><ins>+from caldavclientlibrary.protocol.url import URL
</ins><span class="cx">
</span><del>-from calendarserver.push.amppush import subscribeToIDs
-from calendarserver.tools.notifications import PubSubClientFactory
-
-from contrib.performance.httpauth import AuthHandlerAgent
-from contrib.performance.httpclient import StringProducer, readBody
</del><ins>+from contrib.performance.httpclient import readBody
</ins><span class="cx"> from contrib.performance.loadtest.subscribe import Periodical
</span><ins>+from contrib.performance.loadtest.resources import Event, Calendar
+from contrib.performance.loadtest.requester import Requester, IncorrectResponseCode
+from contrib.performance.loadtest.push import PushMonitor
</ins><span class="cx">
</span><span class="cx"> from pycalendar.datetime import DateTime
</span><span class="cx"> from pycalendar.duration import Duration
</span><span class="cx"> from pycalendar.timezone import Timezone
</span><span class="cx">
</span><del>-from twext.internet.adaptendpoint import connect
-from twext.internet.gaiendpoint import GAIEndpoint
-from twisted.internet.ssl import ClientContextFactory
-
-from twisted.internet.defer import Deferred, inlineCallbacks, returnValue, \
- succeed
</del><span class="cx"> from twisted.internet.task import LoopingCall
</span><del>-from twisted.python.filepath import FilePath
-from twisted.python.log import addObserver, err, msg
-from twisted.python.util import FancyEqMixin
-from twisted.web.client import Agent, ContentDecoderAgent, GzipDecoder, \
- _DeprecatedToCurrentPolicyForHTTPS
-from twisted.web.http import OK, MULTI_STATUS, CREATED, NO_CONTENT, PRECONDITION_FAILED, MOVED_PERMANENTLY, \
- FORBIDDEN, FOUND
</del><ins>+from twisted.internet.defer import succeed, Deferred, inlineCallbacks, returnValue
+from twisted.python.log import err, msg
+from twisted.web.http import OK, MULTI_STATUS, CREATED, NO_CONTENT, FORBIDDEN, PRECONDITION_FAILED, MOVED_PERMANENTLY, FOUND
</ins><span class="cx"> from twisted.web.http_headers import Headers
</span><span class="cx">
</span><span class="cx"> from twistedcaldav.ical import Component, Property
</span><span class="cx">
</span><del>-from urlparse import urlparse, urlunparse, urlsplit, urljoin
</del><ins>+from urlparse import urlparse, urlsplit
</ins><span class="cx"> from uuid import uuid4
</span><span class="cx"> from xml.etree import ElementTree
</span><span class="cx">
</span><span class="lines">@@ -59,193 +47,32 @@
</span><span class="cx"> import os
</span><span class="cx"> import random
</span><span class="cx">
</span><del>-ElementTree.QName.__repr__ = lambda self: '<QName %r>' % (self.text,)
</del><ins>+"""
+run
+ startup
+ deserialize
+ _startupPropfindWellKnown
+ _principalPropfindInitial
+ _extractPrincipalDetails
+ _checkCalendarsForEvents
+"""
</ins><span class="cx">
</span><del>-def loadRequestBody(clientType, label):
- return FilePath(__file__).sibling('request-data').child(clientType).child(label + '.request').getContent()
</del><span class="cx">
</span><ins>+ElementTree.QName.__repr__ = lambda self: '<QName %r>' % (self.text,)
</ins><span class="cx">
</span><del>-SUPPORTED_REPORT_SET = '{DAV:}supported-report-set'
</del><ins>+SUPPORTED_REPORT_SET = davxml.supported_report_set.text
</ins><span class="cx">
</span><del>-class IncorrectResponseCode(Exception):
- """
- Raised when a response has a code other than the one expected.
</del><ins>+class Attendee(Property):
+ def __init__(self, ):
+ pass
</ins><span class="cx">
</span><del>- @ivar expected: The response codes which was expected.
- @type expected: C{tuple} of C{int}
</del><span class="cx">
</span><del>- @ivar response: The response which was received
- @type response: L{twisted.web.client.Response}
- """
- def __init__(self, expected, response):
- self.expected = expected
- self.response = response
-
-
-
</del><span class="cx"> class MissingCalendarHome(Exception):
</span><span class="cx"> """
</span><span class="cx"> Raised when the calendar home for a user is 404
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx">
</span><del>-
-class XMPPPush(object, FancyEqMixin):
- """
- This represents an XMPP PubSub location where push notifications for
- particular calendar home might be received.
- """
- compareAttributes = ('server', 'uri', 'pushkey')
-
- def __init__(self, server, uri, pushkey):
- self.server = server
- self.uri = uri
- self.pushkey = pushkey
-
-
-
-def u2str(data):
- return data.encode("utf-8") if type(data) is unicode else data
-
-
-
-class Event(object):
- def __init__(self, serializeBasePath, url, etag, component=None):
- self.serializeBasePath = serializeBasePath
- self.url = url
- self.etag = etag
- self.scheduleTag = None
- if component is not None:
- self.component = component
- self.uid = component.resourceUID() if component is not None else None
-
-
- def getUID(self):
- """
- Return the UID of the calendar resource.
- """
- return self.uid
-
-
- def serializePath(self):
- if self.serializeBasePath:
- calendar = os.path.join(self.serializeBasePath, self.url.split("/")[-2])
- if not os.path.exists(calendar):
- os.makedirs(calendar)
- return os.path.join(calendar, self.url.split("/")[-1])
- else:
- return None
-
-
- def serialize(self):
- """
- Create a dict of the data so we can serialize as JSON.
- """
-
- result = {}
- for attr in ("url", "etag", "scheduleTag", "uid",):
- result[attr] = getattr(self, attr)
- return result
-
-
- @staticmethod
- def deserialize(serializeLocation, data):
- """
- Convert dict (deserialized from JSON) into an L{Event}.
- """
-
- event = Event(serializeLocation, None, None)
- for attr in ("url", "etag", "scheduleTag", "uid",):
- setattr(event, attr, u2str(data[attr]))
- return event
-
-
- @property
- def component(self):
- """
- Data always read from disk - never cached in the object.
- """
- path = self.serializePath()
- if path and os.path.exists(path):
- f = open(path)
- comp = Component.fromString(f.read())
- f.close()
- return comp
- else:
- return None
-
-
- @component.setter
- def component(self, component):
- """
- Data always written to disk - never cached on the object.
- """
- path = self.serializePath()
- if path:
- if component is None:
- os.remove(path)
- else:
- f = open(path, "w")
- f.write(str(component))
- f.close()
- self.uid = component.resourceUID() if component is not None else None
-
-
- def removed(self):
- """
- Resource no longer exists on the server - remove associated data.
- """
- path = self.serializePath()
- if path and os.path.exists(path):
- os.remove(path)
-
-
-
-class Calendar(object):
- def __init__(self, resourceType, componentTypes, name, url, changeToken):
- self.resourceType = resourceType
- self.componentTypes = componentTypes
- self.name = name
- self.url = url
- self.changeToken = changeToken
- self.events = {}
-
-
- def serialize(self):
- """
- Create a dict of the data so we can serialize as JSON.
- """
-
- result = {}
- for attr in ("resourceType", "name", "url", "changeToken"):
- result[attr] = getattr(self, attr)
- result["componentTypes"] = list(sorted(self.componentTypes))
- result["events"] = sorted(self.events.keys())
- return result
-
-
- @staticmethod
- def deserialize(data, events):
- """
- Convert dict (deserialized from JSON) into an L{Calendar}.
- """
-
- calendar = Calendar(None, None, None, None, None)
- for attr in ("resourceType", "name", "url", "changeToken"):
- setattr(calendar, attr, u2str(data[attr]))
- calendar.componentTypes = set(map(u2str, data["componentTypes"]))
-
- for event in data["events"]:
- url = urljoin(calendar.url, event)
- if url in events:
- calendar.events[event] = events[url]
- else:
- # Ughh - an event is missing - force changeToken to empty to trigger full resync
- calendar.changeToken = ""
- return calendar
-
-
-
</del><span class="cx"> class BaseClient(object):
</span><span class="cx"> """
</span><span class="cx"> Base interface for all simulated clients.
</span><span class="lines">@@ -259,7 +86,7 @@
</span><span class="cx"> _client_id = None # Unique id for the client itself
</span><span class="cx">
</span><span class="cx">
</span><del>- def _setEvent(self, href, event):
</del><ins>+ def _cacheEvent(self, href, event):
</ins><span class="cx"> """
</span><span class="cx"> Cache the provided event
</span><span class="cx"> """
</span><span class="lines">@@ -268,7 +95,7 @@
</span><span class="cx"> self._calendars[calendar + '/'].events[basePath] = event
</span><span class="cx">
</span><span class="cx">
</span><del>- def _removeEvent(self, href):
</del><ins>+ def _invalidateEvent(self, href):
</ins><span class="cx"> """
</span><span class="cx"> Remove event from local cache.
</span><span class="cx"> """
</span><span class="lines">@@ -278,6 +105,21 @@
</span><span class="cx"> del self._calendars[calendar + '/'].events[basePath]
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def _cacheCalendar(self, href, calendar):
+ """
+ Cache the provided L{Calendar}
+ """
+ self._calendars[href] = calendar
+
+
+ def _invalidateCalendar(self, href):
+ """
+ Remove calendar from the local cache
+ """
+ if href in self._calendars:
+ del self._calendars[href]
+
+
</ins><span class="cx"> def addEvent(self, href, calendar):
</span><span class="cx"> """
</span><span class="cx"> Called when a profile needs to add an event (no scheduling).
</span><span class="lines">@@ -321,46 +163,29 @@
</span><span class="cx"> """
</span><span class="cx"> raise NotImplementedError("%r does not implement changeEventAttendee" % (self.__class__,))
</span><span class="cx">
</span><ins>+ def addCalendar(self, href, calendar):
+ """
+ Called when a profile needs to add a new calendar.
+ """
+ raise NotImplementedError("%r does not implement addCalendar" % (self.__class__,))
</ins><span class="cx">
</span><ins>+ def changeCalendar(self, href, calendar):
+ """
+ Called when a profile needs to change a calendar.
+ """
+ raise NotImplementedError("%r does not implement changeCalendar" % (self.__class__,))
</ins><span class="cx">
</span><del>-class _PubSubClientFactory(PubSubClientFactory):
- """
- Factory for XMPP pubsub functionality.
- """
- def __init__(self, client, *args, **kwargs):
- PubSubClientFactory.__init__(self, *args, **kwargs)
- self._client = client
</del><span class="cx">
</span><ins>+ def deleteCalendar(self, href):
+ """
+ Called when a profile needs to delete a calendar.
+ """
+ raise NotImplementedError("%r does not implement deleteCalendar" % (self.__class__,))
</ins><span class="cx">
</span><del>- def initFailed(self, reason):
- print('XMPP initialization failed', reason)
</del><span class="cx">
</span><span class="cx">
</span><del>- def authFailed(self, reason):
- print('XMPP Authentication failed', reason)
</del><span class="cx">
</span><span class="cx">
</span><del>- def handleMessageEventItems(self, iq):
- item = iq.firstChildElement().firstChildElement()
- if item:
- node = item.getAttribute("node")
- if node:
- url, _ignore_name, _ignore_kind = self.nodes.get(node, (None, None, None))
- if url is not None:
- self._client._checkCalendarsForEvents(url, push=True)
-
-
-
-class WebClientContextFactory(ClientContextFactory):
- """
- A web context factory which ignores the hostname and port and does no
- certificate verification.
- """
- def getContext(self, hostname, port):
- return ClientContextFactory.getContext(self)
-
-
-
</del><span class="cx"> class BaseAppleClient(BaseClient):
</span><span class="cx"> """
</span><span class="cx"> Implementation of common OS X/iOS client behavior.
</span><span class="lines">@@ -368,7 +193,8 @@
</span><span class="cx">
</span><span class="cx"> _client_type = "Generic"
</span><span class="cx">
</span><del>- USER_AGENT = None # Override this for specific clients
</del><ins>+ # Override this for specific clients
+ USER_AGENT = None
</ins><span class="cx">
</span><span class="cx"> # The default interval, used if none is specified in external
</span><span class="cx"> # configuration.
</span><span class="lines">@@ -380,9 +206,6 @@
</span><span class="cx"> # Override and turn on if client supports Sync REPORT
</span><span class="cx"> _SYNC_REPORT = False
</span><span class="cx">
</span><del>- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = False
-
</del><span class="cx"> # Override and turn off if client does not support attendee lookups
</span><span class="cx"> _ATTENDEE_LOOKUPS = True
</span><span class="cx">
</span><span class="lines">@@ -416,32 +239,23 @@
</span><span class="cx"> self,
</span><span class="cx"> reactor,
</span><span class="cx"> root,
</span><del>- principalPathTemplate,
</del><span class="cx"> serializePath,
</span><span class="cx"> record,
</span><span class="cx"> auth,
</span><span class="cx"> title=None,
</span><span class="cx"> calendarHomePollInterval=None,
</span><del>- supportPush=True,
</del><span class="cx"> supportAmpPush=True,
</span><span class="cx"> ampPushHost=None,
</span><span class="cx"> ampPushPort=62311,
</span><span class="cx"> ):
</span><del>-
</del><span class="cx"> self._client_id = str(uuid4())
</span><span class="cx">
</span><span class="cx"> self.reactor = reactor
</span><span class="cx">
</span><del>- # The server might use gzip encoding
- agent = Agent(
- self.reactor,
- contextFactory=_DeprecatedToCurrentPolicyForHTTPS(WebClientContextFactory()),
</del><ins>+ self.requester = Requester(
+ root, self.getDefaultHeaders(), title,
+ record.uid, self._client_id, auth, self.reactor
</ins><span class="cx"> )
</span><del>- agent = ContentDecoderAgent(agent, [("gzip", GzipDecoder)])
- self.agent = AuthHandlerAgent(agent, auth)
-
- self.root = root
- self.principalPathTemplate = principalPathTemplate
</del><span class="cx"> self.record = record
</span><span class="cx">
</span><span class="cx"> self.title = title if title else self._client_type
</span><span class="lines">@@ -450,208 +264,62 @@
</span><span class="cx"> calendarHomePollInterval = self.CALENDAR_HOME_POLL_INTERVAL
</span><span class="cx"> self.calendarHomePollInterval = calendarHomePollInterval
</span><span class="cx">
</span><del>- self.supportPush = supportPush
</del><ins>+ if supportAmpPush:
+ if ampPushHost is None:
+ ampPushHost = urlparse(root)[1].split(":")[0]
+ self.monitor = PushMonitor(self.reactor, ampPushHost, ampPushPort, self.updateCalendarHomeFromPush)
+ else:
+ self.monitor = None
</ins><span class="cx">
</span><del>- self.supportAmpPush = supportAmpPush
- if ampPushHost is None:
- ampPushHost = urlparse(self.root)[1].split(":")[0]
- self.ampPushHost = ampPushHost
- self.ampPushPort = ampPushPort
-
</del><span class="cx"> self.serializePath = serializePath
</span><span class="cx">
</span><span class="cx"> self.supportSync = self._SYNC_REPORT
</span><span class="cx">
</span><del>- # Keep track of the calendars on this account, keys are
- # Calendar URIs, values are Calendar instances.
- self._calendars = {}
-
</del><span class="cx"> # The principalURL found during discovery
</span><span class="cx"> self.principalURL = None
</span><span class="cx">
</span><span class="cx"> # The principal collection found during startup
</span><span class="cx"> self.principalCollection = None
</span><span class="cx">
</span><ins>+ # Keep track of the calendars on this account, keys are
+ # Calendar URIs, values are Calendar instances.
+ self._calendars = {}
+
</ins><span class="cx"> # Keep track of the events on this account, keys are event
</span><span class="cx"> # URIs (which are unambiguous across different calendars
</span><span class="cx"> # because they start with the uri of the calendar they are
</span><span class="cx"> # part of), values are Event instances.
</span><span class="cx"> self._events = {}
</span><span class="cx">
</span><del>- # Keep track of which calendar homes are being polled
- self._checking = set()
-
- # Keep track of XMPP parameters for calendar homes we encounter. This
- # dictionary has calendar home URLs as keys and XMPPPush instances as
- # values.
- self.xmpp = {}
-
- self.ampPushKeys = {}
-
- # Keep track of push factories so we can unsubscribe at shutdown
- self._pushFactories = []
-
</del><span class="cx"> # Allow events to go out into the world.
</span><span class="cx"> self.catalog = {
</span><span class="cx"> "eventChanged": Periodical(),
</span><span class="cx"> }
</span><span class="cx">
</span><ins>+ self._checking = set()
</ins><span class="cx">
</span><del>- def _addDefaultHeaders(self, headers):
- """
- Add the clients default set of headers to ones being used in a request.
- Default is to add User-Agent, sub-classes should override to add other
- client specific things, Accept etc.
- """
- headers.setRawHeaders('User-Agent', [self.USER_AGENT])
-
-
- @inlineCallbacks
- def _request(self, expectedResponseCodes, method, url, headers=None, body=None, method_label=None):
- """
- Execute a request and check against the expected response codes.
- """
- if type(expectedResponseCodes) is int:
- expectedResponseCodes = (expectedResponseCodes,)
- if headers is None:
- headers = Headers({})
- self._addDefaultHeaders(headers)
- msg(
- type="request",
- method=method_label if method_label else method,
- url=url,
- user=self.record.uid,
- client_type=self.title,
- client_id=self._client_id,
- )
-
- before = self.reactor.seconds()
- response = yield self.agent.request(method, url, headers, body)
-
- # XXX This is time to receive response headers, not time
- # to receive full response. Should measure the latter, if
- # not both.
- after = self.reactor.seconds()
-
- success = response.code in expectedResponseCodes
-
- msg(
- type="response",
- success=success,
- method=method_label if method_label else method,
- headers=headers,
- body=body,
- code=response.code,
- user=self.record.uid,
- client_type=self.title,
- client_id=self._client_id,
- duration=(after - before),
- url=url,
- )
-
- if success:
- returnValue(response)
-
- raise IncorrectResponseCode(expectedResponseCodes, response)
-
-
- def _parseMultiStatus(self, response, otherTokens=False):
- """
- Parse a <multistatus> - might need to return other top-level elements
- in the response - e.g. DAV:sync-token
- I{PROPFIND} request for the principal URL.
-
- @type response: C{str}
- @rtype: C{cls}
- """
- parser = PropFindParser()
- parser.parseData(response)
- if otherTokens:
- return (parser.getResults(), parser.getOthers(),)
- else:
- return parser.getResults()
-
</del><span class="cx"> _CALENDAR_TYPES = set([
</span><span class="cx"> caldavxml.calendar,
</span><span class="cx"> caldavxml.schedule_inbox,
</span><span class="cx"> ])
</span><span class="cx">
</span><del>- @inlineCallbacks
- def _propfind(self, url, body, depth='0', allowedStatus=(MULTI_STATUS,), method_label=None):
- """
- Issue a PROPFIND on the chosen URL
- """
- hdrs = Headers({'content-type': ['text/xml']})
- if depth is not None:
- hdrs.addRawHeader('depth', depth)
- response = yield self._request(
- allowedStatus,
- 'PROPFIND',
- self.root + url.encode('utf-8'),
- hdrs,
- StringProducer(body),
- method_label=method_label,
- )
</del><ins>+ def getDefaultHeaders(self):
+ return {
+ 'User-Agent': [self.USER_AGENT],
+ 'Accept': ['*/*'],
+ 'Accept-Language': ['en-us'],
+ 'Accept-Encoding': ['gzip,deflate'],
+ 'Connection': ['keep-alive']
+ }
</ins><span class="cx">
</span><del>- body = yield readBody(response)
- result = self._parseMultiStatus(body) if response.code == MULTI_STATUS else None
-
- returnValue((response, result,))
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def _proppatch(self, url, body, method_label=None):
- """
- Issue a PROPPATCH on the chosen URL
- """
- hdrs = Headers({'content-type': ['text/xml']})
- response = yield self._request(
- (OK, MULTI_STATUS,),
- 'PROPPATCH',
- self.root + url.encode('utf-8'),
- hdrs,
- StringProducer(body),
- method_label=method_label,
- )
- if response.code == MULTI_STATUS:
- body = yield readBody(response)
- result = self._parseMultiStatus(body)
- returnValue(result)
- else:
- returnValue(None)
-
-
- @inlineCallbacks
- def _report(self, url, body, depth='0', allowedStatus=(MULTI_STATUS,), otherTokens=False, method_label=None):
- """
- Issue a REPORT on the chosen URL
- """
- hdrs = Headers({'content-type': ['text/xml']})
- if depth is not None:
- hdrs.addRawHeader('depth', depth)
- response = yield self._request(
- allowedStatus,
- 'REPORT',
- self.root + url.encode('utf-8'),
- hdrs,
- StringProducer(body),
- method_label=method_label,
- )
-
- body = yield readBody(response)
- result = self._parseMultiStatus(body, otherTokens) if response.code == MULTI_STATUS else None
-
- returnValue(result)
-
-
- @inlineCallbacks
</del><span class="cx"> def _startupPropfindWellKnown(self):
</span><span class="cx"> """
</span><span class="cx"> Issue a PROPFIND on the /.well-known/caldav/ URL
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> location = "/.well-known/caldav/"
</span><del>- response, result = yield self._propfind(
</del><ins>+ response, result = yield self.requester.propfind(
</ins><span class="cx"> location,
</span><span class="cx"> self._STARTUP_WELL_KNOWN,
</span><span class="cx"> allowedStatus=(MULTI_STATUS, MOVED_PERMANENTLY, FOUND,),
</span><span class="lines">@@ -662,7 +330,7 @@
</span><span class="cx"> if response.code in (MOVED_PERMANENTLY, FOUND,):
</span><span class="cx"> location = response.headers.getRawHeaders("location")[0]
</span><span class="cx"> location = urlsplit(location)[2]
</span><del>- response, result = yield self._propfind(
</del><ins>+ response, result = yield self.requester.propfind(
</ins><span class="cx"> location,
</span><span class="cx"> self._STARTUP_WELL_KNOWN,
</span><span class="cx"> allowedStatus=(MULTI_STATUS),
</span><span class="lines">@@ -678,8 +346,8 @@
</span><span class="cx"> Issue a PROPFIND on the /principals/users/<uid> URL to retrieve
</span><span class="cx"> the /principals/__uids__/<guid> principal URL
</span><span class="cx"> """
</span><del>- principalPath = self.principalPathTemplate % (user,)
- _ignore_response, result = yield self._propfind(
</del><ins>+ principalPath = '/principals/users/%s' % (user,)
+ _ignore_response, result = yield self.requester.propfind(
</ins><span class="cx"> principalPath,
</span><span class="cx"> self._STARTUP_PRINCIPAL_PROPFIND_INITIAL,
</span><span class="cx"> method_label="PROPFIND{find-principal}",
</span><span class="lines">@@ -694,7 +362,7 @@
</span><span class="cx"> user and return a L{Principal} instance constructed from the
</span><span class="cx"> response.
</span><span class="cx"> """
</span><del>- _ignore_response, result = yield self._propfind(
</del><ins>+ _ignore_response, result = yield self.requester.propfind(
</ins><span class="cx"> self.principalURL,
</span><span class="cx"> self._STARTUP_PRINCIPAL_PROPFIND,
</span><span class="cx"> method_label="PROPFIND{principal}",
</span><span class="lines">@@ -706,7 +374,7 @@
</span><span class="cx"> """
</span><span class="cx"> Issue a principal-search-property-set REPORT against the chosen URL
</span><span class="cx"> """
</span><del>- return self._report(
</del><ins>+ return self.requester.report(
</ins><span class="cx"> principalCollectionSet,
</span><span class="cx"> self._STARTUP_PRINCIPALS_REPORT,
</span><span class="cx"> allowedStatus=(OK,),
</span><span class="lines">@@ -715,28 +383,14 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def _calendarHomePropfind(self, calendarHomeSet):
- """
- Do the poll Depth:1 PROPFIND on the calendar home.
- """
- if not calendarHomeSet.endswith('/'):
- calendarHomeSet = calendarHomeSet + '/'
- _ignore_response, result = yield self._propfind(
- calendarHomeSet,
- self._POLL_CALENDARHOME_PROPFIND,
- depth='1',
- method_label="PROPFIND{home}",
- )
- calendars = self._extractCalendars(result, calendarHomeSet)
- returnValue((calendars, result,))
-
-
- @inlineCallbacks
</del><span class="cx"> def _extractPrincipalDetails(self):
</span><span class="cx"> # Using the actual principal URL, retrieve principal information
</span><ins>+ # XXX We could be recording more information here
</ins><span class="cx"> principal = yield self._principalPropfind()
</span><span class="cx">
</span><span class="cx"> hrefs = principal.getHrefProperties()
</span><ins>+ # from pprint import pprint
+ # pprint(hrefs)
</ins><span class="cx">
</span><span class="cx"> # Remember our outbox and ignore notifications
</span><span class="cx"> self.outbox = hrefs[caldavxml.schedule_outbox_URL].toString()
</span><span class="lines">@@ -764,14 +418,114 @@
</span><span class="cx">
</span><span class="cx"> returnValue(principal)
</span><span class="cx">
</span><ins>+ def startup(self):
+ """
+ Overridden by subclasses of BaseAppleClient.
+ """
+ raise NotImplementedError
</ins><span class="cx">
</span><ins>+ def calendarCheckLoop(self, calendarHome):
+ """
+ Periodically check the calendar home for changes to calendars.
+ """
+ pollCalendarHome = LoopingCall(
+ self.checkCalendarsForEvents, calendarHome)
+ return pollCalendarHome.start(self.calendarHomePollInterval, now=False)
+
+ @inlineCallbacks
+ def updateCalendarHomeFromPush(self, calendarHomeSet):
+ """
+ Emulate the client behavior upon receiving a notification that the
+ given calendar home has changed.
+ """
+ # Todo - ensure that the self._checking set is properly cleared even if there is an error
+ self._checking.add(calendarHomeSet)
+ result = yield self._poll(calendarHomeSet, firstTime=False)
+
+ # Todo - should this be a returnValue?
+ yield self._newOperation("push", result)
+
+ @inlineCallbacks
+ def checkCalendarsForEvents(self, calendarHomeSet, firstTime=False):
+ """
+ The actions a client does when polling for changes, or in response to a
+ push notification of a change. There are some actions done on the first poll
+ we should emulate.
+ """
+
+ result = True
+ try:
+ result = yield self._newOperation("poll", self._poll(calendarHomeSet, firstTime))
+ finally:
+ if result:
+ try:
+ self._checking.remove(calendarHomeSet)
+ except KeyError:
+ pass
+ returnValue(result)
+
+ """
+ REFRESH UTILITIES
+ """
+
+ @inlineCallbacks
+ def _poll(self, calendarHomeSet, firstTime):
+ if calendarHomeSet in self._checking:
+ returnValue(False)
+ self._checking.add(calendarHomeSet)
+
+ calendars, results = yield self._calendarHomePropfind(calendarHomeSet)
+
+ # First time operations
+ if firstTime:
+ yield self._pollFirstTime1(results[calendarHomeSet], calendars)
+
+ # Normal poll
+ for cal in calendars:
+ newToken = cal.changeToken
+ if cal.url not in self._calendars:
+ # Calendar seen for the first time - reload it
+ self._calendars[cal.url] = cal
+ cal.changeToken = ""
+ yield self._updateCalendar(self._calendars[cal.url], newToken)
+ elif self._calendars[cal.url].changeToken != newToken:
+ # Calendar changed - reload it
+ yield self._updateCalendar(self._calendars[cal.url], newToken)
+
+ # When there is no sync REPORT, clients have to do a full PROPFIND
+ # on the notification collection because there is no ctag
+ if self.notificationURL is not None and not self.supportSync:
+ yield self._notificationPropfind(self.notificationURL)
+ yield self._notificationChangesPropfind(self.notificationURL)
+
+ # One time delegate expansion
+ if firstTime:
+ # yield self._pollFirstTime2()
+ pass
+
+ returnValue(True)
+
+ @inlineCallbacks
+ def _calendarHomePropfind(self, calendarHomeSet):
+ """
+ Do the poll Depth:1 PROPFIND on the calendar home.
+ """
+ if not calendarHomeSet.endswith('/'):
+ calendarHomeSet = calendarHomeSet + '/'
+ _ignore_response, result = yield self.requester.propfind(
+ calendarHomeSet,
+ self._POLL_CALENDARHOME_PROPFIND,
+ depth='1',
+ method_label="PROPFIND{home}",
+ )
+ calendars = self._extractCalendars(result, calendarHomeSet)
+ returnValue((calendars, result,))
+
+
</ins><span class="cx"> def _extractCalendars(self, results, calendarHome=None):
</span><span class="cx"> """
</span><span class="cx"> Parse a calendar home PROPFIND response and create local state
</span><span class="cx"> representing the calendars it contains.
</span><del>-
- If XMPP push is enabled, also look for and record information about
- that from the response.
</del><span class="cx"> """
</span><span class="cx"> calendars = []
</span><span class="cx"> for href in results:
</span><span class="lines">@@ -785,18 +539,8 @@
</span><span class="cx"> pass
</span><span class="cx"> else:
</span><span class="cx"> if pushkey:
</span><del>- self.ampPushKeys[href] = pushkey
</del><ins>+ self.monitor.addPushkey(href, pushkey)
</ins><span class="cx">
</span><del>- try:
- server = text[csxml.xmpp_server]
- uri = text[csxml.xmpp_uri]
- pushkey = text[csxml.pushkey]
- except KeyError:
- pass
- else:
- if server and uri:
- self.xmpp[href] = XMPPPush(server, uri, pushkey)
-
</del><span class="cx"> nodes = results[href].getNodeProperties()
</span><span class="cx"> for nodeType in nodes[davxml.resourcetype]:
</span><span class="cx"> if nodeType.tag in self._CALENDAR_TYPES:
</span><span class="lines">@@ -819,6 +563,40 @@
</span><span class="cx"> return calendars
</span><span class="cx">
</span><span class="cx">
</span><ins>+
+
+ @inlineCallbacks
+ def _pollFirstTime1(self, homeNode, calendars):
+ # Detect sync report if needed
+ if self.supportSync:
+ nodes = homeNode.getNodeProperties()
+ syncnodes = nodes[davxml.supported_report_set].findall(
+ str(davxml.supported_report) + "/" +
+ str(davxml.report) + "/" +
+ str(davxml.sync_collection)
+ )
+ self.supportSync = len(syncnodes) != 0
+
+ # Patch calendar properties
+ for cal in calendars:
+ if cal.name != "inbox":
+ # yield self.requester.proppatch(
+ # cal.url,
+ # self._STARTUP_PROPPATCH_CALENDAR_COLOR,
+ # method_label="PROPPATCH{calendar}",
+ # )
+ yield self.requester.proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_ORDER,
+ method_label="PROPPATCH{calendar}",
+ )
+ yield self.requester.proppatch(
+ cal.url,
+ self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE,
+ method_label="PROPPATCH{calendar}",
+ )
+
+
</ins><span class="cx"> def _updateCalendar(self, calendar, newToken):
</span><span class="cx"> """
</span><span class="cx"> Update the local cached data for a calendar in an appropriate manner.
</span><span class="lines">@@ -840,11 +618,11 @@
</span><span class="cx"> # the sim can fire a PUT between the PROPFIND and when process the removals.
</span><span class="cx"> old_hrefs = set([calendar.url + child for child in calendar.events.keys()])
</span><span class="cx">
</span><del>- _ignore_response, result = yield self._propfind(
</del><ins>+ _ignore_response, result = yield self.requester.propfind(
</ins><span class="cx"> calendar.url,
</span><span class="cx"> self._POLL_CALENDAR_PROPFIND_D1,
</span><ins>+ method_label="PROPFIND{calendar}",
</ins><span class="cx"> depth='1',
</span><del>- method_label="PROPFIND{calendar}"
</del><span class="cx"> )
</span><span class="cx">
</span><span class="cx"> yield self._updateApplyChanges(calendar, result, old_hrefs)
</span><span class="lines">@@ -868,7 +646,7 @@
</span><span class="cx"> # Get changes from sync REPORT (including the other nodes at the top-level
</span><span class="cx"> # which will have the new sync token.
</span><span class="cx"> fullSync = not calendar.changeToken
</span><del>- result = yield self._report(
</del><ins>+ result = yield self.requester.report(
</ins><span class="cx"> calendar.url,
</span><span class="cx"> self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': calendar.changeToken},
</span><span class="cx"> depth='1',
</span><span class="lines">@@ -879,7 +657,7 @@
</span><span class="cx"> if result is None:
</span><span class="cx"> if not fullSync:
</span><span class="cx"> fullSync = True
</span><del>- result = yield self._report(
</del><ins>+ result = yield self.requester.report(
</ins><span class="cx"> calendar.url,
</span><span class="cx"> self._POLL_CALENDAR_SYNC_REPORT % {'sync-token': ''},
</span><span class="cx"> depth='1',
</span><span class="lines">@@ -905,13 +683,13 @@
</span><span class="cx"> # Differentiate a remove vs new/update result
</span><span class="cx"> if result[responseHref].getStatus() / 100 == 2:
</span><span class="cx"> if responseHref not in self._events:
</span><del>- self._setEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
</del><ins>+ self._cacheEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
</ins><span class="cx">
</span><span class="cx"> event = self._events[responseHref]
</span><span class="cx"> if event.etag != etag:
</span><span class="cx"> changed.append(responseHref)
</span><span class="cx"> elif result[responseHref].getStatus() == 404:
</span><del>- self._removeEvent(responseHref)
</del><ins>+ self._invalidateEvent(responseHref)
</ins><span class="cx">
</span><span class="cx"> yield self._updateChangedEvents(calendar, changed)
</span><span class="cx">
</span><span class="lines">@@ -920,7 +698,7 @@
</span><span class="cx"> # Detect removed items and purge them
</span><span class="cx"> remove_hrefs = old_hrefs - set(changed)
</span><span class="cx"> for href in remove_hrefs:
</span><del>- self._removeEvent(href)
</del><ins>+ self._invalidateEvent(href)
</ins><span class="cx">
</span><span class="cx"> # Now update calendar to the new token taken from the report
</span><span class="cx"> for node in others:
</span><span class="lines">@@ -951,7 +729,7 @@
</span><span class="cx"> continue
</span><span class="cx">
</span><span class="cx"> if responseHref not in self._events:
</span><del>- self._setEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
</del><ins>+ self._cacheEvent(responseHref, Event(self.serializeLocation(), responseHref, None))
</ins><span class="cx">
</span><span class="cx"> event = self._events[responseHref]
</span><span class="cx"> if event.etag != etag:
</span><span class="lines">@@ -963,7 +741,7 @@
</span><span class="cx"> # Detect removed items and purge them
</span><span class="cx"> remove_hrefs = old_hrefs - set(all_hrefs)
</span><span class="cx"> for href in remove_hrefs:
</span><del>- self._removeEvent(href)
</del><ins>+ self._invalidateEvent(href)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="lines">@@ -979,9 +757,9 @@
</span><span class="cx"> changed = changed[self.MULTIGET_BATCH_SIZE:]
</span><span class="cx">
</span><span class="cx"> multistatus = yield self._eventReport(calendar.url, batchedHrefs)
</span><del>- for responseHref in batchedHrefs:
</del><ins>+ for href in batchedHrefs:
</ins><span class="cx"> try:
</span><del>- res = multistatus[responseHref]
</del><ins>+ res = multistatus[href]
</ins><span class="cx"> except KeyError:
</span><span class="cx"> # Resource might have been deleted
</span><span class="cx"> continue
</span><span class="lines">@@ -993,18 +771,10 @@
</span><span class="cx"> except KeyError:
</span><span class="cx"> scheduleTag = None
</span><span class="cx"> body = text[caldavxml.calendar_data]
</span><del>- self.eventChanged(responseHref, etag, scheduleTag, body)
</del><ins>+ component = Component.fromString(body)
+ self._updateEventCache(href, etag, scheduleTag, component)
</ins><span class="cx">
</span><span class="cx">
</span><del>- def eventChanged(self, href, etag, scheduleTag, body):
- event = self._events[href]
- event.etag = etag
- if scheduleTag is not None:
- event.scheduleTag = scheduleTag
- event.component = Component.fromString(body)
- self.catalog["eventChanged"].issue(href)
-
-
</del><span class="cx"> def _eventReport(self, calendar, events):
</span><span class="cx"> # Next do a REPORT on events that might have information
</span><span class="cx"> # we don't know about.
</span><span class="lines">@@ -1018,7 +788,7 @@
</span><span class="cx"> if len(events) > 75:
</span><span class="cx"> label_suffix = "huge"
</span><span class="cx">
</span><del>- return self._report(
</del><ins>+ return self.requester.report(
</ins><span class="cx"> calendar,
</span><span class="cx"> self._POLL_CALENDAR_MULTIGET_REPORT % {'hrefs': hrefs},
</span><span class="cx"> depth=None,
</span><span class="lines">@@ -1027,101 +797,8 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def _checkCalendarsForEvents(self, calendarHomeSet, firstTime=False, push=False):
- """
- The actions a client does when polling for changes, or in response to a
- push notification of a change. There are some actions done on the first poll
- we should emulate.
- """
-
- result = True
- try:
- result = yield self._newOperation("push" if push else "poll", self._poll(calendarHomeSet, firstTime))
- finally:
- if result:
- try:
- self._checking.remove(calendarHomeSet)
- except KeyError:
- pass
- returnValue(result)
-
-
- @inlineCallbacks
- def _poll(self, calendarHomeSet, firstTime):
- if calendarHomeSet in self._checking:
- returnValue(False)
- self._checking.add(calendarHomeSet)
-
- calendars, results = yield self._calendarHomePropfind(calendarHomeSet)
-
- # First time operations
- if firstTime:
- yield self._pollFirstTime1(results[calendarHomeSet], calendars)
-
- # Normal poll
- for cal in calendars:
- newToken = cal.changeToken
- if cal.url not in self._calendars:
- # Calendar seen for the first time - reload it
- self._calendars[cal.url] = cal
- cal.changeToken = ""
- yield self._updateCalendar(self._calendars[cal.url], newToken)
- elif self._calendars[cal.url].changeToken != newToken:
- # Calendar changed - reload it
- yield self._updateCalendar(self._calendars[cal.url], newToken)
-
- # When there is no sync REPORT, clients have to do a full PROPFIND
- # on the notification collection because there is no ctag
- if self.notificationURL is not None and not self.supportSync:
- yield self._notificationPropfind(self.notificationURL)
- yield self._notificationChangesPropfind(self.notificationURL)
-
- # One time delegate expansion
- if firstTime:
- yield self._pollFirstTime2()
-
- returnValue(True)
-
-
- @inlineCallbacks
- def _pollFirstTime1(self, homeNode, calendars):
- # Detect sync report if needed
- if self.supportSync:
- nodes = homeNode.getNodeProperties()
- syncnodes = nodes[davxml.supported_report_set].findall(
- str(davxml.supported_report) + "/" +
- str(davxml.report) + "/" +
- str(davxml.sync_collection)
- )
- self.supportSync = len(syncnodes) != 0
-
- # Patch calendar properties
- for cal in calendars:
- if cal.name != "inbox":
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_COLOR,
- method_label="PROPPATCH{calendar}",
- )
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_ORDER,
- method_label="PROPPATCH{calendar}",
- )
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_TIMEZONE,
- method_label="PROPPATCH{calendar}",
- )
-
-
- def _pollFirstTime2(self):
- return self._principalExpand(self.principalURL)
-
-
- @inlineCallbacks
</del><span class="cx"> def _notificationPropfind(self, notificationURL):
</span><del>- _ignore_response, result = yield self._propfind(
</del><ins>+ _ignore_response, result = yield self.requester.propfind(
</ins><span class="cx"> notificationURL,
</span><span class="cx"> self._POLL_NOTIFICATION_PROPFIND,
</span><span class="cx"> method_label="PROPFIND{notification}",
</span><span class="lines">@@ -1131,7 +808,7 @@
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _notificationChangesPropfind(self, notificationURL):
</span><del>- _ignore_response, result = yield self._propfind(
</del><ins>+ _ignore_response, result = yield self.requester.propfind(
</ins><span class="cx"> notificationURL,
</span><span class="cx"> self._POLL_NOTIFICATION_PROPFIND_D1,
</span><span class="cx"> depth='1',
</span><span class="lines">@@ -1139,10 +816,12 @@
</span><span class="cx"> )
</span><span class="cx"> returnValue(result)
</span><span class="cx">
</span><ins>+ def _pollFirstTime2(self):
+ return self._principalExpand(self.principalURL)
</ins><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _principalExpand(self, principalURL):
</span><del>- result = yield self._report(
</del><ins>+ result = yield self.requester.report(
</ins><span class="cx"> principalURL,
</span><span class="cx"> self._STARTUP_PRINCIPAL_EXPAND,
</span><span class="cx"> depth=None,
</span><span class="lines">@@ -1151,19 +830,7 @@
</span><span class="cx"> returnValue(result)
</span><span class="cx">
</span><span class="cx">
</span><del>- def startup(self):
- raise NotImplementedError
</del><span class="cx">
</span><del>-
- def _calendarCheckLoop(self, calendarHome):
- """
- Periodically check the calendar home for changes to calendars.
- """
- pollCalendarHome = LoopingCall(
- self._checkCalendarsForEvents, calendarHome)
- return pollCalendarHome.start(self.calendarHomePollInterval, now=False)
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def _newOperation(self, label, deferred):
</span><span class="cx"> before = self.reactor.seconds()
</span><span class="lines">@@ -1202,55 +869,7 @@
</span><span class="cx"> returnValue(result)
</span><span class="cx">
</span><span class="cx">
</span><del>- def _monitorPubSub(self, home, params):
- """
- Start monitoring the
- """
- host, port = params.server.split(':')
- port = int(port)
-
- service, _ignore_stuff = params.uri.split('?')
- service = service.split(':', 1)[1]
-
- # XXX What is the domain of the 2nd argument supposed to be? The
- # hostname we use to connect, or the same as the email address in the
- # user record?
- factory = _PubSubClientFactory(
- self, "%s@%s" % (self.record.uid, host),
- self.record.password, service,
- {params.pushkey: (home, home, "Calendar home")}, False,
- sigint=False)
- self._pushFactories.append(factory)
- connect(GAIEndpoint(self.reactor, host, port), factory)
-
-
- def _receivedPush(self, inboundID, dataChangedTimestamp, priority=5):
- for href, id in self.ampPushKeys.iteritems():
- if inboundID == id:
- self._checkCalendarsForEvents(href, push=True)
- break
- else:
- # somehow we are not subscribed to this id
- pass
-
-
- def _monitorAmpPush(self, home, pushKeys):
- """
- Start monitoring for AMP-based push notifications
- """
- subscribeToIDs(
- self.ampPushHost, self.ampPushPort, pushKeys,
- self._receivedPush, self.reactor
- )
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def _unsubscribePubSub(self):
- for factory in self._pushFactories:
- yield factory.unsubscribeAll()
-
-
- @inlineCallbacks
</del><span class="cx"> def run(self):
</span><span class="cx"> """
</span><span class="cx"> Emulate a CalDAV client.
</span><span class="lines">@@ -1262,37 +881,28 @@
</span><span class="cx"> calendarHome = hrefs[caldavxml.calendar_home_set].toString()
</span><span class="cx"> if calendarHome is None:
</span><span class="cx"> raise MissingCalendarHome
</span><del>- yield self._checkCalendarsForEvents(calendarHome, firstTime=True)
</del><ins>+ yield self.checkCalendarsForEvents(calendarHome, firstTime=True)
</ins><span class="cx"> returnValue(calendarHome)
</span><span class="cx"> calendarHome = yield self._newOperation("startup: %s" % (self.title,), startup())
</span><del>-
</del><span class="cx"> self.started = True
</span><span class="cx">
</span><del>- # Start monitoring PubSub notifications, if possible.
- # _checkCalendarsForEvents populates self.xmpp if it finds
- # anything.
- if self.supportPush and calendarHome in self.xmpp:
- self._monitorPubSub(calendarHome, self.xmpp[calendarHome])
</del><ins>+ # Start monitoring AMP push notifications, if possible
+ if self.monitor and self.monitor.isSubscribedTo(calendarHome):
+ self.monitor.begin()
</ins><span class="cx"> # Run indefinitely.
</span><span class="cx"> yield Deferred()
</span><del>- elif self.supportAmpPush and calendarHome in self.ampPushKeys:
- pushKeys = self.ampPushKeys.values()
- self._monitorAmpPush(calendarHome, pushKeys)
- # Run indefinitely.
- yield Deferred()
</del><span class="cx"> else:
</span><span class="cx"> # This completes when the calendar home poll loop completes, which
</span><span class="cx"> # currently it never will except due to an unexpected error.
</span><del>- yield self._calendarCheckLoop(calendarHome)
</del><ins>+ yield self.calendarCheckLoop(calendarHome)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def stop(self):
</span><span class="cx"> """
</span><span class="cx"> Called before connections are closed, giving a chance to clean up
</span><span class="cx"> """
</span><del>-
</del><span class="cx"> self.serialize()
</span><del>- return self._unsubscribePubSub()
</del><ins>+ return self.monitor.unsubscribeFromAll()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def serializeLocation(self):
</span><span class="lines">@@ -1384,94 +994,67 @@
</span><span class="cx"> )
</span><span class="cx"> return organizer
</span><span class="cx">
</span><ins>+ def _getEventSizeDescription(self, numAttendees):
+ if numAttendees > 75:
+ return "huge"
+ if numAttendees > 20:
+ return "large"
+ if numAttendees > 5:
+ return "medium"
+ return "small"
</ins><span class="cx">
</span><ins>+ """ literally wtf is this event stuff
+submitEvent(event: Event, )
+
+"""
</ins><span class="cx"> @inlineCallbacks
</span><span class="cx"> def addEventAttendee(self, href, attendee):
</span><ins>+ individual = attendee.parameterValue('CUTYPE') == 'INDIVIDUAL'
</ins><span class="cx">
</span><span class="cx"> event = self._events[href]
</span><span class="cx"> component = event.component
</span><ins>+ vevent = component.mainComponent()
</ins><span class="cx">
</span><ins>+ query = attendee.parameterValue('CN')
+
+ from pprint import pprint
</ins><span class="cx"> # Trigger auto-complete behavior
</span><del>- yield self._attendeeAutoComplete(component, attendee)
</del><ins>+ matchingPrincipals = yield self._principalSearchReport(query, isAttendeeSearch=individual)
+ for k, v in matchingPrincipals.items():
+ pprint(k)
+ for prop, val in v.getNodeProperties().items():
+ print("%s %s" % (prop, val.__dict__))
+ for child in val._children:
+ print(child.text)
</ins><span class="cx">
</span><del>- # If the event has no attendees, add ourselves as an attendee.
- attendees = list(component.mainComponent().properties('ATTENDEE'))
- if len(attendees) == 0:
- # First add ourselves as a participant and as the
- # organizer. In the future for this event we should
- # already have those roles.
- component.mainComponent().addProperty(self._makeSelfOrganizer())
- component.mainComponent().addProperty(self._makeSelfAttendee())
- attendees.append(attendee)
- component.mainComponent().addProperty(attendee)
</del><ins>+ uuids = []
</ins><span class="cx">
</span><del>- label_suffix = "small"
- if len(attendees) > 5:
- label_suffix = "medium"
- if len(attendees) > 20:
- label_suffix = "large"
- if len(attendees) > 75:
- label_suffix = "huge"
</del><ins>+ for principal_url, propfindresult in matchingPrincipals.items():
+ props = propfindresult.getNodeProperties()
+ for cuaddr in props.get(caldavxml.calendar_user_address_set):
+ # print(cuaddr)
+ uuids.append(cuaddr.text)
+ break
</ins><span class="cx">
</span><del>- # At last, upload the new event definition
- response = yield self._request(
- (NO_CONTENT, PRECONDITION_FAILED,),
- 'PUT',
- self.root + href.encode('utf-8'),
- Headers({
- 'content-type': ['text/calendar'],
- 'if-match': [event.etag]}),
- StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{organizer-%s}" % (label_suffix,)
- )
</del><ins>+ print(uuids)
</ins><span class="cx">
</span><del>- # Finally, re-retrieve the event to update the etag
- yield self._updateEvent(response, href)
</del><span class="cx">
</span><ins>+ start = vevent.getStartDateUTC()
+ end = vevent.getEndDateUTC()
</ins><span class="cx">
</span><del>- @inlineCallbacks
- def _attendeeAutoComplete(self, component, attendee):
</del><ins>+ yield self.requestAvailability(start, end, uuids)
</ins><span class="cx">
</span><del>- if self._ATTENDEE_LOOKUPS:
- # Temporarily use some non-test names (some which will return
- # many results, and others which will return fewer) because the
- # test account names are all too similar
- # name = attendee.parameterValue('CN').encode("utf-8")
- # prefix = name[:4].lower()
- prefix = random.choice([
- "chris", "cyru", "dre", "eric", "morg",
- "well", "wilfr", "witz"
- ])
</del><ins>+ # # Do free-busy lookups
+ # if individual:
+ # # When adding individual attendees, we only look up the availability
+ # # of the specific attendee
+ # yield self.checkAvailability()
+ # else:
+ # # When adding a location, we look up the availability of each location
+ # # returned by the principal search.
+ # yield self.checkAvailability()
</ins><span class="cx">
</span><del>- email = attendee.value()
- if email.startswith("mailto:"):
- email = email[7:]
- elif attendee.hasParameter('EMAIL'):
- email = attendee.parameterValue('EMAIL').encode("utf-8")
</del><span class="cx">
</span><del>- # First try to discover some names to supply to the
- # auto-completion
- yield self._report(
- self.principalCollection,
- self._USER_LIST_PRINCIPAL_PROPERTY_SEARCH % {
- 'displayname': prefix,
- 'email': prefix,
- 'firstname': prefix,
- 'lastname': prefix,
- },
- depth=None,
- method_label="REPORT{psearch}",
- )
-
- # Now learn about the attendee's availability
- yield self.requestAvailability(
- component.mainComponent().getStartDateUTC(),
- component.mainComponent().getEndDateUTC(),
- [self.email, u'mailto:' + email],
- [component.resourceUID()]
- )
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def changeEventAttendee(self, href, oldAttendee, newAttendee):
</span><span class="cx"> event = self._events[href]
</span><span class="lines">@@ -1480,149 +1063,197 @@
</span><span class="cx"> # Change the event to have the new attendee instead of the old attendee
</span><span class="cx"> component.mainComponent().removeProperty(oldAttendee)
</span><span class="cx"> component.mainComponent().addProperty(newAttendee)
</span><del>- okCodes = NO_CONTENT
- headers = Headers({
- 'content-type': ['text/calendar'],
- })
</del><ins>+
+ headers = Headers()
</ins><span class="cx"> if event.scheduleTag is not None:
</span><span class="cx"> headers.addRawHeader('if-schedule-tag-match', event.scheduleTag)
</span><del>- okCodes = (NO_CONTENT, PRECONDITION_FAILED,)
-
</del><ins>+ event.component = component
</ins><span class="cx"> attendees = list(component.mainComponent().properties('ATTENDEE'))
</span><del>- label_suffix = "small"
- if len(attendees) > 5:
- label_suffix = "medium"
- if len(attendees) > 20:
- label_suffix = "large"
- if len(attendees) > 75:
- label_suffix = "huge"
</del><ins>+ label_suffix = self._getEventSizeDescription(len(attendees))
+ method_label = "PUT{attendee-%s}" % (label_suffix,)
</ins><span class="cx">
</span><del>- response = yield self._request(
- okCodes,
- 'PUT',
- self.root + href.encode('utf-8'),
- headers, StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{attendee-%s}" % (label_suffix,),
- )
</del><ins>+ yield self.putEvent(href, event, headers=headers, method_label=method_label, new=False)
</ins><span class="cx">
</span><del>- # Finally, re-retrieve the event to update the etag
- yield self._updateEvent(response, href)
</del><span class="cx">
</span><del>-
</del><span class="cx"> @inlineCallbacks
</span><del>- def deleteEvent(self, href):
</del><ins>+ def addInvite(self, event):
</ins><span class="cx"> """
</span><del>- Issue a DELETE for the given URL and remove local state
- associated with that event.
</del><ins>+ Add an event that is an invite - i.e., has attendees. Presumably the appropriate principal searches and
+ free-busy lookups have already been accounted for (in addEventAttendee)
</ins><span class="cx"> """
</span><ins>+ vevent = event.component.mainComponent()
+ # If the event has no attendees, add ourselves as an attendee.
+ attendees = list(vevent.properties('ATTENDEE'))
+ if len(attendees) == 0:
+ # First add ourselves as a participant and as the
+ # organizer. In the future for this event we should
+ # already have those roles.
+ vevent.addProperty(self._makeSelfOrganizer())
+ vevent.addProperty(self._makeSelfAttendee())
</ins><span class="cx">
</span><del>- self._removeEvent(href)
</del><ins>+ label_suffix = self._getEventSizeDescription(len(attendees))
+ method_label = "PUT{organizer-%s}" % (label_suffix,)
</ins><span class="cx">
</span><del>- response = yield self._request(
- NO_CONTENT,
- 'DELETE',
- self.root + href.encode('utf-8'),
- method_label="DELETE{event}",
- )
- returnValue(response)
</del><ins>+ yield self.updateEvent(event, method_label=method_label)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def addEvent(self, href, component, invite=False):
</del><ins>+ def addEvent(self, href, event):
+ """
+ client.addEvent(
+ Event e
+ """
</ins><span class="cx"> headers = Headers({
</span><del>- 'content-type': ['text/calendar'],
</del><ins>+ 'if-none-match': ['*']
</ins><span class="cx"> })
</span><ins>+ yield self.putEvent(
+ href,
+ event,
+ headers=headers,
+ method_label="PUT{event}"
+ )
</ins><span class="cx">
</span><del>- attendees = list(component.mainComponent().properties('ATTENDEE'))
- label_suffix = "small"
- if len(attendees) > 5:
- label_suffix = "medium"
- if len(attendees) > 20:
- label_suffix = "large"
- if len(attendees) > 75:
- label_suffix = "huge"
</del><ins>+ # attendees = list(component.mainComponent().properties('ATTENDEE'))
+ # label_suffix = self._getEventSizeDescription(len(attendees))
+ # method_label = "PUT{organizer-%s}" % (label_suffix,) if invite else "PUT{event}"
</ins><span class="cx">
</span><del>- response = yield self._request(
- CREATED,
- 'PUT',
- self.root + href.encode('utf-8'),
- headers,
- StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{organizer-%s}" % (label_suffix,) if invite else "PUT{event}",
- )
- self._localUpdateEvent(response, href, component)
</del><ins>+ @inlineCallbacks
+ def updateEvent(self, event, method_label="PUT{event}"):
+ headers = Headers({
+ 'if-match': [event.etag]
+ })
+ yield self.putEvent(event.url, event, headers=headers, method_label=method_label)
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def addInvite(self, href, component):
</del><ins>+ def putEvent(self, href, event, headers=None, method_label=None):
</ins><span class="cx"> """
</span><del>- Add an event that is an invite - i.e., has attendees. We will do attendee lookups and freebusy
- checks on each attendee to simulate what happens when an organizer creates a new invite.
</del><ins>+ PUT an event to the server
</ins><span class="cx"> """
</span><ins>+ if headers == None:
+ headers = Headers()
+ headers.addRawHeader('content-type', 'text/calendar')
</ins><span class="cx">
</span><del>- # Do lookup and free busy of each attendee (not self)
- attendees = list(component.mainComponent().properties('ATTENDEE'))
- for attendee in attendees:
- if attendee.value() in (self.uuid, self.email):
- continue
- yield self._attendeeAutoComplete(component, attendee)
</del><ins>+ okCodes = (CREATED, NO_CONTENT, PRECONDITION_FAILED)
</ins><span class="cx">
</span><del>- # Now do a normal PUT
- yield self.addEvent(href, component, invite=True)
</del><ins>+ # At last, upload the new event definition
+ response = yield self.requester.put(
+ okCodes,
+ href,
+ event.component,
+ headers=headers,
+ method_label=method_label
+ )
+ # If the server doesn't return an etag, it has changed the resource
+ # and we need to refetch it
+ if not response.headers.hasHeader('etag'):
+ yield self._refreshEvent(href)
+ else:
+ etag, scheduleTag = self.extractTags(response)
+ yield succeed(self._updateEventCache(href, etag=etag, scheduleTag=scheduleTag, component=event.component))
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def changeEvent(self, href):
</del><ins>+ def _refreshEvent(self, href):
+ """
+ Issues a GET to the specified href (representing an event that already exists on the server)
+ and uses the response to update local state associated with that event
+ """
+ response = yield self.requester.get(href, method_label="GET{event}")
+ etag, scheduleTag = self.extractTags(response)
+ body = yield readBody(response)
+ component = Component.fromString(body)
+ self._updateEventCache(href, etag=etag, scheduleTag=scheduleTag, component=component)
</ins><span class="cx">
</span><del>- event = self._events[href]
- component = event.component
</del><ins>+ def _updateEventCache(self, href, etag=None, scheduleTag=None, component=None):
+ """
+ Update local state associated with the event at href
+ """
</ins><span class="cx">
</span><del>- # At last, upload the new event definition
- response = yield self._request(
- (NO_CONTENT, PRECONDITION_FAILED,),
- 'PUT',
- self.root + href.encode('utf-8'),
- Headers({
- 'content-type': ['text/calendar'],
- 'if-match': [event.etag]
- }),
- StringProducer(component.getTextWithTimezones(includeTimezones=True)),
- method_label="PUT{update}"
- )
</del><ins>+ if href in self._events:
+ event = self._events[href]
+ else: # This is a new resource
+ event = Event(self.serializeLocation(), href, None, None)
</ins><span class="cx">
</span><del>- # Finally, re-retrieve the event to update the etag
- yield self._updateEvent(response, href)
</del><ins>+ if etag:
+ event.etag = etag
+ if scheduleTag:
+ event.scheduleTag = scheduleTag
+ if component:
+ event.component = component
</ins><span class="cx">
</span><ins>+ if True: # XXX some other test
+ self.catalog["eventChanged"].issue(href)
+ self._cacheEvent(href, event)
</ins><span class="cx">
</span><del>- def _localUpdateEvent(self, response, href, component):
</del><ins>+ @inlineCallbacks
+ def deleteEvent(self, href):
+ """
+ Issue a DELETE for the given URL and remove local state
+ associated with that event.
+ """
+ self._invalidateEvent(href)
+ yield self.requester.delete(href, method_label="DELETE{event}")
+
+ def extractTags(self, response):
</ins><span class="cx"> headers = response.headers
</span><span class="cx"> etag = headers.getRawHeaders("etag", [None])[0]
</span><span class="cx"> scheduleTag = headers.getRawHeaders("schedule-tag", [None])[0]
</span><ins>+ return etag, scheduleTag
</ins><span class="cx">
</span><del>- event = Event(self.serializeLocation(), href, etag, component)
- event.scheduleTag = scheduleTag
- self._setEvent(href, event)
</del><ins>+ # @inlineCallbacks
+ # def _attendeeAutoComplete(self, component, attendee):
</ins><span class="cx">
</span><ins>+ # if self._ATTENDEE_LOOKUPS:
+ # # Temporarily use some non-test names (some which will return
+ # # many results, and others which will return fewer) because the
+ # # test account names are all too similar
+ # # name = attendee.parameterValue('CN').encode("utf-8")
+ # # prefix = name[:4].lower()
+ # prefix = random.choice([
+ # "chris", "cyru", "dre", "eric", "morg",
+ # "well", "wilfr", "witz"
+ # ])
</ins><span class="cx">
</span><del>- def updateEvent(self, href):
- return self._updateEvent(None, href)
</del><ins>+ # email = attendee.value()
+ # if email.startswith("mailto:"):
+ # email = email[7:]
+ # elif attendee.hasParameter('EMAIL'):
+ # email = attendee.parameterValue('EMAIL').encode("utf-8")
</ins><span class="cx">
</span><ins>+ # # First try to discover some names to supply to the
+ # # auto-completion
+ # yield self.requester.report(
+ # self.principalCollection,
+ # self._USER_LIST_PRINCIPAL_PROPERTY_SEARCH % {
+ # 'displayname': prefix,
+ # 'email': prefix,
+ # 'firstname': prefix,
+ # 'lastname': prefix,
+ # },
+ # depth=None,
+ # method_label="REPORT{psearch}",
+ # )
</ins><span class="cx">
</span><ins>+ # # Now learn about the attendee's availability
+ # yield self.requestAvailability(
+ # component.mainComponent().getStartDateUTC(),
+ # component.mainComponent().getEndDateUTC(),
+ # [self.email, u'mailto:' + email],
+ # [component.resourceUID()]
+ # )
+
</ins><span class="cx"> @inlineCallbacks
</span><del>- def _updateEvent(self, ignored, href):
- response = yield self._request(
- OK,
- 'GET',
- self.root + href.encode('utf-8'),
- method_label="GET{event}",
- )
- headers = response.headers
- etag = headers.getRawHeaders('etag')[0]
- scheduleTag = headers.getRawHeaders('schedule-tag', [None])[0]
- body = yield readBody(response)
- self.eventChanged(href, etag, scheduleTag, body)
</del><ins>+ def _principalSearchReport(self, query, isAttendeeSearch):
+ """ context = attendee if isAttendeeSearch else location """
+ context = "attendee" if isAttendeeSearch else "location"
+ tokens = query.split()
+ search = '\n'.join(["<C:search-token>%s</C:search-token>" % (token, ) for token in tokens])
+ body = self._CALENDARSERVER_PRINCIPAL_SEARCH_REPORT.format(context=context, searchTokens=search)
+ principals = yield self.requester.report('/principals/', body, depth=None)
+ print("Found some principals:")
+ returnValue(principals)
</ins><span class="cx">
</span><del>-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def requestAvailability(self, start, end, users, mask=set()):
</span><span class="cx"> """
</span><span class="lines">@@ -1644,8 +1275,6 @@
</span><span class="cx"> @return: A C{Deferred} which fires with a C{dict}. Keys in the dict
</span><span class="cx"> are user UUIDs (those requested) and values are something else.
</span><span class="cx"> """
</span><del>- outbox = self.root + self.outbox
-
</del><span class="cx"> if mask:
</span><span class="cx"> maskStr = u'\r\n'.join(['X-CALENDARSERVER-MASK-UID:' + uid
</span><span class="cx"> for uid in mask]) + u'\r\n'
</span><span class="lines">@@ -1669,21 +1298,16 @@
</span><span class="cx"> end = end.getText()
</span><span class="cx"> now = DateTime.getNowUTC().getText()
</span><span class="cx">
</span><del>- label_suffix = "small"
- if len(users) > 5:
- label_suffix = "medium"
- if len(users) > 20:
- label_suffix = "large"
- if len(users) > 75:
- label_suffix = "huge"
</del><ins>+ label_suffix = self._getEventSizeDescription(len(users))
</ins><span class="cx">
</span><del>- response = yield self._request(
- OK, 'POST', outbox,
- Headers({
- 'content-type': ['text/calendar'],
- 'originator': [self.email],
- 'recipient': [u', '.join(users).encode('utf-8')]}),
- StringProducer(self._POST_AVAILABILITY % {
</del><ins>+ headers = Headers({
+ 'content-type': ['text/calendar'],
+ 'originator': [self.email],
+ 'recipient': [u', '.join(users).encode('utf-8')]
+ })
+ response = yield self.requester.post(
+ self.outbox,
+ self._POST_AVAILABILITY % {
</ins><span class="cx"> 'attendees': attendeeStr,
</span><span class="cx"> 'summary': (u'Availability for %s' % (', '.join(users),)).encode('utf-8'),
</span><span class="cx"> 'organizer': self.email.encode('utf-8'),
</span><span class="lines">@@ -1692,385 +1316,103 @@
</span><span class="cx"> 'start': start,
</span><span class="cx"> 'end': end,
</span><span class="cx"> 'now': now,
</span><del>- }),
</del><ins>+ },
+ headers=headers,
</ins><span class="cx"> method_label="POST{fb-%s}" % (label_suffix,),
</span><span class="cx"> )
</span><ins>+
</ins><span class="cx"> body = yield readBody(response)
</span><span class="cx"> returnValue(body)
</span><span class="cx">
</span><del>-
-
-class OS_X_10_6(BaseAppleClient):
- """
- Implementation of the OS X 10.6 iCal network behavior.
-
- Anything OS X 10.6 iCal does on its own, or any particular
- network behaviors it takes in response to a user action, belong on
- this class.
-
- Usage-profile based behaviors ("the user modifies an event every
- 3.2 minutes") belong elsewhere.
- """
-
- _client_type = "OS X 10.6"
-
- USER_AGENT = "DAVKit/4.0.3 (732); CalendarStore/4.0.3 (991); iCal/4.0.3 (1388); Mac OS X/10.6.4 (10F569)"
-
- # The default interval, used if none is specified in external
- # configuration. This is also the actual value used by Snow
- # Leopard iCal.
- CALENDAR_HOME_POLL_INTERVAL = 15 * 60
-
- # The maximum number of resources to retrieve in a single multiget
- MULTIGET_BATCH_SIZE = 200
-
- # Override and turn on if client supports Sync REPORT
- _SYNC_REPORT = False
-
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = False
-
- # Override and turn off if client does not support attendee lookups
- _ATTENDEE_LOOKUPS = True
-
- # Request body data
- _LOAD_PATH = "OS_X_10_6"
-
- _STARTUP_WELL_KNOWN = loadRequestBody(_LOAD_PATH, 'startup_well_known')
- _STARTUP_PRINCIPAL_PROPFIND_INITIAL = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind_initial')
- _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind')
- _STARTUP_PRINCIPALS_REPORT = loadRequestBody(_LOAD_PATH, 'startup_principals_report')
- _STARTUP_PRINCIPAL_EXPAND = loadRequestBody(_LOAD_PATH, 'startup_principal_expand')
- _STARTUP_PROPPATCH_CALENDAR_COLOR = loadRequestBody(_LOAD_PATH, 'startup_calendar_color_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_ORDER = loadRequestBody(_LOAD_PATH, 'startup_calendar_order_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_TIMEZONE = loadRequestBody(_LOAD_PATH, 'startup_calendar_timezone_proppatch')
-
- _POLL_CALENDARHOME_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_propfind')
- _POLL_CALENDAR_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_CALENDAR_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind_d1')
- _POLL_CALENDAR_MULTIGET_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget')
- _POLL_CALENDAR_MULTIGET_REPORT_HREF = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget_hrefs')
- _POLL_CALENDAR_SYNC_REPORT = None
- _POLL_NOTIFICATION_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_NOTIFICATION_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_notification_propfind_d1')
-
- _USER_LIST_PRINCIPAL_PROPERTY_SEARCH = loadRequestBody(_LOAD_PATH, 'user_list_principal_property_search')
- _POST_AVAILABILITY = loadRequestBody(_LOAD_PATH, 'post_availability')
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def startup(self):
</del><ins>+ def postAttachment(self, href, content):
+ url = "{0}?{1}".format(href, "action=attachment-add")
+ filename = 'file-{}.txt'.format(len(content))
+ headers = Headers({
+ # 'Transfer-Encoding': ['Chunked'],
+ 'Content-Disposition': ['attachment; filename="{}"'.format(filename)]
+ })
+ l = len(content)
+ # lengthPrefix = hex(l)[2:].upper() # For some reason, this attachment is length-prefixed in hex
+ label_suffix = self._getEventSizeDescription(l / 1024)
+ # body = "{0}\n{1}\n0\n".format(lengthPrefix, content) # XXX There HAS to be a better way to do this
+ yield self.requester.post(
+ url,
+ content,
+ headers=headers,
+ method_label="POST{attach-%s}" % (label_suffix,)
+ )
</ins><span class="cx">
</span><del>- # Try to read data from disk - if it succeeds self.principalURL will be set
- self.deserialize()
-
- if self.principalURL is None:
- # PROPFIND principal path to retrieve actual principal-URL
- response = yield self._principalPropfindInitial(self.record.uid)
- hrefs = response.getHrefProperties()
- self.principalURL = hrefs[davxml.principal_URL].toString()
-
- # Using the actual principal URL, retrieve principal information
- principal = (yield self._extractPrincipalDetails())
- returnValue(principal)
-
-
-
-class OS_X_10_7(BaseAppleClient):
- """
- Implementation of the OS X 10.7 iCal network behavior.
- """
-
- _client_type = "OS X 10.7"
-
- USER_AGENT = "CalendarStore/5.0.2 (1166); iCal/5.0.2 (1571); Mac OS X/10.7.3 (11D50)"
-
- # The default interval, used if none is specified in external
- # configuration. This is also the actual value used by Snow
- # Leopard iCal.
- CALENDAR_HOME_POLL_INTERVAL = 15 * 60
-
- # The maximum number of resources to retrieve in a single multiget
- MULTIGET_BATCH_SIZE = 50
-
- # Override and turn on if client supports Sync REPORT
- _SYNC_REPORT = True
-
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = False
-
- # Override and turn off if client does not support attendee lookups
- _ATTENDEE_LOOKUPS = True
-
- # Request body data
- _LOAD_PATH = "OS_X_10_7"
-
- _STARTUP_WELL_KNOWN = loadRequestBody(_LOAD_PATH, 'startup_well_known')
- _STARTUP_PRINCIPAL_PROPFIND_INITIAL = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind_initial')
- _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind')
- _STARTUP_PRINCIPALS_REPORT = loadRequestBody(_LOAD_PATH, 'startup_principals_report')
- _STARTUP_PRINCIPAL_EXPAND = loadRequestBody(_LOAD_PATH, 'startup_principal_expand')
- _STARTUP_PROPPATCH_CALENDAR_COLOR = loadRequestBody(_LOAD_PATH, 'startup_calendar_color_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_ORDER = loadRequestBody(_LOAD_PATH, 'startup_calendar_order_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_TIMEZONE = loadRequestBody(_LOAD_PATH, 'startup_calendar_timezone_proppatch')
-
- _POLL_CALENDARHOME_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_propfind')
- _POLL_CALENDAR_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_CALENDAR_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind_d1')
- _POLL_CALENDAR_MULTIGET_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget')
- _POLL_CALENDAR_MULTIGET_REPORT_HREF = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget_hrefs')
- _POLL_CALENDAR_SYNC_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_sync')
- _POLL_NOTIFICATION_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_NOTIFICATION_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_notification_propfind_d1')
-
- _USER_LIST_PRINCIPAL_PROPERTY_SEARCH = loadRequestBody(_LOAD_PATH, 'user_list_principal_property_search')
- _POST_AVAILABILITY = loadRequestBody(_LOAD_PATH, 'post_availability')
-
-
- def _addDefaultHeaders(self, headers):
- """
- Add the clients default set of headers to ones being used in a request.
- Default is to add User-Agent, sub-classes should override to add other
- client specific things, Accept etc.
- """
-
- super(OS_X_10_7, self)._addDefaultHeaders(headers)
- headers.setRawHeaders('Accept', ['*/*'])
- headers.setRawHeaders('Accept-Language', ['en-us'])
- headers.setRawHeaders('Accept-Encoding', ['gzip,deflate'])
- headers.setRawHeaders('Connection', ['keep-alive'])
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def startup(self):
-
- # Try to read data from disk - if it succeeds self.principalURL will be set
- self.deserialize()
-
- if self.principalURL is None:
- # PROPFIND well-known with redirect
- response = yield self._startupPropfindWellKnown()
- hrefs = response.getHrefProperties()
- if davxml.current_user_principal in hrefs:
- self.principalURL = hrefs[davxml.current_user_principal].toString()
- elif davxml.principal_URL in hrefs:
- self.principalURL = hrefs[davxml.principal_URL].toString()
- else:
- # PROPFIND principal path to retrieve actual principal-URL
- response = yield self._principalPropfindInitial(self.record.uid)
- hrefs = response.getHrefProperties()
- self.principalURL = hrefs[davxml.principal_URL].toString()
-
- # Using the actual principal URL, retrieve principal information
- principal = yield self._extractPrincipalDetails()
- returnValue(principal)
-
-
-
-class iOS_5(BaseAppleClient):
- """
- Implementation of the iOS 5 network behavior.
- """
-
- _client_type = "iOS 5"
-
- USER_AGENT = "iOS/5.1 (9B179) dataaccessd/1.0"
-
- # The default interval, used if none is specified in external
- # configuration. This is also the actual value used by Snow
- # Leopard iCal.
- CALENDAR_HOME_POLL_INTERVAL = 15 * 60
-
- # The maximum number of resources to retrieve in a single multiget
- MULTIGET_BATCH_SIZE = 50
-
- # Override and turn on if client supports Sync REPORT
- _SYNC_REPORT = False
-
- # Override and turn on if client syncs using time-range queries
- _SYNC_TIMERANGE = True
-
- # Override and turn off if client does not support attendee lookups
- _ATTENDEE_LOOKUPS = False
-
- # Request body data
- _LOAD_PATH = "iOS_5"
-
- _STARTUP_WELL_KNOWN = loadRequestBody(_LOAD_PATH, 'startup_well_known')
- _STARTUP_PRINCIPAL_PROPFIND_INITIAL = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind_initial')
- _STARTUP_PRINCIPAL_PROPFIND = loadRequestBody(_LOAD_PATH, 'startup_principal_propfind')
- _STARTUP_PRINCIPALS_REPORT = loadRequestBody(_LOAD_PATH, 'startup_principals_report')
- _STARTUP_PROPPATCH_CALENDAR_COLOR = loadRequestBody(_LOAD_PATH, 'startup_calendar_color_proppatch')
- _STARTUP_PROPPATCH_CALENDAR_ORDER = loadRequestBody(_LOAD_PATH, 'startup_calendar_order_proppatch')
-
- _POLL_CALENDARHOME_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendarhome_propfind')
- _POLL_CALENDAR_PROPFIND = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind')
- _POLL_CALENDAR_VEVENT_TR_QUERY = loadRequestBody(_LOAD_PATH, 'poll_calendar_vevent_tr_query')
- _POLL_CALENDAR_VTODO_QUERY = loadRequestBody(_LOAD_PATH, 'poll_calendar_vtodo_query')
- _POLL_CALENDAR_PROPFIND_D1 = loadRequestBody(_LOAD_PATH, 'poll_calendar_propfind_d1')
- _POLL_CALENDAR_MULTIGET_REPORT = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget')
- _POLL_CALENDAR_MULTIGET_REPORT_HREF = loadRequestBody(_LOAD_PATH, 'poll_calendar_multiget_hrefs')
-
-
- def _addDefaultHeaders(self, headers):
</del><ins>+ def addCalendar(self, href, calendar_xml):
</ins><span class="cx"> """
</span><del>- Add the clients default set of headers to ones being used in a request.
- Default is to add User-Agent, sub-classes should override to add other
- client specific things, Accept etc.
</del><ins>+ client.addCalendar(
+ '/calendars/__uids__/10000000-0000-0000-0000-000000000001/1C1A8475-2671-4B97-AC58-DD9777B5CD93/',
+ # <Component: 'BEGIN:VCALENDAR\r\n...END:VCALENDAR\r\n'>)
+ )
</ins><span class="cx"> """
</span><ins>+ response = yield self.requester.mkcalendar(
+ href,
+ calendar_xml,
+ method_label="MK{calendar}",
+ )
+ # self._cacheCalendar(href, calendar)
</ins><span class="cx">
</span><del>- super(iOS_5, self)._addDefaultHeaders(headers)
- headers.setRawHeaders('Accept', ['*/*'])
- headers.setRawHeaders('Accept-Language', ['en-us'])
- headers.setRawHeaders('Accept-Encoding', ['gzip,deflate'])
- headers.setRawHeaders('Connection', ['keep-alive'])
</del><span class="cx">
</span><del>-
</del><span class="cx"> @inlineCallbacks
</span><del>- def _pollFirstTime1(self, homeNode, calendars):
- # Patch calendar properties
- for cal in calendars:
- if cal.name != "inbox":
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_COLOR,
- method_label="PROPPATCH{calendar}",
- )
- yield self._proppatch(
- cal.url,
- self._STARTUP_PROPPATCH_CALENDAR_ORDER,
- method_label="PROPPATCH{calendar}",
- )
</del><ins>+ def changeCalendar(self, href, body):
</ins><span class="cx">
</span><ins>+ calendar = self._calendars[href]
+ headers = Headers({
+ 'content-type': ['text/xml']
+ })
</ins><span class="cx">
</span><del>- def _pollFirstTime2(self):
- # Nothing here
- return succeed(None)
</del><ins>+ # At last, upload the new event definition
+ response = yield self.requester.proppatch(
+ href,
+ body,
+ headers=headers,
+ method_label="PATCH{calendar}"
+ )
</ins><span class="cx">
</span><ins>+ # Finally, re-retrieve the event to update the etag
+ # yield self._updateEvent(response, href)
</ins><span class="cx">
</span><del>- def _updateCalendar(self, calendar, newToken):
- """
- Update the local cached data for a calendar in an appropriate manner.
- """
- if calendar.name == "inbox":
- # Inbox is done as a PROPFIND Depth:1
- return self._updateCalendar_PROPFIND(calendar, newToken)
- elif "VEVENT" in calendar.componentTypes:
- # VEVENTs done as time-range VEVENT-only queries
- return self._updateCalendar_VEVENT(calendar, newToken)
- elif "VTODO" in calendar.componentTypes:
- # VTODOs done as VTODO-only queries
- return self._updateCalendar_VTODO(calendar, newToken)
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def _updateCalendar_VEVENT(self, calendar, newToken):
- """
- Sync all locally cached VEVENTs using a VEVENT-only time-range query.
- """
-
- # Grab old hrefs prior to the PROPFIND so we sync with the old state. We need this because
- # the sim can fire a PUT between the PROPFIND and when process the removals.
- old_hrefs = set([calendar.url + child for child in calendar.events.keys()])
-
- now = DateTime.getNowUTC()
- now.setDateOnly(True)
- now.offsetMonth(-1) # 1 month back default
- result = yield self._report(
- calendar.url,
- self._POLL_CALENDAR_VEVENT_TR_QUERY % {"start-date": now.getText()},
- depth='1',
- method_label="REPORT{vevent}",
</del><ins>+ def postXML(self, href, xml):
+ headers = Headers({
+ 'content-type': ['text/xml']
+ })
+ response = yield self.requester.post(
+ href,
+ xml,
+ headers=headers,
+ method_label="SHARE{calendar}"
</ins><span class="cx"> )
</span><span class="cx">
</span><del>- yield self._updateApplyChanges(calendar, result, old_hrefs)
</del><span class="cx">
</span><del>- # Now update calendar to the new token
- self._calendars[calendar.url].changeToken = newToken
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>- def _updateCalendar_VTODO(self, calendar, newToken):
</del><ins>+ def deleteCalendar(self, href):
</ins><span class="cx"> """
</span><del>- Sync all locally cached VTODOs using a VTODO-only query.
</del><ins>+ Issue a DELETE for the given URL and remove local state
+ associated with that calendar.
+
+ Usage: client.deleteCalendar('/calendars/__uids__/<user-uid>/<calendar-uid>/')
</ins><span class="cx"> """
</span><span class="cx">
</span><del>- # Grab old hrefs prior to the PROPFIND so we sync with the old state. We need this because
- # the sim can fire a PUT between the PROPFIND and when process the removals.
- old_hrefs = set([calendar.url + child for child in calendar.events.keys()])
</del><ins>+ self._invalidateCalendar(href)
</ins><span class="cx">
</span><del>- result = yield self._report(
- calendar.url,
- self._POLL_CALENDAR_VTODO_QUERY,
- depth='1',
- method_label="REPORT{vtodo}",
- )
</del><ins>+ response = yield self.requester.delete(href, method_label="DELETE{calendar}")
+ returnValue(response)
</ins><span class="cx">
</span><del>- yield self._updateApplyChanges(calendar, result, old_hrefs)
</del><span class="cx">
</span><del>- # Now update calendar to the new token
- self._calendars[calendar.url].changeToken = newToken
</del><span class="cx">
</span><del>-
- @inlineCallbacks
- def startup(self):
-
- # Try to read data from disk - if it succeeds self.principalURL will be set
- self.deserialize()
-
- if self.principalURL is None:
- # PROPFIND well-known with redirect
- response = yield self._startupPropfindWellKnown()
- hrefs = response.getHrefProperties()
- if davxml.current_user_principal in hrefs:
- self.principalURL = hrefs[davxml.current_user_principal].toString()
- elif davxml.principal_URL in hrefs:
- self.principalURL = hrefs[davxml.principal_URL].toString()
- else:
- # PROPFIND principal path to retrieve actual principal-URL
- response = yield self._principalPropfindInitial(self.record.uid)
- hrefs = response.getHrefProperties()
- self.principalURL = hrefs[davxml.principal_URL].toString()
-
- # Using the actual principal URL, retrieve principal information
- principal = yield self._extractPrincipalDetails()
- returnValue(principal)
-
-
-
-class RequestLogger(object):
- format = u"%(user)s request %(code)s%(success)s[%(duration)5.2f s] %(method)8s %(url)s"
- success = u"\N{CHECK MARK}"
- failure = u"\N{BALLOT X}"
-
- def observe(self, event):
- if event.get("type") == "response":
- formatArgs = dict(
- user=event['user'],
- method=event['method'],
- url=urlunparse(('', '') + urlparse(event['url'])[2:]),
- code=event['code'],
- duration=event['duration'],
- )
-
- if event['success']:
- formatArgs['success'] = self.success
- else:
- formatArgs['success'] = self.failure
- print((self.format % formatArgs).encode('utf-8'))
-
-
- def report(self, output):
- pass
-
-
- def failures(self):
- return []
-
-
-
</del><span class="cx"> def main():
</span><span class="cx"> from urllib2 import HTTPDigestAuthHandler
</span><span class="cx"> from twisted.internet import reactor
</span><ins>+ from twisted.python.log import addObserver
+ from contrib.performance.loadtest.logger import RequestLogger
+ from contrib.performance.loadtest.clients import OS_X_10_11
</ins><span class="cx"> auth = HTTPDigestAuthHandler()
</span><span class="cx"> auth.add_password(
</span><span class="cx"> realm="Test Realm",
</span><span class="lines">@@ -2080,14 +1422,17 @@
</span><span class="cx">
</span><span class="cx"> addObserver(RequestLogger().observe)
</span><span class="cx">
</span><del>- from sim import _DirectoryRecord
- client = OS_X_10_6(
- reactor, 'http://127.0.0.1:8008/',
- _DirectoryRecord(
- u'user01', u'user01', u'User 01', u'user01@example.org'),
- auth)
</del><ins>+ from contrib.performance.loadtest.records import DirectoryRecord
+ client = OS_X_10_11(
+ reactor,
+ 'http://127.0.0.1:8008/', # root
+ '/tmp/sim', # serializePath
+ DirectoryRecord(u'user01', u'user01', u'User 01', u'user01@example.org', u'10000000-0000-0000-0000-000000000001'),
+ auth,
+ title='OS X 10.11 Client Simulator'
+ )
</ins><span class="cx"> d = client.run()
</span><del>- d.addErrback(err, "10.6 client run() problem")
</del><ins>+ d.addErrback(err, "10.11 client run() problem")
</ins><span class="cx"> d.addCallback(lambda ignored: reactor.stop())
</span><span class="cx"> reactor.run()
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestloggerpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/logger.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/logger.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -14,9 +14,27 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> #
</span><span class="cx"> ##
</span><ins>+import json
+import collections
+import os
+import sys
+from datetime import datetime
</ins><span class="cx">
</span><del>-from contrib.performance.stats import mean, median, stddev
</del><ins>+from urlparse import urlparse, urlunparse
</ins><span class="cx">
</span><ins>+from contrib.performance.stats import mean, median, stddev, mad
+
+class TerminalColors:
+ HEADER = '\033[95m'
+ OKBLUE = '\033[94m'
+ OKGREEN = '\033[92m'
+ WARNING = '\033[93m'
+ LIGHTBLUE = '\033[36m'
+ FAIL = '\033[91m'
+ ENDC = '\033[0m'
+ BOLD = '\033[1m'
+ UNDERLINE = '\033[4m'
+
</ins><span class="cx"> class SummarizingMixin(object):
</span><span class="cx">
</span><span class="cx"> def printHeader(self, output, fields):
</span><span class="lines">@@ -87,3 +105,467 @@
</span><span class="cx"> """
</span><span class="cx"> for method, data in perOperationTimes:
</span><span class="cx"> self._printRow(output, formats, self._summarizeData(method, data))
</span><ins>+
+
+
+class RequestLogger(object):
+ format = u"%(user)s request %(code)s%(success)s[%(duration)5.2f s] %(method)8s %(url)s"
+ success = u"\N{CHECK MARK}"
+ failure = u"\N{BALLOT X}"
+
+ def observe(self, event):
+ if event.get("type") == "response":
+ formatArgs = dict(
+ user=event['user'],
+ method=event['method'],
+ url=urlunparse(('', '') + urlparse(event['url'])[2:]),
+ code=event['code'],
+ duration=event['duration'],
+ )
+
+ if event['success']:
+ formatArgs['success'] = self.success
+ start = TerminalColors.OKGREEN
+ else:
+ formatArgs['success'] = self.failure
+ start = TerminalColors.FAIL
+ print(start + (self.format % formatArgs).encode('utf-8') + TerminalColors.ENDC)
+
+
+ def report(self, output):
+ pass
+
+
+ def failures(self):
+ return []
+
+
+
+class OperationLogger(SummarizingMixin):
+ """
+ Profiles will initiate operations which may span multiple requests. Start
+ and stop log messages are emitted for these operations and logged by this
+ logger.
+ """
+ formats = {
+ u"start" : u"%(user)s - - - - - - - - - - - %(label)8s BEGIN %(lag)s",
+ u"end" : u"%(user)s - - - - - - - - - - - %(label)8s END [%(duration)5.2f s]",
+ u"failed": u"%(user)s x x x x x x x x x x x %(label)8s FAILED %(reason)s",
+ }
+
+ lagFormat = u'{lag %5.2f ms}'
+
+ # the response time thresholds to display together with failing % count threshold
+ _thresholds_default = {
+ "operations": {
+ "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds": {
+ "default": [100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
+ }
+ }
+ }
+ _lag_cut_off = 1.0 # Maximum allowed median scheduling latency, seconds
+ _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
+
+ _fields_init = [
+ ('operation', -25, '%-25s'),
+ ('count', 8, '%8s'),
+ ('failed', 8, '%8s'),
+ ]
+
+ _fields_extend = [
+ ('mean', 8, '%8.4f'),
+ ('median', 8, '%8.4f'),
+ ('stddev', 8, '%8.4f'),
+ ('avglag (ms)', 12, '%12.4f'),
+ ('STATUS', 8, '%8s'),
+ ]
+
+ def __init__(self, outfile=None, **params):
+ self._perOperationTimes = {}
+ self._perOperationLags = {}
+ if outfile is None:
+ outfile = sys.stdout
+ self._outfile = outfile
+
+ # Load parameters from config
+ if "thresholdsPath" in params:
+ jsondata = json.load(open(params["thresholdsPath"]))
+ elif "thresholds" in params:
+ jsondata = params["thresholds"]
+ else:
+ jsondata = self._thresholds_default
+ self._thresholds = [[limit, {}] for limit in jsondata["operations"]["limits"]]
+ for ctr, item in enumerate(self._thresholds):
+ for k, v in jsondata["operations"]["thresholds"].items():
+ item[1][k] = v[ctr]
+
+ self._fields = self._fields_init[:]
+ for threshold, _ignore_fail_at in self._thresholds:
+ self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
+ self._fields.extend(self._fields_extend)
+
+ if "lagCutoff" in params:
+ self._lag_cut_off = params["lagCutoff"]
+
+ if "failCutoff" in params:
+ self._fail_cut_off = params["failCutoff"]
+
+
+ def observe(self, event):
+ if event.get("type") == "operation":
+ event = event.copy()
+ lag = event.get('lag')
+ if lag is None:
+ event['lag'] = ''
+ else:
+ event['lag'] = self.lagFormat % (lag * 1000.0,)
+
+ self._outfile.write(
+ TerminalColors.LIGHTBLUE +
+ (self.formats[event[u'phase']] % event).encode('utf-8') + TerminalColors.ENDC + '\n')
+
+ if event[u'phase'] == u'end':
+ dataset = self._perOperationTimes.setdefault(event[u'label'], [])
+ dataset.append((event[u'success'], event[u'duration']))
+ elif lag is not None:
+ dataset = self._perOperationLags.setdefault(event[u'label'], [])
+ dataset.append(lag)
+
+
+ def _summarizeData(self, operation, data):
+ avglag = mean(self._perOperationLags.get(operation, [0.0])) * 1000.0
+ data = SummarizingMixin._summarizeData(self, operation, data)
+ return data[:-1] + (avglag,) + data[-1:]
+
+
+ def report(self, output):
+ output.write("\n")
+ self.printHeader(output, [
+ (label, width)
+ for (label, width, _ignore_fmt) in self._fields
+ ])
+ self.printData(
+ output,
+ [fmt for (label, width, fmt) in self._fields],
+ sorted(self._perOperationTimes.items())
+ )
+
+ _LATENCY_REASON = "Median %(operation)s scheduling lag greater than %(cutoff)sms"
+ _FAILED_REASON = "Greater than %(cutoff).0f%% %(operation)s failed"
+
+ def failures(self):
+ reasons = []
+
+ for operation, lags in self._perOperationLags.iteritems():
+ if median(lags) > self._lag_cut_off:
+ reasons.append(self._LATENCY_REASON % dict(
+ operation=operation.upper(), cutoff=self._lag_cut_off * 1000))
+
+ for operation, times in self._perOperationTimes.iteritems():
+ failures = len([success for (success, _ignore_duration) in times if not success])
+ if failures * 100.0 / len(times) > self._fail_cut_off:
+ reasons.append(self._FAILED_REASON % dict(
+ operation=operation.upper(), cutoff=self._fail_cut_off))
+
+ return reasons
+
+
+
+class StatisticsBase(object):
+ def observe(self, event):
+ if event.get('type') == 'response':
+ self.eventReceived(event)
+ elif event.get('type') == 'client-failure':
+ self.clientFailure(event)
+ elif event.get('type') == 'sim-failure':
+ self.simFailure(event)
+
+
+ def report(self, output):
+ pass
+
+
+ def failures(self):
+ return []
+
+
+
+class SimpleStatistics(StatisticsBase):
+ def __init__(self):
+ self._times = []
+ self._failures = collections.defaultdict(int)
+ self._simFailures = collections.defaultdict(int)
+
+
+ def eventReceived(self, event):
+ self._times.append(event['duration'])
+ if len(self._times) == 200:
+ print('mean:', mean(self._times))
+ print('median:', median(self._times))
+ print('stddev:', stddev(self._times))
+ print('mad:', mad(self._times))
+ del self._times[:100]
+
+
+ def clientFailure(self, event):
+ self._failures[event] += 1
+
+
+ def simFailure(self, event):
+ self._simFailures[event] += 1
+
+
+
+class ReportStatistics(StatisticsBase, SummarizingMixin):
+ """
+
+ @ivar _users: A C{set} containing all user UIDs which have been observed in
+ events. When generating the final report, the size of this set is
+ reported as the number of users in the simulation.
+
+ """
+
+ # the response time thresholds to display together with failing % count threshold
+ _thresholds_default = {
+ "requests": {
+ "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
+ "thresholds": {
+ "default": [100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ }
+ }
+ }
+ _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
+
+ _fields_init = [
+ ('request', -25, '%-25s'),
+ ('count', 8, '%8s'),
+ ('failed', 8, '%8s'),
+ ]
+
+ _fields_extend = [
+ ('mean', 8, '%8.4f'),
+ ('median', 8, '%8.4f'),
+ ('stddev', 8, '%8.4f'),
+ ('QoS', 8, '%8.4f'),
+ ('STATUS', 8, '%8s'),
+ ]
+
+ def __init__(self, **params):
+ self._perMethodTimes = {}
+ self._users = set()
+ self._clients = set()
+ self._failed_clients = []
+ self._failed_sim = collections.defaultdict(int)
+ self._startTime = datetime.now()
+ self._expired_data = None
+
+ # Load parameters from config
+ if "thresholdsPath" in params:
+ jsondata = json.load(open(params["thresholdsPath"]))
+ elif "thresholds" in params:
+ jsondata = params["thresholds"]
+ else:
+ jsondata = self._thresholds_default
+ self._thresholds = [[limit, {}] for limit in jsondata["requests"]["limits"]]
+ for ctr, item in enumerate(self._thresholds):
+ for k, v in jsondata["requests"]["thresholds"].items():
+ item[1][k] = v[ctr]
+
+ self._fields = self._fields_init[:]
+ for threshold, _ignore_fail_at in self._thresholds:
+ self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
+ self._fields.extend(self._fields_extend)
+
+ if "benchmarksPath" in params:
+ self.benchmarks = json.load(open(params["benchmarksPath"]))
+ else:
+ self.benchmarks = {}
+
+ if "failCutoff" in params:
+ self._fail_cut_off = params["failCutoff"]
+
+
+ def observe(self, event):
+ if event.get('type') == 'sim-expired':
+ self.simExpired(event)
+ else:
+ super(ReportStatistics, self).observe(event)
+
+
+ def countUsers(self):
+ return len(self._users)
+
+
+ def countClients(self):
+ return len(self._clients)
+
+
+ def countClientFailures(self):
+ return len(self._failed_clients)
+
+
+ def countSimFailures(self):
+ return len(self._failed_sim)
+
+
+ def eventReceived(self, event):
+ dataset = self._perMethodTimes.setdefault(event['method'], [])
+ dataset.append((event['success'], event['duration']))
+ self._users.add(event['user'])
+ self._clients.add(event['client_id'])
+
+
+ def clientFailure(self, event):
+ self._failed_clients.append(event['reason'])
+
+
+ def simFailure(self, event):
+ self._failed_sim[event['reason']] += 1
+
+
+ def simExpired(self, event):
+ self._expired_data = event['reason']
+
+
+ def printMiscellaneous(self, output, items):
+ maxColumnWidth = str(len(max(items.iterkeys(), key=len)))
+ fmt = "%" + maxColumnWidth + "s : %-s\n"
+ for k in sorted(items.iterkeys()):
+ output.write(fmt % (k.title(), items[k],))
+
+
+ def qos(self):
+ """
+ Determine a "quality of service" value that can be used for comparisons between runs. This value
+ is based on the percentage deviation of means of each request from a set of "benchmarks" for each
+ type of request.
+ """
+
+ # Get means for each type of method
+ means = {}
+ for method, results in self._perMethodTimes.items():
+ means[method] = mean([duration for success, duration in results if success])
+
+ # Determine percentage differences with weighting
+ differences = []
+ for method, value in means.items():
+ result = self.qos_value(method, value)
+ if result is not None:
+ differences.append(result)
+
+ return ("%-8.4f" % mean(differences)) if differences else "None"
+
+
+ def qos_value(self, method, value):
+ benchmark = self.benchmarks.get(method)
+ if benchmark is None:
+ return None
+ test_mean, weight = (benchmark["mean"], benchmark["weight"],)
+ return ((value / test_mean) - 1.0) * weight + 1.0
+
+
+ def _summarizeData(self, operation, data):
+ data = SummarizingMixin._summarizeData(self, operation, data)
+ value = self.qos_value(operation, data[-4])
+ if value is None:
+ value = 0.0
+ return data[:-1] + (value,) + data[-1:]
+
+
+ def report(self, output):
+ output.write("\n")
+ output.write("** REPORT **\n")
+ output.write("\n")
+ runtime = datetime.now() - self._startTime
+ cpu = os.times()
+ cpuUser = cpu[0] + cpu[2]
+ cpuSys = cpu[1] + cpu[3]
+ cpuTotal = cpuUser + cpuSys
+ runHours, remainder = divmod(runtime.seconds, 3600)
+ runMinutes, runSeconds = divmod(remainder, 60)
+ cpuHours, remainder = divmod(cpuTotal, 3600)
+ cpuMinutes, cpuSeconds = divmod(remainder, 60)
+ items = {
+ 'Users': self.countUsers(),
+ 'Clients': self.countClients(),
+ 'Start time': self._startTime.strftime('%m/%d %H:%M:%S'),
+ 'Run time': "%02d:%02d:%02d" % (runHours, runMinutes, runSeconds),
+ 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,),
+ 'QoS': self.qos(),
+ }
+ if self.countClientFailures() > 0:
+ items['Failed clients'] = self.countClientFailures()
+ for ctr, reason in enumerate(self._failed_clients, 1):
+ items['Failure #%d' % (ctr,)] = reason
+ if self.countSimFailures() > 0:
+ for reason, count in self._failed_sim.items():
+ items['Failed operation'] = "%s : %d times" % (reason, count,)
+ output.write("* Client\n")
+ self.printMiscellaneous(output, items)
+ output.write("\n")
+
+ if self._expired_data is not None:
+ items = {
+ "Req/sec" : "%.1f" % (self._expired_data[0],),
+ "Response": "%.1f (ms)" % (self._expired_data[1],),
+ "Slots": "%.2f" % (self._expired_data[2],),
+ "CPU": "%.1f%%" % (self._expired_data[3],),
+ }
+ output.write("* Server (Last 5 minutes)\n")
+ self.printMiscellaneous(output, items)
+ output.write("\n")
+ output.write("* Details\n")
+
+ self.printHeader(output, [
+ (label, width)
+ for (label, width, _ignore_fmt)
+ in self._fields
+ ])
+ self.printData(
+ output,
+ [fmt for (label, width, fmt) in self._fields],
+ sorted(self._perMethodTimes.items())
+ )
+
+ _FAILED_REASON = "Greater than %(cutoff)g%% %(method)s failed"
+
+ _REASON_1 = "Greater than %(cutoff)g%% %(method)s exceeded "
+ _REASON_2 = "%g second response time"
+
+ def failures(self):
+ # TODO
+ reasons = []
+
+ for (method, times) in self._perMethodTimes.iteritems():
+ failures = 0
+ overDurations = [0] * len(self._thresholds)
+
+ for success, duration in times:
+ if not success:
+ failures += 1
+ for ctr, item in enumerate(self._thresholds):
+ threshold, _ignore_fail_at = item
+ if duration > threshold:
+ overDurations[ctr] += 1
+
+ checks = [
+ (failures, self._fail_cut_off, self._FAILED_REASON),
+ ]
+
+ for ctr, item in enumerate(self._thresholds):
+ threshold, fail_at = item
+ fail_at = fail_at.get(method, fail_at["default"])
+ checks.append(
+ (overDurations[ctr], fail_at, self._REASON_1 + self._REASON_2 % (threshold,))
+ )
+
+ for count, cutoff, reason in checks:
+ if count * 100.0 / len(times) > cutoff:
+ reasons.append(reason % dict(method=method, cutoff=cutoff))
+
+ if self.countClientFailures() != 0:
+ reasons.append("Client failures: %d" % (self.countClientFailures(),))
+ if self.countSimFailures() != 0:
+ reasons.append("Overall failures: %d" % (self.countSimFailures(),))
+ return reasons
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestpopulationpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/population.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/population.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -25,13 +25,10 @@
</span><span class="cx">
</span><span class="cx"> from tempfile import mkdtemp
</span><span class="cx"> from itertools import izip
</span><del>-from datetime import datetime
</del><ins>+
</ins><span class="cx"> from urllib2 import HTTPBasicAuthHandler
</span><span class="cx"> from urllib2 import HTTPDigestAuthHandler
</span><span class="cx"> from urllib2 import HTTPPasswordMgrWithDefaultRealm
</span><del>-import collections
-import json
-import os
</del><span class="cx">
</span><span class="cx"> from twisted.internet.defer import DeferredList
</span><span class="cx"> from twisted.python.failure import Failure
</span><span class="lines">@@ -41,10 +38,8 @@
</span><span class="cx">
</span><span class="cx"> from twistedcaldav.timezones import TimezoneCache
</span><span class="cx">
</span><del>-from contrib.performance.stats import mean, median, stddev, mad
</del><span class="cx"> from contrib.performance.loadtest.trafficlogger import loggedReactor
</span><del>-from contrib.performance.loadtest.logger import SummarizingMixin
-from contrib.performance.loadtest.ical import OS_X_10_6, RequestLogger
</del><ins>+
</ins><span class="cx"> from contrib.performance.loadtest.profiles import Eventer, Inviter, Accepter
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -64,10 +59,16 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def __call__(self, reactor, simulator, client, number):
</span><del>- return self.profileType(reactor, simulator, client, number, **self.params)
</del><ins>+ base = self.profileType(**self.params)
+ base.setUp(reactor, simulator, client, number)
+ return base
</ins><span class="cx">
</span><span class="cx">
</span><ins>+ def __repr__(self):
+ return "ProfileType(%s, params=%s)" % (self.profileType.__name__, self.params)
</ins><span class="cx">
</span><ins>+
+
</ins><span class="cx"> class ClientType(object, FancyEqMixin):
</span><span class="cx"> """
</span><span class="cx"> @ivar clientType: An L{ICalendarClient} implementation
</span><span class="lines">@@ -81,14 +82,15 @@
</span><span class="cx"> self.profileTypes = profileTypes
</span><span class="cx">
</span><span class="cx">
</span><del>- def new(self, reactor, serverAddress, principalPathTemplate, serializationPath, userRecord, authInfo):
</del><ins>+ def new(self, reactor, serverAddress, serializationPath, userRecord, authInfo):
</ins><span class="cx"> """
</span><span class="cx"> Create a new instance of this client type.
</span><span class="cx"> """
</span><ins>+ # print(self.clientType)
+ # print(self.clientParams)
</ins><span class="cx"> return self.clientType(
</span><del>- reactor, serverAddress, principalPathTemplate,
- serializationPath, userRecord, authInfo,
- **self.clientParams
</del><ins>+ reactor, serverAddress, serializationPath,
+ userRecord, authInfo, **self.clientParams
</ins><span class="cx"> )
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -163,12 +165,13 @@
</span><span class="cx">
</span><span class="cx"> class CalendarClientSimulator(object):
</span><span class="cx"> def __init__(self, records, populator, parameters, reactor, server,
</span><del>- principalPathTemplate, serializationPath, workerIndex=0, workerCount=1):
</del><ins>+ serializationPath, workerIndex=0, workerCount=1):
+ from pprint import pprint
+ pprint(records)
</ins><span class="cx"> self._records = records
</span><span class="cx"> self.populator = populator
</span><span class="cx"> self.reactor = reactor
</span><span class="cx"> self.server = server
</span><del>- self.principalPathTemplate = principalPathTemplate
</del><span class="cx"> self.serializationPath = serializationPath
</span><span class="cx"> self._pop = self.populator.populate(parameters)
</span><span class="cx"> self._user = 0
</span><span class="lines">@@ -177,7 +180,7 @@
</span><span class="cx"> self.workerCount = workerCount
</span><span class="cx"> self.clients = []
</span><span class="cx">
</span><del>- TimezoneCache.create()
</del><ins>+ # TimezoneCache.create()
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def getUserRecord(self, index):
</span><span class="lines">@@ -242,7 +245,6 @@
</span><span class="cx"> client = clientType.new(
</span><span class="cx"> reactor,
</span><span class="cx"> self.server,
</span><del>- self.principalPathTemplate,
</del><span class="cx"> self.serializationPath,
</span><span class="cx"> self.getUserRecord(number),
</span><span class="cx"> auth,
</span><span class="lines">@@ -252,6 +254,7 @@
</span><span class="cx"> d.addErrback(self._clientFailure, reactor)
</span><span class="cx">
</span><span class="cx"> for profileType in clientType.profileTypes:
</span><ins>+ print(profileType)
</ins><span class="cx"> profile = profileType(reactor, self, client, number)
</span><span class="cx"> if profile.enabled:
</span><span class="cx"> d = profile.initialize()
</span><span class="lines">@@ -265,7 +268,23 @@
</span><span class="cx"> # really used by much anyway.
</span><span class="cx"> msg(type="status", clientCount=self._user - 1)
</span><span class="cx">
</span><ins>+ # for i in range(numClients):
+ # for j in range(clientsPerUser):
+ # client = self._pop.next()
+ # # Reactor magic goes here
+ # client.setUp(
+ # self.reactor,
+ # self.server,
+ # self.serializationPath,
+ # self.getUserRecord(number),
+ # auth
+ # )
+ # for profile in client.profiles:
+ # profile.setUp(self.reactor, self, client, number)
</ins><span class="cx">
</span><ins>+
+
+
</ins><span class="cx"> def _dumpLogs(self, loggingReactor, reason):
</span><span class="cx"> path = FilePath(mkdtemp())
</span><span class="cx"> logstate = loggingReactor.getLogFiles()
</span><span class="lines">@@ -278,6 +297,13 @@
</span><span class="cx"> return path
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def _profileFailure(self, reason, profileType, reactor):
+ if not self._stopped:
+ where = self._dumpLogs(reactor, reason)
+ err(reason, "Profile stopped with error; recent traffic in %r" % (
+ where.path,))
+
+
</ins><span class="cx"> def _clientFailure(self, reason, reactor):
</span><span class="cx"> if not self._stopped:
</span><span class="cx"> where = self._dumpLogs(reactor, reason)
</span><span class="lines">@@ -288,13 +314,6 @@
</span><span class="cx"> msg(type="client-failure", reason="%s: %s" % (reason.type, reason.value,))
</span><span class="cx">
</span><span class="cx">
</span><del>- def _profileFailure(self, reason, profileType, reactor):
- if not self._stopped:
- where = self._dumpLogs(reactor, reason)
- err(reason, "Profile stopped with error; recent traffic in %r" % (
- where.path,))
-
-
</del><span class="cx"> def _simFailure(self, reason, reactor):
</span><span class="cx"> if not self._stopped:
</span><span class="cx"> msg(type="sim-failure", reason=reason)
</span><span class="lines">@@ -317,306 +336,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-class StatisticsBase(object):
- def observe(self, event):
- if event.get('type') == 'response':
- self.eventReceived(event)
- elif event.get('type') == 'client-failure':
- self.clientFailure(event)
- elif event.get('type') == 'sim-failure':
- self.simFailure(event)
-
-
- def report(self, output):
- pass
-
-
- def failures(self):
- return []
-
-
-
-class SimpleStatistics(StatisticsBase):
- def __init__(self):
- self._times = []
- self._failures = collections.defaultdict(int)
- self._simFailures = collections.defaultdict(int)
-
-
- def eventReceived(self, event):
- self._times.append(event['duration'])
- if len(self._times) == 200:
- print('mean:', mean(self._times))
- print('median:', median(self._times))
- print('stddev:', stddev(self._times))
- print('mad:', mad(self._times))
- del self._times[:100]
-
-
- def clientFailure(self, event):
- self._failures[event] += 1
-
-
- def simFailure(self, event):
- self._simFailures[event] += 1
-
-
-
-class ReportStatistics(StatisticsBase, SummarizingMixin):
- """
-
- @ivar _users: A C{set} containing all user UIDs which have been observed in
- events. When generating the final report, the size of this set is
- reported as the number of users in the simulation.
-
- """
-
- # the response time thresholds to display together with failing % count threshold
- _thresholds_default = {
- "requests": {
- "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
- "thresholds": {
- "default": [100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
- }
- }
- }
- _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
-
- _fields_init = [
- ('request', -25, '%-25s'),
- ('count', 8, '%8s'),
- ('failed', 8, '%8s'),
- ]
-
- _fields_extend = [
- ('mean', 8, '%8.4f'),
- ('median', 8, '%8.4f'),
- ('stddev', 8, '%8.4f'),
- ('QoS', 8, '%8.4f'),
- ('STATUS', 8, '%8s'),
- ]
-
- def __init__(self, **params):
- self._perMethodTimes = {}
- self._users = set()
- self._clients = set()
- self._failed_clients = []
- self._failed_sim = collections.defaultdict(int)
- self._startTime = datetime.now()
- self._expired_data = None
-
- # Load parameters from config
- if "thresholdsPath" in params:
- jsondata = json.load(open(params["thresholdsPath"]))
- elif "thresholds" in params:
- jsondata = params["thresholds"]
- else:
- jsondata = self._thresholds_default
- self._thresholds = [[limit, {}] for limit in jsondata["requests"]["limits"]]
- for ctr, item in enumerate(self._thresholds):
- for k, v in jsondata["requests"]["thresholds"].items():
- item[1][k] = v[ctr]
-
- self._fields = self._fields_init[:]
- for threshold, _ignore_fail_at in self._thresholds:
- self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
- self._fields.extend(self._fields_extend)
-
- if "benchmarksPath" in params:
- self.benchmarks = json.load(open(params["benchmarksPath"]))
- else:
- self.benchmarks = {}
-
- if "failCutoff" in params:
- self._fail_cut_off = params["failCutoff"]
-
-
- def observe(self, event):
- if event.get('type') == 'sim-expired':
- self.simExpired(event)
- else:
- super(ReportStatistics, self).observe(event)
-
-
- def countUsers(self):
- return len(self._users)
-
-
- def countClients(self):
- return len(self._clients)
-
-
- def countClientFailures(self):
- return len(self._failed_clients)
-
-
- def countSimFailures(self):
- return len(self._failed_sim)
-
-
- def eventReceived(self, event):
- dataset = self._perMethodTimes.setdefault(event['method'], [])
- dataset.append((event['success'], event['duration']))
- self._users.add(event['user'])
- self._clients.add(event['client_id'])
-
-
- def clientFailure(self, event):
- self._failed_clients.append(event['reason'])
-
-
- def simFailure(self, event):
- self._failed_sim[event['reason']] += 1
-
-
- def simExpired(self, event):
- self._expired_data = event['reason']
-
-
- def printMiscellaneous(self, output, items):
- maxColumnWidth = str(len(max(items.iterkeys(), key=len)))
- fmt = "%" + maxColumnWidth + "s : %-s\n"
- for k in sorted(items.iterkeys()):
- output.write(fmt % (k.title(), items[k],))
-
-
- def qos(self):
- """
- Determine a "quality of service" value that can be used for comparisons between runs. This value
- is based on the percentage deviation of means of each request from a set of "benchmarks" for each
- type of request.
- """
-
- # Get means for each type of method
- means = {}
- for method, results in self._perMethodTimes.items():
- means[method] = mean([duration for success, duration in results if success])
-
- # Determine percentage differences with weighting
- differences = []
- for method, value in means.items():
- result = self.qos_value(method, value)
- if result is not None:
- differences.append(result)
-
- return ("%-8.4f" % mean(differences)) if differences else "None"
-
-
- def qos_value(self, method, value):
- benchmark = self.benchmarks.get(method)
- if benchmark is None:
- return None
- test_mean, weight = (benchmark["mean"], benchmark["weight"],)
- return ((value / test_mean) - 1.0) * weight + 1.0
-
-
- def _summarizeData(self, operation, data):
- data = SummarizingMixin._summarizeData(self, operation, data)
- value = self.qos_value(operation, data[-4])
- if value is None:
- value = 0.0
- return data[:-1] + (value,) + data[-1:]
-
-
- def report(self, output):
- output.write("\n")
- output.write("** REPORT **\n")
- output.write("\n")
- runtime = datetime.now() - self._startTime
- cpu = os.times()
- cpuUser = cpu[0] + cpu[2]
- cpuSys = cpu[1] + cpu[3]
- cpuTotal = cpuUser + cpuSys
- runHours, remainder = divmod(runtime.seconds, 3600)
- runMinutes, runSeconds = divmod(remainder, 60)
- cpuHours, remainder = divmod(cpuTotal, 3600)
- cpuMinutes, cpuSeconds = divmod(remainder, 60)
- items = {
- 'Users': self.countUsers(),
- 'Clients': self.countClients(),
- 'Start time': self._startTime.strftime('%m/%d %H:%M:%S'),
- 'Run time': "%02d:%02d:%02d" % (runHours, runMinutes, runSeconds),
- 'CPU Time': "user %-5.2f sys %-5.2f total %02d:%02d:%02d" % (cpuUser, cpuSys, cpuHours, cpuMinutes, cpuSeconds,),
- 'QoS': self.qos(),
- }
- if self.countClientFailures() > 0:
- items['Failed clients'] = self.countClientFailures()
- for ctr, reason in enumerate(self._failed_clients, 1):
- items['Failure #%d' % (ctr,)] = reason
- if self.countSimFailures() > 0:
- for reason, count in self._failed_sim.items():
- items['Failed operation'] = "%s : %d times" % (reason, count,)
- output.write("* Client\n")
- self.printMiscellaneous(output, items)
- output.write("\n")
-
- if self._expired_data is not None:
- items = {
- "Req/sec" : "%.1f" % (self._expired_data[0],),
- "Response": "%.1f (ms)" % (self._expired_data[1],),
- "Slots": "%.2f" % (self._expired_data[2],),
- "CPU": "%.1f%%" % (self._expired_data[3],),
- }
- output.write("* Server (Last 5 minutes)\n")
- self.printMiscellaneous(output, items)
- output.write("\n")
- output.write("* Details\n")
-
- self.printHeader(output, [
- (label, width)
- for (label, width, _ignore_fmt)
- in self._fields
- ])
- self.printData(
- output,
- [fmt for (label, width, fmt) in self._fields],
- sorted(self._perMethodTimes.items())
- )
-
- _FAILED_REASON = "Greater than %(cutoff)g%% %(method)s failed"
-
- _REASON_1 = "Greater than %(cutoff)g%% %(method)s exceeded "
- _REASON_2 = "%g second response time"
-
- def failures(self):
- # TODO
- reasons = []
-
- for (method, times) in self._perMethodTimes.iteritems():
- failures = 0
- overDurations = [0] * len(self._thresholds)
-
- for success, duration in times:
- if not success:
- failures += 1
- for ctr, item in enumerate(self._thresholds):
- threshold, _ignore_fail_at = item
- if duration > threshold:
- overDurations[ctr] += 1
-
- checks = [
- (failures, self._fail_cut_off, self._FAILED_REASON),
- ]
-
- for ctr, item in enumerate(self._thresholds):
- threshold, fail_at = item
- fail_at = fail_at.get(method, fail_at["default"])
- checks.append(
- (overDurations[ctr], fail_at, self._REASON_1 + self._REASON_2 % (threshold,))
- )
-
- for count, cutoff, reason in checks:
- if count * 100.0 / len(times) > cutoff:
- reasons.append(reason % dict(method=method, cutoff=cutoff))
-
- if self.countClientFailures() != 0:
- reasons.append("Client failures: %d" % (self.countClientFailures(),))
- if self.countSimFailures() != 0:
- reasons.append("Overall failures: %d" % (self.countSimFailures(),))
- return reasons
-
-
-
</del><span class="cx"> def main():
</span><span class="cx"> import random
</span><span class="cx">
</span><span class="lines">@@ -626,6 +345,9 @@
</span><span class="cx"> from twisted.python.failure import startDebugMode
</span><span class="cx"> startDebugMode()
</span><span class="cx">
</span><ins>+ from contrib.performance.loadtest.clients import OS_X_10_6
+ from contrib.performance.loadtest.logger import ReportStatistics, SimpleStatistics, RequestLogger
+
</ins><span class="cx"> report = ReportStatistics()
</span><span class="cx"> addObserver(SimpleStatistics().observe)
</span><span class="cx"> addObserver(report.observe)
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestprofilespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/profiles.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/profiles.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -21,9 +21,7 @@
</span><span class="cx">
</span><span class="cx"> from __future__ import division
</span><span class="cx">
</span><del>-import json
</del><span class="cx"> import random
</span><del>-import sys
</del><span class="cx"> from uuid import uuid4
</span><span class="cx">
</span><span class="cx"> from caldavclientlibrary.protocol.caldav.definitions import caldavxml
</span><span class="lines">@@ -31,19 +29,21 @@
</span><span class="cx"> from twisted.python import context
</span><span class="cx"> from twisted.python.log import msg
</span><span class="cx"> from twisted.python.failure import Failure
</span><del>-from twisted.internet.defer import Deferred, succeed, fail
</del><ins>+from twisted.internet.defer import Deferred, DeferredList, succeed, fail
</ins><span class="cx"> from twisted.internet.task import LoopingCall
</span><span class="cx"> from twisted.web.http import PRECONDITION_FAILED
</span><span class="cx">
</span><del>-from twistedcaldav.ical import Property, Component
</del><ins>+from twistedcaldav.ical import Property
</ins><span class="cx">
</span><del>-from contrib.performance.stats import NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, mean, median
</del><ins>+from contrib.performance.stats import NearFutureDistribution, NormalDistribution, UniformDiscreteDistribution, BernoulliDistribution
</ins><span class="cx"> from contrib.performance.stats import LogNormalDistribution, RecurrenceDistribution
</span><del>-from contrib.performance.loadtest.logger import SummarizingMixin
</del><span class="cx"> from contrib.performance.loadtest.ical import IncorrectResponseCode
</span><ins>+from contrib.performance.loadtest.resources import Calendar, Event
+from contrib.performance.loadtest.templates import eventTemplate, alarmTemplate, taskTemplate
</ins><span class="cx">
</span><span class="cx"> from pycalendar.datetime import DateTime
</span><span class="cx"> from pycalendar.duration import Duration
</span><ins>+from pycalendar.value import Value
</ins><span class="cx">
</span><span class="cx"> class ProfileBase(object):
</span><span class="cx"> """
</span><span class="lines">@@ -52,18 +52,53 @@
</span><span class="cx"> """
</span><span class="cx"> random = random
</span><span class="cx">
</span><del>- def __init__(self, reactor, simulator, client, userNumber, **params):
</del><ins>+ def __init__(self, enabled, interval, **params):
+ self.enabled = enabled
+ self._interval = interval
+ self.setParameters(**params)
+ self._initialized = False
+
+ def setUp(self, reactor, simulator, client, record):
</ins><span class="cx"> self._reactor = reactor
</span><span class="cx"> self._sim = simulator
</span><span class="cx"> self._client = client
</span><del>- self._number = userNumber
- self.setParameters(**params)
</del><ins>+ self._record = record
+ self._initialized = True
</ins><span class="cx">
</span><del>-
</del><span class="cx"> def setParameters(self):
</span><span class="cx"> pass
</span><span class="cx">
</span><ins>+ def run(self):
+ # def action(self):
+ # if self.enabled:
+ # return self.action()
</ins><span class="cx">
</span><ins>+ print("Hello from run")
+ _call = LoopingCall(self.action)
+ _call.clock = self._reactor
+ return _call.start(self._interval)
+
+ # OR
+
+ # return self._loopWithDistribution(
+ # self._reactor, self._sendInvitationDistribution, self._invite)
+
+ # def _loopWithDistribution(reactor, distribution, function):
+ # result = Deferred()
+
+ # def repeat(ignored):
+ # reactor.callLater(distribution.sample(), iterate)
+
+ # def iterate():
+ # d = function()
+ # if d is not None:
+ # d.addCallbacks(repeat, result.errback)
+ # else:
+ # repeat(None)
+
+ # repeat(None)
+ # return result
+
</ins><span class="cx"> def initialize(self):
</span><span class="cx"> """
</span><span class="cx"> Called before the profile runs for real. Can be used to initialize client state.
</span><span class="lines">@@ -90,6 +125,46 @@
</span><span class="cx"> return attendee.parameterValue('EMAIL') == self._client.email[len('mailto:'):]
</span><span class="cx">
</span><span class="cx">
</span><ins>+ def _getRandomCalendar(self):
+ """
+ Return a random L{Calendar} object from the current user
+ or C{None} if there are no calendars to work with
+ """
+ calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
+ if not calendars: # Oh no! There are no calendars to play with
+ return None
+ # Choose a random calendar
+ calendar = self.random.choice(calendars)
+ return calendar
+
+
+ def _getRandomEvent(self):
+ """
+ Return a random L{Event} object from the current user
+ or C{None} if there are no events to work with
+ """
+ calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
+ while calendars:
+ calendar = self.random.choice(calendars)
+ calendars.remove(calendar)
+ if not calendar.events:
+ continue
+
+ events = calendar.events.keys()
+ while events:
+ href = self.random.choice(events)
+ events.remove(href)
+ event = calendar.events[href]
+ if not event.component:
+ continue
+ return event
+ return None
+
+
+ def _getRandomLocation(self):
+ pass
+
+
</ins><span class="cx"> def _newOperation(self, label, deferred):
</span><span class="cx"> """
</span><span class="cx"> Helper to emit a log event when a new operation is started and
</span><span class="lines">@@ -155,47 +230,695 @@
</span><span class="cx"> """
</span><span class="cx"> pass
</span><span class="cx">
</span><ins>+ """ Event-Interaction Profiles
+Event Creation - Eventer
+Event Changing - EventUpdaterBase
+ TitlerMixin
+ RelocaterMixin
+ ReschedulerMixin
+ RepeaterMixin
+ AlerterMixin
+ InviterMixin
+ NoterMixin
+ InviterMixin
+ LinkerMixin
+ AttacherMixin
+Event Deletion - EventerDeleter
+"""
</ins><span class="cx">
</span><ins>+#####################
+# Eventer Behaviors #
+#####################
</ins><span class="cx">
</span><del>-def loopWithDistribution(reactor, distribution, function):
- result = Deferred()
</del><ins>+class EventerBase(ProfileBase):
+ """
+ Base profile for a calendar user who interacts with events
+ """
+ def setParameters(
+ self,
+ enabled=True,
+ interval=25,
+ **params
+ ):
+ self.enabled = enabled
+ self._interval = interval
+ self.setDistributions(**params)
</ins><span class="cx">
</span><del>- def repeat(ignored):
- reactor.callLater(distribution.sample(), iterate)
</del><span class="cx">
</span><span class="cx">
</span><del>- def iterate():
- d = function()
- if d is not None:
- d.addCallbacks(repeat, result.errback)
</del><ins>+class Eventer(EventerBase):
+ """
+ A Calendar user who creates new events.
+ """
+ def initialize(self):
+ self.action = self._addEvent
+ return succeed(None)
+
+ def setDistributions(
+ self,
+ eventStartDistribution=NearFutureDistribution(),
+ eventDurationDistribution=UniformDiscreteDistribution([
+ 15 * 60, 30 * 60,
+ 45 * 60, 60 * 60,
+ 120 * 60
+ ])
+ ):
+ self._eventStartDistribution = eventStartDistribution
+ self._eventDurationDistribution = eventDurationDistribution
+
+ def _addEvent(self):
+ print "Hello a bit"
+
+ if not self._client.started:
+ return succeed(None)
+
+
+
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
+
+ print "Made it"
+
+ # Copy the template event and fill in some of its fields
+ # to make a new event to create on the calendar.
+ vcalendar = eventTemplate.duplicate()
+ vevent = vcalendar.mainComponent()
+ uid = str(uuid4())
+ dtstart = self._eventStartDistribution.sample()
+ dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
+
+ vevent.replaceProperty(Property("UID", uid))
+ vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create", d)
+
+# Could have better handling for not changing events once they're modified
+# esp re: repeating
+class EventUpdaterBase(EventerBase):
+ """Superclass of all event mixins.
+ Accepts two parameters
+ enabled: bool on or off
+ interval: distibution that generates integers representing delays
+ """
+ # COMPONENT_TYPE = None
+ def action(self):
+ event = self._getRandomEvent()
+ if not event:
+ return succeed(None)
+ component = event.component
+ vevent = component.mainComponent()
+
+ label = self.modifyEvent(event.url, vevent)
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ event.component = component
+ return self._client.updateEvent(event)
+ # d.addCallback(finish)
+
+ return self._newOperation(label, d)
+
+ def modifyEvent(self):
+ """Overriden by subclasses"""
+ pass
+
+class Titler(EventUpdaterBase):
+ def setDistributions(
+ self,
+ titleLengthDistribution=NormalDistribution(10, 2)
+ ):
+ self._titleLength = titleLengthDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ length = max(5, int(self._titleLength.sample()))
+ vevent.replaceProperty(Property("SUMMARY", "Event" + "." * (length - 5)))
+ return "update{title}"
+
+class Transparenter(EventUpdaterBase):
+ def setDistributions(
+ self,
+ transparentLikelihoodDistribution=BernoulliDistribution(0.95)
+ ):
+ self._transparentLikelihood = transparentLikelihoodDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ if self._transparentLikelihood.sample():
+ transparency = "TRANSPARENT"
</ins><span class="cx"> else:
</span><del>- repeat(None)
</del><ins>+ transparency = "OPAQUE"
+ vevent.replaceProperty(Property("TRANSP", transparency))
+ return "update{transp}"
</ins><span class="cx">
</span><del>- repeat(None)
- return result
</del><ins>+class Hider(EventUpdaterBase):
+ def setDistributions(
+ self,
+ publicLikelihoodDistribution=BernoulliDistribution(0.95)
+ ):
+ self._publicLikelihood = publicLikelihoodDistribution
</ins><span class="cx">
</span><ins>+ def modifyEvent(self, _ignore_href, vevent):
+ if self._publicLikelihood.sample():
+ privacy = "PUBLIC"
+ else:
+ privacy = "CONFIDENTIAL"
+ vevent.replaceProperty(Property("X-CALENDARSERVER-ACCESS", privacy))
+ return "update{privacy}"
</ins><span class="cx">
</span><ins>+class Noter(EventUpdaterBase):
+ def setDistributions(
+ self,
+ noteLengthDistribution=NormalDistribution(10, 2)
+ ):
+ self._noteLength = noteLengthDistribution
</ins><span class="cx">
</span><del>-class Inviter(ProfileBase):
</del><ins>+ def modifyEvent(self, _ignore_href, vevent):
+ length = max(5, int(self._noteLength.sample()))
+ vevent.replaceProperty(Property("DESCRIPTION", "." * length))
+ return "update{notes}"
+
+class Linker(EventUpdaterBase):
+ def setDistributions(
+ self,
+ urlLengthDistribution=NormalDistribution(10, 2)
+ ):
+ self._urlLength = urlLengthDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ length = max(5, int(self._urlLength.sample()))
+ vevent.replaceProperty(Property("URL", 'https://bit.ly/' + '*' * length, valuetype=Value.VALUETYPE_URI))
+ return "update{url}"
+
+class Repeater(EventUpdaterBase):
+ def setDistributions(
+ self,
+ recurrenceDistribution=RecurrenceDistribution(False)
+ ):
+ self._recurrence = recurrenceDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ rrule = self._recurrence.sample()
+ if rrule is not None:
+ vevent.replaceProperty(Property(None, None, None, pycalendar=rrule))
+ return "update{rrule}"
+
+class Rescheduler(EventUpdaterBase):
+ def setDistributions(
+ self,
+ eventStartDistribution=NearFutureDistribution(),
+ eventDurationDistribution=UniformDiscreteDistribution([
+ 15 * 60, 30 * 60,
+ 45 * 60, 60 * 60,
+ 120 * 60
+ ])
+ ):
+ self._eventStart = eventStartDistribution
+ self._eventDuration = eventDurationDistribution
+
+ def modifyEvent(self, _ignore_href, vevent):
+ dtstart = self._eventStart.sample()
+ dtend = dtstart + Duration(seconds=self._eventDuration.sample())
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+ return "reschedule{event}"
+
+# class Alerter(EventUpdaterBase):
+# pass
+
+class Attacher(EventUpdaterBase):
+ def setDistributions(
+ self,
+ ):
+ # filesizeDistribution=NormalDistribution(24, 3),
+ # numAttachmentsDistribution=LogNormalDistribution(2, 1),
+ # attachLikelihoodDistribution=BernoulliDistribution(0.9)
+ # self._filesize = filesizeDistribution
+ # self._numAttachments = numAttachmentsDistribution
+ # self._attachLikelihood = attachLikelihoodDistribution
+ pass
+
+ def modifyEvent(self, href, vevent):
+ d = self._client.postAttachment(href, 'x' * 1024)
+ return "attach{files}"
+
+ def handleAttachments(self):
+ pass
+
+ # if True: # attachLikelihoodDistribution.sample():
+ # # size = max(0, int(self._filesize.sample()))
+ # numAttachments()
+ # self.attachFiles(event, filesizeDistribution.sample())
+ # else:
+ # pass
+
+ def attachFile(self, event):
+ # PUT new event information (nothing has actually changed)
+ # POST attachment (with Content-Disposition header, and response location)
+ # GET updated event
+ pass
+
+ def unattachFile(self):
+ pass
+
+class InviterBase(EventerBase):
</ins><span class="cx"> """
</span><del>- A Calendar user who invites and de-invites other users to events.
</del><ins>+ Base profile for a calendar user that invites and deinvites other principals to events
</ins><span class="cx"> """
</span><span class="cx"> def setParameters(
</span><span class="cx"> self,
</span><span class="cx"> enabled=True,
</span><span class="cx"> sendInvitationDistribution=NormalDistribution(600, 60),
</span><del>- inviteeDistribution=UniformDiscreteDistribution(range(-10, 11))
</del><ins>+ inviteeDistribution=UniformDiscreteDistribution(range(-10, 11)),
+ **params
</ins><span class="cx"> ):
</span><span class="cx"> self.enabled = enabled
</span><span class="cx"> self._sendInvitationDistribution = sendInvitationDistribution
</span><span class="cx"> self._inviteeDistribution = inviteeDistribution
</span><ins>+ if len(params) > 0:
+ pass
</ins><span class="cx">
</span><ins>+ def getAttendees():
+ pass
</ins><span class="cx">
</span><del>- def run(self):
- return loopWithDistribution(
- self._reactor, self._sendInvitationDistribution, self._invite)
</del><ins>+ # def _invitePrincipal(self, ...):
</ins><span class="cx">
</span><ins>+ # def _uninvitePrincipal(self, ...):
</ins><span class="cx">
</span><ins>+
+
+
+ def _loopWithDistribution(reactor, distribution, function):
+ result = Deferred()
+
+ def repeat(ignored):
+ reactor.callLater(distribution.sample(), iterate)
+
+ def iterate():
+ d = function()
+ if d is not None:
+ d.addCallbacks(repeat, result.errback)
+ else:
+ repeat(None)
+
+ repeat(None)
+ return result
+
+ # def _didSelfOrganize(self, vevent):
+
+
+ # def _buildIndividualAttendee(self, commonName, record, ):
+
+ # # ATTENDEE;CN="Super User";CUTYPE=INDIVIDUAL;EMAIL="admin@example.com":mailto:admin@example.com
+ # # ATTENDEE;CN="User 04";CUTYPE=INDIVIDUAL;EMAIL="user04@example.com":mailto:user04@example.com
+
+ # role = ['REQ-PARTICIPANT', '']
+ # attendee = Property(
+ # name=u'ATTENDEE',
+ # value=cuaddr.encode("utf-8"),
+ # params={
+ # 'CN': commonName,
+ # 'CUTYPE': 'INDIVIDUAL',
+ # 'PARTSTAT': 'NEEDS-ACTION',
+ # 'ROLE': 'REQ-PARTICIPANT',
+ # 'RSVP': 'TRUE',
+ # },
+ # )
+ # return attendee
+
+ # def _buildLocationAttendee(self, cn, cuaddr):
+ # """
+ # Example Usage: profile._buildLocationAttendee("Location 01", "urn:uuid:...")
+ # """
+ # role = ['REQ-PARTICIPANT', '']
+ # attendee = Property(
+ # name=u'ATTENDEE',
+ # value=cuaddr.encode("utf-8"),
+ # params={
+ # 'CN': commonName,
+ # 'CUTYPE': 'ROOM',
+ # 'PARTSTAT': 'NEEDS-ACTION',
+ # 'ROLE': 'REQ-PARTICIPANT',
+ # 'RSVP': 'TRUE',
+ # },
+ # )
+ # return attendee
+
+ def _invite():
+ raise NotImplementedError
+
+ def _addAttendee():
+ raise NotImplementedError
+
+# TODO - invite groups
+# class Inviter(EventUpdaterBase): pass
+
+class Relocater(EventUpdaterBase):
+ def setParameters(
+ self,
+ ):
+ pass
+
+class AppleEventer(EventerBase):
+ """
+ Calendar user who makes events in the form of Apple logo
+ """
+ def initialize(self):
+ self.action = self._addEvent
+ return succeed(None)
+
+ def setDistributions(
+ self
+ ):
+ pass
+
+ def _addEvent(self):
+ if not self._client.started:
+ return succeed(None)
+
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
+
+ # Copy the template event and fill in some of its fields
+ # to make a new event to create on the calendar.
+ vcalendar = eventTemplate.duplicate()
+ vevent = vcalendar.mainComponent()
+ uid = str(uuid4())
+ dtstart = self._eventStartDistribution.sample()
+ dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
+
+ vevent.replaceProperty(Property("UID", uid))
+ vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create", d)
+
+class HappyEventer(EventerBase):
+ """
+ Calendar user who makes events in the form of Apple logo
+ """
+ def initialize(self):
+ self.action = self._addEvent
+ return succeed(None)
+
+ def setDistributions(
+ self
+ ):
+ self._eventStart = UniformDiscreteDistribution([
+ # August 3
+ DateTime(year=2015, month=8, day=3, hours=15, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=3, hours=15, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=3, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=3, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ # August 4
+ DateTime(year=2015, month=8, day=4, hours=10, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=10, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=11, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=11, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ DateTime(year=2015, month=8, day=4, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=4, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
+
+ # August 5
+ DateTime(year=2015, month=8, day=5, hours=13, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=5, hours=13, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=5, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=5, hours=17, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ # August 6
+ DateTime(year=2015, month=8, day=6, hours=10, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=10, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=11, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=11, minutes=30, seconds=0, tzid=None, utcoffset=None),
+
+ DateTime(year=2015, month=8, day=6, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=6, hours=17, minutes=0, seconds=0, tzid=None, utcoffset=None),
+
+ # August 7
+ DateTime(year=2015, month=8, day=7, hours=15, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=7, hours=15, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=7, hours=16, minutes=0, seconds=0, tzid=None, utcoffset=None),
+ DateTime(year=2015, month=8, day=7, hours=16, minutes=30, seconds=0, tzid=None, utcoffset=None),
+ ])
+ self._eventDuration = UniformDiscreteDistribution([
+ 30 * 60,
+ 60 * 60
+ ])
+
+ def _addEvent(self):
+ if not self._client.started:
+ return succeed(None)
+
+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return succeed(None)
+
+ # Copy the template event and fill in some of its fields
+ # to make a new event to create on the calendar.
+ vcalendar = eventTemplate.duplicate()
+ vevent = vcalendar.mainComponent()
+ uid = str(uuid4())
+ dtstart = self._eventStart.sample()
+ dtend = dtstart + Duration(seconds=self._eventDuration.sample())
+
+ vevent.replaceProperty(Property("UID", uid))
+ vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ vevent.replaceProperty(Property("DTSTART", dtstart))
+ vevent.replaceProperty(Property("DTEND", dtend))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ event = Event(self._client.serializeLocation(), href, None, component=vcalendar)
+ d = self._client.addEvent(href, event)
+ return self._newOperation("create", d)
+
+
+
+# class EventDeleter(ProfileBase):
+
+
+
+""" TEST """
+# class Intern(object):
+# def __init__(self):
+# self.behaviors = [
+# Eventer(asdfjadsf),
+# Attacher(asjadsfjasdf),
+# Inviter(enabled=True, **params)
+# ]
+
+# def run(self):
+# deferreds = []
+# for behavior in self.behaviors:
+# deferreds.append(behavior.run())
+# return DeferredList(deferreds)
+
+
+
+####################
+# Class Hierarchy
+# ---------------
+# TaskBase
+# Tasker
+# TaskUpdaterBase
+# Titler
+# Alerter
+# Noter
+# Prioritizer
+# Completer
+# TaskDeleter
+####################
+class TaskBase(ProfileBase):
+ """
+ Base profile for a calendar user who interacts with tasks
+ """
+ def setParameters(self, enabled=True, interval=25, **params):
+ self.enabled = enabled
+ self._interval = interval
+
+class Tasker(TaskBase):
+ """
+ A Calendar user who creates new tasks.
+ """
+ def initialize(self):
+ self.action = self._addTask
+ return succeed(None)
+
+ def _addTask(self, title="Simple Task"):
+ if not self._client.started:
+ return succeed(None)
+
+ calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
+
+ while calendars:
+ calendar = self.random.choice(calendars)
+ calendars.remove(calendar)
+
+ # Copy the template task and fill in some of its fields
+ # to make a new task to create on the calendar.
+ vcalendar = taskTemplate.duplicate()
+ vtodo = vcalendar.mainComponent()
+ uid = str(uuid4())
+ vtodo.replaceProperty(Property("UID", uid))
+ vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ d = self._client.addEvent(href, vcalendar)
+ return self._newOperation("create", d)
+
+
+class TaskUpdaterBase(TaskBase):
+ def action(self):
+ task = self._getRandomTask()
+ if not task:
+ return succeed(None)
+ component = task.component
+ vtodo = component.mainComponent()
+
+ label = self.modifyEvent(task.url, vtodo)
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+
+ task.component = component
+ d = self._client.updateEvent(task)
+ return self._newOperation(label, d)
+
+ def modifyEvent(self):
+ """Overriden by subclasses"""
+ pass
+
+
+
+
+class TaskUpdater(TaskBase):
+ """
+ A Calendar user who creates and updates complex tasks with:
+ Mark as completed/not completed
+ Change name
+ Change priority
+ Change notes
+ Sets due dates
+ """
+
+ def setParameters(
+ self,
+ taskDueDistribution=NearFutureDistribution(),
+ ):
+ pass
+
+
+ def _addTask(self):
+ if not self._client.started:
+ return succeed(None)
+
+ calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
+
+ while calendars:
+ calendar = self.random.choice(calendars)
+ calendars.remove(calendar)
+
+ # Copy the template task and fill in some of its fields
+ # to make a new task to create on the calendar.
+ vcalendar = taskTemplate.duplicate()
+ vtodo = vcalendar.mainComponent()
+ uid = str(uuid4())
+ due = self._taskStartDistribution.sample()
+ vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("UID", uid))
+
+ # vtodo.replaceProperty(Property("SUMMARY", title))
+
+ href = '%s%s.ics' % (calendar.url, uid)
+ d = self._client.addEvent(href, vcalendar)
+ return self._newOperation("create", d)
+
+
+
+# class TaskTitlerMixin = TitlerMixin
+# class TaskAlerterMixin = AlerterMixin
+# self._taskStartDistribution = taskDueDistribution
+# vtodo.replaceProperty(Property("DUE", due))
+# class TaskNoterMixin = NoterMixin
+
+class Prioritizer(TaskBase):
+ PRIORITY_NONE = 0
+ PRIORITY_HIGH = 1
+ PRIORITY_MEDIUM = 5
+ PRIORITY_LOW = 9
+
+ def _setPriority(self, priority, vtodo):
+ if priority == self.PRIORITY_NONE:
+ vtodo.removeProperty("PRIORITY")
+ else:
+ vtodo.replaceProperty(Property("PRIORITY", priority))
+
+class Completer(TaskBase):
+ def _markTaskComplete(vtodo):
+ """ task is a Component representing a VTODO """
+ vtodo.replaceProperty(Property("COMPLETED", DateTime.getNowUTC()))
+ vtodo.replaceProperty(Property("PERCENT-COMPLETE", 100))
+ vtodo.replaceProperty(Property("STATUS", "COMPLETED"))
+
+
+ def _markTaskIncomplete(vtodo):
+ """ mark a VTODO as incomplete """
+ vtodo.removeProperty("COMPLETED")
+ vtodo.removeProperty("PERCENT-COMPLETE")
+ vtodo.replaceProperty(Property("STATUS", "NEEDS-ACTION"))
+
+# class TaskDeleter(ProfileBase): pass
+
+
+class Inviter(ProfileBase):
+
+ def initialize(self):
+ self.action = self.test
+ return succeed(None)
+
+ def test(self):
+ event = self._getRandomEvent()
+ if not event:
+ return succeed(None)
+ print("Found event: " + str(event))
+ href = event.url
+
+ attendee = Property(
+ name=u'ATTENDEE',
+ value='urn:uuid:30000000-0000-0000-0000-000000000002',
+ params={
+ 'CN': 'Location 02',
+ 'CUTYPE': 'ROOM',
+ 'PARTSTAT': 'NEEDS-ACTION',
+ 'ROLE': 'REQ-PARTICIPANT',
+ 'RSVP': 'TRUE',
+ },
+ )
+
+ d = self._client.addEventAttendee(href, attendee)
+
+ component = event.component
+ component.mainComponent().addProperty(attendee)
+ event.component = component
+
+ d2 = self._client.addInvite(event)
+ return self._newOperation("add attendee", DeferredList([d, d2]))
+
+
</ins><span class="cx"> def _addAttendee(self, event, attendees):
</span><span class="cx"> """
</span><span class="cx"> Create a new attendee to add to the list of attendees for the
</span><span class="lines">@@ -280,7 +1003,8 @@
</span><span class="cx"> # Find out who might attend
</span><span class="cx"> attendees = tuple(component.properties('ATTENDEE'))
</span><span class="cx">
</span><del>- d = self._addAttendee(event, attendees)
</del><ins>+ # d = self._addAttendee(event, attendees)
+ d = self._addLocation(event, "Location 05", "urn:uuid:30000000-0000-0000-0000-000000000005")
</ins><span class="cx"> d.addCallbacks(
</span><span class="cx"> lambda attendee:
</span><span class="cx"> self._client.addEventAttendee(
</span><span class="lines">@@ -297,25 +1021,6 @@
</span><span class="cx"> """
</span><span class="cx"> A Calendar user who invites other users to new events.
</span><span class="cx"> """
</span><del>- _eventTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VEVENT
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-DTEND;TZID=America/New_York:20101021T130000
-TRANSP:OPAQUE
-SUMMARY:Simple event
-DTSTART;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-SEQUENCE:2
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-
-
</del><span class="cx"> def setParameters(
</span><span class="cx"> self,
</span><span class="cx"> enabled=True,
</span><span class="lines">@@ -341,11 +1046,6 @@
</span><span class="cx"> self._recurrenceDistribution = recurrenceDistribution
</span><span class="cx">
</span><span class="cx">
</span><del>- def run(self):
- return loopWithDistribution(
- self._reactor, self._sendInvitationDistribution, self._invite)
-
-
</del><span class="cx"> def _addAttendee(self, event, attendees):
</span><span class="cx"> """
</span><span class="cx"> Create a new attendee to add to the list of attendees for the
</span><span class="lines">@@ -413,7 +1113,7 @@
</span><span class="cx">
</span><span class="cx"> # Copy the template event and fill in some of its fields
</span><span class="cx"> # to make a new event to create on the calendar.
</span><del>- vcalendar = self._eventTemplate.duplicate()
</del><ins>+ vcalendar = eventTemplate.duplicate()
</ins><span class="cx"> vevent = vcalendar.mainComponent()
</span><span class="cx"> uid = str(uuid4())
</span><span class="cx"> dtstart = self._eventStartDistribution.sample()
</span><span class="lines">@@ -444,7 +1144,9 @@
</span><span class="cx"> return self._newOperation("invite", d)
</span><span class="cx">
</span><span class="cx">
</span><del>-
</del><ins>+##########################
+# Notification Behaviors #
+##########################
</ins><span class="cx"> class Accepter(ProfileBase):
</span><span class="cx"> """
</span><span class="cx"> A Calendar user who accepts invitations to events. As well as accepting requests, this
</span><span class="lines">@@ -533,7 +1235,7 @@
</span><span class="cx">
</span><span class="cx"> # Download the event again and attempt to make the change
</span><span class="cx"> # to the attendee list again.
</span><del>- d = self._client.updateEvent(href)
</del><ins>+ d = self._client._refreshEvent(href)
</ins><span class="cx"> def cbUpdated(ignored):
</span><span class="cx"> d = change()
</span><span class="cx"> d.addErrback(scheduleError)
</span><span class="lines">@@ -597,149 +1299,20 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-class Eventer(ProfileBase):
</del><ins>+class EventUpdater(EventerBase):
</ins><span class="cx"> """
</span><del>- A Calendar user who creates new events.
- """
- _eventTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VEVENT
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-DTEND;TZID=America/New_York:20101021T130000
-TRANSP:OPAQUE
-SUMMARY:Simple event
-DTSTART;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-SEQUENCE:2
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-
- def setParameters(
- self,
- enabled=True,
- interval=25,
- eventStartDistribution=NearFutureDistribution(),
- eventDurationDistribution=UniformDiscreteDistribution([
- 15 * 60, 30 * 60,
- 45 * 60, 60 * 60,
- 120 * 60
- ]),
- recurrenceDistribution=RecurrenceDistribution(False),
- ):
- self.enabled = enabled
- self._interval = interval
- self._eventStartDistribution = eventStartDistribution
- self._eventDurationDistribution = eventDurationDistribution
- self._recurrenceDistribution = recurrenceDistribution
-
-
- def run(self):
- self._call = LoopingCall(self._addEvent)
- self._call.clock = self._reactor
- return self._call.start(self._interval)
-
-
- def _addEvent(self):
- if not self._client.started:
- return succeed(None)
-
- calendars = self._calendarsOfType(caldavxml.calendar, "VEVENT")
-
- while calendars:
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
-
- # Copy the template event and fill in some of its fields
- # to make a new event to create on the calendar.
- vcalendar = self._eventTemplate.duplicate()
- vevent = vcalendar.mainComponent()
- uid = str(uuid4())
- dtstart = self._eventStartDistribution.sample()
- dtend = dtstart + Duration(seconds=self._eventDurationDistribution.sample())
- vevent.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vevent.replaceProperty(Property("DTSTART", dtstart))
- vevent.replaceProperty(Property("DTEND", dtend))
- vevent.replaceProperty(Property("UID", uid))
-
- rrule = self._recurrenceDistribution.sample()
- if rrule is not None:
- vevent.addProperty(Property(None, None, None, pycalendar=rrule))
-
- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
-
-
-
-class EventUpdater(ProfileBase):
- """
</del><span class="cx"> A Calendar user who creates a new event, and then updates its alarm.
</span><span class="cx"> """
</span><del>- _eventTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VEVENT
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-DTEND;TZID=America/New_York:20101021T130000
-TRANSP:OPAQUE
-SUMMARY:Simple event
-DTSTART;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-SEQUENCE:2
-BEGIN:VALARM
-X-WR-ALARMUID:D9D1AC84-F629-4B9D-9B6B-4A6CA9A11FEF
-UID:D9D1AC84-F629-4B9D-9B6B-4A6CA9A11FEF
-DESCRIPTION:Event reminder
-TRIGGER:-PT8M
-ACTION:DISPLAY
-END:VALARM
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n"))
-
- def setParameters(
- self,
- enabled=True,
- interval=25,
- eventStartDistribution=NearFutureDistribution(),
- eventDurationDistribution=UniformDiscreteDistribution([
- 15 * 60, 30 * 60,
- 45 * 60, 60 * 60,
- 120 * 60
- ]),
- recurrenceDistribution=RecurrenceDistribution(False),
- ):
- self.enabled = enabled
- self._interval = interval
- self._eventStartDistribution = eventStartDistribution
- self._eventDurationDistribution = eventDurationDistribution
- self._recurrenceDistribution = recurrenceDistribution
-
-
</del><span class="cx"> def initialize(self):
</span><span class="cx"> """
</span><span class="cx"> Called before the profile runs for real. Can be used to initialize client state.
</span><span class="cx">
</span><span class="cx"> @return: a L{Deferred} that fires when initialization is done
</span><span class="cx"> """
</span><ins>+ self.action = self._updateEvent
</ins><span class="cx"> return self._initEvent()
</span><span class="cx">
</span><span class="cx">
</span><del>- def run(self):
- self._call = LoopingCall(self._updateEvent)
- self._call.clock = self._reactor
- return self._call.start(self._interval)
-
-
</del><span class="cx"> def _initEvent(self):
</span><span class="cx"> if not self._client.started:
</span><span class="cx"> return succeed(None)
</span><span class="lines">@@ -801,195 +1374,175 @@
</span><span class="cx"> d = self._client.changeEvent(event.url)
</span><span class="cx"> return self._newOperation("update", d)
</span><span class="cx">
</span><ins>+ # def _changeEventTitle(self, event, title):
+ # event.component = self._setEventTitle(event.component, title)
+ # event.
</ins><span class="cx">
</span><span class="cx">
</span><del>-class Tasker(ProfileBase):
</del><ins>+
+
+
+class EventDeleter(EventerBase):
</ins><span class="cx"> """
</span><del>- A Calendar user who creates new tasks.
</del><ins>+ A calendar user who deletes events at random
</ins><span class="cx"> """
</span><del>- _taskTemplate = Component.fromString("""\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VTODO
-CREATED:20101018T155431Z
-UID:C98AD237-55AD-4F7D-9009-0D355D835822
-SUMMARY:Simple task
-DUE;TZID=America/New_York:20101021T120000
-DTSTAMP:20101018T155438Z
-END:VTODO
-END:VCALENDAR
-""".replace("\n", "\r\n"))
</del><ins>+ def initialize(self):
+ self.action = self._deleteEvent
</ins><span class="cx">
</span><del>- def setParameters(
- self,
- enabled=True,
- interval=25,
- taskDueDistribution=NearFutureDistribution(),
- ):
</del><ins>+ def _deleteEvent(self):
+ event = self._getRandomEvent()
+ if event is None:
+ return succeed(None)
+ d = self._client.deleteEvent(event)
+ return self._newOperation("delete", d)
+
+
+
+
+
+######################
+# Calendar Behaviors #
+######################
+class CalendarBase(ProfileBase):
+ """
+ A calendar user who interacts with calendars
+ """
+ def initialize(self):
+ self.action = lambda: None
+ return succeed(None)
+
+
+ def setParameters(self, enabled=True, interval=25):
</ins><span class="cx"> self.enabled = enabled
</span><span class="cx"> self._interval = interval
</span><del>- self._taskStartDistribution = taskDueDistribution
</del><span class="cx">
</span><span class="cx">
</span><del>- def run(self):
- self._call = LoopingCall(self._addTask)
- self._call.clock = self._reactor
- return self._call.start(self._interval)
</del><span class="cx">
</span><ins>+class CalendarMaker(CalendarBase):
+ """ A Calendar user who adds new Calendars """
+ def initialize(self):
+ self.action = self._addCalendar
+ return succeed(None)
</ins><span class="cx">
</span><del>- def _addTask(self):
</del><ins>+
+ def _addCalendar(self):
</ins><span class="cx"> if not self._client.started:
</span><del>- return succeed(None)
</del><ins>+ return None
</ins><span class="cx">
</span><del>- calendars = self._calendarsOfType(caldavxml.calendar, "VTODO")
</del><ins>+ uid = str(uuid4())
</ins><span class="cx">
</span><del>- while calendars:
- calendar = self.random.choice(calendars)
- calendars.remove(calendar)
</del><ins>+ body = Calendar.buildCalendarXML(order=0, component_type="VEVENT", rgba_color='FB524FFF', name='Sample Calendar')
+ print("Making new calendar with uid: " + uid)
+ # XXX Just for testing! remove this soon
+ path = "/calendars/__uids__/" + self._client.record.guid + "/" + uid + "/"
+ d = self._client.addCalendar(path, body)
+ return self._newOperation("create", d)
</ins><span class="cx">
</span><del>- # Copy the template task and fill in some of its fields
- # to make a new task to create on the calendar.
- vcalendar = self._taskTemplate.duplicate()
- vtodo = vcalendar.mainComponent()
- uid = str(uuid4())
- due = self._taskStartDistribution.sample()
- vtodo.replaceProperty(Property("CREATED", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("DTSTAMP", DateTime.getNowUTC()))
- vtodo.replaceProperty(Property("DUE", due))
- vtodo.replaceProperty(Property("UID", uid))
</del><span class="cx">
</span><del>- href = '%s%s.ics' % (calendar.url, uid)
- d = self._client.addEvent(href, vcalendar)
- return self._newOperation("create", d)
</del><span class="cx">
</span><del>-
-
-class OperationLogger(SummarizingMixin):
</del><ins>+class CalendarUpdater(CalendarBase):
</ins><span class="cx"> """
</span><del>- Profiles will initiate operations which may span multiple requests. Start
- and stop log messages are emitted for these operations and logged by this
- logger.
</del><ins>+ A calendar user who updates random calendars
</ins><span class="cx"> """
</span><del>- formats = {
- u"start" : u"%(user)s - - - - - - - - - - - %(label)8s BEGIN %(lag)s",
- u"end" : u"%(user)s - - - - - - - - - - - %(label)8s END [%(duration)5.2f s]",
- u"failed": u"%(user)s x x x x x x x x x x x %(label)8s FAILED %(reason)s",
- }
</del><ins>+ def initialize(self):
+ from collections import defaultdict
+ self.action = self._updateCalendar
+ self._calendarModCount = defaultdict(int) # Map from calendar href to count of modifications
+ return succeed(None)
</ins><span class="cx">
</span><del>- lagFormat = u'{lag %5.2f ms}'
</del><ins>+ def _updateCalendar(self):
+ if not self._client.started:
+ return None
</ins><span class="cx">
</span><del>- # the response time thresholds to display together with failing % count threshold
- _thresholds_default = {
- "operations": {
- "limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
- "thresholds": {
- "default": [100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
- }
- }
- }
- _lag_cut_off = 1.0 # Maximum allowed median scheduling latency, seconds
- _fail_cut_off = 1.0 # % of total count at which failed requests will cause a failure
</del><ins>+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return None
</ins><span class="cx">
</span><del>- _fields_init = [
- ('operation', -25, '%-25s'),
- ('count', 8, '%8s'),
- ('failed', 8, '%8s'),
- ]
</del><ins>+ self._calendarModCount[calendar.url] += 1
+ modcount = self._calendarModCount[calendar.url]
</ins><span class="cx">
</span><del>- _fields_extend = [
- ('mean', 8, '%8.4f'),
- ('median', 8, '%8.4f'),
- ('stddev', 8, '%8.4f'),
- ('avglag (ms)', 12, '%12.4f'),
- ('STATUS', 8, '%8s'),
- ]
</del><ins>+ colors = [
+ "#800000FF", # maroon
+ "#FF0000FF", # red
+ "#008000FF", # green
+ "#00FF00FF", # line
+ "#000080FF", # navy
+ "#0000FFFF", # blue
+ ]
+ color = colors[modcount % len(colors)]
+ self._client.setCalendarDisplayName(calendar, "Calendar ({mods})".format(mods=modcount))
+ self._client.setCalendarColor(calendar, color)
+ # choice = self.random.randint(0, 4)
+ # if choice == 0:
+ # self._client._
+ # return succeed(None)
</ins><span class="cx">
</span><del>- def __init__(self, outfile=None, **params):
- self._perOperationTimes = {}
- self._perOperationLags = {}
- if outfile is None:
- outfile = sys.stdout
- self._outfile = outfile
</del><ins>+ def randomUpdate(self):
+ pass
</ins><span class="cx">
</span><del>- # Load parameters from config
- if "thresholdsPath" in params:
- jsondata = json.load(open(params["thresholdsPath"]))
- elif "thresholds" in params:
- jsondata = params["thresholds"]
- else:
- jsondata = self._thresholds_default
- self._thresholds = [[limit, {}] for limit in jsondata["operations"]["limits"]]
- for ctr, item in enumerate(self._thresholds):
- for k, v in jsondata["operations"]["thresholds"].items():
- item[1][k] = v[ctr]
</del><ins>+class CalendarSharer(CalendarBase, InviterBase):
+ """
+ A calendar user who shares random calendars.
+ Even though the real client allows batch requests (e.g. 10 shares in one HTTP request),
+ we simplify life (TODO: keep it real) by having each HTTP request only add or remove one sharee.
+ """
</ins><span class="cx">
</span><del>- self._fields = self._fields_init[:]
- for threshold, _ignore_fail_at in self._thresholds:
- self._fields.append(('>%g sec' % (threshold,), 10, '%10s'))
- self._fields.extend(self._fields_extend)
</del><ins>+ def initialize(self):
+ self.action = self._shareCalendar
+ return succeed(None)
</ins><span class="cx">
</span><del>- if "lagCutoff" in params:
- self._lag_cut_off = params["lagCutoff"]
</del><ins>+ def _shareCalendar(self):
+ if not self._client.started:
+ return succeed(None)
</ins><span class="cx">
</span><del>- if "failCutoff" in params:
- self._fail_cut_off = params["failCutoff"]
</del><ins>+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return None
</ins><span class="cx">
</span><ins>+ # The decision of who to invite / uninvite should be made here
+ inv = random.randint(0, 1)
+ rem = random.randint(0, 1)
</ins><span class="cx">
</span><del>- def observe(self, event):
- if event.get("type") == "operation":
- event = event.copy()
- lag = event.get('lag')
- if lag is None:
- event['lag'] = ''
- else:
- event['lag'] = self.lagFormat % (lag * 1000.0,)
</del><ins>+ invRecord = self._sim.getUserRecord(inv)
+ remRecord = self._sim.getUserRecord(rem)
</ins><span class="cx">
</span><del>- self._outfile.write(
- (self.formats[event[u'phase']] % event).encode('utf-8') + '\n')
</del><ins>+ print("Sharing " + calendar.url)
+ self._inviteUser(calendar, invRecord)
+ # self._removeUser(calendar, remRecord)
</ins><span class="cx">
</span><del>- if event[u'phase'] == u'end':
- dataset = self._perOperationTimes.setdefault(event[u'label'], [])
- dataset.append((event[u'success'], event[u'duration']))
- elif lag is not None:
- dataset = self._perOperationLags.setdefault(event[u'label'], [])
- dataset.append(lag)
</del><ins>+ return succeed(None)
</ins><span class="cx">
</span><ins>+ def _inviteUser(self, calendar, userRecord):
+ mailto = "mailto:{}".format(userRecord.email)
+ body = Calendar.addInviteeXML(mailto, calendar.name, readwrite=True)
+ d = self._client.postXML(calendar.url, body)
+ # print(body)
</ins><span class="cx">
</span><del>- def _summarizeData(self, operation, data):
- avglag = mean(self._perOperationLags.get(operation, [0.0])) * 1000.0
- data = SummarizingMixin._summarizeData(self, operation, data)
- return data[:-1] + (avglag,) + data[-1:]
</del><ins>+ def _removeUser(self, calendar, userRecord):
+ mailto = "mailto:{}".format(userRecord.email)
</ins><span class="cx">
</span><ins>+ body = Calendar.removeInviteeXML(mailto)
</ins><span class="cx">
</span><del>- def report(self, output):
- output.write("\n")
- self.printHeader(output, [
- (label, width)
- for (label, width, _ignore_fmt) in self._fields
- ])
- self.printData(
- output,
- [fmt for (label, width, fmt) in self._fields],
- sorted(self._perOperationTimes.items())
- )
</del><ins>+ d = self._client.postXML(calendar.url, body)
+ # print(body)
</ins><span class="cx">
</span><del>- _LATENCY_REASON = "Median %(operation)s scheduling lag greater than %(cutoff)sms"
- _FAILED_REASON = "Greater than %(cutoff).0f%% %(operation)s failed"
</del><span class="cx">
</span><del>- def failures(self):
- reasons = []
</del><span class="cx">
</span><del>- for operation, lags in self._perOperationLags.iteritems():
- if median(lags) > self._lag_cut_off:
- reasons.append(self._LATENCY_REASON % dict(
- operation=operation.upper(), cutoff=self._lag_cut_off * 1000))
</del><ins>+class CalendarDeleter(CalendarBase):
+ """
+ A calendar user who deletes entire calendars
+ """
+ def initialize(self):
+ self.action = self._deleteCalendar
+ return succeed(None)
</ins><span class="cx">
</span><del>- for operation, times in self._perOperationTimes.iteritems():
- failures = len([success for (success, _ignore_duration) in times if not success])
- if failures * 100.0 / len(times) > self._fail_cut_off:
- reasons.append(self._FAILED_REASON % dict(
- operation=operation.upper(), cutoff=self._fail_cut_off))
</del><ins>+ def _deleteCalendar(self):
+ if not self._client.started:
+ return succeed(None)
</ins><span class="cx">
</span><del>- return reasons
</del><ins>+ calendar = self._getRandomCalendar()
+ if not calendar:
+ return None
+ print("Deleting " + calendar.url)
+ d = self._client.deleteCalendar(calendar.url)
+ return self._newOperation("delete", d)
</ins></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestrequestdataOS_X_10_7user_list_principal_property_searchrequest"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/request-data/OS_X_10_7/user_list_principal_property_search.request 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -1,2 +1,2 @@
</span><span class="cx"> <?xml version="1.0" encoding="utf-8" ?>
</span><del>-<x0:principal-property-search xmlns:x2="urn:ietf:params:xml:ns:caldav" xmlns:x0="DAV:" xmlns:x1="http://calendarserver.org/ns/" test="anyof"><x0:property-search><x0:prop><x0:displayname/></x0:prop><x0:match match-type="starts-with">%(displayname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:email-address-set/></x0:prop><x0:match match-type="starts-with">%(email)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:first-name/></x0:prop><x0:match match-type="starts-with">%(firstname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:last-name/></x0:prop><x0:match match-type="starts-with">%(lastname)s</x0:match></x0:property-search><x0:prop><x1:email-address-se
t/><x2:calendar-user-address-set/><x2:calendar-user-type/><x0:displayname/><x1:last-name/><x1:first-name/><x1:record-type/><x0:principal-URL/></x0:prop></x0:principal-property-search>
</del><ins>+<x0:principal-property-search xmlns:x2="urn:ietf:params:xml:ns:caldav" xmlns:x0="DAV:" xmlns:x1="http://calendarserver.org/ns/" test="anyof"><x0:property-search><x0:prop><x0:displayname/></x0:prop><x0:match match-type="starts-with">%(displayname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:email-address-set/></x0:prop><x0:match match-type="starts-with">%(email)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:first-name/></x0:prop><x0:match match-type="starts-with">%(firstname)s</x0:match></x0:property-search><x0:property-search><x0:prop><x1:last-name/></x0:prop><x0:match match-type="starts-with">%(lastname)s</x0:match></x0:property-search><x0:prop><x1:email-address-set
/><x2:calendar-user-address-set/><x2:calendar-user-type/><x0:displayname/><x1:last-name/><x1:first-name/><x1:record-type/><x0:principal-URL/></x0:prop></x0:principal-property-search>
</ins><span class="cx">\ No newline at end of file
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestsimpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/sim.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/sim.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -45,58 +45,16 @@
</span><span class="cx">
</span><span class="cx"> from twisted.web.server import Site
</span><span class="cx">
</span><del>-from contrib.performance.loadtest.ical import OS_X_10_6
</del><ins>+from contrib.performance.loadtest.clients import OS_X_10_6
</ins><span class="cx"> from contrib.performance.loadtest.profiles import Eventer, Inviter, Accepter
</span><span class="cx"> from contrib.performance.loadtest.population import (
</span><span class="cx"> Populator, ProfileType, ClientType, PopulationParameters, SmoothRampUp,
</span><span class="cx"> CalendarClientSimulator)
</span><span class="cx"> from contrib.performance.loadtest.webadmin import LoadSimAdminResource
</span><span class="cx">
</span><del>-
-
-class _DirectoryRecord(object):
- def __init__(self, uid, password, commonName, email, guid):
- self.uid = uid
- self.password = password
- self.commonName = commonName
- self.email = email
- self.guid = guid
-
-
-
</del><span class="cx"> def safeDivision(value, total, factor=1):
</span><span class="cx"> return value * factor / total if total else 0
</span><span class="cx">
</span><del>-
-
-def generateRecords(
- count, uidPattern="user%d", passwordPattern="user%d",
- namePattern="User %d", emailPattern="user%d@example.com",
- guidPattern="user%d"
-):
- for i in xrange(count):
- i += 1
- uid = uidPattern % (i,)
- password = passwordPattern % (i,)
- name = namePattern % (i,)
- email = emailPattern % (i,)
- guid = guidPattern % (i,)
- yield _DirectoryRecord(uid, password, name, email, guid)
-
-
-
-def recordsFromCSVFile(path):
- if path:
- pathObj = FilePath(path)
- else:
- pathObj = FilePath(__file__).sibling("accounts.csv")
- return [
- _DirectoryRecord(*line.decode('utf-8').split(u','))
- for line
- in pathObj.getContent().splitlines()]
-
-
-
</del><span class="cx"> class LagTrackingReactor(object):
</span><span class="cx"> """
</span><span class="cx"> This reactor wraps another reactor and proxies all attribute
</span><span class="lines">@@ -129,8 +87,9 @@
</span><span class="cx"> Command line configuration options for the load simulator.
</span><span class="cx"> """
</span><span class="cx"> config = None
</span><del>- _defaultConfig = FilePath(__file__).sibling("config.plist")
- _defaultClients = FilePath(__file__).sibling("clients.plist")
</del><ins>+ settings = FilePath(__file__).sibling("settings")
+ _defaultConfig = settings.child("config.plist")
+ _defaultClients = settings.child("clients.plist")
</ins><span class="cx">
</span><span class="cx"> optParameters = [
</span><span class="cx"> ("runtime", "t", None,
</span><span class="lines">@@ -195,6 +154,8 @@
</span><span class="cx"> configFile.close()
</span><span class="cx">
</span><span class="cx"> try:
</span><ins>+ # from importlib import import_module
+ # client_config = import_module("contrib.performance.loadtest.settings.clients")
</ins><span class="cx"> clientFile = self['clients'].open()
</span><span class="cx"> except IOError, e:
</span><span class="cx"> raise UsageError("--clients %s: %s" % (
</span><span class="lines">@@ -202,13 +163,15 @@
</span><span class="cx"> try:
</span><span class="cx"> try:
</span><span class="cx"> client_config = readPlist(clientFile)
</span><ins>+ # self.config["clients"] = client_config.calendars_only# client_config["clients"]
</ins><span class="cx"> self.config["clients"] = client_config["clients"]
</span><span class="cx"> if "arrivalInterval" in client_config:
</span><span class="cx"> self.config["arrival"]["params"]["interval"] = client_config["arrivalInterval"]
</span><span class="cx"> except ExpatError, e:
</span><span class="cx"> raise UsageError("--clients %s: %s" % (self['clients'].path, e))
</span><span class="cx"> finally:
</span><del>- clientFile.close()
</del><ins>+ # clientFile.close()
+ pass
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> Arrival = namedtuple('Arrival', 'factory parameters')
</span><span class="lines">@@ -224,17 +187,16 @@
</span><span class="cx"> @type arrival: L{Arrival}
</span><span class="cx"> @type parameters: L{PopulationParameters}
</span><span class="cx">
</span><del>- @ivar records: A C{list} of L{_DirectoryRecord} instances giving
</del><ins>+ @ivar records: A C{list} of L{DirectoryRecord} instances giving
</ins><span class="cx"> user information about the accounts on the server being put
</span><span class="cx"> under load.
</span><span class="cx"> """
</span><del>- def __init__(self, server, principalPathTemplate, webadminPort, serverStats, serializationPath, arrival, parameters, observers=None,
</del><ins>+ def __init__(self, server, webadminPort, serverStats, serializationPath, arrival, parameters, observers=None,
</ins><span class="cx"> records=None, reactor=None, runtime=None, workers=None,
</span><span class="cx"> configTemplate=None, workerID=None, workerCount=1):
</span><span class="cx"> if reactor is None:
</span><span class="cx"> from twisted.internet import reactor
</span><span class="cx"> self.server = server
</span><del>- self.principalPathTemplate = principalPathTemplate
</del><span class="cx"> self.webadminPort = webadminPort
</span><span class="cx"> self.serverStats = serverStats
</span><span class="cx"> self.serializationPath = serializationPath
</span><span class="lines">@@ -270,6 +232,8 @@
</span><span class="cx"> """
</span><span class="cx"> Create a L{LoadSimulator} from a parsed instance of a configuration
</span><span class="cx"> property list.
</span><ins>+
+ @type{config} L{Config} object
</ins><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> workers = config.get("workers")
</span><span class="lines">@@ -279,7 +243,6 @@
</span><span class="cx"> workerCount = config.get("workerCount", 1)
</span><span class="cx"> configTemplate = None
</span><span class="cx"> server = config.get('server', 'http://127.0.0.1:8008')
</span><del>- principalPathTemplate = config.get('principalPathTemplate', '/principals/users/%s/')
</del><span class="cx"> serializationPath = None
</span><span class="cx">
</span><span class="cx"> if 'clientDataSerialization' in config:
</span><span class="lines">@@ -306,6 +269,14 @@
</span><span class="cx"> parameters = PopulationParameters()
</span><span class="cx"> if 'clients' in config:
</span><span class="cx"> for clientConfig in config['clients']:
</span><ins>+ # parameters.addClient(
+ # clientConfig["weight"],
+ # ClientType(
+ # clientConfig["software"],
+ # clientConfig["params"],
+ # clientConfig["profiles"]
+ # )
+ # )
</ins><span class="cx"> parameters.addClient(
</span><span class="cx"> clientConfig["weight"],
</span><span class="cx"> ClientType(
</span><span class="lines">@@ -316,7 +287,8 @@
</span><span class="cx"> namedAny(profile["class"]),
</span><span class="cx"> cls._convertParams(profile["params"])
</span><span class="cx"> ) for profile in clientConfig["profiles"]
</span><del>- ]))
</del><ins>+ ])),
+
</ins><span class="cx"> if not parameters.clients:
</span><span class="cx"> parameters.addClient(1,
</span><span class="cx"> ClientType(OS_X_10_6, {},
</span><span class="lines">@@ -324,7 +296,6 @@
</span><span class="cx"> else:
</span><span class="cx"> # Manager / observer process.
</span><span class="cx"> server = ''
</span><del>- principalPathTemplate = ''
</del><span class="cx"> serializationPath = None
</span><span class="cx"> arrival = None
</span><span class="cx"> parameters = None
</span><span class="lines">@@ -359,7 +330,6 @@
</span><span class="cx">
</span><span class="cx"> return cls(
</span><span class="cx"> server,
</span><del>- principalPathTemplate,
</del><span class="cx"> webadminPort,
</span><span class="cx"> serverStats,
</span><span class="cx"> serializationPath,
</span><span class="lines">@@ -377,6 +347,104 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classmethod
</span><ins>+ def fromConfigObject(cls, config, runtime=None, output=stdout):
+ workers = config['workers']
+ if workers is None:
+ # Client / place where the simulator actually runs configuration
+ workerID = config.get("workerID", 0)
+ workerCount = config.get("workerCount", 1)
+ configTemplate = None
+ server = config.get('server', 'http://127.0.0.1:8008')
+ serializationPath = None
+
+ serializationPath = config['serializationPath']
+
+ if 'arrival' in config:
+ arrival = Arrival(
+ namedAny(config['arrival']['factory']),
+ config['arrival']['params'])
+ else:
+ arrival = Arrival(
+ SmoothRampUp, dict(groups=10, groupSize=1, interval=3))
+
+ parameters = PopulationParameters()
+ if 'clients' in config:
+ for clientConfig in config['clients']:
+ parameters.addClient(
+ clientConfig["weight"],
+ ClientType(
+ clientConfig["software"],
+ clientConfig["params"],
+ clientConfig["profiles"]
+ )
+ )
+ # ClientType(
+ # namedAny(clientConfig["software"]),
+ # cls._convertParams(clientConfig["params"]),
+ # [
+ # ProfileType(
+ # namedAny(profile["class"]),
+ # cls._convertParams(profile["params"])
+ # ) for profile in clientConfig["profiles"]
+ # ]))
+ if not parameters.clients:
+ parameters.addClient(1,
+ ClientType(OS_X_10_6, {},
+ [Eventer, Inviter, Accepter]))
+ else:
+ # Manager / observer process.
+ server = ''
+ serializationPath = None
+ arrival = None
+ parameters = None
+ workerID = 0
+ configTemplate = config
+ workerCount = 1
+
+ # webadminPort =
+ webadminPort = None
+ if 'webadmin' in config:
+ if config['webadmin']['enabled']:
+ webadminPort = config['webadmin']['HTTPPort']
+
+ serverStats = None
+ if 'serverStats' in config:
+ if config['serverStats']['enabled']:
+ serverStats = config['serverStats']
+ serverStats['server'] = config['server'] if 'server' in config else ''
+
+ observers = []
+ if 'observers' in config:
+ for observer in config['observers']:
+ observerName = observer["type"]
+ observerParams = observer["params"]
+ observers.append(namedAny(observerName)(**observerParams))
+
+ records = []
+ if 'accounts' in config:
+ loader = config['accounts']['loader']
+ params = config['accounts']['params']
+ records.extend(namedAny(loader)(**params))
+ output.write("Loaded {0} accounts.\n".format(len(records)))
+
+ return cls(
+ server,
+ webadminPort,
+ serverStats,
+ serializationPath,
+ arrival,
+ parameters,
+ observers=observers,
+ records=records,
+ runtime=runtime,
+ reactor=reactor,
+ workers=workers,
+ configTemplate=configTemplate,
+ workerID=workerID,
+ workerCount=workerCount,
+ )
+
+ @classmethod
</ins><span class="cx"> def _convertParams(cls, params):
</span><span class="cx"> """
</span><span class="cx"> Find parameter values which should be more structured than plistlib is
</span><span class="lines">@@ -415,7 +483,6 @@
</span><span class="cx"> self.parameters,
</span><span class="cx"> self.reactor,
</span><span class="cx"> self.server,
</span><del>- self.principalPathTemplate,
</del><span class="cx"> self.serializationPath,
</span><span class="cx"> self.workerID,
</span><span class="cx"> self.workerCount,
</span><span class="lines">@@ -423,6 +490,7 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def createArrivalPolicy(self):
</span><ins>+ # print(self.arrival.parameters)
</ins><span class="cx"> return self.arrival.factory(self.reactor, **self.arrival.parameters)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -431,16 +499,14 @@
</span><span class="cx"> Return a list of L{SimService} subclasses for C{attachServices} to
</span><span class="cx"> instantiate and attach to the reactor.
</span><span class="cx"> """
</span><del>- if self.workers is not None:
- return [
- ObserverService,
- WorkerSpawnerService,
- ReporterService,
- ]
</del><ins>+ if self.workers:
+ PrimaryService = WorkerSpawnerService
+ else:
+ PrimaryService = SimulatorService
</ins><span class="cx"> return [
</span><span class="cx"> ObserverService,
</span><del>- SimulatorService,
</del><span class="cx"> ReporterService,
</span><ins>+ PrimaryService
</ins><span class="cx"> ]
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadteststandardconfigseventsonlyplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/standard-configs/events-only.plist (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/standard-configs/events-only.plist 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/standard-configs/events-only.plist 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -29,23 +29,23 @@
</span><span class="cx">
</span><span class="cx"> <dict>
</span><span class="cx">
</span><del>- <!-- Here is a Lion iCal simulator. -->
</del><ins>+ <!-- Here is an El Captian iCal simulator. -->
</ins><span class="cx"> <key>software</key>
</span><del>- <string>contrib.performance.loadtest.ical.OS_X_10_7</string>
</del><ins>+ <string>contrib.performance.loadtest.ical.OS_X_10_11</string>
</ins><span class="cx">
</span><span class="cx"> <!-- Arguments to use to initialize the client instance. -->
</span><span class="cx"> <key>params</key>
</span><span class="cx"> <dict>
</span><span class="cx"> <!-- Name that appears in logs. -->
</span><span class="cx"> <key>title</key>
</span><del>- <string>10.7</string>
</del><ins>+ <string>10.11</string>
</ins><span class="cx">
</span><span class="cx"> <!-- Client can poll the calendar home at some interval. This is
</span><span class="cx"> in seconds. -->
</span><span class="cx"> <key>calendarHomePollInterval</key>
</span><del>- <integer>300000</integer>
</del><ins>+ <integer>30</integer>
</ins><span class="cx">
</span><del>- <!-- If the server advertises xmpp push, OS X 10.6 can wait for notifications
</del><ins>+ <!-- If the server advertises xmpp push, OS X 10.11 can wait for notifications
</ins><span class="cx"> about calendar home changes instead of polling for them periodically. If
</span><span class="cx"> this option is true, then look for the server advertisement for xmpp push
</span><span class="cx"> and use it if possible. Still fall back to polling if there is no xmpp push
</span><span class="lines">@@ -268,269 +268,6 @@
</span><span class="cx"> </dict>
</span><span class="cx"> </dict>
</span><span class="cx">
</span><del>- <!-- This profile invites some number of new attendees to new events. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.RealisticInviter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define the frequency at which new invitations will be sent out. -->
- <key>sendInvitationDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.NormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mu gives the mean of the normal distribution (in seconds). -->
- <key>mu</key>
- <integer>10</integer>
-
- <!-- and sigma gives its standard deviation. -->
- <key>sigma</key>
- <integer>5</integer>
- </dict>
- </dict>
-
- <!-- Define the distribution of who will be invited to an event.
-
- When inviteeClumping is turned on each invitee is based on a sample of
- users "close to" the organizer based on account index. If the clumping
- is too "tight" for the requested number of attendees, then invites for
- those larger numbers will simply fail (the sim will report that situation).
-
- When inviteeClumping is off invitees will be sampled across an entire
- range of account indexes. In this case the distribution ought to be a
- UniformIntegerDistribution with min=0 and max set to the number of accounts.
- -->
- <key>inviteeDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.UniformIntegerDistribution</string>
- <key>params</key>
- <dict>
- <!-- The minimum value (inclusive) of the uniform distribution. -->
- <key>min</key>
- <integer>0</integer>
- <!-- The maximum value (exclusive) of the uniform distribution. -->
- <key>max</key>
- <integer>99</integer>
- </dict>
- </dict>
-
- <key>inviteeClumping</key>
- <true/>
-
- <!-- Define the distribution of how many attendees will be invited to an event.
-
- LogNormal is the best fit to observed data.
-
-
- For LogNormal "mode" is the peak, "mean" is the mean value. For invites,
- mode should typically be 1, and mean whatever matches the user behavior.
- Our typical mean is 6.
- -->
- <key>inviteeCountDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>1</integer>
- <!-- mean - average-->
- <key>median</key>
- <integer>6</integer>
- <!-- maximum -->
- <key>maximum</key>
- <real>100</real>
- </dict>
- </dict>
-
- <!-- Define how start times (DTSTART) for the randomly generated events
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>eventStartDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
-
- <!-- Define how recurrences are created. -->
- <key>recurrenceDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. We have a fixed set of
- RRULEs defined for this distribution and pick each based on a
- weight. -->
- <key>type</key>
- <string>contrib.performance.stats.RecurrenceDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- False to disable RRULEs -->
- <key>allowRecurrence</key>
- <true/>
-
- <!-- These are the weights for the specific set of RRULEs. -->
- <key>weights</key>
- <dict>
- <!-- Half of all events will be non-recurring -->
- <key>none</key>
- <integer>50</integer>
-
- <!-- Daily and weekly are pretty common -->
- <key>daily</key>
- <integer>10</integer>
- <key>weekly</key>
- <integer>20</integer>
-
- <!-- Monthly, yearly, daily & weekly limit not so common -->
- <key>monthly</key>
- <integer>2</integer>
- <key>yearly</key>
- <integer>1</integer>
- <key>dailylimit</key>
- <integer>2</integer>
- <key>weeklylimit</key>
- <integer>5</integer>
-
- <!-- Work days pretty common -->
- <key>workdays</key>
- <integer>10</integer>
- </dict>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- This profile accepts invitations to events, handles cancels, and
- handles replies received. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Accepter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define how long to wait after seeing a new invitation before
- accepting it.
-
- For LogNormal "mode" is the peak, "median" is the 50% cummulative value
- (i.e., half of the user have accepted by that time).
- -->
- <key>acceptDelayDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.LogNormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mode - peak-->
- <key>mode</key>
- <integer>300</integer>
- <!-- median - 50% done-->
- <key>median</key>
- <integer>1800</integer>
- </dict>
- </dict>
- </dict>
- </dict>
-
- <!-- A task-creating profile, which will periodically create
- new tasks at a random time on a random calendar. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Tasker</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <false/>
-
- <!-- Define the interval (in seconds) at which this profile will use
- its client to create a new task. -->
- <key>interval</key>
- <integer>300</integer>
-
- <!-- Define how due times (DUE) for the randomly generated tasks
- will be selected. This is an example of a "Distribution" parameter. The value
- for most "Distribution" parameters are interchangeable and extensible. -->
- <key>taskDueDistribution</key>
- <dict>
-
- <!-- This distribution is pretty specialized. It produces timestamps
- in the near future, limited to certain days of the week and certain hours
- of the day. -->
- <key>type</key>
- <string>contrib.performance.stats.WorkDistribution</string>
-
- <key>params</key>
- <dict>
- <!-- These are the days of the week the distribution will use. -->
- <key>daysOfWeek</key>
- <array>
- <string>mon</string>
- <string>tue</string>
- <string>wed</string>
- <string>thu</string>
- <string>fri</string>
- </array>
-
- <!-- The earliest hour of a day at which an event might be scheduled. -->
- <key>beginHour</key>
- <integer>8</integer>
-
- <!-- And the latest hour of a day (at which an event will be scheduled
- to begin!). -->
- <key>endHour</key>
- <integer>16</integer>
-
- <!-- The timezone in which the event is scheduled. (XXX Does this
- really work right?) -->
- <key>tzname</key>
- <string>America/Los_Angeles</string>
- </dict>
- </dict>
- </dict>
- </dict>
-
</del><span class="cx"> </array>
</span><span class="cx">
</span><span class="cx"> <!-- Determine the frequency at which this client configuration will
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformanceloadtestwebadminpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/webadmin.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/loadtest/webadmin.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/loadtest/webadmin.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -39,6 +39,7 @@
</span><span class="cx"> HEAD = """\
</span><span class="cx"> <html>
</span><span class="cx"> <head>
</span><ins>+<meta http-equiv="refresh" content="1" >
</ins><span class="cx"> <style type="text/css">
</span><span class="cx"> body {color:#000000;}
</span><span class="cx"> h1 h2 h3 {color:#333333;}
</span><span class="lines">@@ -80,9 +81,11 @@
</span><span class="cx"> def __init__(self, loadsim):
</span><span class="cx"> self.loadsim = loadsim
</span><span class="cx"> self.token = str(uuid.uuid4())
</span><ins>+ self.count = 0
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> def render_GET(self, request):
</span><ins>+ self.count += 1
</ins><span class="cx"> return self._renderReport()
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -109,5 +112,5 @@
</span><span class="cx"> html = self.HEAD + self.BODY_RESULTS_STOPPED
</span><span class="cx"> return html % (None, report.getvalue(), ms)
</span><span class="cx"> else:
</span><del>- html = self.HEAD + self.BODY_RESULTS
</del><ins>+ html = self.HEAD + "<h3>Request #" + str(self.count) + "</h3>" + self.BODY_RESULTS
</ins><span class="cx"> return html % (self.token, report.getvalue(), ms)
</span></span></pre></div>
<a id="CalendarServerbranchesuserssredmondclientsimcontribperformancestatspy"></a>
<div class="modfile"><h4>Modified: CalendarServer/branches/users/sredmond/clientsim/contrib/performance/stats.py (15042 => 15046)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/contrib/performance/stats.py 2015-08-15 02:19:49 UTC (rev 15042)
+++ CalendarServer/branches/users/sredmond/clientsim/contrib/performance/stats.py 2015-08-17 20:29:08 UTC (rev 15046)
</span><span class="lines">@@ -14,26 +14,30 @@
</span><span class="cx"> # limitations under the License.
</span><span class="cx"> ##
</span><span class="cx">
</span><ins>+"""
+Implementation of a statistics library for Calendar performance analysis.
+Exports:
+
+_Statistic object
+ Duration
+ SQLDuration
+ Bytes
+"""
+
</ins><span class="cx"> from __future__ import print_function
</span><span class="cx">
</span><del>-from math import log, sqrt
-from time import mktime
-import random
</del><span class="cx"> import sqlparse
</span><span class="cx">
</span><del>-from pycalendar.datetime import DateTime
-from pycalendar.duration import Duration as PyDuration
-from pycalendar.icalendar.property import Property
-from pycalendar.timezone import Timezone
-
-from zope.interface import Interface, implements
-from twisted.python.util import FancyEqMixin
-
-
</del><span class="cx"> NANO = 1000000000.0
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> def mean(samples):
</span><ins>+ """
+ Computes the mean value of a list of numbers
+ @type samples list of numerics
+
+ Raises ZeroDivisionError if `samples` is empty
+ """
</ins><span class="cx"> return sum(samples) / len(samples)
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -48,13 +52,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-def stddev(samples):
- m = mean(samples)
- variance = sum([datum ** 2 for datum in residuals(samples, m)]) / len(samples)
- return variance ** 0.5
-
-
-
</del><span class="cx"> def mad(samples):
</span><span class="cx"> """
</span><span class="cx"> Return the median absolute deviation of the given data set.
</span><span class="lines">@@ -65,6 +62,13 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><ins>+def stddev(samples):
+ m = mean(samples)
+ variance = sum([datum ** 2 for datum in residuals(samples, m)]) / len(samples)
+ return variance ** 0.5
+
+
+
</ins><span class="cx"> class _Statistic(object):
</span><span class="cx"> commands = ['summarize']
</span><span class="cx">
</span><span class="lines">@@ -221,272 +225,3 @@
</span><span class="cx">
</span><span class="cx"> def summarize(self, samples):
</span><span class="cx"> return _Statistic.summarize(self, self.squash(samples))
</span><del>-
-
-
-def quantize(data):
- """
- Given some continuous data, quantize it into appropriately sized
- discrete buckets (eg, as would be suitable for constructing a
- histogram of the values).
- """
- # buckets = {}
- return []
-
-
-
-class IPopulation(Interface):
- def sample(): #@NoSelf
- pass
-
-
-
-class UniformDiscreteDistribution(object, FancyEqMixin):
- """
-
- """
- implements(IPopulation)
-
- compareAttributes = ['_values']
-
- def __init__(self, values, randomize=True):
- self._values = values
- self._randomize = randomize
- self._refill()
-
-
- def _refill(self):
- self._remaining = self._values[:]
- if self._randomize:
- random.shuffle(self._remaining)
-
-
- def sample(self):
- if not self._remaining:
- self._refill()
- return self._remaining.pop()
-
-
-
-class LogNormalDistribution(object, FancyEqMixin):
- """
- """
- implements(IPopulation)
-
- compareAttributes = ['_mu', '_sigma', '_maximum']
-
- def __init__(self, mu=None, sigma=None, mean=None, mode=None, median=None, maximum=None):
-
- if mu is not None and sigma is not None:
- scale = 1.0
- elif not (mu is None and sigma is None):
- raise ValueError("mu and sigma must both be defined or both not defined")
- elif mode is None:
- raise ValueError("When mu and sigma are not defined, mode must be defined")
- elif median is not None:
- scale = mode
- median /= mode
- mode = 1.0
- mu = log(median)
- sigma = sqrt(log(median) - log(mode))
- elif mean is not None:
- scale = mode
- mean /= mode
- mode = 1.0
- mu = log(mean) + log(mode) / 2.0
- sigma = sqrt(log(mean) - log(mode) / 2.0)
- else:
- raise ValueError("When using mode one of median or mean must be defined")
-
- self._mu = mu
- self._sigma = sigma
- self._scale = scale
- self._maximum = maximum
-
-
- def sample(self):
- result = self._scale * random.lognormvariate(self._mu, self._sigma)
- if self._maximum is not None and result > self._maximum:
- for _ignore in range(10):
- result = self._scale * random.lognormvariate(self._mu, self._sigma)
- if result <= self._maximum:
- break
- else:
- raise ValueError("Unable to generate LogNormalDistribution sample within required range")
- return result
-
-
-
-class FixedDistribution(object, FancyEqMixin):
- """
- """
- implements(IPopulation)
-
- compareAttributes = ['_value']
-
- def __init__(self, value):
- self._value = value
-
-
- def sample(self):
- return self._value
-
-
-
-class NearFutureDistribution(object, FancyEqMixin):
- compareAttributes = ['_offset']
-
- def __init__(self):
- self._offset = LogNormalDistribution(7, 0.8)
-
-
- def sample(self):
- now = DateTime.getNowUTC()
- now.offsetSeconds(int(self._offset.sample()))
- return now
-
-
-
-class NormalDistribution(object, FancyEqMixin):
- compareAttributes = ['_mu', '_sigma']
-
- def __init__(self, mu, sigma):
- self._mu = mu
- self._sigma = sigma
-
-
- def sample(self):
- # Only return positive values or zero
- v = random.normalvariate(self._mu, self._sigma)
- while v < 0:
- v = random.normalvariate(self._mu, self._sigma)
- return v
-
-
-
-class UniformIntegerDistribution(object, FancyEqMixin):
- compareAttributes = ['_min', '_max']
-
- def __init__(self, min, max):
- self._min = min
- self._max = max
-
-
- def sample(self):
- return int(random.uniform(self._min, self._max))
-
-
-NUM_WEEKDAYS = 7
-
-class WorkDistribution(object, FancyEqMixin):
- compareAttributes = ["_daysOfWeek", "_beginHour", "_endHour"]
-
- _weekdayNames = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"]
-
- def __init__(self, daysOfWeek=["mon", "tue", "wed", "thu", "fri"], beginHour=8, endHour=17, tzname="UTC"):
- self._daysOfWeek = [self._weekdayNames.index(day) for day in daysOfWeek]
- self._beginHour = beginHour
- self._endHour = endHour
- self._tzname = tzname
- self._helperDistribution = NormalDistribution(
- # Mean 6 workdays in the future
- 60 * 60 * 8 * 6,
- # Standard deviation of 4 workdays
- 60 * 60 * 8 * 4)
- self.now = DateTime.getNow
-
-
- def astimestamp(self, dt):
- return mktime(dt.timetuple())
-
-
- def _findWorkAfter(self, when):
- """
- Return a two-tuple of the start and end of work hours following
- C{when}. If C{when} falls within work hours, then the start time will
- be equal to when.
- """
- # Find a workday that follows the timestamp
- weekday = when.getDayOfWeek()
- for i in range(NUM_WEEKDAYS):
- day = when + PyDuration(days=i)
- if (weekday + i) % NUM_WEEKDAYS in self._daysOfWeek:
- # Joy, a day on which work might occur. Find the first hour on
- # this day when work may start.
- day.setHHMMSS(self._beginHour, 0, 0)
- begin = day
- end = begin.duplicate()
- end.setHHMMSS(self._endHour, 0, 0)
- if end > when:
- return begin, end
-
-
- def sample(self):
- offset = PyDuration(seconds=int(self._helperDistribution.sample()))
- beginning = self.now(Timezone(tzid=self._tzname))
- while offset:
- start, end = self._findWorkAfter(beginning)
- if end - start > offset:
- result = start + offset
- result.setMinutes(result.getMinutes() // 15 * 15)
- result.setSeconds(0)
- return result
- offset.setDuration(offset.getTotalSeconds() - (end - start).getTotalSeconds())
- beginning = end
-
-
-
-class RecurrenceDistribution(object, FancyEqMixin):
- compareAttributes = ["_allowRecurrence", "_weights"]
-
- _model_rrules = {
- "none": None,
- "daily": "RRULE:FREQ=DAILY",
- "weekly": "RRULE:FREQ=WEEKLY",
- "monthly": "RRULE:FREQ=MONTHLY",
- "yearly": "RRULE:FREQ=YEARLY",
- "dailylimit": "RRULE:FREQ=DAILY;COUNT=14",
- "weeklylimit": "RRULE:FREQ=WEEKLY;COUNT=4",
- "workdays": "RRULE:FREQ=DAILY;BYDAY=MO,TU,WE,TH,FR"
- }
-
- def __init__(self, allowRecurrence, weights={}):
- self._allowRecurrence = allowRecurrence
- self._rrules = []
- if self._allowRecurrence:
- for rrule, count in sorted(weights.items(), key=lambda x: x[0]):
- for _ignore in range(count):
- self._rrules.append(self._model_rrules[rrule])
- self._helperDistribution = UniformIntegerDistribution(0, len(self._rrules) - 1)
-
-
- def sample(self):
-
- if self._allowRecurrence:
- index = self._helperDistribution.sample()
- rrule = self._rrules[index]
- if rrule:
- prop = Property.parseText(rrule)
- return prop
-
- return None
-
-if __name__ == '__main__':
-
- from collections import defaultdict
- mu = 1.5
- sigma = 1.22
- distribution = LogNormalDistribution(mu, sigma, 100)
- result = defaultdict(int)
- for i in range(100000):
- s = int(distribution.sample())
- if s > 300:
- continue
- result[s] += 1
-
- total = 0
- for k, v in sorted(result.items(), key=lambda x: x[0]):
- print("%d\t%.5f" % (k, float(v) / result[1]))
- total += k * v
-
- print("Average: %.2f" % (float(total) / sum(result.values()),))
</del></span></pre>
</div>
</div>
</body>
</html>