[CalendarServer-changes] [15749] CalendarServer/branches/users/cdaboo/better-next-job
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jul 7 09:14:45 PDT 2016
Revision: 15749
http://trac.calendarserver.org//changeset/15749
Author: cdaboo at apple.com
Date: 2016-07-07 09:14:45 -0700 (Thu, 07 Jul 2016)
Log Message:
-----------
Merge from trunk.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/better-next-job/bin/_build.sh
CalendarServer/branches/users/cdaboo/better-next-job/bin/testpodssim
CalendarServer/branches/users/cdaboo/better-next-job/bin/testsim
CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/caldav.py
CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/util.py
CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-apple.plist
CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-stdconfig.plist
CalendarServer/branches/users/cdaboo/better-next-job/lib-patches/Twisted/securetransport.patch
CalendarServer/branches/users/cdaboo/better-next-job/requirements-cs.txt
CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-default.txt
CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-osx.txt
CalendarServer/branches/users/cdaboo/better-next-job/setup.py
CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/dumpconfig.py
CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/base/datastore/dbapiclient.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_dump.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_tables.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_sql_dump.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_trash.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/util.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/upgrade.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist
CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist
Property Changed:
----------------
CalendarServer/branches/users/cdaboo/better-next-job/
Property changes on: CalendarServer/branches/users/cdaboo/better-next-job
___________________________________________________________________
Modified: svn:mergeinfo
- /CalDAVTester/trunk:11193-11198
/CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
/CalendarServer/branches/release/CalendarServer-5.1-dev:11846
/CalendarServer/branches/release/CalendarServer-5.2-dev:11972,12357-12358,12794,12814
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/cross-pod-sharing:12038-12191
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/fix-no-ischedule:11607-11871
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/json:11622-11912
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/newlog:15301
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/performance-tweaks:11824-11836
/CalendarServer/branches/users/cdaboo/pod2pod-migration:14338-14520
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/reverse-proxy-pods:11875-11900
/CalendarServer/branches/users/cdaboo/scheduling-queue-refresh:11783-12557
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/sharing-in-the-store:11935-12016
/CalendarServer/branches/users/cdaboo/store-scheduling:10876-11129
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/cdaboo/update-packages:15273-15284
/CalendarServer/branches/users/gaya/cleanrevisions:12152-12334
/CalendarServer/branches/users/gaya/groupsharee2:13669-13773
/CalendarServer/branches/users/gaya/sharedgroupfixes:12120-12142
/CalendarServer/branches/users/gaya/sharedgroups-3:11088-11204
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/enforce-max-requests:11640-11643
/CalendarServer/branches/users/glyph/hang-fix:11465-11491
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/launchd-wrapper-bis:11413-11436
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/log-cleanups:11691-11731
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/start-service-start-loop:11060-11065
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/warning-cleanups:11347-11357
/CalendarServer/branches/users/glyph/whenNotProposed:11881-11897
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/move2who:12819-12860
/CalendarServer/branches/users/sagen/move2who-2:12861-12898
/CalendarServer/branches/users/sagen/move2who-3:12899-12913
/CalendarServer/branches/users/sagen/move2who-4:12914-13157
/CalendarServer/branches/users/sagen/move2who-5:13158-13163
/CalendarServer/branches/users/sagen/newcua:13309-13327
/CalendarServer/branches/users/sagen/newcua-1:13328-13330
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/recordtypes:13648-13656
/CalendarServer/branches/users/sagen/recordtypes-2:13657
/CalendarServer/branches/users/sagen/request-socket:14748-14767
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/sagen/testing:10827-10851,10853-10855
/CalendarServer/branches/users/sagen/trashcan:14185-14269
/CalendarServer/branches/users/sagen/trashcan-2:14270-14324
/CalendarServer/branches/users/sagen/trashcan-3:14325-14450
/CalendarServer/branches/users/sagen/trashcan-4:14451-14471
/CalendarServer/branches/users/sagen/trashcan-5:14471-14555
/CalendarServer/branches/users/wsanchez/psycopg2cffi:14427-14439
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:15717-15725
+ /CalDAVTester/trunk:11193-11198
/CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/release/CalendarServer-4.3-dev:10180-10190,10192
/CalendarServer/branches/release/CalendarServer-5.1-dev:11846
/CalendarServer/branches/release/CalendarServer-5.2-dev:11972,12357-12358,12794,12814
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/cross-pod-sharing:12038-12191
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/fix-no-ischedule:11607-11871
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/json:11622-11912
/CalendarServer/branches/users/cdaboo/managed-attachments:9985-10145
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/newlog:15301
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/performance-tweaks:11824-11836
/CalendarServer/branches/users/cdaboo/pod2pod-migration:14338-14520
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/reverse-proxy-pods:11875-11900
/CalendarServer/branches/users/cdaboo/scheduling-queue-refresh:11783-12557
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/sharing-in-the-store:11935-12016
/CalendarServer/branches/users/cdaboo/store-scheduling:10876-11129
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/cdaboo/update-packages:15273-15284
/CalendarServer/branches/users/gaya/cleanrevisions:12152-12334
/CalendarServer/branches/users/gaya/groupsharee2:13669-13773
/CalendarServer/branches/users/gaya/sharedgroupfixes:12120-12142
/CalendarServer/branches/users/gaya/sharedgroups-3:11088-11204
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/digest-auth-redux:10624-10635
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/enforce-max-requests:11640-11643
/CalendarServer/branches/users/glyph/hang-fix:11465-11491
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/launchd-wrapper-bis:11413-11436
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/log-cleanups:11691-11731
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/queue-locking-and-timing:10204-10289
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/start-service-start-loop:11060-11065
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/unshare-when-access-revoked:10562-10595
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/warning-cleanups:11347-11357
/CalendarServer/branches/users/glyph/whenNotProposed:11881-11897
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/move2who:12819-12860
/CalendarServer/branches/users/sagen/move2who-2:12861-12898
/CalendarServer/branches/users/sagen/move2who-3:12899-12913
/CalendarServer/branches/users/sagen/move2who-4:12914-13157
/CalendarServer/branches/users/sagen/move2who-5:13158-13163
/CalendarServer/branches/users/sagen/newcua:13309-13327
/CalendarServer/branches/users/sagen/newcua-1:13328-13330
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/recordtypes:13648-13656
/CalendarServer/branches/users/sagen/recordtypes-2:13657
/CalendarServer/branches/users/sagen/request-socket:14748-14767
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/sagen/testing:10827-10851,10853-10855
/CalendarServer/branches/users/sagen/trashcan:14185-14269
/CalendarServer/branches/users/sagen/trashcan-2:14270-14324
/CalendarServer/branches/users/sagen/trashcan-3:14325-14450
/CalendarServer/branches/users/sagen/trashcan-4:14451-14471
/CalendarServer/branches/users/sagen/trashcan-5:14471-14555
/CalendarServer/branches/users/wsanchez/psycopg2cffi:14427-14439
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:15717-15725,15729-15747
Modified: CalendarServer/branches/users/cdaboo/better-next-job/bin/_build.sh
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/bin/_build.sh 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/bin/_build.sh 2016-07-07 16:14:45 UTC (rev 15749)
@@ -602,7 +602,7 @@
if command -v postgres > /dev/null; then
using_system "Postgres";
else
- local v="9.3.10";
+ local v="9.5.3";
local n="postgresql";
local p="${n}-${v}";
@@ -612,10 +612,10 @@
local enable_dtrace="";
fi;
- c_dependency -m "ec2365548d08f69c8023eddd4f2d1a28" \
+ c_dependency -m "3f0c388566c688c82b01a0edf1e6b7a0" \
"PostgreSQL" "${p}" \
"http://ftp.postgresql.org/pub/source/v${v}/${p}.tar.bz2" \
- --with-python ${enable_dtrace};
+ ${enable_dtrace};
fi;
}
@@ -678,6 +678,12 @@
--no-setuptools \
${virtualenv_opts} \
"${py_virtualenv}";
+ case "$(uname -s)" in
+ Darwin)
+ echo "macOS virtualenv codesign fix."
+ cp "/usr/bin/python" "${py_bindir}/python";
+ ;;
+ esac;
fi;
cd "${wd}";
@@ -711,9 +717,11 @@
ruler "Patching Python requirements";
echo "";
- if [ ! -e "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied" ]; then
+ twisted_version=$("${python}" -c 'from twisted._version import version; print version.base()');
+ if [ ! -e "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied.${twisted_version}" ]; then
apply_patches "Twisted" "${py_virtualenv}/lib/python2.7/site-packages"
- touch "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied";
+ find "${py_virtualenv}/lib/python2.7/site-packages/twisted" -type f -name '.patch_applied*' -print0 | xargs -0 rm -f;
+ touch "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied.${twisted_version}";
fi;
echo "";
@@ -735,37 +743,15 @@
bootstrap_virtualenv () {
mkdir -p "${py_ve_tools}";
- mkdir -p "${py_ve_tools}/lib";
- mkdir -p "${py_ve_tools}/junk";
+ export PYTHONUSERBASE="${py_ve_tools}"
for pkg in \
- setuptools-18.5 \
- pip-7.1.2 \
- virtualenv-13.1.2 \
+ setuptools==18.5 \
+ pip==8.1.2 \
+ virtualenv==15.0.2 \
; do
- local name="${pkg%-*}";
- local version="${pkg#*-}";
- local first="$(echo "${name}" | sed 's|^\(.\).*$|\1|')";
- local url="https://pypi.python.org/packages/source/${first}/${name}/${pkg}.tar.gz";
-
- ruler "Downloading ${pkg}";
-
- local tmp="$(mktemp -d -t ccsXXXXX)";
-
- curl -L "${url}" | tar -C "${tmp}" -xvzf -;
-
- cd "${tmp}/$(basename "${pkg}")";
- PYTHONPATH="${py_ve_tools}/lib" \
- "${bootstrap_python}" setup.py install \
- --install-base="${py_ve_tools}" \
- --install-lib="${py_ve_tools}/lib" \
- --install-headers="${py_ve_tools}/junk" \
- --install-scripts="${py_ve_tools}/junk" \
- --install-data="${py_ve_tools}/junk" \
- ; \
- cd "${wd}";
-
- rm -rf "${tmp}";
+ ruler "Installing ${pkg}";
+ "${bootstrap_python}" -m pip install -I --user "${pkg}";
done;
}
@@ -776,7 +762,7 @@
"${python}" -m pip install \
--disable-pip-version-check \
--download="${dev_home}/pip_downloads" \
- --pre --allow-all-external \
+ --pre \
--no-cache-dir \
--log-file="${dev_home}/pip.log" \
"$@";
@@ -786,7 +772,7 @@
pip_install_from_cache () {
"${python}" -m pip install \
--disable-pip-version-check \
- --pre --allow-all-external \
+ --pre \
--no-index \
--no-cache-dir \
--find-links="${dev_patches}" \
@@ -799,7 +785,7 @@
pip_download_and_install () {
"${python}" -m pip install \
--disable-pip-version-check \
- --pre --allow-all-external \
+ --pre \
--no-cache-dir \
--find-links="${dev_patches}" \
--log-file="${dev_home}/pip.log" \
Modified: CalendarServer/branches/users/cdaboo/better-next-job/bin/testpodssim
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/bin/testpodssim 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/bin/testpodssim 2016-07-07 16:14:45 UTC (rev 15749)
@@ -33,8 +33,8 @@
##
verbose="";
- config="${wd}/contrib/performance/loadtest/config.plist";
- clients="${wd}/contrib/performance/loadtest/clients.plist";
+ config="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist";
+ clients="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist";
runtime="--runtime 300";
logfile="";
Modified: CalendarServer/branches/users/cdaboo/better-next-job/bin/testsim
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/bin/testsim 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/bin/testsim 2016-07-07 16:14:45 UTC (rev 15749)
@@ -33,8 +33,8 @@
##
verbose="";
- config="${wd}/contrib/performance/loadtest/config.plist";
- clients="${wd}/contrib/performance/loadtest/clients.plist";
+ config="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist";
+ clients="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist";
runtime="--runtime 300";
logfile="";
Modified: CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/caldav.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/caldav.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -66,8 +66,7 @@
from twext.internet.ssl import ChainingOpenSSLContextFactory
from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
from twext.enterprise.adbapi2 import ConnectionPool
-from twext.enterprise.ienterprise import ORACLE_DIALECT
-from twext.enterprise.ienterprise import POSTGRES_DIALECT
+from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT, DatabaseType
from twext.enterprise.jobs.queue import NonPerformingQueuer
from twext.enterprise.jobs.queue import ControllerQueue
from twext.enterprise.jobs.queue import WorkerFactory as QueueWorkerFactory
@@ -1565,7 +1564,7 @@
@rtype: L{IService}
"""
- def createSubServiceFactory(dbtype):
+ def createSubServiceFactory(dbtype, dbfeatures=()):
if dbtype == "":
dialect = POSTGRES_DIALECT
paramstyle = "pyformat"
@@ -1579,8 +1578,8 @@
def subServiceFactory(connectionFactory, storageService):
ms = MultiService()
cp = ConnectionPool(
- connectionFactory, dialect=dialect,
- paramstyle=paramstyle,
+ connectionFactory,
+ dbtype=DatabaseType(dialect, paramstyle, dbfeatures),
maxConnections=config.MaxDBConnectionsPerPool
)
cp.setServiceParent(ms)
@@ -1680,13 +1679,13 @@
# to it.
pgserv = pgServiceFromConfig(
config,
- createSubServiceFactory(""),
+ createSubServiceFactory("", config.DBFeatures),
uid=overrideUID, gid=overrideGID
)
return pgserv
else:
# Connect to a database that is already running.
- return createSubServiceFactory(config.DBType)(
+ return createSubServiceFactory(config.DBType, config.DBFeatures)(
DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect, None
)
else:
Modified: CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/util.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/calendarserver/tap/util.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -44,8 +44,7 @@
from twext.enterprise.adbapi2 import ConnectionPool, ConnectionPoolConnection
from twext.enterprise.adbapi2 import ConnectionPoolClient
-from twext.enterprise.ienterprise import ORACLE_DIALECT
-from twext.enterprise.ienterprise import POSTGRES_DIALECT
+from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT, DatabaseType
from twext.internet.ssl import ChainingOpenSSLContextFactory
from twext.python.filepath import CachingFilePath
from twext.python.filepath import CachingFilePath as FilePath
@@ -179,14 +178,14 @@
-def transactionFactoryFromFD(dbampfd, dialect, paramstyle):
+def transactionFactoryFromFD(dbampfd, dbtype):
"""
Create a transaction factory from an inherited file descriptor, such as one
created by L{ConnectionDispenser}.
"""
skt = fromfd(dbampfd, AF_UNIX, SOCK_STREAM)
os.close(dbampfd)
- protocol = ConnectionPoolClient(dialect=dialect, paramstyle=paramstyle)
+ protocol = ConnectionPoolClient(dbtype=dbtype)
transport = ConnectionWithPeer(skt, protocol)
protocol.makeConnection(transport)
transport.startReading()
@@ -830,7 +829,7 @@
pool = None
if config.DBAMPFD:
txnFactory = transactionFactoryFromFD(
- int(config.DBAMPFD), dialect, paramstyle
+ int(config.DBAMPFD), DatabaseType(dialect, paramstyle, config.DBFeatures)
)
elif not config.UseDatabase:
txnFactory = None
@@ -843,8 +842,7 @@
else:
connectionFactory = DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect
- pool = ConnectionPool(connectionFactory, dialect=dialect,
- paramstyle=paramstyle,
+ pool = ConnectionPool(connectionFactory, dbtype=DatabaseType(dialect, paramstyle, config.DBFeatures),
maxConnections=config.MaxDBConnectionsPerPool)
txnFactory = pool.connection
else:
Modified: CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-apple.plist 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-apple.plist 2016-07-07 16:14:45 UTC (rev 15749)
@@ -108,6 +108,11 @@
<!-- Database connection -->
<key>DBType</key>
<string></string>
+
+ <!-- No features - when macOS supports postgres 9.5 and up change
+ this to include 'skip-locked'. -->
+ <key>DBFeatures</key>
+ <array></array>
<key>Postgres</key>
<dict>
Modified: CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-stdconfig.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-stdconfig.plist 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/conf/caldavd-stdconfig.plist 2016-07-07 16:14:45 UTC (rev 15749)
@@ -118,35 +118,43 @@
<array>
</array>
+ <!-- Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes. -->
+ <key>UseMetaFD</key>
+ <true/>
+
<!-- Inherited file descriptor to call recvmsg() on to receive sockets (none =
don't inherit) -->
<key>MetaFD</key>
<integer>0</integer>
- <!-- Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes. -->
- <key>UseMetaFD</key>
- <true/>
+ <!-- Database configuration information.
- <!-- True: database; False: files -->
+ Defines what kind of database to use: file (deprecated) or SQL. File-based
+ DB is only supported for migration purposes - it cannot be used for a real
+ service.
+
+ For an SQL-based DB, configuration of connection parameters and various
+ timeouts is provided. -->
+
+ <!-- True: database; False: files (deprecated) -->
<key>UseDatabase</key>
<true/>
- <!-- Timeout transactions that take longer than the specified number of
- seconds. Zero means no timeouts. 5 minute default. -->
- <key>TransactionTimeoutSeconds</key>
- <integer>300</integer>
-
- <!-- When a transactions times out tell HTTP clients clients to retry after
- this amount of time -->
- <key>TransactionHTTPRetrySeconds</key>
- <integer>300</integer>
-
- <!-- 2 possible values: empty, meaning 'spawn postgres yourself', or
- 'postgres', meaning 'connect to a postgres database as specified by the
- 'DSN' configuration key. Will support more values in the future. -->
+ <!-- Possible values: empty, meaning 'spawn postgres yourself', or 'postgres'
+ or 'oracle', meaning 'connect to a postgres or Oracle database as
+ specified by the 'DSN' configuration key. -->
<key>DBType</key>
<string></string>
+ <!-- Features supported by the database
+
+ 'skip-locked': SKIP LOCKED available with SELECT (remove if using postgres
+ < v9.5) -->
+ <key>DBFeatures</key>
+ <array>
+ <string>skip-locked</string>
+ </array>
+
<!-- The username to use when DBType is empty -->
<key>SpawnedDBUser</key>
<string>caldav</string>
@@ -175,17 +183,17 @@
<false/>
</dict>
+ <!-- Use a shared database connection pool in the master process, rather than
+ having each client make its connections directly. -->
+ <key>SharedConnectionPool</key>
+ <false/>
+
<!-- Internally used by database to tell slave processes to inherit a file
descriptor and use it as an AMP connection over a UNIX socket; see
twext.enterprise.adbapi2.ConnectionPoolConnection -->
<key>DBAMPFD</key>
<integer>0</integer>
- <!-- Use a shared database connection pool in the master process, rather than
- having each client make its connections directly. -->
- <key>SharedConnectionPool</key>
- <false/>
-
<!-- Set to True to prevent the server or utility tools from running if the
database needs a schema upgrade. -->
<key>FailIfUpgradeNeeded</key>
@@ -203,6 +211,16 @@
<key>UpgradeHomePrefix</key>
<string></string>
+ <!-- Timeout transactions that take longer than the specified number of
+ seconds. Zero means no timeouts. 5 minute default. -->
+ <key>TransactionTimeoutSeconds</key>
+ <integer>300</integer>
+
+ <!-- When a transactions times out tell HTTP clients clients to retry after
+ this amount of time -->
+ <key>TransactionHTTPRetrySeconds</key>
+ <integer>300</integer>
+
<!-- Work queue configuration information -->
<key>WorkQueue</key>
Copied: CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist (from rev 15745, CalendarServer/trunk/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist)
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist (rev 0)
+++ CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist 2016-07-07 16:14:45 UTC (rev 15749)
@@ -0,0 +1,835 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Copyright (c) 2011-2016 Apple Inc. All rights reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+ <dict>
+ <!-- Define the kinds of software and user behavior the load simulation
+ will simulate. -->
+ <key>clients</key>
+
+ <!-- Have as many different kinds of software and user behavior configurations
+ as you want. Each is a dict -->
+ <array>
+
+ <dict>
+
+ <!-- Here is a OS X client simulator. -->
+ <key>software</key>
+ <string>contrib.performance.loadtest.ical.OS_X_10_11</string>
+
+ <!-- Arguments to use to initialize the OS_X_10_11 instance. -->
+ <key>params</key>
+ <dict>
+ <!-- Name that appears in logs. -->
+ <key>title</key>
+ <string>main</string>
+
+ <!-- OS_X_10_11 can poll the calendar home at some interval. This is
+ in seconds. -->
+ <key>calendarHomePollInterval</key>
+ <integer>30</integer>
+
+ <!-- If the server advertises xmpp push, OS_X_10_7 can wait for notifications
+ about calendar home changes instead of polling for them periodically. If
+ this option is true, then look for the server advertisement for xmpp push
+ and use it if possible. Still fall back to polling if there is no xmpp push
+ advertised. -->
+ <key>supportPush</key>
+ <true/>
+
+ <key>supportAmpPush</key>
+ <true/>
+
+ <key>unauthenticatedPercentage</key>
+ <integer>40</integer>
+
+ </dict>
+
+ <!-- The profiles define certain types of user behavior on top of the
+ client software being simulated. -->
+ <key>profiles</key>
+ <array>
+
+ <!-- First an event-creating profile, which will periodically create
+ new events at a random time on a random calendar. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Eventer</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to create a new event. -->
+ <key>interval</key>
+ <integer>120</integer>
+
+ <!-- Define how start times (DTSTART) for the randomly generated events
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>eventStartDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+
+ <!-- Define how recurrences are created. -->
+ <key>recurrenceDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. We have a fixed set of
+ RRULEs defined for this distribution and pick each based on a
+ weight. -->
+ <key>type</key>
+ <string>contrib.performance.stats.RecurrenceDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- False to disable RRULEs -->
+ <key>allowRecurrence</key>
+ <true/>
+
+ <!-- These are the weights for the specific set of RRULEs. -->
+ <key>weights</key>
+ <dict>
+ <!-- Half of all events will be non-recurring -->
+ <key>none</key>
+ <integer>50</integer>
+
+ <!-- Daily and weekly are pretty common -->
+ <key>daily</key>
+ <integer>10</integer>
+ <key>weekly</key>
+ <integer>20</integer>
+
+ <!-- Monthly, yearly, daily & weekly limit not so common -->
+ <key>monthly</key>
+ <integer>2</integer>
+ <key>yearly</key>
+ <integer>1</integer>
+ <key>dailylimit</key>
+ <integer>2</integer>
+ <key>weeklylimit</key>
+ <integer>5</integer>
+
+ <!-- Work days pretty common -->
+ <key>workdays</key>
+ <integer>10</integer>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- Percentage of time to attach a file -->
+ <key>fileAttachPercentage</key>
+ <integer>1</integer>
+
+ <!-- Define the attachment size distribution. -->
+ <key>fileSizeDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.NormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mu gives the mean of the normal distribution (in bytes). -->
+ <key>mu</key>
+ <integer>500000</integer>
+
+ <!-- and sigma gives its standard deviation. -->
+ <key>sigma</key>
+ <integer>100000</integer>
+ </dict>
+ </dict>
+
+ </dict>
+ </dict>
+
+ <!-- This profile will create a new event, and then periodically update the ACKNOWLEDGED property. -->
+
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.AlarmAcknowledger</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <false/>
+
+ <!-- Define how often to check (in seconds) whether it's
+ time to do an alarm acknowledgement -->
+ <key>interval</key>
+ <integer>15</integer>
+
+ <!-- Acknowledgements will only actually occur at one of these
+ minutes past the hour: -->
+ <key>pastTheHour</key>
+ <array>
+ <integer>0</integer>
+ <integer>15</integer>
+ <integer>30</integer>
+ <integer>45</integer>
+ </array>
+
+ <!-- Define how start times (DTSTART) for the randomly generated events
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>eventStartDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+
+ <!-- Define how recurrences are created. -->
+ <key>recurrenceDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. We have a fixed set of
+ RRULEs defined for this distribution and pick each based on a
+ weight. -->
+ <key>type</key>
+ <string>contrib.performance.stats.RecurrenceDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- False to disable RRULEs -->
+ <key>allowRecurrence</key>
+ <true/>
+
+ <!-- These are the weights for the specific set of RRULEs. -->
+ <key>weights</key>
+ <dict>
+ <!-- Half of all events will be non-recurring -->
+ <key>none</key>
+ <integer>50</integer>
+
+ <!-- Daily and weekly are pretty common -->
+ <key>daily</key>
+ <integer>25</integer>
+ <key>weekly</key>
+ <integer>25</integer>
+
+ <!-- Monthly, yearly, daily & weekly limit not so common -->
+ <key>monthly</key>
+ <integer>0</integer>
+ <key>yearly</key>
+ <integer>0</integer>
+ <key>dailylimit</key>
+ <integer>0</integer>
+ <key>weeklylimit</key>
+ <integer>0</integer>
+
+ <!-- Work days pretty common -->
+ <key>workdays</key>
+ <integer>0</integer>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- Picks a random event and changes the title -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.TitleChanger</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to create a new event. -->
+ <key>interval</key>
+ <integer>120</integer>
+
+ </dict>
+ </dict>
+
+ <!-- Picks a random event and changes the description -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.DescriptionChanger</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to create a new event. -->
+ <key>interval</key>
+ <integer>120</integer>
+
+ <!-- Define the description length distribution. -->
+ <key>descriptionLengthDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.LogNormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mode - peak-->
+ <key>mode</key>
+ <integer>450</integer>
+ <!-- mean - average-->
+ <key>median</key>
+ <integer>650</integer>
+ <!-- maximum -->
+ <key>maximum</key>
+ <real>1000000</real>
+ </dict>
+ </dict>
+
+ </dict>
+ </dict>
+
+
+ <!-- Removes events from calendars exceeding a threshold -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.EventCountLimiter</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will check
+ for too-large collections. -->
+ <key>interval</key>
+ <integer>60</integer>
+
+ <!-- The upper bound. -->
+ <key>eventCountLimit</key>
+ <integer>100</integer>
+
+ </dict>
+ </dict>
+
+
+ <!-- Shares calendars -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.CalendarSharer</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will share calendars. -->
+ <key>interval</key>
+ <integer>300</integer>
+
+ <!-- Only share a collection with up to this many sharees -->
+ <key>maxSharees</key>
+ <integer>3</integer>
+
+ </dict>
+ </dict>
+
+ <!-- This profile invites some number of new attendees to new events. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Inviter</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the frequency at which new invitations will be sent out. -->
+ <key>sendInvitationDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.NormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mu gives the mean of the normal distribution (in seconds). -->
+ <key>mu</key>
+ <integer>120</integer>
+
+ <!-- and sigma gives its standard deviation. -->
+ <key>sigma</key>
+ <integer>5</integer>
+ </dict>
+ </dict>
+
+ <!-- Define the distribution of who will be invited to an event.
+
+ When inviteeClumping is turned on each invitee is based on a sample of
+ users "close to" the organizer based on account index. If the clumping
+ is too "tight" for the requested number of attendees, then invites for
+ those larger numbers will simply fail (the sim will report that situation).
+
+ When inviteeClumping is off invitees will be sampled across an entire
+ range of account indexes. In this case the distribution ought to be a
+ UniformIntegerDistribution with min=0 and max set to the number of accounts.
+ -->
+ <key>inviteeDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.UniformIntegerDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- The minimum value (inclusive) of the uniform distribution. -->
+ <key>min</key>
+ <integer>0</integer>
+ <!-- The maximum value (exclusive) of the uniform distribution. -->
+ <key>max</key>
+ <integer>99</integer>
+ </dict>
+ </dict>
+
+ <key>inviteeClumping</key>
+ <true/>
+
+ <!-- Define the distribution of how many attendees will be invited to an event.
+
+ LogNormal is the best fit to observed data.
+
+
+ For LogNormal "mode" is the peak, "mean" is the mean value. For invites,
+ mode should typically be 1, and mean whatever matches the user behavior.
+ Our typical mean is 6.
+ -->
+ <key>inviteeCountDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.LogNormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mode - peak-->
+ <key>mode</key>
+ <integer>6</integer>
+ <!-- mean - average-->
+ <key>median</key>
+ <integer>8</integer>
+ <!-- maximum -->
+ <key>maximum</key>
+ <real>60</real>
+ </dict>
+ </dict>
+
+ <!-- Percentage of time to do a lookup for invitee -->
+ <key>inviteeLookupPercentage</key>
+ <integer>50</integer>
+
+ <!-- Define how start times (DTSTART) for the randomly generated events
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>eventStartDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+
+ <!-- Define how recurrences are created. -->
+ <key>recurrenceDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. We have a fixed set of
+ RRULEs defined for this distribution and pick each based on a
+ weight. -->
+ <key>type</key>
+ <string>contrib.performance.stats.RecurrenceDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- False to disable RRULEs -->
+ <key>allowRecurrence</key>
+ <true/>
+
+ <!-- These are the weights for the specific set of RRULEs. -->
+ <key>weights</key>
+ <dict>
+ <!-- Half of all events will be non-recurring -->
+ <key>none</key>
+ <integer>50</integer>
+
+ <!-- Daily and weekly are pretty common -->
+ <key>daily</key>
+ <integer>10</integer>
+ <key>weekly</key>
+ <integer>20</integer>
+
+ <!-- Monthly, yearly, daily & weekly limit not so common -->
+ <key>monthly</key>
+ <integer>2</integer>
+ <key>yearly</key>
+ <integer>1</integer>
+ <key>dailylimit</key>
+ <integer>2</integer>
+ <key>weeklylimit</key>
+ <integer>5</integer>
+
+ <!-- Work days pretty common -->
+ <key>workdays</key>
+ <integer>10</integer>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- Percentage of time to attach a file -->
+ <key>fileAttachPercentage</key>
+ <integer>1</integer>
+
+ <!-- Define the attachment size distribution. -->
+ <key>fileSizeDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.NormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mu gives the mean of the normal distribution (in seconds). -->
+ <key>mu</key>
+ <integer>500000</integer>
+
+ <!-- and sigma gives its standard deviation. -->
+ <key>sigma</key>
+ <integer>100000</integer>
+ </dict>
+ </dict>
+
+ </dict>
+ </dict>
+
+ <!-- This profile accepts invitations to events, handles cancels, and
+ handles replies received. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Accepter</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define how long to wait after seeing a new invitation before
+ accepting it.
+
+ For LogNormal "mode" is the peak, "median" is the 50% cummulative value
+ (i.e., half of the user have accepted by that time).
+ -->
+ <key>acceptDelayDistribution</key>
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.stats.LogNormalDistribution</string>
+ <key>params</key>
+ <dict>
+ <!-- mode - peak-->
+ <key>mode</key>
+ <integer>300</integer>
+ <!-- median - 50% done-->
+ <key>median</key>
+ <integer>1800</integer>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+
+ <!-- This profile downloads attachments when an event changes. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.AttachmentDownloader</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+ </dict>
+ </dict>
+
+ <!-- A task-creating profile, which will periodically create
+ new tasks at a random time on a random calendar. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Tasker</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to create a new task. -->
+ <key>interval</key>
+ <integer>300</integer>
+
+ <!-- Define how due times (DUE) for the randomly generated tasks
+ will be selected. This is an example of a "Distribution" parameter. The value
+ for most "Distribution" parameters are interchangeable and extensible. -->
+ <key>taskDueDistribution</key>
+ <dict>
+
+ <!-- This distribution is pretty specialized. It produces timestamps
+ in the near future, limited to certain days of the week and certain hours
+ of the day. -->
+ <key>type</key>
+ <string>contrib.performance.stats.WorkDistribution</string>
+
+ <key>params</key>
+ <dict>
+ <!-- These are the days of the week the distribution will use. -->
+ <key>daysOfWeek</key>
+ <array>
+ <string>mon</string>
+ <string>tue</string>
+ <string>wed</string>
+ <string>thu</string>
+ <string>fri</string>
+ </array>
+
+ <!-- The earliest hour of a day at which an event might be scheduled. -->
+ <key>beginHour</key>
+ <integer>8</integer>
+
+ <!-- And the latest hour of a day (at which an event will be scheduled
+ to begin!). -->
+ <key>endHour</key>
+ <integer>16</integer>
+
+ <!-- The timezone in which the event is scheduled. (XXX Does this
+ really work right?) -->
+ <key>tzname</key>
+ <string>America/Los_Angeles</string>
+ </dict>
+ </dict>
+ </dict>
+ </dict>
+
+
+ <!-- A profile which periodically does calendar-query reports
+ on a random calendar. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.TimeRanger</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to run a report. -->
+ <key>interval</key>
+ <integer>300</integer>
+
+ </dict>
+ </dict>
+
+
+ <!-- A profile which periodically does a "deep refresh" of PROPFINDs
+ on each calendar. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.DeepRefresher</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Define the interval (in seconds) at which this profile will use
+ its client to refresh. -->
+ <key>interval</key>
+ <integer>60</integer>
+
+ </dict>
+ </dict>
+
+ <!-- A profile which will periodically reset itself, as if a user removed
+ and re-added their account. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.Resetter</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <false/>
+
+ <!-- Define the interval (in seconds) at which this profile will
+ reset its client. -->
+ <key>interval</key>
+ <integer>600</integer>
+ </dict>
+ </dict>
+
+ </array>
+
+ <!-- Determine the frequency at which this client configuration will
+ appear in the clients which are created by the load tester. -->
+ <key>weight</key>
+ <integer>1</integer>
+ </dict>
+
+ <dict>
+
+ <!-- Here is a more passive OS X client simulator. -->
+ <key>software</key>
+ <string>contrib.performance.loadtest.ical.OS_X_10_11</string>
+
+ <!-- Arguments to use to initialize the OS_X_10_11 instance. -->
+ <key>params</key>
+ <dict>
+ <!-- Name that appears in logs. -->
+ <key>title</key>
+ <string>idle</string>
+
+ <!-- OS_X_10_11 can poll the calendar home at some interval. This is
+ in seconds. -->
+ <key>calendarHomePollInterval</key>
+ <integer>30</integer>
+
+ <!-- If the server advertises xmpp push, OS_X_10_7 can wait for notifications
+ about calendar home changes instead of polling for them periodically. If
+ this option is true, then look for the server advertisement for xmpp push
+ and use it if possible. Still fall back to polling if there is no xmpp push
+ advertised. -->
+ <key>supportPush</key>
+ <true/>
+
+ <key>supportAmpPush</key>
+ <true/>
+
+ <key>unauthenticatedPercentage</key>
+ <integer>40</integer>
+
+ </dict>
+
+ <!-- The profiles define certain types of user behavior on top of the
+ client software being simulated. -->
+ <key>profiles</key>
+ <array>
+ <!-- This profile downloads attachments when an event changes. -->
+ <dict>
+ <key>class</key>
+ <string>contrib.performance.loadtest.profiles.AttachmentDownloader</string>
+
+ <key>params</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+ </dict>
+ </dict>
+ </array>
+ <!-- Determine the frequency at which this client configuration will
+ appear in the clients which are created by the load tester. -->
+ <key>weight</key>
+ <integer>1</integer>
+ </dict>
+
+ </array>
+ </dict>
+</plist>
Copied: CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist (from rev 15745, CalendarServer/trunk/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist)
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist (rev 0)
+++ CalendarServer/branches/users/cdaboo/better-next-job/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist 2016-07-07 16:14:45 UTC (rev 15749)
@@ -0,0 +1,241 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Copyright (c) 2011-2016 Apple Inc. All rights reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+ <dict>
+ <!-- Identify all the servers to be load tested. -->
+ <key>servers</key>
+ <dict>
+ <key>PodA</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <!-- Identify the server to be load tested. -->
+ <key>uri</key>
+ <string>https://localhost:8443</string>
+
+ <key>ampPushHosts</key>
+ <array>
+ <string>localhost</string>
+ </array>
+ <key>ampPushPort</key>
+ <integer>62311</integer>
+
+ <!-- Define whether server supports stats socket. -->
+ <key>stats</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+ <key>Port</key>
+ <integer>8100</integer>
+ </dict>
+ </dict>
+ <key>PodB</key>
+ <dict>
+ <key>enabled</key>
+ <false/>
+
+ <!-- Identify the server to be load tested. -->
+ <key>uri</key>
+ <string>https://localhost:8543</string>
+
+ <key>ampPushHosts</key>
+ <array>
+ <string>localhost</string>
+ </array>
+ <key>ampPushPort</key>
+ <integer>62312</integer>
+
+ <!-- Define whether server supports stats socket. -->
+ <key>stats</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+ <key>Port</key>
+ <integer>8101</integer>
+ </dict>
+ </dict>
+ </dict>
+
+ <!-- The template URI for doing initial principal lookup on. -->
+ <key>principalPathTemplate</key>
+ <string>/principals/users/%s/</string>
+
+ <!-- Configure Admin Web UI. -->
+ <key>webadmin</key>
+ <dict>
+ <key>enabled</key>
+ <true/>
+
+ <key>HTTPPort</key>
+ <integer>8080</integer>
+ </dict>
+
+ <!-- Define whether client data should be re-used. It will always be saved to the specified path.-->
+ <key>clientDataSerialization</key>
+ <dict>
+ <key>UseOldData</key>
+ <true/>
+ <key>Path</key>
+ <string>/tmp/sim</string>
+ </dict>
+
+ <!-- Define the credentials of the clients which will be used to load test
+ the server. These credentials must already be valid on the server. -->
+ <key>accounts</key>
+ <dict>
+ <!-- The loader is the fully-qualified Python name of a callable which
+ returns a list of directory service records defining all of the client accounts
+ to use. contrib.performance.loadtest.sim.recordsFromCSVFile reads username,
+ password, mailto triples from a CSV file and returns them as a list of faked
+ directory service records. -->
+ <key>loader</key>
+ <string>contrib.performance.loadtest.sim.recordsFromCSVFile</string>
+
+ <!-- Keyword arguments may be passed to the loader. -->
+ <key>params</key>
+ <dict>
+ <!-- recordsFromCSVFile interprets the path relative to the config.plist,
+ to make it independent of the script's working directory while still allowing
+ a relative path. This isn't a great solution. -->
+ <key>path</key>
+ <string>contrib/performance/loadtest/accounts.csv</string>
+
+ <!-- When there are accounts for multiple pods, interleave the accounts for each
+ pod so that the arrival mechanism will cycle clients between each pod. -->
+ <key>interleavePods</key>
+ <true/>
+ </dict>
+ </dict>
+
+ <!-- Define how many clients will participate in the load test and how
+ they will show up. -->
+ <key>arrival</key>
+ <dict>
+
+ <!-- Specify a class which creates new clients and introduces them into
+ the test. contrib.performance.loadtest.population.SmoothRampUp introduces
+ groups of new clients at fixed intervals up to a maximum. The size of the
+ group, interval, and maximum are configured by the parameters below. The
+ total number of clients is groups * groupSize, which needs to be no larger
+ than the number of credentials created in the accounts section. -->
+ <key>factory</key>
+ <string>contrib.performance.loadtest.population.SmoothRampUp</string>
+
+ <key>params</key>
+ <dict>
+ <!-- groups gives the total number of groups of clients to introduce. -->
+ <key>groups</key>
+ <integer>10</integer>
+
+ <!-- groupSize is the number of clients in each group of clients. It's
+ really only a "smooth" ramp up if this is pretty small. -->
+ <key>groupSize</key>
+ <integer>3</integer>
+
+ <!-- Number of seconds between the introduction of each group. -->
+ <key>interval</key>
+ <integer>3</integer>
+
+ <!-- Number of clients each user is assigned to. -->
+ <!-- Set weight of clients to 1 if this is > 1. Number of clients must match this value if > 1. -->
+ <key>clientsPerUser</key>
+ <integer>2</integer>
+ </dict>
+
+ </dict>
+
+ <!-- Define some log observers to report on the load test. -->
+ <key>observers</key>
+ <array>
+ <!-- ReportStatistics generates an end-of-run summary of the HTTP requests
+ made, their timings, and their results. -->
+ <dict>
+ <key>type</key>
+
+ <string>contrib.performance.loadtest.population.ReportStatistics</string>
+ <key>params</key>
+ <dict>
+ <!-- The thresholds for each request type -->
+ <key>thresholdsPath</key>
+ <string>contrib/performance/loadtest/thresholds.json</string>
+
+ <!-- The benchmarks for overall QoS -->
+ <key>benchmarksPath</key>
+ <string>contrib/performance/loadtest/benchmarks.json</string>
+
+ <!-- The % of failures that constitute a failed test -->
+ <key>failCutoff</key>
+ <real>1.0</real>
+ </dict>
+ </dict>
+
+ <!-- RequestLogger generates a realtime log of all HTTP requests made
+ during the load test. -->
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.ical.RequestLogger</string>
+ <key>params</key>
+ <dict>
+ </dict>
+ </dict>
+
+ <!-- ErrorLogger logs the request/response bodies whenever an incorrect
+ response code is received, or if the duration exceeds threshold. -->
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.ical.ErrorLogger</string>
+ <key>params</key>
+ <dict>
+ <key>directory</key>
+ <string>/tmp/sim_errors</string>
+ <key>durationThreshold</key>
+ <real>10.0</real>
+ </dict>
+ </dict>
+
+ <!-- OperationLogger generates an end-of-run summary of the gross operations
+ performed (logical operations which may span more than one HTTP request,
+ such as inviting an attendee to an event). -->
+ <dict>
+ <key>type</key>
+ <string>contrib.performance.loadtest.profiles.OperationLogger</string>
+ <key>params</key>
+ <dict>
+ <!-- The thresholds for each operation type -->
+ <key>thresholdsPath</key>
+ <string>contrib/performance/loadtest/thresholds.json</string>
+
+ <!-- The % of operations beyond the lag cut-off that constitute a failed test -->
+ <key>lagCutoff</key>
+ <real>1.0</real>
+
+ <!-- The % of failures that constitute a failed test -->
+ <key>failCutoff</key>
+ <real>1.0</real>
+
+ <!-- There must be at least one push operation -->
+ <key>failIfNoPush</key>
+ <true/>
+ </dict>
+ </dict>
+ </array>
+ </dict>
+</plist>
Modified: CalendarServer/branches/users/cdaboo/better-next-job/lib-patches/Twisted/securetransport.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/lib-patches/Twisted/securetransport.patch 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/lib-patches/Twisted/securetransport.patch 2016-07-07 16:14:45 UTC (rev 15749)
@@ -2,14 +2,14 @@
===================================================================
--- twisted/internet/_sslverify.py (revision 45115)
+++ twisted/internet/_sslverify.py (working copy)
-@@ -162,7 +162,9 @@
+@@ -174,7 +174,9 @@
+ "rejected."
+ )
- major, minor = list(int(part) for part in lib.__version__.split("."))[:2]
-
-- if (major, minor) >= (0, 12):
+- if _usablePyOpenSSL(lib.__version__):
+ if hasattr(lib, "__SecureTransport__"):
+ pass
-+ elif (major, minor) >= (0, 12):
++ elif _usablePyOpenSSL(lib.__version__):
try:
from service_identity import VerificationError
from service_identity.pyopenssl import verify_hostname
@@ -18,7 +18,7 @@
===================================================================
--- twisted/protocols/tls.py (revision 45115)
+++ twisted/protocols/tls.py (working copy)
-@@ -660,7 +660,9 @@
+@@ -705,7 +705,9 @@
@rtype: L{OpenSSL.SSL.Connection}
"""
context = self._oldStyleContextFactory.getContext()
Modified: CalendarServer/branches/users/cdaboo/better-next-job/requirements-cs.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/requirements-cs.txt 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/requirements-cs.txt 2016-07-07 16:14:45 UTC (rev 15749)
@@ -7,23 +7,23 @@
zope.interface==4.1.3
setuptools==18.5
- --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/branches/users/cdaboo/better-next-job@15730#egg=twextpy
- cffi==1.3.0
+ --editable svn+http://svn.calendarserver.org/repository/calendarserver/twext/branches/users/cdaboo/better-next-job@15748#egg=twextpy
+ cffi==1.7.0
pycparser==2.14
#twisted
# [LDAP] extra
- python-ldap==2.4.22
+ python-ldap==2.4.25
#setuptools
# [DAL] extra
- sqlparse==0.1.18
+ sqlparse==0.1.19
# [OpenDirectory] extra
#pyobjc-framework-OpenDirectory # Use system module
# [Postgres] extra
- pg8000==1.10.2
+ pg8000==1.10.6
# [Oracle] extra
#cx_Oracle==5.2 # Needs manual patch
@@ -31,10 +31,10 @@
--editable svn+http://svn.calendarserver.org/repository/calendarserver/PyKerberos/trunk@15423#egg=kerberos
--editable svn+http://svn.calendarserver.org/repository/calendarserver/PyCalendar/trunk@15423#egg=pycalendar
- python-dateutil==2.4.2
- pytz==2015.7
+ python-dateutil==2.5.3
+ pytz==2016.4
- psutil==3.4.2
- setproctitle==1.1.9
+ psutil==4.3.0
+ setproctitle==1.1.10
# xattr==0.7.5 # Only needed for upgrades from ancient versions. Added in _cache_deps.
#cffi
Modified: CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-default.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-default.txt 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-default.txt 2016-07-07 16:14:45 UTC (rev 15749)
@@ -3,27 +3,27 @@
##
-Twisted==15.4.0
+Twisted==16.3.0
#zope.interface
# NOTE: Twisted also uses pyOpenSSL, pycrypto and service_identity,
# but doesn't specify them as dependencies, so that are explicitly
# added to calendarserver.
#pyOpenSSL
- service_identity==14.0.0
+ service_identity==16.0.0
characteristic==14.3.0
pyasn1==0.1.9
pyasn1-modules==0.0.8
#pyOpenSSL
pycrypto==2.6.1
- pyOpenSSL==0.15.1
+ pyOpenSSL==16.0.0
cryptography==1.1
idna
#pyasn1
#cffi
- enum34==1.0.4
+ enum34==1.1.6
ipaddress
- #setuptools==17.0
+ #setuptools
#six
six==1.10.0
Modified: CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-osx.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-osx.txt 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/requirements-twisted-osx.txt 2016-07-07 16:14:45 UTC (rev 15749)
@@ -2,13 +2,13 @@
# Twisted dependency for OS X systems (uses pySecureTransport).
##
-Twisted==15.4.0
+Twisted==16.3.0
#zope.interface
pycrypto==2.6.1
--editable svn+http://svn.calendarserver.org/repository/calendarserver/OSXFrameworks/trunk@15423#egg=osxframeworks
- #cffi==1.3.0
- # pycparser==2.14
+ #cffi
+ # pycparser
--editable svn+http://svn.calendarserver.org/repository/calendarserver/PySecureTransport/trunk@15554#egg=pysecuretransport
Modified: CalendarServer/branches/users/cdaboo/better-next-job/setup.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/setup.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/setup.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -328,7 +328,7 @@
install_requirements = [
# Core frameworks
"zope.interface",
- "Twisted==15.4.0",
+ "Twisted==16.3.0",
"twextpy",
# Security frameworks
Modified: CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/dumpconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/dumpconfig.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/dumpconfig.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -23,6 +23,8 @@
import re
import textwrap
+DEBUG = False
+
COPYRIGHT = """
<!--
Copyright (c) 2006-2016 Apple Inc. All rights reserved.
@@ -313,12 +315,14 @@
}
for item in maps.keys():
+ if DEBUG:
+ print(item)
lines = parseConfigItem(item)
- maps[item] = processConfig(lines, with_comments=True, verbose=False)
+ maps[item] = processConfig(lines, with_comments=True, verbose=DEBUG)
# Generate the plist for the default config, substituting for the *_PARAMS items
lines = parseConfigItem("DEFAULT_CONFIG")
- j = processConfig(lines, with_comments=True, verbose=False, substitutions=maps)
+ j = processConfig(lines, with_comments=True, verbose=DEBUG, substitutions=maps)
return writeOrderedPlistToString(j)
Modified: CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/stdconfig.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/twistedcaldav/stdconfig.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -195,24 +195,36 @@
"BindSSLPorts": [], # List of port numbers to bind to for SSL [empty = same as "SSLPort"]
"InheritFDs": [], # File descriptors to inherit for HTTP requests [empty = don't inherit]
"InheritSSLFDs": [], # File descriptors to inherit for HTTPS requests [empty = don't inherit]
- "MetaFD": 0, # Inherited file descriptor to call recvmsg() on to receive sockets (none = don't inherit)
"UseMetaFD": True, # Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes.
+ "MetaFD": 0, # Inherited file descriptor to call recvmsg() on to receive sockets (none = don't inherit)
- "UseDatabase": True, # True: database; False: files
+ #
+ # Database configuration information.
+ #
+ # Defines what kind of database to use: file (deprecated) or SQL.
+ # File-based DB is only supported for migration purposes - it
+ # cannot be used for a real service.
+ #
+ # For an SQL-based DB, configuration of connection parameters and various
+ # timeouts is provided.
+ #
- "TransactionTimeoutSeconds": 300, # Timeout transactions that take longer than
- # the specified number of seconds. Zero means
- # no timeouts. 5 minute default.
- "TransactionHTTPRetrySeconds": 300, # When a transactions times out tell HTTP clients
- # clients to retry after this amount of time
+ "UseDatabase": True, # True: database; False: files (deprecated)
- "DBType": "", # 2 possible values: empty, meaning 'spawn postgres
- # yourself', or 'postgres', meaning 'connect to a
- # postgres database as specified by the 'DSN'
- # configuration key. Will support more values in
- # the future.
+ "DBType": "", # Possible values: empty, meaning 'spawn postgres
+ # yourself', or 'postgres' or 'oracle', meaning
+ # 'connect to a postgres or Oracle database as
+ # specified by the 'DSN' configuration key.
+
+ "DBFeatures": [ # Features supported by the database
+ #
+ # 'skip-locked': SKIP LOCKED available with SELECT (remove if using postgres < v9.5)
+ #
+ "skip-locked",
+ ],
+
"SpawnedDBUser": "caldav", # The username to use when DBType is empty
"DatabaseConnection": { # Used to connect to an external database if DBType is non-empty
@@ -223,16 +235,16 @@
"ssl": False, # Set to True to require SSL (pg8000 only).
},
+ "SharedConnectionPool": False, # Use a shared database connection pool in
+ # the master process, rather than having
+ # each client make its connections directly.
+
"DBAMPFD": 0, # Internally used by database to tell slave
# processes to inherit a file descriptor and use it
# as an AMP connection over a UNIX socket; see
# twext.enterprise.adbapi2.ConnectionPoolConnection
- "SharedConnectionPool": False, # Use a shared database connection pool in
- # the master process, rather than having
- # each client make its connections directly.
-
- "FailIfUpgradeNeeded": True, # Set to True to prevent the server or utility
+ "FailIfUpgradeNeeded": True, # Set to True to prevent the server or utility
# tools from running if the database needs a schema
# upgrade.
@@ -246,6 +258,12 @@
# need to be run again without this prefix set to complete the overall
# upgrade.
+ "TransactionTimeoutSeconds": 300, # Timeout transactions that take longer than
+ # the specified number of seconds. Zero means
+ # no timeouts. 5 minute default.
+ "TransactionHTTPRetrySeconds": 300, # When a transactions times out tell HTTP clients
+ # clients to retry after this amount of time
+
#
# Work queue configuration information
#
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/base/datastore/dbapiclient.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/base/datastore/dbapiclient.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/base/datastore/dbapiclient.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -26,6 +26,7 @@
import datetime
import pg8000 as postgres
+import six
try:
import os
@@ -452,7 +453,7 @@
return v.encode("utf-8") if isinstance(v, unicode) else str(v)
connection.realConnection.py_types[str] = (705, postgres.core.FC_TEXT, my_text_out)
- connection.realConnection.py_types[postgres.six.text_type] = (705, postgres.core.FC_TEXT, my_text_out)
+ connection.realConnection.py_types[six.text_type] = (705, postgres.core.FC_TEXT, my_text_out)
def my_text_recv(data, offset, length):
return str(data[offset: offset + length])
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -643,8 +643,7 @@
directlyProvides(self, *extraInterfaces)
self._sqlTxn = sqlTxn
- self.paramstyle = sqlTxn.paramstyle
- self.dialect = sqlTxn.dialect
+ self.dbtype = sqlTxn.dbtype
self._stats = (
TransactionStatsCollector(self._label, self._store.logStatsLogFile)
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_dump.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_dump.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_dump.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -54,9 +54,9 @@
Generate the L{Schema}.
"""
- if txn.dialect == POSTGRES_DIALECT:
+ if txn.dbtype.dialect == POSTGRES_DIALECT:
return dumpSchema_postgres(txn, title, schemaname)
- elif txn.dialect == ORACLE_DIALECT:
+ elif txn.dbtype.dialect == ORACLE_DIALECT:
return dumpSchema_oracle(txn, title, schemaname)
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_tables.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_tables.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/sql_tables.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -25,7 +25,8 @@
from twext.enterprise.dal.model import Sequence, ProcedureCall
from twext.enterprise.dal.parseschema import schemaFromPath
from twext.enterprise.dal.syntax import FixedPlaceholder
-from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT
+from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT, \
+ DatabaseType
from twext.enterprise.dal.syntax import Insert
from twext.enterprise.ienterprise import ORACLE_TABLE_NAME_MAX
import hashlib
@@ -314,7 +315,7 @@
@return: the generated SQL string.
@rtype: C{str}
"""
- qgen = QueryGenerator(ORACLE_DIALECT, FixedPlaceholder('%s'))
+ qgen = QueryGenerator(DatabaseType(ORACLE_DIALECT, "pyformat"), FixedPlaceholder('%s'))
if doquote:
qgen.shouldQuote = lambda name: True
if hasattr(sql, 'subSQL'):
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_sql_dump.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_sql_dump.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_sql_dump.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -53,7 +53,7 @@
@inlineCallbacks
def cleanUp(self):
startTxn = self.store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
else:
@@ -69,7 +69,7 @@
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = self.store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("create schema test_dbUpgrades")
yield startTxn.execSQL("set search_path to test_dbUpgrades")
yield startTxn.execSQLBlock(schema)
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_trash.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_trash.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/test_trash.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -1814,6 +1814,7 @@
yield txn.commit()
+
@inlineCallbacks
def test_trashCalendarRestoreWithFailures(self):
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/util.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/test/util.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -36,7 +36,7 @@
from twext.python.filepath import CachingFilePath as FilePath
from twext.enterprise.adbapi2 import ConnectionPool
from twext.enterprise.ienterprise import AlreadyFinishedError, POSTGRES_DIALECT, \
- ORACLE_DIALECT
+ ORACLE_DIALECT, DatabaseType
from twext.enterprise.jobs.jobitem import JobItem
from twext.enterprise.jobs.queue import ControllerQueue
from twext.who.directory import DirectoryRecord
@@ -162,8 +162,7 @@
cp = ConnectionPool(
stubsvc.produceConnection,
maxConnections=1,
- dialect=DB_TYPE[0],
- paramstyle=DB_TYPE[1],
+ dbtype=DatabaseType(DB_TYPE[0], DB_TYPE[1]),
)
# Attach the service to the running reactor.
cp.startService()
@@ -251,8 +250,7 @@
cp = ConnectionPool(
self.sharedService.produceConnection,
maxConnections=4,
- dialect=DB_TYPE[0],
- paramstyle=DB_TYPE[1],
+ dbtype=DatabaseType(DB_TYPE[0], DB_TYPE[1]),
)
quota = deriveQuota(testCase)
store = CommonDataStore(
@@ -340,10 +338,10 @@
# Change the starting values of sequences to random values
for sequence in schema.model.sequences: #@UndefinedVariable
try:
- if cleanupTxn.dialect == POSTGRES_DIALECT:
+ if cleanupTxn.dbtype.dialect == POSTGRES_DIALECT:
curval = (yield cleanupTxn.execSQL("select nextval('{}')".format(sequence.name), []))[0][0]
yield cleanupTxn.execSQL("select setval('{}', {})".format(sequence.name, curval + randint(1, 10000)), [])
- elif cleanupTxn.dialect == ORACLE_DIALECT:
+ elif cleanupTxn.dbtype.dialect == ORACLE_DIALECT:
yield cleanupTxn.execSQL("alter sequence {} increment by {}".format(sequence.name, randint(1, 10000)), [])
yield cleanupTxn.execSQL("select {}.nextval from dual".format(sequence.name), [])
yield cleanupTxn.execSQL("alter sequence {} increment by {}".format(sequence.name, 1), [])
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -171,7 +171,7 @@
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("create schema test_dbUpgrades")
yield startTxn.execSQL("set search_path to test_dbUpgrades")
yield startTxn.execSQLBlock(path.getContent())
@@ -194,20 +194,20 @@
@inlineCallbacks
def _unloadOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
- elif startTxn.dialect == ORACLE_DIALECT:
+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
yield cleanDatabase(startTxn)
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade")
- elif startTxn.dialect == ORACLE_DIALECT:
+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
yield cleanDatabase(startTxn)
yield startTxn.commit()
@@ -286,7 +286,7 @@
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("create schema test_dbUpgrades")
yield startTxn.execSQL("set search_path to test_dbUpgrades")
yield startTxn.execSQLBlock(path.getContent())
@@ -303,20 +303,20 @@
@inlineCallbacks
def _unloadOldData():
startTxn = store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
- elif startTxn.dialect == ORACLE_DIALECT:
+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
yield cleanDatabase(startTxn)
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldData():
startTxn = store.newTransaction("test_dbUpgrades")
- if startTxn.dialect == POSTGRES_DIALECT:
+ if startTxn.dbtype.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade")
- elif startTxn.dialect == ORACLE_DIALECT:
+ elif startTxn.dbtype.dialect == ORACLE_DIALECT:
yield cleanDatabase(startTxn)
yield startTxn.commit()
Modified: CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/upgrade.py 2016-07-07 16:12:19 UTC (rev 15748)
+++ CalendarServer/branches/users/cdaboo/better-next-job/txdav/common/datastore/upgrade/sql/upgrade.py 2016-07-07 16:14:45 UTC (rev 15749)
@@ -190,7 +190,7 @@
# Get the schema version in the current database
sqlTxn = self.sqlStore.newTransaction(label="UpgradeDatabaseCoreStep.getVersions")
- dialect = sqlTxn.dialect
+ dialect = sqlTxn.dbtype.dialect
try:
actual_version = yield sqlTxn.calendarserverValue(self.versionKey)
actual_version = int(actual_version)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160707/648ee9be/attachment-0001.html>
More information about the calendarserver-changes
mailing list