[CalendarServer-changes] [9547] CalendarServer/branches/users/gaya/inviteclean
source_changes at macosforge.org
source_changes at macosforge.org
Thu Aug 9 12:03:16 PDT 2012
Revision: 9547
http://trac.macosforge.org/projects/calendarserver/changeset/9547
Author: gaya at apple.com
Date: 2012-08-09 12:03:13 -0700 (Thu, 09 Aug 2012)
Log Message:
-----------
merge in trunk
Modified Paths:
--------------
CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/applepush.py
CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/test/test_applepush.py
CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/caldav.py
CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/util.py
CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/bootstrapdatabase.py
CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/principals.py
CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/test/test_principals.py
CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/calendarcertupdate.py
CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/test/test_certupdate.py
CalendarServer/branches/users/gaya/inviteclean/contrib/migration/calendarmigrator.py
CalendarServer/branches/users/gaya/inviteclean/contrib/migration/test/test_migrator.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/loadtest/thresholds.json
CalendarServer/branches/users/gaya/inviteclean/contrib/tools/protocolanalysis.py
CalendarServer/branches/users/gaya/inviteclean/contrib/tools/request_monitor.py
CalendarServer/branches/users/gaya/inviteclean/doc/Admin/ExtendedLogItems.txt
CalendarServer/branches/users/gaya/inviteclean/support/build.sh
CalendarServer/branches/users/gaya/inviteclean/support/submit
CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/syntax.py
CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/test/test_sqlsyntax.py
CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/method/report.py
CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/noneprops.py
CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/resource.py
CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/test_xattrprops.py
CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/util.py
CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/xattrprops.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/__init__.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/customxml.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/directory.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/ldapdirectory.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_directory.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_ldapdirectory.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/extensions.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/memcacheprops.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/propfind.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_addressbook_query.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_calendar_query.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_multiget_common.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_sync_collection.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/notify.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/resource.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/implicit.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/ischedule.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/test/test_implicit.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/stdconfig.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/storebridge.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_icalendar.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_memcacheprops.py
CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_validation.py
CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/base.py
CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/sql.py
CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/xattr.py
CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/common.py
CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/test_sql.py
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/file.py
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql.py
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_legacy.py
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current.sql
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/test/test_sql.py
CalendarServer/branches/users/gaya/inviteclean/txdav/common/icommondatastore.py
Added Paths:
-----------
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py
CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/nclob-fixes-and-prefetch.patch
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/oracle-dialect/v10.sql
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/postgres-dialect/v10.sql
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_10_to_11.sql
CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_10_to_11.sql
Removed Paths:
-------------
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py
CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py
CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/bytes-per-nclob-character.patch
Property Changed:
----------------
CalendarServer/branches/users/gaya/inviteclean/
Property changes on: CalendarServer/branches/users/gaya/inviteclean
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:9408-9546
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/applepush.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/applepush.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/applepush.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -28,6 +28,7 @@
from twisted.internet import protocol
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
from twisted.internet.protocol import ClientFactory, ReconnectingClientFactory
+from twisted.internet.task import LoopingCall
from twistedcaldav.extensions import DAVResource, DAVResourceWithoutChildrenMixin
from twistedcaldav.resource import ReadOnlyNoCopyResourceMixIn
import OpenSSL
@@ -87,6 +88,9 @@
service.providers = {}
service.feedbacks = {}
service.dataHost = settings["DataHost"]
+ service.purgeCall = None
+ service.purgeIntervalSeconds = settings["SubscriptionPurgeIntervalSeconds"]
+ service.purgeSeconds = settings["SubscriptionPurgeSeconds"]
for protocol in ("CalDAV", "CardDAV"):
@@ -134,7 +138,44 @@
return service
+ def startService(self):
+ """
+ In addition to starting the provider and feedback sub-services, start a
+ LoopingCall whose job it is to purge old subscriptions
+ """
+ service.MultiService.startService(self)
+ self.log_debug("ApplePushNotifierService startService")
+ self.purgeCall = LoopingCall(self.purgeOldSubscriptions, self.purgeSeconds)
+ self.purgeCall.start(self.purgeIntervalSeconds, now=False)
+
+
+ def stopService(self):
+ """
+ In addition to stopping the provider and feedback sub-services, stop the
+ LoopingCall
+ """
+ service.MultiService.stopService(self)
+ self.log_debug("ApplePushNotifierService stopService")
+ if self.purgeCall is not None:
+ self.purgeCall.stop()
+ self.purgeCall = None
+
+
@inlineCallbacks
+ def purgeOldSubscriptions(self, purgeSeconds):
+ """
+ Remove any subscriptions that registered more than purgeSeconds ago
+
+ @param purgeSeconds: The cutoff given in seconds
+ @type purgeSeconds: C{int}
+ """
+ self.log_debug("ApplePushNotifierService purgeOldSubscriptions")
+ txn = self.store.newTransaction()
+ yield txn.purgeOldAPNSubscriptions(int(time.time()) - purgeSeconds)
+ yield txn.commit()
+
+
+ @inlineCallbacks
def enqueue(self, op, id):
"""
Sends an Apple Push Notification to any device token subscribed to
@@ -340,6 +381,7 @@
self.store = store
self.noisy = True
self.maxDelay = 30 # max seconds between connection attempts
+ self.shuttingDown = False
def clientConnectionMade(self):
self.log_warn("Connection to APN server made")
@@ -347,7 +389,8 @@
self.delay = 1.0
def clientConnectionLost(self, connector, reason):
- self.log_warn("Connection to APN server lost: %s" % (reason,))
+ if not self.shuttingDown:
+ self.log_warn("Connection to APN server lost: %s" % (reason,))
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
@@ -360,8 +403,12 @@
self.log_warn("Reconnecting to APN server")
ReconnectingClientFactory.retry(self, connector)
+ def stopTrying(self):
+ self.shuttingDown = True
+ ReconnectingClientFactory.stopTrying(self)
+
class APNConnectionService(service.Service, LoggingMixIn):
def __init__(self, host, port, certPath, keyPath, chainPath="",
@@ -578,6 +625,7 @@
yield txn.commit()
+
class APNFeedbackFactory(ClientFactory, LoggingMixIn):
protocol = APNFeedbackProtocol
@@ -594,8 +642,8 @@
class APNFeedbackService(APNConnectionService):
- def __init__(self, store, updateSeconds, host, port, certPath, keyPath,
- chainPath="", passphrase="", sslMethod="TLSv1_METHOD",
+ def __init__(self, store, updateSeconds, host, port,
+ certPath, keyPath, chainPath="", passphrase="", sslMethod="TLSv1_METHOD",
testConnector=None, reactor=None):
APNConnectionService.__init__(self, host, port, certPath, keyPath,
@@ -623,6 +671,7 @@
self.checkForFeedback)
+
class APNSubscriptionResource(ReadOnlyNoCopyResourceMixIn,
DAVResourceWithoutChildrenMixin, DAVResource, LoggingMixIn):
"""
@@ -721,6 +770,9 @@
token = request.args.get("token", ("",))[0].replace(" ", "").lower()
key = request.args.get("key", ("",))[0]
+ userAgent = request.headers.getHeader("user-agent", "-")
+ host = request.remoteAddr.host
+
if not (key and token):
code = responsecode.BAD_REQUEST
msg = "Invalid request: both 'token' and 'key' must be provided"
@@ -733,7 +785,7 @@
principal = self.principalFromRequest(request)
uid = principal.record.uid
try:
- yield self.addSubscription(token, key, uid)
+ yield self.addSubscription(token, key, uid, userAgent, host)
code = responsecode.OK
msg = None
except InvalidSubscriptionValues:
@@ -743,7 +795,7 @@
returnValue((code, msg))
@inlineCallbacks
- def addSubscription(self, token, key, uid):
+ def addSubscription(self, token, key, uid, userAgent, host):
"""
Add a subscription (or update its timestamp if already there).
@@ -755,10 +807,16 @@
@param uid: The uid of the subscriber principal
@type uid: C{str}
+
+ @param userAgent: The user-agent requesting the subscription
+ @type key: C{str}
+
+ @param host: The host requesting the subscription
+ @type key: C{str}
"""
now = int(time.time()) # epoch seconds
txn = self.store.newTransaction()
- yield txn.addAPNSubscription(token, key, now, uid)
+ yield txn.addAPNSubscription(token, key, now, uid, userAgent, host)
yield txn.commit()
def renderResponse(self, code, body=None):
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/test/test_applepush.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/test/test_applepush.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/push/test/test_applepush.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -14,6 +14,8 @@
# limitations under the License.
##
+import struct
+import time
from calendarserver.push.applepush import (
ApplePushNotifierService, APNProviderProtocol
)
@@ -21,7 +23,6 @@
from twistedcaldav.test.util import TestCase
from twisted.internet.defer import inlineCallbacks, succeed
from twisted.internet.task import Clock
-import struct
from txdav.common.datastore.test.util import buildStore, CommonCommonTests
from txdav.common.icommondatastore import InvalidSubscriptionValues
@@ -39,6 +40,8 @@
"Service" : "calendarserver.push.applepush.ApplePushNotifierService",
"Enabled" : True,
"SubscriptionURL" : "apn",
+ "SubscriptionPurgeSeconds" : 24 * 60 * 60,
+ "SubscriptionPurgeIntervalSeconds" : 24 * 60 * 60,
"DataHost" : "calendars.example.com",
"ProviderHost" : "gateway.push.apple.com",
"ProviderPort" : 2195,
@@ -69,11 +72,11 @@
# Ensure empty values don't get through
try:
- yield txn.addAPNSubscription("", "", "", "")
+ yield txn.addAPNSubscription("", "", "", "", "", "")
except InvalidSubscriptionValues:
pass
try:
- yield txn.addAPNSubscription("", "1", "2", "3")
+ yield txn.addAPNSubscription("", "1", "2", "3", "", "")
except InvalidSubscriptionValues:
pass
@@ -82,12 +85,14 @@
key1 = "/CalDAV/calendars.example.com/user01/calendar/"
timestamp1 = 1000
uid = "D2256BCC-48E2-42D1-BD89-CBA1E4CCDFFB"
- yield txn.addAPNSubscription(token, key1, timestamp1, uid)
- yield txn.addAPNSubscription(token2, key1, timestamp1, uid)
+ userAgent = "test agent"
+ ipAddr = "127.0.0.1"
+ yield txn.addAPNSubscription(token, key1, timestamp1, uid, userAgent, ipAddr)
+ yield txn.addAPNSubscription(token2, key1, timestamp1, uid, userAgent, ipAddr)
key2 = "/CalDAV/calendars.example.com/user02/calendar/"
timestamp2 = 3000
- yield txn.addAPNSubscription(token, key2, timestamp2, uid)
+ yield txn.addAPNSubscription(token, key2, timestamp2, uid, userAgent, ipAddr)
subscriptions = (yield txn.apnSubscriptionsBySubscriber(uid))
self.assertTrue([token, key1, timestamp1] in subscriptions)
@@ -98,14 +103,14 @@
# the new uid
timestamp3 = 5000
uid2 = "D8FFB335-9D36-4CE8-A3B9-D1859E38C0DA"
- yield txn.addAPNSubscription(token, key2, timestamp3, uid2)
+ yield txn.addAPNSubscription(token, key2, timestamp3, uid2, userAgent, ipAddr)
subscriptions = (yield txn.apnSubscriptionsBySubscriber(uid))
self.assertTrue([token, key1, timestamp1] in subscriptions)
self.assertFalse([token, key2, timestamp3] in subscriptions)
subscriptions = (yield txn.apnSubscriptionsBySubscriber(uid2))
self.assertTrue([token, key2, timestamp3] in subscriptions)
# Change it back
- yield txn.addAPNSubscription(token, key2, timestamp2, uid)
+ yield txn.addAPNSubscription(token, key2, timestamp2, uid, userAgent, ipAddr)
yield txn.commit()
@@ -260,7 +265,31 @@
yield txn.commit()
self.assertEquals(subscriptions, [])
+ #
+ # Verify purgeOldAPNSubscriptions
+ #
+ # Create two subscriptions, one old and one new
+ txn = self.store.newTransaction()
+ now = int(time.time())
+ yield txn.addAPNSubscription(token2, key1, now - 2 * 24 * 60 * 60, uid, userAgent, ipAddr) # old
+ yield txn.addAPNSubscription(token2, key2, now, uid, userAgent, ipAddr) # recent
+ yield txn.commit()
+
+ # Purge old subscriptions
+ txn = self.store.newTransaction()
+ yield txn.purgeOldAPNSubscriptions(now - 60 * 60)
+ yield txn.commit()
+
+ # Check that only the recent subscription remains
+ txn = self.store.newTransaction()
+ subscriptions = (yield txn.apnSubscriptionsByToken(token2))
+ yield txn.commit()
+ self.assertEquals(len(subscriptions), 1)
+ self.assertEquals(subscriptions[0][0], key2)
+
+ service.stopService()
+
def test_validToken(self):
self.assertTrue(validToken("2d0d55cd7f98bcb81c6e24abcdc35168254c7846a43e2828b1ba5a8f82e219df"))
self.assertFalse(validToken("d0d55cd7f98bcb81c6e24abcdc35168254c7846a43e2828b1ba5a8f82e219df"))
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/caldav.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/caldav.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -557,6 +557,7 @@
"--reactor=%s" % (config.Twisted.reactor,),
"-n", self.maker.groupMembershipCacherTapName,
"-f", self.configPath,
+ "-o", "PIDFile=groupcacher.pid",
))
self.monitor.addProcess("groupcacher", groupMembershipCacherArgv,
@@ -585,6 +586,7 @@
Removes pidfile, registers an exec to happen after shutdown, then
stops the reactor.
"""
+ self.log_info("SIGHUP received - restarting")
try:
self.log_info("Removing pidfile: %s" % (self.pidfilePath,))
os.remove(self.pidfilePath)
@@ -595,7 +597,6 @@
self.reactor.stop()
def sighupHandler(self, num, frame):
- self.log_info("SIGHUP received - restarting")
self.reactor.callFromThread(self.reExec)
def startService(self):
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/util.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/tap/util.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -233,6 +233,7 @@
quota=quota,
logLabels=config.LogDatabase.LabelsInSQL,
logStats=config.LogDatabase.Statistics,
+ logStatsLogFile=config.LogDatabase.StatisticsLogFile,
logSQL=config.LogDatabase.SQLStatements,
logTransactionWaits=config.LogDatabase.TransactionWaitSeconds,
timeoutTransactions=config.TransactionTimeoutSeconds,
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/bootstrapdatabase.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/bootstrapdatabase.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/bootstrapdatabase.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -239,7 +239,7 @@
except IOError:
print "Unable to open the schema file: %s" % (SCHEMAFILE,)
else:
- found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", data)
+ found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d+)'\);", data)
if found is None:
print "Schema is missing required schema VERSION insert statement: %s" % (SCHEMAFILE,)
else:
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/principals.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/principals.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/principals.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -19,6 +19,7 @@
import sys
import os
import operator
+import signal
from getopt import getopt, GetoptError
from uuid import UUID
from pwd import getpwnam
@@ -593,7 +594,9 @@
(yield action_removeProxyPrincipal(principal, proxyPrincipal, proxyTypes=proxyTypes))
+ triggerGroupCacherUpdate(config)
+
@inlineCallbacks
def setProxies(principal, readProxyPrincipals, writeProxyPrincipals, directory=None):
"""
@@ -621,6 +624,7 @@
memberURLs.append(davxml.HRef(proxyURL))
membersProperty = davxml.GroupMemberSet(*memberURLs)
(yield subPrincipal.writeProperty(membersProperty, None))
+ triggerGroupCacherUpdate(config)
@inlineCallbacks
@@ -697,7 +701,9 @@
membersProperty = davxml.GroupMemberSet(*memberURLs)
(yield subPrincipal.writeProperty(membersProperty, None))
- returnValue(removed)
+ if removed:
+ triggerGroupCacherUpdate(config)
+ returnValue(removed)
@inlineCallbacks
@@ -898,7 +904,28 @@
returnValue(record)
+def triggerGroupCacherUpdate(config, killMethod=None):
+ """
+ Look up the pid of the group cacher sidecar and HUP it to trigger an update
+ """
+ if killMethod is None:
+ killMethod = os.kill
+ pidFilename = os.path.join(config.RunRoot, "groupcacher.pid")
+ if os.path.exists(pidFilename):
+ pidFile = open(pidFilename, "r")
+ pid = pidFile.read().strip()
+ pidFile.close()
+ try:
+ pid = int(pid)
+ except ValueError:
+ return
+ try:
+ killMethod(pid, signal.SIGHUP)
+ except OSError:
+ pass
+
+
if __name__ == "__main__":
main()
Modified: CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/test/test_principals.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/test/test_principals.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/calendarserver/tools/test/test_principals.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -15,6 +15,7 @@
##
import os
+import signal
import sys
from twext.python.filepath import CachingFilePath as FilePath
@@ -29,7 +30,9 @@
ErrorOutput
from calendarserver.tap.util import directoryFromConfig
-from calendarserver.tools.principals import parseCreationArgs, matchStrings, updateRecord, principalForPrincipalID, getProxies, setProxies
+from calendarserver.tools.principals import (parseCreationArgs, matchStrings,
+ updateRecord, principalForPrincipalID, getProxies, setProxies,
+ triggerGroupCacherUpdate)
class ManagePrincipalsTestCase(TestCase):
@@ -347,3 +350,25 @@
self.assertEquals(readProxies, []) # now empty
self.assertEquals(set(writeProxies), set(["user05"])) # unchanged
+
+ def test_triggerGroupCacherUpdate(self):
+ """
+ Verify triggerGroupCacherUpdate can read a pidfile and send a SIGHUP
+ """
+
+ self.calledArgs = None
+ def killMethod(pid, sig):
+ self.calledArgs = (pid, sig)
+
+ class StubConfig(object):
+ def __init__(self, runRootPath):
+ self.RunRoot = runRootPath
+
+ runRootDir = FilePath(self.mktemp())
+ runRootDir.createDirectory()
+ pidFile = runRootDir.child("groupcacher.pid")
+ pidFile.setContent("1234")
+ testConfig = StubConfig(runRootDir.path)
+ triggerGroupCacherUpdate(testConfig, killMethod=killMethod)
+ self.assertEquals(self.calledArgs, (1234, signal.SIGHUP))
+ runRootDir.remove()
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/calendarcertupdate.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/calendarcertupdate.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/calendarcertupdate.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -27,6 +27,7 @@
SERVICE_NAME = "calendar"
CALDAVD_PLIST = "/Library/Server/Calendar and Contacts/Config/caldavd.plist"
SERVER_ADMIN = "/Applications/Server.app/Contents/ServerRoot/usr/sbin/serveradmin"
+CERT_ADMIN = "/Applications/Server.app/Contents/ServerRoot/usr/sbin/certadmin"
def main():
@@ -36,7 +37,14 @@
if sys.argv[1] != "remove":
die("Bad command line; 'remove' expected", 2)
if isThisMyCert(CALDAVD_PLIST, sys.argv[2]):
- die("%s is in use by calendar" % (sys.argv[2],), 1)
+ defaultCert = getDefaultCert()
+ if defaultCert:
+ replaceCert(CALDAVD_PLIST, defaultCert)
+ restartService(CALDAVD_PLIST)
+ die("Replaced calendar cert with default: %s" % (defaultCert,), 0)
+ else:
+ removeCert(CALDAVD_PLIST)
+ die("No default, so removing calendar cert", 0)
else:
die("%s is not in use by calendar" % (sys.argv[2],), 0)
@@ -75,6 +83,49 @@
return otherCert == myCert
+def getDefaultCert():
+ """
+ Ask certadmin for default cert
+ @returns: path to default certificate, or empty string if no default
+ @rtype: C{str}
+ """
+ child = subprocess.Popen(
+ args=[CERT_ADMIN, "--default-certificate-path"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+ output, error = child.communicate()
+ if child.returncode:
+ log("Error looking up default certificate (%d): %s" % (child.returncode, error))
+ return ""
+ else:
+ certPath = output.strip()
+ log("Default certificate is: %s" % (certPath,))
+ return certPath
+
+
+def removeCert(plistPath):
+ """
+ Remove SSL settings in plist at plistPath
+ """
+ log("Reading plist %s" % (plistPath,))
+ plist = readPlist(plistPath)
+ log("Read in plist %s" % (plistPath,))
+
+ log("Clearing SSLCertificate")
+ plist["SSLCertificate"] = ""
+ log("Clearing SSLAuthorityChain")
+ plist["SSLAuthorityChain"] = ""
+ log("Clearing SSLPrivateKey")
+ plist["SSLPrivateKey"] = ""
+
+ log("Disabling SSL")
+ plist["EnableSSL"] = False
+
+ log("Writing plist %s" % (plistPath,))
+ writePlist(plist, plistPath)
+
+
def replaceCert(plistPath, otherCert):
"""
Replace SSL settings in plist at plistPath based on otherCert path
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/test/test_certupdate.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/test/test_certupdate.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/certupdate/test/test_certupdate.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -19,7 +19,7 @@
import twistedcaldav.test.util
from plistlib import readPlist
from contrib.certupdate.calendarcertupdate import (
- getMyCert, isThisMyCert, replaceCert
+ getMyCert, isThisMyCert, replaceCert, removeCert
)
samplePlist = """<?xml version="1.0" encoding="UTF-8"?>
@@ -32,6 +32,8 @@
<string>/etc/certificates/original.cert.pem</string>
<key>SSLPrivateKey</key>
<string>/etc/certificates/original.key.pem</string>
+ <key>EnableSSL</key>
+ <true/>
</dict>
</plist>
"""
@@ -63,3 +65,10 @@
self.assertEquals(plist["SSLAuthorityChain"], "/etc/certificates/new.chain.pem")
self.assertEquals(plist["SSLCertificate"], "/etc/certificates/new.cert.pem")
self.assertEquals(plist["SSLPrivateKey"], "/etc/certificates/new.key.pem")
+
+ def test_removeCert(self):
+ removeCert(self.path)
+ plist = readPlist(self.path)
+ self.assertEquals(plist["SSLAuthorityChain"], "")
+ self.assertEquals(plist["SSLCertificate"], "")
+ self.assertEquals(plist["SSLPrivateKey"], "")
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/migration/calendarmigrator.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/migration/calendarmigrator.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/migration/calendarmigrator.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -104,9 +104,9 @@
help='path to the root of the new system',
default='/')
- optionParser.add_option('--language', choices=('en', 'fr', 'de', 'ja'),
- metavar='[en|fr|de|ja]',
- help='language identifier (IGNORED)')
+ optionParser.add_option('--language',
+ help='language identifier (IGNORED)',
+ default="en")
(options, args) = optionParser.parse_args()
log("Options: %s" % (options,))
@@ -590,13 +590,13 @@
# /Library/Server/Calendar and Contacts will be new ServerRoot no matter what.
if oldCalDocumentRootValueProcessed:
- if diskAccessor.exists(oldCalDocumentRootValueProcessed): # external volume
+ if oldCalDocumentRootValueProcessed.startswith("/Volumes/"): # external volume
# The old external calendar DocumentRoot becomes the new DataRoot
newDataRoot = newDataRootValue = os.path.join(os.path.dirname(oldCalDocumentRootValue.rstrip("/")), "Calendar and Contacts Data")
newDocumentRoot = os.path.join(newDataRoot, newDocumentRootValue)
# Move aside whatever is there
if diskAccessor.exists(newDataRoot):
- renameTo = nextAvailable(newDataRoot + ".bak", diskAccessor=diskAccessor)
+ renameTo = nextAvailable(newDataRoot, "bak", diskAccessor=diskAccessor)
diskAccessor.rename(newDataRoot, renameTo)
if diskAccessor.exists(absolutePathWithRoot(sourceRoot, oldCalDataRootValueProcessed)):
@@ -629,8 +629,7 @@
# Old AddressBook DocumentRoot
if oldABDocumentRootValue:
newAddressBooks = os.path.join(newDocumentRoot, "addressbooks")
- if diskAccessor.exists(oldABDocumentRootValue):
- # Must be on an external volume if we see it existing at the point
+ if oldABDocumentRootValue.startswith("/Volumes/"): # external volume
diskAccessor.ditto(
os.path.join(oldABDocumentRootValue, "addressbooks"),
newAddressBooks
@@ -651,7 +650,7 @@
# Before 10.8, DocumentRoot and DataRoot were relative to ServerRoot
if oldServerRootValue:
- if oldServerRootValue.rstrip("/") != NEW_SERVER_ROOT: # external volume
+ if oldServerRootValue.rstrip("/").startswith("/Volumes/"): # external volume
log("Using external calendar server root: %s" % (oldServerRootValue,))
# ServerRoot needs to be /Library/Server/Calendar and Contacts
# Since DocumentRoot is now relative to DataRoot, move DocumentRoot into DataRoot
@@ -699,7 +698,7 @@
else: # 10.8 -> 10.8
if oldServerRootValue:
- if oldServerRootValue.rstrip("/") != NEW_SERVER_ROOT: # external volume
+ if oldServerRootValue.rstrip("/").startswith("/Volumes/"): # external volume
log("Using external calendar server root: %s" % (oldServerRootValue,))
elif diskAccessor.exists(absolutePathWithRoot(sourceRoot, oldServerRootValue)):
log("Copying calendar server root: %s" % (newServerRoot,))
@@ -769,26 +768,30 @@
return os.path.join(root, path)
-def nextAvailable(path, diskAccessor=None):
+def nextAvailable(path, ext, diskAccessor=None):
"""
- If path doesn't exist, return path. Otherwise return the first path name
- following the path.NNN pattern that doesn't exist, where NNN starts at 1
+ If path.ext doesn't exist, return path.ext. Otherwise return the first path name
+ following the path.N.ext pattern that doesn't exist, where N starts at 1
and increments until a non-existent path name is determined.
@param path: path to examine
@type path: C{str}
- @returns: non-existent path name C{str}
+ @param ext: filename extension to append (don't include ".")
+ @type ext: C{str}
+ @returns: non-existent path name
+ @rtype: C{str}
"""
if diskAccessor is None:
diskAccessor = DiskAccessor()
- if not diskAccessor.exists(path):
- return path
+ newPath = "%s.%s" % (path, ext)
+ if not diskAccessor.exists(newPath):
+ return newPath
i = 1
while(True):
- newPath = "%s.%d" % (path, i)
+ newPath = "%s.%d.%s" % (path, i, ext)
if not diskAccessor.exists(newPath):
return newPath
i += 1
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/migration/test/test_migrator.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/migration/test/test_migrator.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/migration/test/test_migrator.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1040,8 +1040,8 @@
"/Volumes/External/CalendarServer/Documents/calendars/" : True,
"/Volumes/External/CalendarServer/Calendar and Contacts Data/" : True,
"/Volumes/External/CalendarServer/Calendar and Contacts Data.bak/" : True,
- "/Volumes/External/CalendarServer/Calendar and Contacts Data.bak.1/" : True,
- "/Volumes/External/CalendarServer/Calendar and Contacts Data.bak.2/" : True,
+ "/Volumes/External/CalendarServer/Calendar and Contacts Data.1.bak/" : True,
+ "/Volumes/External/CalendarServer/Calendar and Contacts Data.2.bak/" : True,
"/Library/Server/Previous/Library/CalendarServer/Data/" : True,
"/Volumes/External/AddressBookServer/Documents/addressbooks/" : True,
"/Library/Server/Previous/Library/AddressBookServer/Data/" : True,
@@ -1064,7 +1064,7 @@
[ # expected DiskAccessor history
('rename',
'/Volumes/External/CalendarServer/Calendar and Contacts Data',
- '/Volumes/External/CalendarServer/Calendar and Contacts Data.bak.3'),
+ '/Volumes/External/CalendarServer/Calendar and Contacts Data.3.bak'),
('ditto', '/Library/Server/Previous/Library/CalendarServer/Data', '/Volumes/External/CalendarServer/Calendar and Contacts Data'),
('rename', '/Volumes/External/CalendarServer/Documents', '/Volumes/External/CalendarServer/Calendar and Contacts Data/Documents'),
('chown-recursive', '/Volumes/External/CalendarServer/Calendar and Contacts Data', FakeUser.pw_uid, FakeGroup.gr_gid),
@@ -1415,6 +1415,53 @@
),
(
+ "Lion -> Mountain Lion Migration, ServerRoot is non-standard but also not on an external volume, e.g. /Library/CalendarServer/Documents",
+ {
+ "/Library/Server/Previous/private/etc/caldavd/caldavd.plist" : """
+ <plist version="1.0">
+ <dict>
+ <key>ServerRoot</key>
+ <string>/Library/CalendarServer/Documents</string>
+ <key>DocumentRoot</key>
+ <string>Documents</string>
+ <key>DataRoot</key>
+ <string>Data</string>
+ <key>UserName</key>
+ <string>calendar</string>
+ <key>GroupName</key>
+ <string>calendar</string>
+ </dict>
+ </plist>
+ """,
+
+ "/Library/Server/Previous/Library/CalendarServer/Documents/" : True,
+ "/Library/Server/Previous/Library/CalendarServer/Documents/Documents/" : True,
+ "/Library/Server/Previous/Library/CalendarServer/Documents/Data/" : True,
+ },
+ ( # args
+ "/Library/Server/Previous", # sourceRoot
+ "/Volumes/new", # targetRoot
+ "10.7.4", # sourceVersion
+ "/Library/CalendarServer/Documents", # oldServerRootValue
+ "Documents", # oldCalDocumentRootValue
+ "Data", # oldCalDataRootValue
+ None, # oldABDocumentRootValue
+ FakeUser.pw_uid, FakeGroup.gr_gid, # user id, group id
+ ),
+ ( # expected return values
+ "/Volumes/new/Library/Server/Calendar and Contacts",
+ "/Library/Server/Calendar and Contacts",
+ "Data",
+ ),
+ [
+ ('ditto', '/Library/Server/Previous/Library/CalendarServer/Documents', '/Volumes/new/Library/Server/Calendar and Contacts'),
+ ('mkdir', '/Volumes/new/Library/Server/Calendar and Contacts/Data'),
+ ('mkdir', '/Volumes/new/Library/Server/Calendar and Contacts/Data/Documents'),
+ ('chown-recursive', '/Volumes/new/Library/Server/Calendar and Contacts', FakeUser.pw_uid, FakeGroup.gr_gid),
+ ]
+ ),
+
+ (
"Mountain Lion -> Mountain Lion Migration, all in default locations",
{
"/Library/Server/Previous/private/etc/caldavd/caldavd.plist" : """
@@ -1511,12 +1558,12 @@
def test_nextAvailable(self):
data = [
( { }, "a.bak" ),
- ( { "a.bak": True }, "a.bak.1" ),
- ( { "a.bak": True, "a.bak.1" : True }, "a.bak.2" ),
+ ( { "a.bak": True }, "a.1.bak" ),
+ ( { "a.bak": True, "a.1.bak" : True }, "a.2.bak" ),
]
for paths, expected in data:
accessor = StubDiskAccessor(paths)
- actual = nextAvailable("a.bak", diskAccessor=accessor)
+ actual = nextAvailable("a", "bak", diskAccessor=accessor)
self.assertEquals(actual, expected)
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/loadtest/thresholds.json
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/loadtest/thresholds.json 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/loadtest/thresholds.json 2012-08-09 19:03:13 UTC (rev 9547)
@@ -5,26 +5,26 @@
"thresholds": {
"default" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
- "GET{event}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "GET{event}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 5.0],
- "PUT{event}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
- "PUT{attendee-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
- "PUT{attendee-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.0],
- "PUT{attendee-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
- "PUT{organizer-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
- "PUT{organizer-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
- "PUT{organizer-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
+ "PUT{event}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 0.5],
+ "PUT{attendee-small}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 0.5],
+ "PUT{attendee-medium}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 5.0],
+ "PUT{attendee-large}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 5.0],
+ "PUT{organizer-small}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 0.5],
+ "PUT{organizer-medium}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 0.5],
+ "PUT{organizer-large}" : [ 100.0, 100.0, 100.0, 100.0, 100.0, 75.0, 25.0],
"DELETE{event}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
- "POST{fb-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
- "POST{fb-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
- "POST{fb-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
+ "POST{fb-small}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 5.0],
+ "POST{fb-medium}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 5.0],
+ "POST{fb-large}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 25.0, 5.0],
"PROPFIND{well-known}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"PROPFIND{find-principal}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"PROPFIND{principal}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
- "PROPFIND{home}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
+ "PROPFIND{home}" : [ 100.0, 100.0, 100.0, 50.0, 10.0, 0.5, 0.0],
"PROPFIND{calendar}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"PROPFIND{notification}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"PROPFIND{notification-items}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
@@ -40,14 +40,21 @@
"REPORT{vtodo}" : [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"REPORT{multiget-small}" : [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
"REPORT{multiget-medium}" : [ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
- "REPORT{multiget-large}" : [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0]
+ "REPORT{multiget-large}" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 10.0, 5.0]
}
},
"operations": {
"limits" : [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
- "thresholds": {
- "default" : [ 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]
+ "thresholds": {
+ "default" : [ 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0],
+
+ "accept" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 10.0, 5.0],
+ "create" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 10.0, 5.0],
+ "invite" : [ 100.0, 100.0, 100.0, 100.0, 75.0, 50.0, 25.0],
+ "poll" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 10.0, 5.0],
+ "push" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 10.0, 5.0],
+ "reply done" : [ 100.0, 100.0, 100.0, 75.0, 50.0, 10.0, 5.0]
}
}
}
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/__init__.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,19 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-SQL usage analysis tool.
-"""
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/__init__.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/__init__.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,19 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+SQL usage analysis tool.
+"""
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/__init__.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,19 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-SQL usage requests.
-"""
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/requests/__init__.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/__init__.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,19 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+SQL usage requests.
+"""
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/httpTests.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,98 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-Defines a set of HTTP requests to execute and return results.
-"""
-
-class HTTPTestBase(object):
- """
- Base class for an HTTP request that executes and results are returned for.
- """
-
- class SQLResults(object):
-
- def __init__(self, count, rows, timing):
- self.count = count
- self.rows = rows
- self.timing = timing
-
- def __init__(self, label, session, href, logFilePath):
- """
- @param label: label used to identify the test
- @type label: C{str}
- """
- self.label = label
- self.session = session
- self.baseHref = href
- self.logFilePath = logFilePath
- self.result = None
-
- def execute(self):
- """
- Execute the HTTP request and read the results.
- """
-
- self.prepare()
- self.clearLog()
- self.doRequest()
- self.collectResults()
- self.cleanup()
- return self.result
-
- def prepare(self):
- """
- Do some setup prior to the real request.
- """
- pass
-
- def clearLog(self):
- """
- Clear the server's SQL log file.
- """
- open(self.logFilePath, "w").write("")
-
- def doRequest(self):
- """
- Execute the actual HTTP request. Sub-classes override.
- """
- raise NotImplementedError
-
- def collectResults(self):
- """
- Parse the server log file to extract the details we need.
- """
-
- def extractInt(line):
- pos = line.find(": ")
- return int(line[pos+2:])
-
- def extractFloat(line):
- pos = line.find(": ")
- return float(line[pos+2:])
-
- data = open(self.logFilePath).read()
- lines = data.splitlines()
- count = extractInt(lines[4])
- rows = extractInt(lines[5])
- timing = extractFloat(lines[6])
- self.result = HTTPTestBase.SQLResults(count, rows, timing)
-
- def cleanup(self):
- """
- Do some cleanup after the real request.
- """
- pass
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/requests/httpTests.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/httpTests.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,98 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Defines a set of HTTP requests to execute and return results.
+"""
+
+class HTTPTestBase(object):
+ """
+ Base class for an HTTP request that executes and results are returned for.
+ """
+
+ class SQLResults(object):
+
+ def __init__(self, count, rows, timing):
+ self.count = count
+ self.rows = rows
+ self.timing = timing
+
+ def __init__(self, label, session, href, logFilePath):
+ """
+ @param label: label used to identify the test
+ @type label: C{str}
+ """
+ self.label = label
+ self.session = session
+ self.baseHref = href
+ self.logFilePath = logFilePath
+ self.result = None
+
+ def execute(self):
+ """
+ Execute the HTTP request and read the results.
+ """
+
+ self.prepare()
+ self.clearLog()
+ self.doRequest()
+ self.collectResults()
+ self.cleanup()
+ return self.result
+
+ def prepare(self):
+ """
+ Do some setup prior to the real request.
+ """
+ pass
+
+ def clearLog(self):
+ """
+ Clear the server's SQL log file.
+ """
+ open(self.logFilePath, "w").write("")
+
+ def doRequest(self):
+ """
+ Execute the actual HTTP request. Sub-classes override.
+ """
+ raise NotImplementedError
+
+ def collectResults(self):
+ """
+ Parse the server log file to extract the details we need.
+ """
+
+ def extractInt(line):
+ pos = line.find(": ")
+ return int(line[pos+2:])
+
+ def extractFloat(line):
+ pos = line.find(": ")
+ return float(line[pos+2:])
+
+ data = open(self.logFilePath).read()
+ lines = data.splitlines()
+ count = extractInt(lines[4])
+ rows = extractInt(lines[5])
+ timing = extractFloat(lines[6])
+ self.result = HTTPTestBase.SQLResults(count, rows, timing)
+
+ def cleanup(self):
+ """
+ Do some cleanup after the real request.
+ """
+ pass
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/multiget.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,56 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from caldavclientlibrary.protocol.caldav.definitions import caldavxml
-from caldavclientlibrary.protocol.caldav.multiget import Multiget
-from caldavclientlibrary.protocol.http.data.string import ResponseDataString
-from caldavclientlibrary.protocol.webdav.definitions import davxml, statuscodes
-from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
-from twext.web2.dav.util import joinURL
-
-class MultigetTest(HTTPTestBase):
- """
- A multiget operation
- """
-
- def __init__(self, label, session, href, logFilePath, count):
- super(MultigetTest, self).__init__(label, session, href, logFilePath)
- self.count = count
-
- def doRequest(self):
- """
- Execute the actual HTTP request.
- """
- hrefs = [joinURL(self.baseHref, "%d.ics" % (i+1,)) for i in range(self.count)]
- props = (
- davxml.getetag,
- caldavxml.calendar_data,
- caldavxml.schedule_tag,
- )
-
- # Create CalDAV multiget
- request = Multiget(self.session, self.baseHref, hrefs, props)
- result = ResponseDataString()
- request.setOutput(result)
-
- # Process it
- self.session.runSession(request)
-
- # If its a 207 we want to parse the XML
- if request.getStatusCode() == statuscodes.MultiStatus:
- pass
- else:
- raise RuntimeError("Muliget request failed: %s" % (request.getStatusCode(),))
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/requests/multiget.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/multiget.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,56 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from caldavclientlibrary.protocol.caldav.definitions import caldavxml
+from caldavclientlibrary.protocol.caldav.multiget import Multiget
+from caldavclientlibrary.protocol.http.data.string import ResponseDataString
+from caldavclientlibrary.protocol.webdav.definitions import davxml, statuscodes
+from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
+from twext.web2.dav.util import joinURL
+
+class MultigetTest(HTTPTestBase):
+ """
+ A multiget operation
+ """
+
+ def __init__(self, label, session, href, logFilePath, count):
+ super(MultigetTest, self).__init__(label, session, href, logFilePath)
+ self.count = count
+
+ def doRequest(self):
+ """
+ Execute the actual HTTP request.
+ """
+ hrefs = [joinURL(self.baseHref, "%d.ics" % (i+1,)) for i in range(self.count)]
+ props = (
+ davxml.getetag,
+ caldavxml.calendar_data,
+ caldavxml.schedule_tag,
+ )
+
+ # Create CalDAV multiget
+ request = Multiget(self.session, self.baseHref, hrefs, props)
+ result = ResponseDataString()
+ request.setOutput(result)
+
+ # Process it
+ self.session.runSession(request)
+
+ # If its a 207 we want to parse the XML
+ if request.getStatusCode() == statuscodes.MultiStatus:
+ pass
+ else:
+ raise RuntimeError("Muliget request failed: %s" % (request.getStatusCode(),))
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/propfind.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,53 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from caldavclientlibrary.protocol.http.data.string import ResponseDataString
-from caldavclientlibrary.protocol.webdav.definitions import davxml, statuscodes,\
- headers
-from caldavclientlibrary.protocol.webdav.propfind import PropFind
-from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
-
-class PropfindTest(HTTPTestBase):
- """
- A propfind operation
- """
-
- def __init__(self, label, session, href, logFilePath, depth=1):
- super(PropfindTest, self).__init__(label, session, href, logFilePath)
- self.depth = headers.Depth1 if depth == 1 else headers.Depth0
-
- def doRequest(self):
- """
- Execute the actual HTTP request.
- """
- props = (
- davxml.getetag,
- davxml.getcontenttype,
- )
-
- # Create WebDAV propfind
- request = PropFind(self.session, self.baseHref, self.depth, props)
- result = ResponseDataString()
- request.setOutput(result)
-
- # Process it
- self.session.runSession(request)
-
- # If its a 207 we want to parse the XML
- if request.getStatusCode() == statuscodes.MultiStatus:
- pass
- else:
- raise RuntimeError("Propfind request failed: %s" % (request.getStatusCode(),))
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/requests/propfind.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/propfind.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,53 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from caldavclientlibrary.protocol.http.data.string import ResponseDataString
+from caldavclientlibrary.protocol.webdav.definitions import davxml, statuscodes,\
+ headers
+from caldavclientlibrary.protocol.webdav.propfind import PropFind
+from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
+
+class PropfindTest(HTTPTestBase):
+ """
+ A propfind operation
+ """
+
+ def __init__(self, label, session, href, logFilePath, depth=1):
+ super(PropfindTest, self).__init__(label, session, href, logFilePath)
+ self.depth = headers.Depth1 if depth == 1 else headers.Depth0
+
+ def doRequest(self):
+ """
+ Execute the actual HTTP request.
+ """
+ props = (
+ davxml.getetag,
+ davxml.getcontenttype,
+ )
+
+ # Create WebDAV propfind
+ request = PropFind(self.session, self.baseHref, self.depth, props)
+ result = ResponseDataString()
+ request.setOutput(result)
+
+ # Process it
+ self.session.runSession(request)
+
+ # If its a 207 we want to parse the XML
+ if request.getStatusCode() == statuscodes.MultiStatus:
+ pass
+ else:
+ raise RuntimeError("Propfind request failed: %s" % (request.getStatusCode(),))
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/query.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,110 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from caldavclientlibrary.protocol.url import URL
-from caldavclientlibrary.protocol.webdav.definitions import davxml, statuscodes
-from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
-from twext.web2.dav.util import joinURL
-from pycalendar.datetime import PyCalendarDateTime
-from caldavclientlibrary.protocol.caldav.query import QueryVEVENTTimeRange
-from caldavclientlibrary.protocol.http.data.string import ResponseDataString
-
-ICAL = """BEGIN:VCALENDAR
-CALSCALE:GREGORIAN
-PRODID:-//Example Inc.//Example Calendar//EN
-VERSION:2.0
-BEGIN:VTIMEZONE
-LAST-MODIFIED:20040110T032845Z
-TZID:US/Eastern
-BEGIN:DAYLIGHT
-DTSTART:20000404T020000
-RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
-TZNAME:EDT
-TZOFFSETFROM:-0500
-TZOFFSETTO:-0400
-END:DAYLIGHT
-BEGIN:STANDARD
-DTSTART:20001026T020000
-RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
-TZNAME:EST
-TZOFFSETFROM:-0400
-TZOFFSETTO:-0500
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-DTSTAMP:20051222T205953Z
-CREATED:20060101T150000Z
-DTSTART:%s
-DURATION:PT1H
-SUMMARY:event 1
-UID:sync-collection-%d-ics
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n")
-
-class QueryTest(HTTPTestBase):
- """
- A sync operation
- """
-
- def __init__(self, label, session, href, logFilePath, count):
- super(QueryTest, self).__init__(label, session, href, logFilePath)
- self.count = count
-
- def prepare(self):
- """
- Do some setup prior to the real request.
- """
- # Add resources to create required number of changes
- self.start = PyCalendarDateTime.getNowUTC()
- self.start.setHHMMSS(12, 0, 0)
- self.end = self.start.duplicate()
- self.end.offsetHours(1)
- for i in range(self.count):
- href = joinURL(self.baseHref, "tr-query-%d.ics" % (i+1,))
- self.session.writeData(URL(path=href), ICAL % (self.start.getText(), i+1,), "text/calendar")
-
- def doRequest(self):
- """
- Execute the actual HTTP request.
- """
- props = (
- davxml.getetag,
- davxml.getcontenttype,
- )
-
- # Create CalDAV query
- request = QueryVEVENTTimeRange(self.session, self.baseHref, self.start.getText(), self.end.getText(), props)
- result = ResponseDataString()
- request.setOutput(result)
-
- # Process it
- self.session.runSession(request)
-
- # If its a 207 we want to parse the XML
- if request.getStatusCode() == statuscodes.MultiStatus:
- pass
- else:
- raise RuntimeError("Query request failed: %s" % (request.getStatusCode(),))
-
- def cleanup(self):
- """
- Do some cleanup after the real request.
- """
- # Remove created resources
- for i in range(self.count):
- href = joinURL(self.baseHref, "tr-query-%d.ics" % (i+1,))
- self.session.deleteResource(URL(path=href))
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/requests/query.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/query.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,110 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from caldavclientlibrary.protocol.url import URL
+from caldavclientlibrary.protocol.webdav.definitions import davxml, statuscodes
+from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
+from twext.web2.dav.util import joinURL
+from pycalendar.datetime import PyCalendarDateTime
+from caldavclientlibrary.protocol.caldav.query import QueryVEVENTTimeRange
+from caldavclientlibrary.protocol.http.data.string import ResponseDataString
+
+ICAL = """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VTIMEZONE
+LAST-MODIFIED:20040110T032845Z
+TZID:US/Eastern
+BEGIN:DAYLIGHT
+DTSTART:20000404T020000
+RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
+TZNAME:EDT
+TZOFFSETFROM:-0500
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20001026T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+TZNAME:EST
+TZOFFSETFROM:-0400
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+DTSTAMP:20051222T205953Z
+CREATED:20060101T150000Z
+DTSTART:%s
+DURATION:PT1H
+SUMMARY:event 1
+UID:sync-collection-%d-ics
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+class QueryTest(HTTPTestBase):
+ """
+ A sync operation
+ """
+
+ def __init__(self, label, session, href, logFilePath, count):
+ super(QueryTest, self).__init__(label, session, href, logFilePath)
+ self.count = count
+
+ def prepare(self):
+ """
+ Do some setup prior to the real request.
+ """
+ # Add resources to create required number of changes
+ self.start = PyCalendarDateTime.getNowUTC()
+ self.start.setHHMMSS(12, 0, 0)
+ self.end = self.start.duplicate()
+ self.end.offsetHours(1)
+ for i in range(self.count):
+ href = joinURL(self.baseHref, "tr-query-%d.ics" % (i+1,))
+ self.session.writeData(URL(path=href), ICAL % (self.start.getText(), i+1,), "text/calendar")
+
+ def doRequest(self):
+ """
+ Execute the actual HTTP request.
+ """
+ props = (
+ davxml.getetag,
+ davxml.getcontenttype,
+ )
+
+ # Create CalDAV query
+ request = QueryVEVENTTimeRange(self.session, self.baseHref, self.start.getText(), self.end.getText(), props)
+ result = ResponseDataString()
+ request.setOutput(result)
+
+ # Process it
+ self.session.runSession(request)
+
+ # If its a 207 we want to parse the XML
+ if request.getStatusCode() == statuscodes.MultiStatus:
+ pass
+ else:
+ raise RuntimeError("Query request failed: %s" % (request.getStatusCode(),))
+
+ def cleanup(self):
+ """
+ Do some cleanup after the real request.
+ """
+ # Remove created resources
+ for i in range(self.count):
+ href = joinURL(self.baseHref, "tr-query-%d.ics" % (i+1,))
+ self.session.deleteResource(URL(path=href))
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/requests/sync.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,102 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from caldavclientlibrary.protocol.url import URL
-from caldavclientlibrary.protocol.webdav.definitions import davxml
-from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
-from twext.web2.dav.util import joinURL
-from pycalendar.datetime import PyCalendarDateTime
-
-ICAL = """BEGIN:VCALENDAR
-CALSCALE:GREGORIAN
-PRODID:-//Example Inc.//Example Calendar//EN
-VERSION:2.0
-BEGIN:VTIMEZONE
-LAST-MODIFIED:20040110T032845Z
-TZID:US/Eastern
-BEGIN:DAYLIGHT
-DTSTART:20000404T020000
-RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
-TZNAME:EDT
-TZOFFSETFROM:-0500
-TZOFFSETTO:-0400
-END:DAYLIGHT
-BEGIN:STANDARD
-DTSTART:20001026T020000
-RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
-TZNAME:EST
-TZOFFSETFROM:-0400
-TZOFFSETTO:-0500
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-DTSTAMP:20051222T205953Z
-CREATED:20060101T150000Z
-DTSTART;TZID=US/Eastern:%d0101T100000
-DURATION:PT1H
-SUMMARY:event 1
-UID:sync-collection-%d-ics
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n")
-
-class SyncTest(HTTPTestBase):
- """
- A sync operation
- """
-
- def __init__(self, label, session, href, logFilePath, full, count):
- super(SyncTest, self).__init__(label, session, href, logFilePath)
- self.full = full
- self.count = count
- self.synctoken = ""
-
- def prepare(self):
- """
- Do some setup prior to the real request.
- """
- if not self.full:
- # Get current sync token
- results, _ignore_bad = self.session.getProperties(URL(path=self.baseHref), (davxml.sync_token,))
- self.synctoken = results[davxml.sync_token]
-
- # Add resources to create required number of changes
- now = PyCalendarDateTime.getNowUTC()
- for i in range(self.count):
- href = joinURL(self.baseHref, "sync-collection-%d.ics" % (i+1,))
- self.session.writeData(URL(path=href), ICAL % (now.getYear() + 1, i+1,), "text/calendar")
-
- def doRequest(self):
- """
- Execute the actual HTTP request.
- """
- props = (
- davxml.getetag,
- davxml.getcontenttype,
- )
-
- # Run sync collection
- self.session.syncCollection(URL(path=self.baseHref), self.synctoken, props)
-
- def cleanup(self):
- """
- Do some cleanup after the real request.
- """
- if not self.full:
- # Remove created resources
- for i in range(self.count):
- href = joinURL(self.baseHref, "sync-collection-%d.ics" % (i+1,))
- self.session.deleteResource(URL(path=href))
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/requests/sync.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/requests/sync.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,102 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from caldavclientlibrary.protocol.url import URL
+from caldavclientlibrary.protocol.webdav.definitions import davxml
+from contrib.performance.sqlusage.requests.httpTests import HTTPTestBase
+from twext.web2.dav.util import joinURL
+from pycalendar.datetime import PyCalendarDateTime
+
+ICAL = """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VTIMEZONE
+LAST-MODIFIED:20040110T032845Z
+TZID:US/Eastern
+BEGIN:DAYLIGHT
+DTSTART:20000404T020000
+RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
+TZNAME:EDT
+TZOFFSETFROM:-0500
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20001026T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+TZNAME:EST
+TZOFFSETFROM:-0400
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+DTSTAMP:20051222T205953Z
+CREATED:20060101T150000Z
+DTSTART;TZID=US/Eastern:%d0101T100000
+DURATION:PT1H
+SUMMARY:event 1
+UID:sync-collection-%d-ics
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+class SyncTest(HTTPTestBase):
+ """
+ A sync operation
+ """
+
+ def __init__(self, label, session, href, logFilePath, full, count):
+ super(SyncTest, self).__init__(label, session, href, logFilePath)
+ self.full = full
+ self.count = count
+ self.synctoken = ""
+
+ def prepare(self):
+ """
+ Do some setup prior to the real request.
+ """
+ if not self.full:
+ # Get current sync token
+ results, _ignore_bad = self.session.getProperties(URL(path=self.baseHref), (davxml.sync_token,))
+ self.synctoken = results[davxml.sync_token]
+
+ # Add resources to create required number of changes
+ now = PyCalendarDateTime.getNowUTC()
+ for i in range(self.count):
+ href = joinURL(self.baseHref, "sync-collection-%d.ics" % (i+1,))
+ self.session.writeData(URL(path=href), ICAL % (now.getYear() + 1, i+1,), "text/calendar")
+
+ def doRequest(self):
+ """
+ Execute the actual HTTP request.
+ """
+ props = (
+ davxml.getetag,
+ davxml.getcontenttype,
+ )
+
+ # Run sync collection
+ self.session.syncCollection(URL(path=self.baseHref), self.synctoken, props)
+
+ def cleanup(self):
+ """
+ Do some cleanup after the real request.
+ """
+ if not self.full:
+ # Remove created resources
+ for i in range(self.count):
+ href = joinURL(self.baseHref, "sync-collection-%d.ics" % (i+1,))
+ self.session.deleteResource(URL(path=href))
Deleted: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py
===================================================================
--- CalendarServer/trunk/contrib/performance/sqlusage/sqlusage.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,214 +0,0 @@
-##
-# Copyright (c) 2012 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from StringIO import StringIO
-from caldavclientlibrary.client.clientsession import CalDAVSession
-from caldavclientlibrary.protocol.url import URL
-from caldavclientlibrary.protocol.webdav.definitions import davxml
-from calendarserver.tools import tables
-from contrib.performance.sqlusage.requests.multiget import MultigetTest
-from contrib.performance.sqlusage.requests.propfind import PropfindTest
-from contrib.performance.sqlusage.requests.query import QueryTest
-from contrib.performance.sqlusage.requests.sync import SyncTest
-from pycalendar.datetime import PyCalendarDateTime
-from twext.web2.dav.util import joinURL
-import getopt
-import sys
-
-"""
-This tool is designed to analyze how SQL is being used for various HTTP requests.
-It will execute a series of HTTP requests against a test server configuration and
-count the total number of SQL statements per request, the total number of rows
-returned per request and the total SQL execution time per request. Each series
-will be repeated against a varying calendar size so the variation in SQL use
-with calendar size can be plotted.
-"""
-
-EVENT_COUNTS = (0, 1, 5, 10, 25, 50, 75, 100, 200, 300, 400, 500, 1000, 2000, 3000, 4000, 5000)
-
-ICAL = """BEGIN:VCALENDAR
-CALSCALE:GREGORIAN
-PRODID:-//Example Inc.//Example Calendar//EN
-VERSION:2.0
-BEGIN:VTIMEZONE
-LAST-MODIFIED:20040110T032845Z
-TZID:US/Eastern
-BEGIN:DAYLIGHT
-DTSTART:20000404T020000
-RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
-TZNAME:EDT
-TZOFFSETFROM:-0500
-TZOFFSETTO:-0400
-END:DAYLIGHT
-BEGIN:STANDARD
-DTSTART:20001026T020000
-RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
-TZNAME:EST
-TZOFFSETFROM:-0400
-TZOFFSETTO:-0500
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-DTSTAMP:20051222T205953Z
-CREATED:20060101T150000Z
-DTSTART;TZID=US/Eastern:%d0101T100000
-DURATION:PT1H
-SUMMARY:event 1
-UID:%d-ics
-END:VEVENT
-END:VCALENDAR
-""".replace("\n", "\r\n")
-
-class SQLUsage(object):
-
- def __init__(self, server, port, user, pswd, logFilePath):
- self.server = server
- self.port = port
- self.user = user
- self.pswd = pswd
- self.logFilePath = logFilePath
- self.requestLabels = []
- self.results = {}
- self.currentCount = 0
-
- self.userhref = "/calendars/users/%s/" % (self.user,)
-
- def runLoop(self):
-
- # Make the session
- session = CalDAVSession(self.server, self.port, user=self.user, pswd=self.pswd, root="/")
-
- # Set of requests to execute
- requests = [
- MultigetTest("multiget-1", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 1),
- MultigetTest("multiget-50", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 50),
- PropfindTest("propfind-cal", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 1),
- SyncTest("sync-full", session, joinURL(self.userhref, "calendar/"), self.logFilePath, True, 0),
- SyncTest("sync-1", session, joinURL(self.userhref, "calendar/"), self.logFilePath, False, 1),
- QueryTest("query-1", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 1),
- QueryTest("query-10", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 10),
- ]
- self.requestLabels = [request.label for request in requests]
-
- # Warm-up server by doing calendar home and calendar propfinds
- props = (davxml.resourcetype,)
- session.getPropertiesOnHierarchy(URL(path=self.userhref), props)
- session.getPropertiesOnHierarchy(URL(path=joinURL(self.userhref, "calendar/")), props)
-
- # Now loop over sets of events
- for count in EVENT_COUNTS:
- print "Testing count = %d" % (count,)
- self.ensureEvents(session, count)
- result = {}
- for request in requests:
- result[request.label] = request.execute()
- self.results[count] = result
-
- def report(self):
-
- self._printReport("SQL Statement Count", "count", "%d")
- self._printReport("SQL Rows Returned", "rows", "%d")
- self._printReport("SQL Time", "timing", "%.3f")
-
- def _printReport(self, title, attr, colFormat):
- table = tables.Table()
-
- print title
- headers = ["Events"] + sorted([label for label in self.requestLabels])
- table.addHeader(headers)
- formats = [tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY)] + \
- [tables.Table.ColumnFormat(colFormat, tables.Table.ColumnFormat.RIGHT_JUSTIFY)] * len(self.requestLabels)
- table.setDefaultColumnFormats(formats)
- for k in sorted(self.results.keys()):
- row = [k] + [getattr(self.results[k][item], attr) for item in sorted(self.results[k].keys())]
- table.addRow(row)
- os = StringIO()
- table.printTable(os=os)
- print os.getvalue()
- print
-
- def ensureEvents(self, session, n):
- """
- Make sure the required number of events are present in the calendar.
-
- @param n: number of events
- @type n: C{int}
- """
- now = PyCalendarDateTime.getNowUTC()
- for i in range(n - self.currentCount):
- index = self.currentCount + i + 1
- href = joinURL(self.userhref, "calendar", "%d.ics" % (index,))
- session.writeData(URL(path=href), ICAL % (now.getYear() + 1, index,), "text/calendar")
-
- self.currentCount = n
-
-def usage(error_msg=None):
- if error_msg:
- print error_msg
-
- print """Usage: sqlusage.py [options] [FILE]
-Options:
- -h Print this help and exit
- --server Server hostname
- --port Server port
- --user User name
- --pswd Password
-
-Arguments:
- FILE File name for sqlstats.log to analyze.
-
-Description:
-This utility will analyze the output of s pg_stat_statement table.
-"""
-
- if error_msg:
- raise ValueError(error_msg)
- else:
- sys.exit(0)
-
-if __name__ == '__main__':
-
- server = "localhost"
- port = 8008
- user = "user01"
- pswd = "user01"
- file = "sqlstats.logs"
-
- options, args = getopt.getopt(sys.argv[1:], "h", ["server", "port", "user", "pswd",])
-
- for option, value in options:
- if option == "-h":
- usage()
- elif option == "--server":
- server = value
- elif option == "--port":
- port = int(value)
- elif option == "--user":
- user = value
- elif option == "--pswd":
- pswd = value
- else:
- usage("Unrecognized option: %s" % (option,))
-
- # Process arguments
- if len(args) == 1:
- file = args[0]
- elif len(args) != 0:
- usage("Must zero or one file arguments")
-
- sql = SQLUsage(server, port, user, pswd, file)
- sql.runLoop()
- sql.report()
Copied: CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py (from rev 9546, CalendarServer/trunk/contrib/performance/sqlusage/sqlusage.py)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/performance/sqlusage/sqlusage.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,214 @@
+##
+# Copyright (c) 2012 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from StringIO import StringIO
+from caldavclientlibrary.client.clientsession import CalDAVSession
+from caldavclientlibrary.protocol.url import URL
+from caldavclientlibrary.protocol.webdav.definitions import davxml
+from calendarserver.tools import tables
+from contrib.performance.sqlusage.requests.multiget import MultigetTest
+from contrib.performance.sqlusage.requests.propfind import PropfindTest
+from contrib.performance.sqlusage.requests.query import QueryTest
+from contrib.performance.sqlusage.requests.sync import SyncTest
+from pycalendar.datetime import PyCalendarDateTime
+from twext.web2.dav.util import joinURL
+import getopt
+import sys
+
+"""
+This tool is designed to analyze how SQL is being used for various HTTP requests.
+It will execute a series of HTTP requests against a test server configuration and
+count the total number of SQL statements per request, the total number of rows
+returned per request and the total SQL execution time per request. Each series
+will be repeated against a varying calendar size so the variation in SQL use
+with calendar size can be plotted.
+"""
+
+EVENT_COUNTS = (0, 1, 5, 10, 25, 50, 75, 100, 200, 300, 400, 500, 1000, 2000, 3000, 4000, 5000)
+
+ICAL = """BEGIN:VCALENDAR
+CALSCALE:GREGORIAN
+PRODID:-//Example Inc.//Example Calendar//EN
+VERSION:2.0
+BEGIN:VTIMEZONE
+LAST-MODIFIED:20040110T032845Z
+TZID:US/Eastern
+BEGIN:DAYLIGHT
+DTSTART:20000404T020000
+RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4
+TZNAME:EDT
+TZOFFSETFROM:-0500
+TZOFFSETTO:-0400
+END:DAYLIGHT
+BEGIN:STANDARD
+DTSTART:20001026T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+TZNAME:EST
+TZOFFSETFROM:-0400
+TZOFFSETTO:-0500
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+DTSTAMP:20051222T205953Z
+CREATED:20060101T150000Z
+DTSTART;TZID=US/Eastern:%d0101T100000
+DURATION:PT1H
+SUMMARY:event 1
+UID:%d-ics
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n")
+
+class SQLUsage(object):
+
+ def __init__(self, server, port, user, pswd, logFilePath):
+ self.server = server
+ self.port = port
+ self.user = user
+ self.pswd = pswd
+ self.logFilePath = logFilePath
+ self.requestLabels = []
+ self.results = {}
+ self.currentCount = 0
+
+ self.userhref = "/calendars/users/%s/" % (self.user,)
+
+ def runLoop(self):
+
+ # Make the session
+ session = CalDAVSession(self.server, self.port, user=self.user, pswd=self.pswd, root="/")
+
+ # Set of requests to execute
+ requests = [
+ MultigetTest("multiget-1", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 1),
+ MultigetTest("multiget-50", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 50),
+ PropfindTest("propfind-cal", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 1),
+ SyncTest("sync-full", session, joinURL(self.userhref, "calendar/"), self.logFilePath, True, 0),
+ SyncTest("sync-1", session, joinURL(self.userhref, "calendar/"), self.logFilePath, False, 1),
+ QueryTest("query-1", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 1),
+ QueryTest("query-10", session, joinURL(self.userhref, "calendar/"), self.logFilePath, 10),
+ ]
+ self.requestLabels = [request.label for request in requests]
+
+ # Warm-up server by doing calendar home and calendar propfinds
+ props = (davxml.resourcetype,)
+ session.getPropertiesOnHierarchy(URL(path=self.userhref), props)
+ session.getPropertiesOnHierarchy(URL(path=joinURL(self.userhref, "calendar/")), props)
+
+ # Now loop over sets of events
+ for count in EVENT_COUNTS:
+ print "Testing count = %d" % (count,)
+ self.ensureEvents(session, count)
+ result = {}
+ for request in requests:
+ result[request.label] = request.execute()
+ self.results[count] = result
+
+ def report(self):
+
+ self._printReport("SQL Statement Count", "count", "%d")
+ self._printReport("SQL Rows Returned", "rows", "%d")
+ self._printReport("SQL Time", "timing", "%.3f")
+
+ def _printReport(self, title, attr, colFormat):
+ table = tables.Table()
+
+ print title
+ headers = ["Events"] + sorted([label for label in self.requestLabels])
+ table.addHeader(headers)
+ formats = [tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY)] + \
+ [tables.Table.ColumnFormat(colFormat, tables.Table.ColumnFormat.RIGHT_JUSTIFY)] * len(self.requestLabels)
+ table.setDefaultColumnFormats(formats)
+ for k in sorted(self.results.keys()):
+ row = [k] + [getattr(self.results[k][item], attr) for item in sorted(self.results[k].keys())]
+ table.addRow(row)
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+ print
+
+ def ensureEvents(self, session, n):
+ """
+ Make sure the required number of events are present in the calendar.
+
+ @param n: number of events
+ @type n: C{int}
+ """
+ now = PyCalendarDateTime.getNowUTC()
+ for i in range(n - self.currentCount):
+ index = self.currentCount + i + 1
+ href = joinURL(self.userhref, "calendar", "%d.ics" % (index,))
+ session.writeData(URL(path=href), ICAL % (now.getYear() + 1, index,), "text/calendar")
+
+ self.currentCount = n
+
+def usage(error_msg=None):
+ if error_msg:
+ print error_msg
+
+ print """Usage: sqlusage.py [options] [FILE]
+Options:
+ -h Print this help and exit
+ --server Server hostname
+ --port Server port
+ --user User name
+ --pswd Password
+
+Arguments:
+ FILE File name for sqlstats.log to analyze.
+
+Description:
+This utility will analyze the output of s pg_stat_statement table.
+"""
+
+ if error_msg:
+ raise ValueError(error_msg)
+ else:
+ sys.exit(0)
+
+if __name__ == '__main__':
+
+ server = "localhost"
+ port = 8008
+ user = "user01"
+ pswd = "user01"
+ file = "sqlstats.logs"
+
+ options, args = getopt.getopt(sys.argv[1:], "h", ["server", "port", "user", "pswd",])
+
+ for option, value in options:
+ if option == "-h":
+ usage()
+ elif option == "--server":
+ server = value
+ elif option == "--port":
+ port = int(value)
+ elif option == "--user":
+ user = value
+ elif option == "--pswd":
+ pswd = value
+ else:
+ usage("Unrecognized option: %s" % (option,))
+
+ # Process arguments
+ if len(args) == 1:
+ file = args[0]
+ elif len(args) != 0:
+ usage("Must zero or one file arguments")
+
+ sql = SQLUsage(server, port, user, pswd, file)
+ sql.runLoop()
+ sql.report()
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/tools/protocolanalysis.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/tools/protocolanalysis.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/tools/protocolanalysis.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -24,6 +24,7 @@
import sys
import tables
import traceback
+import glob
def safePercent(x, y, multiplier=100):
return ((multiplier * x) / y) if y else 0
@@ -118,6 +119,7 @@
METHOD_PROPFIND_CALENDAR_HOME = "PROPFIND Calendar Home"
METHOD_PROPFIND_CACHED_CALENDAR_HOME = "PROPFIND cached Calendar Home"
METHOD_PROPFIND_CALENDAR = "PROPFIND Calendar"
+METHOD_PROPFIND_INBOX = "PROPFIND Inbox"
METHOD_PROPFIND_ADDRESSBOOK_HOME = "PROPFIND Adbk Home"
METHOD_PROPFIND_CACHED_ADDRESSBOOK_HOME = "PROPFIND cached Adbk Home"
METHOD_PROPFIND_ADDRESSBOOK = "PROPFIND Adbk"
@@ -184,6 +186,9 @@
METHOD_DELETE_ADDRESSBOOK = "DELETE Adbk"
METHOD_DELETE_VCF = "DELETE vcf"
+# 401s
+METHOD_401 = "Z 401s"
+
class CalendarServerLogAnalyzer(object):
"""
@@ -225,6 +230,7 @@
filterByUser=None,
filterByClient=None,
ignoreNonHTTPMethods=True,
+ separate401s=True,
):
self.startHour = startHour
@@ -234,6 +240,7 @@
self.filterByUser = filterByUser
self.filterByClient = filterByClient
self.ignoreNonHTTPMethods = ignoreNonHTTPMethods
+ self.separate401s = separate401s
self.startTime = datetime.datetime.now().replace(microsecond=0)
@@ -258,7 +265,9 @@
self.hourlyByMethodCount = collections.defaultdict(lambda:[0,] * self.timeBucketCount)
self.hourlyByOKMethodCount = collections.defaultdict(lambda:[0,] * self.timeBucketCount)
self.hourlyByMethodTime = collections.defaultdict(lambda:[0.0,] * self.timeBucketCount)
+ self.hourlyByOKMethodTime = collections.defaultdict(lambda:[0.0,] * self.timeBucketCount)
self.averagedHourlyByMethodTime = collections.defaultdict(lambda:[0.0,] * self.timeBucketCount)
+ self.averagedHourlyByOKMethodTime = collections.defaultdict(lambda:[0.0,] * self.timeBucketCount)
self.hourlyPropfindByResponseCount = collections.defaultdict(lambda:[0,] * self.timeBucketCount)
self.hourlyByStatus = collections.defaultdict(lambda:[0,] * self.timeBucketCount)
@@ -293,8 +302,10 @@
f = GzipFile(fpath)
else:
f = open(fpath)
-
- self.maxIndex = (self.endHour - self.startHour + 1) * 60 / self.resolutionMinutes
+
+ effectiveStart = self.startHour if self.startHour is not None else 0
+ effectiveEnd = self.endHour if self.endHour is not None else 23
+ self.maxIndex = (effectiveEnd - effectiveStart + 1) * 60 / self.resolutionMinutes
try:
lineCtr = 0
for line in f:
@@ -314,7 +325,7 @@
# Filter method
if self.ignoreNonHTTPMethods and not self.currentLine.method.startswith("REPORT(") and self.currentLine.method not in httpMethods:
self.currentLine.method = "???"
-
+
# Do hour ranges
logHour = int(self.currentLine.logTime[0:2])
logMinute = int(self.currentLine.logTime[3:5])
@@ -324,9 +335,9 @@
hourFromStart = logHour - self.adjustHour
if hourFromStart < 0:
hourFromStart += 24
- if logHour < self.startHour:
+ if self.startHour is not None and logHour < self.startHour:
continue
- elif logHour > self.endHour:
+ elif self.endHour is not None and logHour > self.endHour:
continue
timeBucketIndex = (hourFromStart * 60 + logMinute) / self.resolutionMinutes
@@ -391,12 +402,13 @@
# Method counts, timing and status
self.hourlyByMethodCount[" TOTAL"][timeBucketIndex] += 1
self.hourlyByMethodCount[adjustedMethod][timeBucketIndex] += 1
- if isOK:
- self.hourlyByOKMethodCount[" TOTAL"][timeBucketIndex] += 1
- self.hourlyByOKMethodCount[adjustedMethod][timeBucketIndex] += 1
+ self.hourlyByOKMethodCount[" TOTAL"][timeBucketIndex] += 1
+ self.hourlyByOKMethodCount[adjustedMethod if isOK else METHOD_401][timeBucketIndex] += 1
self.hourlyByMethodTime[" TOTAL"][timeBucketIndex] += responseTime
self.hourlyByMethodTime[adjustedMethod][timeBucketIndex] += responseTime
+ self.hourlyByOKMethodTime[" TOTAL"][timeBucketIndex] += responseTime
+ self.hourlyByOKMethodTime[adjustedMethod if isOK else METHOD_401][timeBucketIndex] += responseTime
self.hourlyByStatus[" TOTAL"][timeBucketIndex] += 1
self.hourlyByStatus[self.currentLine.status][timeBucketIndex] += 1
@@ -481,6 +493,15 @@
else:
newValue = hours[hour]
self.averagedHourlyByMethodTime[method][hour] = newValue
+ self.averagedHourlyByOKMethodTime.clear()
+ for method, hours in self.hourlyByOKMethodTime.iteritems():
+ counts = self.hourlyByOKMethodCount[method]
+ for hour in xrange(self.timeBucketCount):
+ if counts[hour]:
+ newValue = hours[hour] / counts[hour]
+ else:
+ newValue = hours[hour]
+ self.averagedHourlyByOKMethodTime[method][hour] = newValue
self.averagedResponseTimeVsQueueDepth.clear()
for k, v in self.responseTimeVsQueueDepth.iteritems():
@@ -648,7 +669,10 @@
if uribits[3] in calendar_specials:
return "PROPFIND %s" % (uribits[3],)
elif len(uribits) == 4:
- return METHOD_PROPFIND_CALENDAR
+ if uribits[3] == "inbox":
+ return METHOD_PROPFIND_INBOX
+ else:
+ return METHOD_PROPFIND_CALENDAR
elif uribits[0] == "addressbooks":
@@ -926,11 +950,25 @@
self.printClientTotals(doTabs)
print "Protocol Analysis Count"
- self.printHourlyByXXXDetails(self.hourlyByMethodCount, doTabs)
+ self.printHourlyByXXXDetails(
+ self.hourlyByOKMethodCount if self.separate401s else self.hourlyByMethodCount,
+ doTabs,
+ )
print "Protocol Analysis Average Response Time (ms)"
- self.printHourlyByXXXDetails(self.averagedHourlyByMethodTime, doTabs, showAverages=True)
+ self.printHourlyByXXXDetails(
+ self.averagedHourlyByOKMethodTime if self.separate401s else self.averagedHourlyByMethodTime,
+ doTabs,
+ showAverages=True,
+ )
+ print "Protocol Analysis Total Response Time (ms)"
+ self.printHourlyByXXXDetails(
+ self.hourlyByOKMethodTime if self.separate401s else self.hourlyByMethodTime,
+ doTabs,
+ showFloatPercent=True,
+ )
+
print "Status Code Analysis"
self.printHourlyByXXXDetails(self.hourlyByStatus, doTabs)
@@ -1109,7 +1147,7 @@
table.printTabDelimitedData() if doTabs else table.printTable()
print ""
- def printHourlyByXXXDetails(self, hourlyByXXX, doTabs, showTotals=True, showAverages=False):
+ def printHourlyByXXXDetails(self, hourlyByXXX, doTabs, showTotals=True, showAverages=False, showFloatPercent=False):
totals = [(0, 0,)] * len(hourlyByXXX)
table = tables.Table()
@@ -1173,7 +1211,10 @@
if data:
totals[colctr] = (totals[colctr][0] + data, totals[colctr][1] + 1,)
elif type(data) is float:
- row[colctr + 1] = "%.1f" % (data,)
+ if total is not None and showFloatPercent:
+ row[colctr + 1] = "%.1f (%2d%%)" % (data, safePercent(data, total),)
+ else:
+ row[colctr + 1] = "%.1f" % (data,)
if data:
totals[colctr] = (totals[colctr][0] + data, totals[colctr][1] + 1,)
table.addRow(row)
@@ -1187,7 +1228,10 @@
row[colctr + 1] = "%d (%2d%%)" % (data, safePercent(data, totals[0][0]),)
elif type(data) is float:
data = ((data / count) if count else 0.0) if showAverages else data
- row[colctr + 1] = "%.1f" % (data,)
+ if showFloatPercent:
+ row[colctr + 1] = "%.1f (%2d%%)" % (data, safePercent(data, totals[0][0]),)
+ else:
+ row[colctr + 1] = "%.1f" % (data,)
table.addFooter(row)
table.printTabDelimitedData() if doTabs else table.printTable()
@@ -1998,21 +2042,27 @@
pwd = os.getcwd()
- analyzers = []
+ logs = []
for arg in args:
arg = os.path.expanduser(arg)
if not arg.startswith("/"):
arg = os.path.join(pwd, arg)
if arg.endswith("/"):
arg = arg[:-1]
- if not os.path.exists(arg):
- print "Path does not exist: '%s'. Ignoring." % (arg,)
- continue
+ if "*" in arg:
+ logs.extend(glob.iglob(arg))
+ else:
+ if not os.path.exists(arg):
+ print "Path does not exist: '%s'. Ignoring." % (arg,)
+ continue
+ logs.append(arg)
+ analyzers = []
+ for log in logs:
if diffMode or not analyzers:
analyzers.append(CalendarServerLogAnalyzer(startHour, endHour, utcoffset, resolution, filterByUser, filterByClient))
- print "Analyzing: %s" % (arg,)
- analyzers[-1].analyzeLogFile(arg)
+ print "Analyzing: %s" % (log,)
+ analyzers[-1].analyzeLogFile(log)
if diffMode and len(analyzers) > 1:
Differ(analyzers).printAll(doTabDelimited, summary)
Modified: CalendarServer/branches/users/gaya/inviteclean/contrib/tools/request_monitor.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/contrib/tools/request_monitor.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/contrib/tools/request_monitor.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -354,14 +354,17 @@
numProxied = collections.defaultdict(int)
slotCount = collections.defaultdict(int)
totalRespTime = collections.defaultdict(float)
+ totalRespWithoutWRTime = collections.defaultdict(float)
maxRespTime = collections.defaultdict(float)
- under10ms = 0
- over10ms = 0
- over100ms = 0
- over1s = 0
- over10s = 0
- over30s = 0
- over60s = 0
+ under10ms = [0, 0]
+ over10ms = [0, 0]
+ over100ms = [0, 0]
+ over1s = [0, 0]
+ over10s = [0, 0]
+ over30s = [0, 0]
+ over60s = [0, 0]
+ totalOver1s = [0, 0]
+ totalOver10s = [0, 0]
requests = []
users = { }
startTime = None
@@ -414,26 +417,33 @@
slotCount[filename] += outstanding
respTime = float(extended['t'])
+ wrTime = float(extended.get('t-resp-wr', 0.0))
timeSpent = timesSpent.get(logId, 0.0) + respTime
timesSpent[logId] = timeSpent
totalRespTime[filename] += respTime
+ totalRespWithoutWRTime[filename] += respTime - wrTime
if respTime > maxRespTime[filename]:
maxRespTime[filename] = respTime
- if respTime >= 60000.0:
- over60s += 1
- elif respTime >= 30000.0:
- over30s +=1
- elif respTime >= 10000.0:
- over10s +=1
- elif respTime >= 1000.0:
- over1s +=1
- elif respTime >= 100.0:
- over100ms +=1
- elif respTime >= 10.0:
- over10ms +=1
- else:
- under10ms +=1
+ for index, testTime in enumerate((respTime, respTime - wrTime,)):
+ if testTime >= 60000.0:
+ over60s[index] += 1
+ elif testTime >= 30000.0:
+ over30s[index] +=1
+ elif testTime >= 10000.0:
+ over10s[index] +=1
+ elif testTime >= 1000.0:
+ over1s[index] +=1
+ elif testTime >= 100.0:
+ over100ms[index] +=1
+ elif testTime >= 10.0:
+ over10ms[index] +=1
+ else:
+ under10ms[index] +=1
+ if testTime >= 1000.0:
+ totalOver1s[index] +=1
+ if testTime >= 10000.0:
+ totalOver10s[index] +=1
ext = []
@@ -486,10 +496,10 @@
table = tables.Table()
table.addHeader(
- ("Instance", "Requests", "Av. Requests", "Av. Response", "Max. Response", "Slot", "Start", "End", "File Name"),
+ ("Instance", "Requests", "Av. Requests", "Av. Response", "Av. Response", "Max. Response", "Slot", "Start", "End", "File Name"),
)
table.addHeader(
- ( "", "", "per second", "(ms)", "(ms)", "Average", "Time", "Time", ""),
+ ( "", "", "per second", "(ms)", "no write(ms)", "(ms)", "Average", "Time", "Time", ""),
)
table.setDefaultColumnFormats(
(
@@ -498,6 +508,7 @@
tables.Table.ColumnFormat("%.1f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.1f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.1f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.2f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
@@ -507,6 +518,7 @@
totalAverage = 0.0
totalResponseTime = 0.0
+ totalResponseWithoutWRTime = 0.0
maxResponseTime = 0.0
totalSlots = 0
minStartTime = None
@@ -528,6 +540,9 @@
avgResponse = totalRespTime[filename] / len(times)
totalResponseTime += totalRespTime[filename]
+ avgResponseWithWR = totalRespWithoutWRTime[filename] / len(times)
+ totalResponseWithoutWRTime += totalRespWithoutWRTime[filename]
+
maxResponseTime = max(maxResponseTime, maxRespTime[filename])
totalSlots += slotCount[filename]
@@ -537,6 +552,7 @@
len(times),
avgRequests,
avgResponse,
+ avgResponseWithWR,
maxRespTime[filename],
float(slotCount[filename]) / len(times),
startTime,
@@ -550,6 +566,7 @@
totalRequests,
totalAverage,
totalResponseTime / totalRequests,
+ totalResponseWithoutWRTime / totalRequests,
maxResponseTime,
float(totalSlots) / totalRequests,
minStartTime,
@@ -567,7 +584,41 @@
lqlatency[0],
lqlatency[1],
)
- print "<10ms: %d >10ms: %d >100ms: %d >1s: %d >10s: %d >30s: %d >60s: %d" % (under10ms, over10ms, over100ms, over1s, over10s, over30s, over60s)
+
+ table = tables.Table()
+ table.addHeader(
+ ("", "<10ms", "10ms<->100ms", "100ms<->1s", "1s<->10s", "10s<->30s", "30s<->60s", ">60s", "Over 1s", "Over 10s"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.CENTER_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+ for i in xrange(2):
+ table.addRow((
+ "Overall Response" if i == 0 else "Response without Write",
+ (under10ms[i], safePercent(under10ms[i], totalRequests)),
+ (over10ms[i], safePercent(over10ms[i], totalRequests)),
+ (over100ms[i], safePercent(over100ms[i], totalRequests)),
+ (over1s[i], safePercent(over1s[i], totalRequests)),
+ (over10s[i], safePercent(over10s[i], totalRequests)),
+ (over30s[i], safePercent(over30s[i], totalRequests)),
+ (over60s[i], safePercent(over60s[i], totalRequests)),
+ safePercent(totalOver1s[i], totalRequests),
+ safePercent(totalOver10s[i], totalRequests),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
print
if errorCount:
print "Number of 500 errors: %d" % (errorCount,)
Modified: CalendarServer/branches/users/gaya/inviteclean/doc/Admin/ExtendedLogItems.txt
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/doc/Admin/ExtendedLogItems.txt 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/doc/Admin/ExtendedLogItems.txt 2012-08-09 19:03:13 UTC (rev 9547)
@@ -79,3 +79,32 @@
::
17.108.160.37 - scastillo [15/Sep/2009:20:10:23 +0000] "REPORT(CalDAV:calendar-multiget) /calendars/__uids__/B8CE9430-965B-11DE-B626-EC2E9DB52B69/calendar/ HTTP/1.1" 207 149285 "-" "DAVKit/4.0 (729); CalendarStore/4.0 (965); iCal/4.0 (1362); Mac OS X/10.6.1 (10B504)" i=8459 t=183.0 or=1 rcount=32
+
+
+
+**Fine-grained request time logging**
+
+If the configuration key EnableExtendedTimingAccessLog is set to true, additional key-value pairs will be logged with each request. The overall request time "t" is broken into four phases, and the elapsed time for each phase is logged. The new keys representing the four request phases are:
+
+ ``t-req-proc``
+
+ time elapsed from when a request object is created up until renderHTTP is about to be called.
+ This is the overhead of parsing the request headers and locating the target resource.
+
+ ``t-resp-gen``
+
+ time elapsed from t-req-proc up until the response is ready to write
+
+ ``t-resp-wr``
+
+ time elapsed from t-resp-gen up until response is written
+
+ ``t-log``
+
+ time from t-resp-wr up until log entry is ready to write to master
+
+A sample log line with EnableExtendedTimingAccessLog enabled is shown below:
+
+::
+
+ 17.209.103.42 - wsanchez [24/Jul/2012:17:51:29 +0000] "REPORT(CalDAV:calendar-multiget) /calendars/__uids__/F114CA1D-295F-42A5-A5BD-D1A1B19FC049/60E68E32-4C87-4E63-9BF2-12A25E8F2623/ HTTP/1.1" 207 114349 "-" "CalendarStore/5.0.2 (1166); iCal/5.0.2 (1571); Mac OS X/10.7.3 (11D50d)" i=7 or=1 t=764.7 t-req-proc=4.8 t-resp-gen=754.5 t-resp-wr=5.1 t-log=0.2 rcount=2
\ No newline at end of file
Deleted: CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/bytes-per-nclob-character.patch
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/bytes-per-nclob-character.patch 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/bytes-per-nclob-character.patch 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1,17 +0,0 @@
-Index: ExternalLobVar.c
-===================================================================
---- ExternalLobVar.c (revision 355)
-+++ ExternalLobVar.c (working copy)
-@@ -259,10 +259,9 @@
- amount = 1;
- }
- length = amount;
-- if (var->lobVar->type == &vt_CLOB)
-+ if ((var->lobVar->type == &vt_CLOB) ||
-+ (var->lobVar->type == &vt_NCLOB))
- bufferSize = amount * var->lobVar->environment->maxBytesPerCharacter;
-- else if (var->lobVar->type == &vt_NCLOB)
-- bufferSize = amount * 2;
- else bufferSize = amount;
-
- // create a string for retrieving the value
Copied: CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/nclob-fixes-and-prefetch.patch (from rev 9546, CalendarServer/trunk/lib-patches/cx_Oracle/nclob-fixes-and-prefetch.patch)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/nclob-fixes-and-prefetch.patch (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/lib-patches/cx_Oracle/nclob-fixes-and-prefetch.patch 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,97 @@
+Index: Connection.c
+===================================================================
+--- Connection.c 2011-03-19 16:05:30.000000000 -0700
++++ Connection.c 2012-08-01 09:22:17.000000000 -0700
+@@ -713,6 +713,19 @@
+ if (newPasswordObj)
+ return Connection_ChangePassword(self, self->password, newPasswordObj);
+
++ /* set lob prefetch attribute to session */
++ ub4 default_lobprefetch_size = 4096; /* Set default size to 4K */
++ status = OCIAttrSet (self->sessionHandle, (ub4) OCI_HTYPE_SESSION,
++ (void *)&default_lobprefetch_size, /* attribute value */
++ 0, /* attribute size; not required to specify; */
++ (ub4) OCI_ATTR_DEFAULT_LOBPREFETCH_SIZE,
++ self->environment->errorHandle);
++ if (Environment_CheckForError(self->environment, status,
++ "Connection_Connect(): OCI_ATTR_DEFAULT_LOBPREFETCH_SIZE") < 0) {
++ self->sessionHandle = NULL;
++ return -1;
++ }
++
+ // begin the session
+ Py_BEGIN_ALLOW_THREADS
+ status = OCISessionBegin(self->handle, self->environment->errorHandle,
+
+Index: Cursor.c
+===================================================================
+--- Cursor.c 2011-03-19 16:05:30.000000000 -0700
++++ Cursor.c 2012-08-01 09:15:53.000000000 -0700
+@@ -1813,8 +1813,8 @@
+ }
+ }
+ Py_BEGIN_ALLOW_THREADS
+- status = OCIStmtFetch(self->handle, self->environment->errorHandle,
+- numRows, OCI_FETCH_NEXT, OCI_DEFAULT);
++ status = OCIStmtFetch2(self->handle, self->environment->errorHandle,
++ numRows, OCI_FETCH_NEXT, 0, OCI_DEFAULT);
+ Py_END_ALLOW_THREADS
+ if (status != OCI_NO_DATA) {
+ if (Environment_CheckForError(self->environment, status,
+
+Index: ExternalLobVar.c
+===================================================================
+--- ExternalLobVar.c 2011-03-19 16:05:30.000000000 -0700
++++ ExternalLobVar.c 2012-07-31 14:26:16.000000000 -0700
+@@ -170,6 +170,8 @@
+ int offset) // offset
+ {
+ sword status;
++ oraub8 blength = 0;
++ oraub8 clength = *length;
+
+ if (var->lobVar->isFile) {
+ Py_BEGIN_ALLOW_THREADS
+@@ -183,11 +185,13 @@
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+- status = OCILobRead(var->lobVar->connection->handle,
++ status = OCILobRead2(var->lobVar->connection->handle,
+ var->lobVar->environment->errorHandle,
+- var->lobVar->data[var->pos], length, offset, buffer,
+- bufferSize, NULL, NULL, 0, var->lobVar->type->charsetForm);
++ var->lobVar->data[var->pos], &blength, &clength, offset, buffer,
++ bufferSize, OCI_ONE_PIECE, NULL, NULL, 0, var->lobVar->type->charsetForm);
+ Py_END_ALLOW_THREADS
++ *length = blength;
++
+ if (Environment_CheckForError(var->lobVar->environment, status,
+ "ExternalLobVar_LobRead()") < 0) {
+ OCILobFileClose(var->lobVar->connection->handle,
+@@ -219,10 +223,10 @@
+ udt_ExternalLobVar *var) // variable to return the size of
+ {
+ sword status;
+- ub4 length;
++ oraub8 length;
+
+ Py_BEGIN_ALLOW_THREADS
+- status = OCILobGetLength(var->lobVar->connection->handle,
++ status = OCILobGetLength2(var->lobVar->connection->handle,
+ var->lobVar->environment->errorHandle,
+ var->lobVar->data[var->pos], &length);
+ Py_END_ALLOW_THREADS
+@@ -259,10 +263,9 @@
+ amount = 1;
+ }
+ length = amount;
+- if (var->lobVar->type == &vt_CLOB)
++ if ((var->lobVar->type == &vt_CLOB) || (var->lobVar->type == &vt_NCLOB))
++ // Always use environment setting for character LOBs
+ bufferSize = amount * var->lobVar->environment->maxBytesPerCharacter;
+- else if (var->lobVar->type == &vt_NCLOB)
+- bufferSize = amount * 2;
+ else bufferSize = amount;
+
+ // create a string for retrieving the value
Modified: CalendarServer/branches/users/gaya/inviteclean/support/build.sh
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/support/build.sh 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/support/build.sh 2012-08-09 19:03:13 UTC (rev 9547)
@@ -698,7 +698,7 @@
"http://pypi.python.org/packages/source/p/pyOpenSSL/${po}.tar.gz";
if type -P krb5-config > /dev/null; then
- py_dependency -r 8679 \
+ py_dependency -r 9409 \
"PyKerberos" "kerberos" "PyKerberos" \
"${svn_uri_base}/PyKerberos/trunk";
fi;
@@ -743,7 +743,7 @@
"${pypi}/p/python-ldap/${ld}.tar.gz";
# XXX actually PyCalendar should be imported in-place.
- py_dependency -fe -i "src" -r 204 \
+ py_dependency -fe -i "src" -r 209 \
"pycalendar" "pycalendar" "pycalendar" \
"http://svn.mulberrymail.com/repos/PyCalendar/branches/server";
Modified: CalendarServer/branches/users/gaya/inviteclean/support/submit
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/support/submit 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/support/submit 2012-08-09 19:03:13 UTC (rev 9547)
@@ -74,15 +74,15 @@
if "${install}"; then usage "-i flag requires -b"; fi;
if "${package}"; then usage "-p flag requires -b"; fi;
- if [ $# == 0 ]; then usage "No releases specified"; fi;
- releases="$@"; shift $#;
-
if ! "${submission_enabled}"; then
echo "Submissions from this branch are not enabled.";
exit 1;
fi;
fi;
+if [ $# == 0 ]; then usage "No releases specified"; fi;
+releases="$@"; shift $#;
+
if [ $# != 0 ]; then usage "Unrecognized arguments:" "$@"; fi;
project="CalendarServer";
@@ -174,9 +174,14 @@
merge_flags="";
fi;
+ release_flags="";
+ for release in "${releases}"; do
+ release_flags="${release_flags} -release ${release}";
+ done;
+
sudo buildit "${wc}" \
$(file /System/Library/Frameworks/Python.framework/Versions/Current/Python | sed -n -e 's|^.*(for architecture \([^)][^)]*\).*$|-arch \1|p' | sed 's|ppc7400|ppc|') \
- ${merge_flags};
+ ${merge_flags}${release_flags};
if "${package}"; then
package_file="${project_version}.tgz";
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/syntax.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/syntax.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/syntax.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -324,12 +324,24 @@
"SQL expressions should not be tested for truth value in Python.")
- def In(self, subselect):
- # Can't be Select.__contains__ because __contains__ gets __nonzero__
- # called on its result by the 'in' syntax.
- return CompoundComparison(self, 'in', subselect)
+ def In(self, other):
+ """
+ We support two forms of the SQL "IN" syntax: one where a list of values is supplied, the other where
+ a sub-select is used to provide a set of values.
+ @param other: a constant parameter or sub-select
+ @type other: L{Parameter} or L{Select}
+ """
+ if isinstance(other, Parameter):
+ if other.count is None:
+ raise DALError("IN expression needs an explicit count of parameters")
+ return CompoundComparison(self, 'in', Constant(other))
+ else:
+ # Can't be Select.__contains__ because __contains__ gets __nonzero__
+ # called on its result by the 'in' syntax.
+ return CompoundComparison(self, 'in', other)
+
def StartsWith(self, other):
return CompoundComparison(self, "like", CompoundComparison(Constant(other), '||', Constant('%')))
@@ -368,6 +380,10 @@
class Constant(ExpressionSyntax):
+ """
+ Generates an expression for a place holder where a value will be bound to the query. If the constant is a Parameter
+ with count > 1 then a parenthesized, comma-separated list of place holders will be generated.
+ """
def __init__(self, value):
self.value = value
@@ -377,7 +393,12 @@
def subSQL(self, queryGenerator, allTables):
- return SQLFragment(queryGenerator.placeholder.placeholder(), [self.value])
+ if isinstance(self.value, Parameter) and self.value.count is not None:
+ return _inParens(_CommaList(
+ [SQLFragment(queryGenerator.placeholder.placeholder(), [self.value] if ctr == 0 else []) for ctr in range(self.value.count)]
+ ).subSQL(queryGenerator, allTables))
+ else:
+ return SQLFragment(queryGenerator.placeholder.placeholder(), [self.value])
@@ -1592,7 +1613,13 @@
params = []
for parameter in self.parameters:
if isinstance(parameter, Parameter):
- params.append(kw[parameter.name])
+ if parameter.count is not None:
+ if parameter.count != len(kw[parameter.name]):
+ raise DALError("Number of place holders does not match number of items to bind")
+ for item in kw[parameter.name]:
+ params.append(item)
+ else:
+ params.append(kw[parameter.name])
else:
params.append(parameter)
return SQLFragment(self.text, params)
@@ -1626,15 +1653,23 @@
class Parameter(object):
+ """
+ Used to represent a place holder for a value to be bound to the query
+ at a later date. If count > 1, then a "set" of parenthesized,
+ comma separate place holders will be generated.
+ """
- def __init__(self, name):
+ def __init__(self, name, count=None):
self.name = name
+ self.count = count
+ if self.count is not None and self.count < 1:
+ raise DALError("Must have Parameter.count > 0")
def __eq__(self, param):
if not isinstance(param, Parameter):
return NotImplemented
- return self.name == param.name
+ return self.name == param.name and self.count == param.count
def __ne__(self, param):
@@ -1647,6 +1682,7 @@
return 'Parameter(%r)' % (self.name,)
+
# Common helpers:
# current timestamp in UTC format. Hack to support standard syntax for this,
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/test/test_sqlsyntax.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/test/test_sqlsyntax.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/enterprise/dal/test/test_sqlsyntax.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -683,7 +683,8 @@
def test_inSubSelect(self):
"""
- L{ColumnSyntax.In} returns a sub-expression using the SQL 'in' syntax.
+ L{ColumnSyntax.In} returns a sub-expression using the SQL 'in' syntax with
+ a sub-select.
"""
wherein = (self.schema.FOO.BAR.In(
Select([self.schema.BOZ.QUX], From=self.schema.BOZ)))
@@ -693,6 +694,79 @@
"select * from FOO where BAR in (select QUX from BOZ)"))
+ def test_inParameter(self):
+ """
+ L{ColumnSyntax.In} returns a sub-expression using the SQL 'in' syntax with
+ parameter list.
+ """
+
+ # One item with IN only
+ items = set(('A',))
+ self.assertEquals(
+ Select(From=self.schema.FOO, Where=self.schema.FOO.BAR.In(Parameter("names", len(items)))).toSQL().bind(names=items),
+ SQLFragment(
+ "select * from FOO where BAR in (?)", ['A']))
+
+ # Two items with IN only
+ items = set(('A', 'B'))
+ self.assertEquals(
+ Select(From=self.schema.FOO, Where=self.schema.FOO.BAR.In(Parameter("names", len(items)))).toSQL().bind(names=items),
+ SQLFragment(
+ "select * from FOO where BAR in (?, ?)", ['A', 'B']))
+
+ # Two items with preceding AND
+ self.assertEquals(
+ Select(
+ From=self.schema.FOO,
+ Where=(self.schema.FOO.BAZ == Parameter('P1')).And(
+ self.schema.FOO.BAR.In(Parameter("names", len(items))
+ ))
+ ).toSQL().bind(P1="P1", names=items),
+ SQLFragment(
+ "select * from FOO where BAZ = ? and BAR in (?, ?)", ['P1', 'A', 'B']),
+ )
+
+ # Two items with following AND
+ self.assertEquals(
+ Select(
+ From=self.schema.FOO,
+ Where=(self.schema.FOO.BAR.In(Parameter("names", len(items))).And(
+ self.schema.FOO.BAZ == Parameter('P2')
+ ))
+ ).toSQL().bind(P2="P2", names=items),
+ SQLFragment(
+ "select * from FOO where BAR in (?, ?) and BAZ = ?", ['A', 'B', 'P2']),
+ )
+
+ # Two items with preceding OR and following AND
+ self.assertEquals(
+ Select(
+ From=self.schema.FOO,
+ Where=(self.schema.FOO.BAZ == Parameter('P1')).Or(
+ self.schema.FOO.BAR.In(Parameter("names", len(items))).And(
+ self.schema.FOO.BAZ == Parameter('P2')
+ ))
+ ).toSQL().bind(P1="P1", P2="P2", names=items),
+ SQLFragment(
+ "select * from FOO where BAZ = ? or BAR in (?, ?) and BAZ = ?", ['P1', 'A', 'B', 'P2']),
+ )
+
+ # Check various error situations
+
+ # No count not allowed
+ self.assertRaises(DALError, self.schema.FOO.BAR.In, Parameter("names"))
+
+ # count=0 not allowed
+ self.assertRaises(DALError, Parameter,"names", 0)
+
+ # Mismatched count and len(items)
+ self.assertRaises(
+ DALError,
+ Select(From=self.schema.FOO, Where=self.schema.FOO.BAR.In(Parameter("names", len(items)))).toSQL().bind,
+ names=["a", "b", "c",]
+ )
+
+
def test_max(self):
"""
L{Max}C{(column)} produces an object in the 'columns' clause that
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/method/report.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/method/report.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/method/report.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -44,6 +44,7 @@
from twext.web2.dav.util import davXMLFromStream
from txdav.xml import element as davxml
from txdav.xml.element import lookupElement
+from txdav.xml.base import encodeXMLName
log = Logger()
@@ -115,7 +116,7 @@
if namespace == davxml.dav_namespace:
request.submethod = "DAV:" + name
else:
- request.submethod = "{%s}%s" % (namespace, name)
+ request.submethod = encodeXMLName(namespace, name)
else:
method_name = to_method(name)
@@ -136,8 +137,8 @@
#
# Requested report is not supported.
#
- log.err("Unsupported REPORT {%s}%s for resource %s (no method %s)"
- % (namespace, name, self, method_name))
+ log.err("Unsupported REPORT %s for resource %s (no method %s)"
+ % (encodeXMLName(namespace, name), self, method_name))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/noneprops.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/noneprops.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/noneprops.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -33,7 +33,9 @@
from twext.web2 import responsecode
from twext.web2.http import HTTPError, StatusResponse
+from txdav.xml.base import encodeXMLName
+
class NonePropertyStore (object):
"""
DAV property store which contains no properties and does not allow
@@ -50,10 +52,16 @@
pass
def get(self, qname, uid=None):
- raise HTTPError(StatusResponse(responsecode.NOT_FOUND, "No such property: {%s}%s" % qname))
+ raise HTTPError(StatusResponse(
+ responsecode.NOT_FOUND,
+ "No such property: %s" % (encodeXMLName(*qname),)
+ ))
def set(self, property, uid=None):
- raise HTTPError(StatusResponse(responsecode.FORBIDDEN, "Permission denied for setting property: %s" % (property,)))
+ raise HTTPError(StatusResponse(
+ responsecode.FORBIDDEN,
+ "Permission denied for setting property: %s" % (property,)
+ ))
def delete(self, qname, uid=None):
# RFC 2518 Section 12.13.1 says that removal of
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/resource.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/resource.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/resource.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -57,6 +57,12 @@
from twisted.internet import reactor
from twext.python.log import Logger
+from txdav.xml import element
+from txdav.xml.base import encodeXMLName
+from txdav.xml.element import WebDAVElement, WebDAVEmptyElement, WebDAVTextElement
+from txdav.xml.element import dav_namespace
+from txdav.xml.element import twisted_dav_namespace, twisted_private_namespace
+from txdav.xml.element import registerElement, lookupElement
from twext.web2 import responsecode
from twext.web2.http import HTTPError, RedirectResponse, StatusResponse
from twext.web2.http_headers import generateContentType
@@ -70,11 +76,6 @@
from twext.web2.dav.noneprops import NonePropertyStore
from twext.web2.dav.util import unimplemented, parentForURL, joinURL
from twext.web2.dav.auth import PrincipalCredentials
-from txdav.xml import element
-from txdav.xml.element import WebDAVElement, WebDAVEmptyElement, WebDAVTextElement
-from txdav.xml.element import dav_namespace
-from txdav.xml.element import twisted_dav_namespace, twisted_private_namespace
-from txdav.xml.element import registerElement, lookupElement
log = Logger()
@@ -217,7 +218,7 @@
def defer():
if type(property) is tuple:
qname = property
- sname = "{%s}%s" % property
+ sname = encodeXMLName(*property)
else:
qname = property.qname()
sname = property.sname()
@@ -416,7 +417,7 @@
def defer():
if type(property) is tuple:
qname = property
- sname = "{%s}%s" % property
+ sname = encodeXMLName(*property)
else:
qname = property.qname()
sname = property.sname()
@@ -680,7 +681,7 @@
@inlineCallbacks
def findChildrenFaster(
- self, depth, request, okcallback, badcallback,
+ self, depth, request, okcallback, badcallback, missingcallback,
names, privileges, inherited_aces
):
"""
@@ -698,6 +699,8 @@
that pass the privilege check, or C{None}
@param badcallback: a callback function used on all resources
that fail the privilege check, or C{None}
+ @param missingcallback: a callback function used on all resources
+ that are missing, or C{None}
@param names: a C{list} of C{str}'s containing the names of
the child resources to lookup. If empty or C{None} all
children will be examined, otherwise only the ones in the
@@ -738,6 +741,10 @@
else:
children.append((child, childpath))
+ if missingcallback:
+ for name in set(names1) - set(childnames):
+ missingcallback(joinURL(basepath, urllib.quote(name)))
+
# Generate (acl,supported_privs) map
aclmap = {}
for resource, url in children:
@@ -779,7 +786,7 @@
yield collection.inheritedACEsforChildren(request)
)
yield collection.findChildrenFaster(
- depth, request, okcallback, badcallback,
+ depth, request, okcallback, badcallback, missingcallback,
child_collections[collection_name] if names else None, privileges,
inherited_aces=collection_inherited_aces
)
@@ -1873,9 +1880,9 @@
return None
if not isinstance(principal, element.Principal):
- log.err("Non-principal value in property {%s}%s "
+ log.err("Non-principal value in property %s "
"referenced by property principal."
- % (namespace, name))
+ % (encodeXMLName(namespace, name),))
return None
if len(principal.children) != 1:
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/test_xattrprops.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/test_xattrprops.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/test_xattrprops.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -132,7 +132,7 @@
attribute = (
self.propertyStore.deadPropertyXattrPrefix +
(uid if uid is not None else "") +
- "{%s}%s" % element.qname())
+ element.sname())
self.attrs[attribute] = value
@@ -141,7 +141,7 @@
attribute = (
self.propertyStore.deadPropertyXattrPrefix +
(uid if uid is not None else "") +
- "{%s}%s" % element.qname())
+ element.sname())
return self.attrs[attribute]
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/util.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/util.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/test/util.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -38,6 +38,7 @@
from twext.web2.dav.static import DAVFile
from twext.web2.dav.util import joinURL
from txdav.xml import element
+from txdav.xml.base import encodeXMLName
log = Logger()
@@ -57,7 +58,7 @@
except KeyError:
raise HTTPError(StatusResponse(
responsecode.NOT_FOUND,
- "No such property: {%s}%s" % qname
+ "No such property: %s" % (encodeXMLName(*qname),)
))
doc = element.WebDAVDocument.fromString(property)
Modified: CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/xattrprops.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/xattrprops.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twext/web2/dav/xattrprops.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -50,10 +50,11 @@
from twisted.python.util import untilConcludes
from twisted.python.failure import Failure
from twisted.python.log import err
+from txdav.xml.base import encodeXMLName
+from txdav.xml.parser import WebDAVDocument
from twext.web2 import responsecode
from twext.web2.http import HTTPError, StatusResponse
from twext.web2.dav.http import statusForFailure
-from txdav.xml.parser import WebDAVDocument
# RFC 2518 Section 12.13.1 says that removal of non-existing property
# is not an error. python-xattr on Linux fails with ENODATA in this
@@ -86,7 +87,7 @@
deadPropertyXattrPrefix = "user."
def _encode(clazz, name, uid=None):
- result = urllib.quote("{%s}%s" % name, safe='{}:')
+ result = urllib.quote(encodeXMLName(*name), safe='{}:')
if uid:
result = uid + result
r = clazz.deadPropertyXattrPrefix + result
@@ -138,17 +139,20 @@
data = self.attrs.get(self._encode(qname, uid))
except KeyError:
raise HTTPError(StatusResponse(
- responsecode.NOT_FOUND,
- "No such property: {%s}%s" % qname))
+ responsecode.NOT_FOUND,
+ "No such property: %s" % (encodeXMLName(*qname),)
+ ))
except IOError, e:
if e.errno in _ATTR_MISSING or e.errno == errno.ENOENT:
raise HTTPError(StatusResponse(
- responsecode.NOT_FOUND,
- "No such property: {%s}%s" % qname))
+ responsecode.NOT_FOUND,
+ "No such property: %s" % (encodeXMLName(*qname),)
+ ))
else:
raise HTTPError(StatusResponse(
- statusForFailure(Failure()),
- "Unable to read property: {%s}%s" % qname))
+ statusForFailure(Failure()),
+ "Unable to read property: %s" % (encodeXMLName(*qname),)
+ ))
#
# Unserialize XML data from an xattr. The storage format has changed
@@ -175,8 +179,8 @@
try:
doc = unpickle(data)
except UnpicklingError:
- format = "Invalid property value stored on server: {%s}%s %s"
- msg = format % (qname[0], qname[1], data)
+ format = "Invalid property value stored on server: %s %s"
+ msg = format % (encodeXMLName(*qname), data)
err(None, msg)
raise HTTPError(
StatusResponse(responsecode.INTERNAL_SERVER_ERROR, msg))
@@ -225,10 +229,10 @@
if e.errno not in _ATTR_MISSING:
raise
except:
- raise HTTPError(
- StatusResponse(
- statusForFailure(Failure()),
- "Unable to delete property: " + key))
+ raise HTTPError(StatusResponse(
+ statusForFailure(Failure()),
+ "Unable to delete property: %s", (key,)
+ ))
def contains(self, qname, uid=None):
@@ -252,10 +256,10 @@
except IOError, e:
if e.errno in _ATTR_MISSING or e.errno == errno.ENOENT:
return False
- raise HTTPError(
- StatusResponse(
- statusForFailure(Failure()),
- "Unable to read property: " + key))
+ raise HTTPError(StatusResponse(
+ statusForFailure(Failure()),
+ "Unable to read property: %s" % (key,)
+ ))
else:
return True
@@ -277,10 +281,10 @@
except IOError, e:
if e.errno == errno.ENOENT:
return []
- raise HTTPError(
- StatusResponse(
- statusForFailure(Failure()),
- "Unable to list properties: " + self.resource.fp.path))
+ raise HTTPError(StatusResponse(
+ statusForFailure(Failure()),
+ "Unable to list properties: %s", (self.resource.fp.path,)
+ ))
else:
results = [
self._decode(name)
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/__init__.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/__init__.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/__init__.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -65,5 +65,5 @@
PyCalendar.setPRODID("-//CALENDARSERVER.ORG//NONSGML Version 1//EN")
# These are properties we use directly and we want the default value type set for TEXT
-PyCalendarProperty.regsiterDefaultValue("X-CALENDARSERVER-PRIVATE-COMMENT", PyCalendarValue.VALUETYPE_TEXT)
-PyCalendarProperty.regsiterDefaultValue("X-CALENDARSERVER-ATTENDEE-COMMENT", PyCalendarValue.VALUETYPE_TEXT)
+PyCalendarProperty.registerDefaultValue("X-CALENDARSERVER-PRIVATE-COMMENT", PyCalendarValue.VALUETYPE_TEXT)
+PyCalendarProperty.registerDefaultValue("X-CALENDARSERVER-ATTENDEE-COMMENT", PyCalendarValue.VALUETYPE_TEXT)
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/customxml.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/customxml.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/customxml.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -415,6 +415,14 @@
@registerElement
+class PubSubAPSRefreshIntervalProperty (WebDAVTextElement):
+ namespace = calendarserver_namespace
+ name = "refresh-interval"
+ protected = True
+ hidden = True
+
+
+ at registerElement
class PubSubXMPPPushKeyProperty (WebDAVTextElement):
namespace = calendarserver_namespace
name = "pushkey"
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/directory.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/directory.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/directory.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -30,6 +30,7 @@
import datetime
import os
+import signal
import sys
import types
import pwd, grp
@@ -827,7 +828,7 @@
def postOptions(self):
config.load(self['config'])
config.updateDefaults(self.overrides)
- self.parent['pidfile'] = None
+ self.parent['pidfile'] = config.PIDFile
@@ -851,36 +852,73 @@
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
+
self.updateSeconds = updateSeconds
self.nextUpdate = None
+ self.updateInProgress = False
+ self.updateAwaiting = False
+
if updateMethod:
self.updateMethod = updateMethod
else:
self.updateMethod = self.updater.updateCache
def startService(self):
+ self.previousHandler = signal.signal(signal.SIGHUP, self.sighupHandler)
self.log_warn("Starting group membership cacher service")
service.Service.startService(self)
return self.update()
- @inlineCallbacks
- def update(self):
- self.nextUpdate = None
- try:
- yield self.updateMethod()
- finally:
- self.log_info("Scheduling next group membership update")
- self.nextUpdate = self.reactor.callLater(self.updateSeconds,
- self.update)
+ def sighupHandler(self, num, frame):
+ self.reactor.callFromThread(self.update)
def stopService(self):
+ signal.signal(signal.SIGHUP, self.previousHandler)
self.log_warn("Stopping group membership cacher service")
service.Service.stopService(self)
if self.nextUpdate is not None:
self.nextUpdate.cancel()
+ self.nextUpdate = None
+ @inlineCallbacks
+ def update(self):
+ """
+ A wrapper around updateCache, this method manages the scheduling of the
+ subsequent update, as well as prevents multiple updates from running
+ simultaneously, which could otherwise happen because SIGHUP now triggers
+ an update on demand. If update is called while an update is in progress,
+ as soon as the first update is finished a new one is started. Otherwise,
+ when an update finishes and there is not another one waiting, the next
+ update is scheduled for updateSeconds in the future.
+ @return: True if an update was already in progress, False otherwise
+ @rtype: C{bool}
+ """
+ self.log_debug("Group membership update called")
+
+ # A call to update while an update is in progress sets the updateAwaiting flag
+ # so that an update happens again right after the current one is complete.
+ if self.updateInProgress:
+ self.updateAwaiting = True
+ returnValue(True)
+
+ self.nextUpdate = None
+ self.updateInProgress = True
+ self.updateAwaiting = False
+ try:
+ yield self.updateMethod()
+ finally:
+ self.updateInProgress = False
+ if self.updateAwaiting:
+ self.log_info("Performing group membership update")
+ yield self.update()
+ else:
+ self.log_info("Scheduling next group membership update")
+ self.nextUpdate = self.reactor.callLater(self.updateSeconds,
+ self.update)
+ returnValue(False)
+
class GroupMembershipCacherServiceMaker(LoggingMixIn):
"""
Configures and returns a GroupMembershipCacherService
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/ldapdirectory.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/ldapdirectory.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -851,7 +851,7 @@
return record
- def queryDirectory(self, recordTypes, indexType, indexKey):
+ def queryDirectory(self, recordTypes, indexType, indexKey, queryMethod=None):
"""
Queries the LDAP directory for the record which has an attribute value
matching the indexType and indexKey parameters.
@@ -864,6 +864,10 @@
Nothing is returned -- the resulting record (if any) is placed in
the cache.
"""
+
+ if queryMethod is None:
+ queryMethod = self.timedSearch
+
self.log_debug("LDAP query for types %s, indexType %s and indexKey %s"
% (recordTypes, indexType, indexKey))
@@ -907,11 +911,19 @@
# emailAddresses can map to multiple LDAP fields
ldapFields = self.rdnSchema[recordType]["mapping"]["emailAddresses"]
if isinstance(ldapFields, str):
- subfilter = "(%s=%s)" % (ldapFields, ldapEsc(email))
+ if ldapFields:
+ subfilter = "(%s=%s)" % (ldapFields, ldapEsc(email))
+ else:
+ continue # No LDAP attribute assigned for emailAddresses
+
else:
subfilter = []
for ldapField in ldapFields:
- subfilter.append("(%s=%s)" % (ldapField, ldapEsc(email)))
+ if ldapField:
+ subfilter.append("(%s=%s)" % (ldapField, ldapEsc(email)))
+ if not subfilter:
+ continue # No LDAP attribute assigned for emailAddresses
+
subfilter = "(|%s)" % ("".join(subfilter))
filterstr = "(&%s%s)" % (filterstr, subfilter)
@@ -921,7 +933,7 @@
# Query the LDAP server
self.log_debug("Retrieving ldap record with base %s and filter %s." %
(ldap.dn.dn2str(base), filterstr))
- result = self.timedSearch(ldap.dn.dn2str(base),
+ result = queryMethod(ldap.dn.dn2str(base),
ldap.SCOPE_SUBTREE, filterstr=filterstr, attrlist=self.attrlist)
if result:
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_directory.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_directory.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_directory.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -144,8 +144,16 @@
service.stopService()
+ service.updateInProgress = True
+ self.assertTrue((yield service.update()))
+ self.assertTrue(service.updateAwaiting)
+ service.updateInProgress = False
+ self.assertFalse((yield service.update()))
+ self.assertFalse(service.updateAwaiting)
+
+
def test_expandedMembers(self):
"""
Make sure expandedMembers( ) returns a complete, flattened set of
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_ldapdirectory.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_ldapdirectory.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/directory/test/test_ldapdirectory.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -153,7 +153,7 @@
"operand" : "or",
"recordType" : "groups",
"expected" : None,
- "optimize" : "(&(uid=*)(generateduid=*)(|(cn=mor*)(cn=sag*)(mail=mor*)(mail=sag*)(givenName=mor*)(givenName=sag*)(sn=mor*)(sn=sag*)))",
+ "optimize" : True,
},
{
"fields" : [
@@ -438,7 +438,7 @@
"mapping": { # maps internal record names to LDAP
"recordName": "cn",
"fullName" : "cn",
- "emailAddresses" : ["mail", "emailAliases"],
+ "emailAddresses" : "", # old style, single string
"firstName" : "givenName",
"lastName" : "sn",
},
@@ -453,7 +453,7 @@
"mapping": { # maps internal record names to LDAP
"recordName": "cn",
"fullName" : "cn",
- "emailAddresses" : ["mail", "emailAliases"],
+ "emailAddresses" : [], # new style, array
"firstName" : "givenName",
"lastName" : "sn",
},
@@ -823,3 +823,32 @@
"uid=foo,cn=us ers,dc=exa mple,dc=com"),
):
self.assertEquals(expected, normalizeDNstr(input))
+
+ def test_queryDirectory(self):
+ """
+ Verify queryDirectory skips LDAP queries where there has been no
+ LDAP attribute mapping provided for the given index type.
+ """
+
+ self.history = []
+
+ def stubSearchMethod(base, scope, filterstr="(objectClass=*)",
+ attrlist=None, timeoutSeconds=-1, resultLimit=0):
+ self.history.append((base, scope, filterstr))
+
+ recordTypes = [
+ self.service.recordType_users,
+ self.service.recordType_groups,
+ self.service.recordType_locations,
+ self.service.recordType_resources,
+ ]
+ self.service.queryDirectory(
+ recordTypes,
+ self.service.INDEX_TYPE_CUA,
+ "mailto:test at example.com",
+ queryMethod=stubSearchMethod
+ )
+ self.assertEquals(
+ self.history,
+ [('cn=users,dc=example,dc=com', 2, '(&(!(objectClass=organizationalUnit))(|(mail=test at example.com)(emailAliases=test at example.com)))'), ('cn=groups,dc=example,dc=com', 2, '(&(!(objectClass=organizationalUnit))(|(mail=test at example.com)(emailAliases=test at example.com)))')]
+ )
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/extensions.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/extensions.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/extensions.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -46,6 +46,7 @@
from twext.web2.stream import FileStream
from twext.web2.static import MetaDataMixin, StaticRenderMixin
from txdav.xml import element
+from txdav.xml.base import encodeXMLName
from txdav.xml.element import dav_namespace
from twext.web2.dav.http import MultiStatusResponse
from twext.web2.dav.static import DAVFile as SuperDAVFile
@@ -339,8 +340,8 @@
whenAllProperties = gatherResults([
maybeDeferred(self.resource.readProperty, qn, request)
.addCallback(lambda p, iqn=qn: (p.sname(), p.toxml())
- if p is not None else ("{%s}%s" % iqn, None) )
- .addErrback(gotError, "{%s}%s" % qn)
+ if p is not None else (encodeXMLName(*iqn), None) )
+ .addErrback(gotError, encodeXMLName(*qn))
for qn in sorted(qnames)
])
@@ -771,7 +772,7 @@
HTTPError.__init__(self,
StatusResponse(
responsecode.NOT_FOUND,
- "No such property: {%s}%s" % qname
+ "No such property: %s" % encodeXMLName(*qname)
)
)
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/memcacheprops.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/memcacheprops.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/memcacheprops.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -36,6 +36,7 @@
from twext.python.memcacheclient import ClientFactory
from twext.python.memcacheclient import MemcacheError, TokenMismatchError
from twext.python.filepath import CachingFilePath as FilePath
+from txdav.xml.base import encodeXMLName
from twext.web2 import responsecode
from twext.web2.http import HTTPError, StatusResponse
@@ -293,11 +294,10 @@
else:
log.error("memcacheprops setProperty had too many failures")
delattr(self, "_propertyCache")
- raise MemcacheError("Unable to %s property %s{%s}%s on %s" % (
+ raise MemcacheError("Unable to %s property %s%s on %s" % (
"delete" if delete else "set",
uid if uid else "",
- qname[0],
- qname[1],
+ encodeXMLName(*qname),
child
))
@@ -345,7 +345,7 @@
else:
raise HTTPError(StatusResponse(
responsecode.NOT_FOUND,
- "No such property: %s{%s}%s" % (uid if uid else "", qname[0], qname[1],)
+ "No such property: %s%s" % (uid if uid else "", encodeXMLName(*qname))
))
self.log_debug("Read for %s%s on %s" % (
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/propfind.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/propfind.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/propfind.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -138,6 +138,7 @@
lambda x, y: resources.append((True, x, y)),
lambda x, y: resources.append((False, x, y)),
None,
+ None,
(davxml.Read(),),
inherited_aces=filtered_aces,
)
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -33,6 +33,7 @@
from twext.web2.http import HTTPError, StatusResponse
from twext.web2.dav.util import davXMLFromStream
from txdav.xml import element as davxml
+from txdav.xml.base import encodeXMLName
from txdav.xml.element import lookupElement
from twext.python.log import Logger
@@ -78,7 +79,7 @@
elif namespace == caldavxml.caldav_namespace:
request.submethod = "CalDAV:" + name
else:
- request.submethod = "{%s}%s" % (namespace, name)
+ request.submethod = encodeXMLName(namespace, name)
else:
request.submethod = name
@@ -114,8 +115,8 @@
#
# Requested report is not supported.
#
- log.err("Unsupported REPORT {%s}%s for resource %s (no method %s)"
- % (namespace, name, self, method_name))
+ log.err("Unsupported REPORT %s for resource %s (no method %s)"
+ % (encodeXMLName(namespace, name), self, method_name))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_addressbook_query.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_addressbook_query.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -248,6 +248,7 @@
request,
lambda x, y: ok_resources.append((x, y)),
None,
+ None,
names,
(davxml.Read(),),
inherited_aces=filteredaces
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_calendar_query.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_calendar_query.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_calendar_query.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -207,6 +207,7 @@
request,
lambda x, y: ok_resources.append((x, y)),
None,
+ None,
names,
(davxml.Read(),),
inherited_aces=filteredaces
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_multiget_common.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_multiget_common.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -28,10 +28,9 @@
from txdav.xml.base import dav_namespace
from twext.web2.dav.http import ErrorResponse, MultiStatusResponse
from twext.web2.dav.resource import AccessDeniedError
-from twext.web2.dav.util import joinURL
from twext.web2.http import HTTPError, StatusResponse
-from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
+from twisted.internet.defer import inlineCallbacks, returnValue
from twistedcaldav import carddavxml
from twistedcaldav.caldavxml import caldav_namespace
@@ -180,37 +179,24 @@
for href in resources:
resource_uri = str(href)
name = unquote(resource_uri[resource_uri.rfind("/") + 1:])
- child = (yield maybeDeferred(self.getChild, name))
- if not self._isChildURI(request, resource_uri) or child is None or not child.exists():
- responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
+ if not self._isChildURI(request, resource_uri):
+ responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.BAD_REQUEST)))
else:
valid_names.append(name)
if not valid_names:
returnValue(None)
- # Verify that valid requested resources are calendar objects
- exists_names = tuple(
- (yield self.index().resourcesExist(valid_names))
- )
- checked_names = []
- for name in valid_names:
- if name not in exists_names:
- href = davxml.HRef.fromString(joinURL(request.uri, name))
- responses.append(davxml.StatusResponse(href, davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
- else:
- checked_names.append(name)
- if not checked_names:
- returnValue(None)
-
# Now determine which valid resources are readable and which are not
ok_resources = []
bad_resources = []
+ missing_resources = []
yield self.findChildrenFaster(
"1",
request,
lambda x, y: ok_resources.append((x, y)),
lambda x, y: bad_resources.append((x, y)),
- checked_names,
+ lambda x: missing_resources.append(x),
+ valid_names,
(davxml.Read(),),
inherited_aces=filteredaces
)
@@ -244,6 +230,10 @@
# Indicate error for all valid non-readable resources
for ignore_resource, href in bad_resources:
responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.FORBIDDEN)))
+
+ # Indicate error for all missing resources
+ for href in missing_resources:
+ responses.append(davxml.StatusResponse(davxml.HRef.fromString(href), davxml.Status.fromResponseCode(responsecode.NOT_FOUND)))
@inlineCallbacks
def doDirectoryAddressBookResponse():
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_sync_collection.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_sync_collection.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/method/report_sync_collection.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -128,6 +128,7 @@
request,
lambda x, y: ok_resources.append((x, y)),
lambda x, y: forbidden_resources.append((x, y)),
+ None,
changed,
(element.Read(),),
inherited_aces=filteredaces
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/notify.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/notify.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/notify.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1310,6 +1310,7 @@
url = "http://%s:%s/%s" % (config.ServerHostName, config.HTTPPort,
applePushSettings.SubscriptionURL)
settings["SubscriptionURL"] = url
+ settings["SubscriptionRefreshIntervalSeconds"] = applePushSettings.SubscriptionRefreshIntervalSeconds
settings["APSEnvironment"] = applePushSettings.Environment
return settings
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/resource.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/resource.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -1908,7 +1908,7 @@
elif name == "calendar-user-address-set":
returnValue(caldavxml.CalendarUserAddressSet(
- *[element.HRef(uri) for uri in self.calendarUserAddresses()]
+ *[element.HRef(uri) for uri in sorted(self.calendarUserAddresses())]
))
elif name == "schedule-inbox-URL":
@@ -2190,7 +2190,7 @@
@inlineCallbacks
def findChildrenFaster(
- self, depth, request, okcallback, badcallback,
+ self, depth, request, okcallback, badcallback, missingcallback,
names, privileges, inherited_aces
):
"""
@@ -2201,7 +2201,7 @@
yield self._newStoreHome.loadChildren()
result = (yield super(CommonHomeResource, self).findChildrenFaster(
- depth, request, okcallback, badcallback, names, privileges, inherited_aces
+ depth, request, okcallback, badcallback, missingcallback, names, privileges, inherited_aces
))
returnValue(result)
@@ -2285,7 +2285,7 @@
if qname == customxml.MaxCollections.qname() and config.MaxCollectionsPerHome:
returnValue(customxml.MaxCollections.fromString(config.MaxCollectionsPerHome))
-
+
elif qname == (customxml.calendarserver_namespace, "push-transports"):
if (config.Notifications.Services.XMPPNotifier.Enabled or
@@ -2312,6 +2312,9 @@
customxml.PubSubAPSEnvironmentProperty(
apsConfiguration["APSEnvironment"]
),
+ customxml.PubSubAPSRefreshIntervalProperty(
+ str(apsConfiguration["SubscriptionRefreshIntervalSeconds"])
+ ),
type="APSD",
)
)
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/implicit.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/implicit.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/implicit.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -404,6 +404,7 @@
# Get the ATTENDEEs
self.attendeesByInstance = self.calendar.getAttendeesByInstance(True, onlyScheduleAgentServer=True)
+ self.instances = set(self.calendar.getComponentInstances())
self.attendees = set()
for attendee, _ignore in self.attendeesByInstance:
self.attendees.add(attendee)
@@ -530,6 +531,7 @@
# Read in existing data
self.oldcalendar = (yield self.resource.iCalendarForUser(self.request))
self.oldAttendeesByInstance = self.oldcalendar.getAttendeesByInstance(True, onlyScheduleAgentServer=True)
+ self.oldInstances = set(self.oldcalendar.getComponentInstances())
self.coerceAttendeesPartstatOnModify()
# Don't allow any SEQUENCE to decrease
@@ -705,10 +707,8 @@
mappedNew = set(self.attendeesByInstance)
# Get missing instances
- oldInstances = set(self.oldcalendar.getComponentInstances())
- newInstances = set(self.calendar.getComponentInstances())
- removedInstances = oldInstances - newInstances
- addedInstances = newInstances - oldInstances
+ removedInstances = self.oldInstances - self.instances
+ addedInstances = self.instances - self.oldInstances
# Also look for new EXDATEs
oldexdates = set()
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/ischedule.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/ischedule.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/ischedule.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -174,10 +174,10 @@
# Generate an HTTP client request
try:
if not hasattr(self.scheduler.request, "extendedLogItems"):
- self.request.extendedLogItems = {}
- if "itip.ischedule" not in self.request.extendedLogItems:
- self.request.extendedLogItems["itip.ischedule"] = 0
- self.request.extendedLogItems["itip.ischedule"] += 1
+ self.scheduler.request.extendedLogItems = {}
+ if "itip.ischedule" not in self.scheduler.request.extendedLogItems:
+ self.scheduler.request.extendedLogItems["itip.ischedule"] = 0
+ self.scheduler.request.extendedLogItems["itip.ischedule"] += 1
from twisted.internet import reactor
f = Factory()
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/test/test_implicit.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/test/test_implicit.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/scheduling/test/test_implicit.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -778,6 +778,7 @@
scheduler.request = None
scheduler.oldcalendar = Component.fromString(calendar1)
scheduler.oldAttendeesByInstance = scheduler.oldcalendar.getAttendeesByInstance(True, onlyScheduleAgentServer=True)
+ scheduler.oldInstances = set(scheduler.oldcalendar.getComponentInstances())
scheduler.calendar = Component.fromString(calendar2)
scheduler.extractCalendarData()
scheduler.findRemovedAttendees()
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/stdconfig.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/stdconfig.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -453,6 +453,7 @@
"LogDatabase" : {
"LabelsInSQL" : False,
"Statistics" : False,
+ "StatisticsLogFile" : "sqlstats.log",
"SQLStatements" : False,
"TransactionWaitSeconds" : 0,
},
@@ -698,6 +699,9 @@
"Service" : "calendarserver.push.applepush.ApplePushNotifierService",
"Enabled" : False,
"SubscriptionURL" : "apns",
+ "SubscriptionRefreshIntervalSeconds" : 2 * 24 * 60 * 60, # How often the client should re-register (2 days)
+ "SubscriptionPurgeIntervalSeconds" : 12 * 60 * 60, # How often a purge is done (12 hours)
+ "SubscriptionPurgeSeconds" : 14 * 24 * 60 * 60, # How old a subscription must be before it's purged (14 days)
"DataHost" : "",
"ProviderHost" : "gateway.push.apple.com",
"ProviderPort" : 2195,
@@ -1021,6 +1025,7 @@
("LogRoot", "AccessLogFile"),
("LogRoot", "ErrorLogFile"),
("LogRoot", ("Postgres", "LogFile",)),
+ ("LogRoot", ("LogDatabase", "StatisticsLogFile",)),
("LogRoot", "AccountingLogRoot"),
("RunRoot", "PIDFile"),
("RunRoot", "GlobalStatsSocket"),
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/storebridge.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/storebridge.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -15,27 +15,39 @@
# limitations under the License.
##
+import time
+import hashlib
+from urlparse import urlsplit
+
+from twisted.python.hashlib import md5
+from twisted.python.log import err as logDefaultException
+from twisted.python.util import FancyEqMixin
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue, maybeDeferred
+from twisted.internet.protocol import Protocol
+
from twext.python.log import Logger
+
+from txdav.xml import element as davxml
+from txdav.xml.base import dav_namespace, WebDAVUnknownElement, encodeXMLName
+from txdav.base.propertystore.base import PropertyName
+from txdav.caldav.icalendarstore import QuotaExceeded
+from txdav.common.icommondatastore import NoSuchObjectResourceError
+from txdav.common.datastore.sql_tables import _BIND_MODE_READ, _BIND_MODE_WRITE
+from txdav.idav import PropertyChangeNotAllowedError
+
from twext.web2 import responsecode
-from txdav.xml import element as davxml
-from txdav.xml.base import dav_namespace, WebDAVUnknownElement
+from twext.web2.stream import ProducerStream, readStream, MemoryStream
+from twext.web2.http import HTTPError, StatusResponse, Response
+from twext.web2.http_headers import ETag, MimeType
from twext.web2.dav.http import ErrorResponse, ResponseQueue, MultiStatusResponse
from twext.web2.dav.noneprops import NonePropertyStore
from twext.web2.dav.resource import TwistedACLInheritable, AccessDeniedError
from twext.web2.dav.util import parentForURL, allDataFromStream, joinURL, davXMLFromStream
-from twext.web2.http import HTTPError, StatusResponse, Response
-from twext.web2.http_headers import ETag, MimeType
from twext.web2.responsecode import (
FORBIDDEN, NO_CONTENT, NOT_FOUND, CREATED, CONFLICT, PRECONDITION_FAILED,
BAD_REQUEST, OK, INSUFFICIENT_STORAGE_SPACE
)
-from twext.web2.stream import ProducerStream, readStream, MemoryStream
-from twisted.internet.defer import succeed, inlineCallbacks, returnValue, maybeDeferred
-from twisted.internet.protocol import Protocol
-from twisted.python.hashlib import md5
-from twisted.python.log import err as logDefaultException
-from twisted.python.util import FancyEqMixin
from twistedcaldav import customxml, carddavxml, caldavxml
from twistedcaldav.cache import CacheStoreNotifier, ResponseCacheMixin,\
DisabledCacheNotifier
@@ -56,16 +68,6 @@
from twistedcaldav.scheduling.implicit import ImplicitScheduler
from twistedcaldav.vcard import Component as VCard, InvalidVCardDataError
-from txdav.base.propertystore.base import PropertyName
-from txdav.caldav.icalendarstore import QuotaExceeded
-from txdav.common.icommondatastore import NoSuchObjectResourceError
-from txdav.common.datastore.sql_tables import _BIND_MODE_READ, _BIND_MODE_WRITE
-from txdav.idav import PropertyChangeNotAllowedError
-
-import time
-import hashlib
-from urlparse import urlsplit
-
"""
Wrappers to translate between the APIs in L{txdav.caldav.icalendarstore} and
L{txdav.carddav.iaddressbookstore} and those in L{twistedcaldav}.
@@ -102,7 +104,7 @@
except KeyError:
raise HTTPError(StatusResponse(
NOT_FOUND,
- "No such property: {%s}%s" % qname
+ "No such property: %s" % (encodeXMLName(*qname),)
))
@@ -112,7 +114,7 @@
except PropertyChangeNotAllowedError:
raise HTTPError(StatusResponse(
FORBIDDEN,
- "Property cannot be changed: {%s}%s" % property.qname(),
+ "Property cannot be changed: %s" % (property.sname(),)
))
@@ -338,7 +340,7 @@
@inlineCallbacks
def findChildrenFaster(
- self, depth, request, okcallback, badcallback,
+ self, depth, request, okcallback, badcallback, missingcallback,
names, privileges, inherited_aces
):
"""
@@ -346,10 +348,13 @@
"""
if depth == "1":
- yield self._newStoreObject.objectResources()
+ if names:
+ yield self._newStoreObject.objectResourcesWithNames(names)
+ else:
+ yield self._newStoreObject.objectResources()
result = (yield super(_CommonHomeChildCollectionMixin, self).findChildrenFaster(
- depth, request, okcallback, badcallback, names, privileges, inherited_aces
+ depth, request, okcallback, badcallback, missingcallback, names, privileges, inherited_aces
))
returnValue(result)
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_icalendar.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_icalendar.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_icalendar.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -5077,6 +5077,169 @@
""",
1, 0,
),
+ (
+ "1.2 - valid with RDATE",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+RDATE:20071115T000000Z
+RRULE:FREQ=WEEKLY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+RDATE:20071115T000000Z
+RRULE:FREQ=WEEKLY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ 0, 0,
+ ),
+ (
+ "2.1 - EXDATEd RECURRENCE-ID",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+EXDATE:20071115T000000Z,20071116T000000Z
+RRULE:FREQ=DAILY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+EXDATE:20071116T000000Z
+RRULE:FREQ=DAILY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ 1, 0,
+ ),
+ (
+ "2.2 - EXDATEd invalid RECURRENCE-ID",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+EXDATE:20071115T000000Z,20071121T000000Z
+RRULE:FREQ=WEEKLY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+EXDATE:20071121T000000Z
+RDATE:20071115T000000Z
+RRULE:FREQ=WEEKLY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ 2, 0,
+ ),
+ (
+ "2.2 - EXDATEd RECURRENCE-ID with RDATE",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+EXDATE:20071115T000000Z,20071121T000000Z
+RDATE:20071115T000000Z
+RRULE:FREQ=WEEKLY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1
+DTSTART:20071114T000000Z
+DTSTAMP:20080601T120000Z
+EXDATE:20071121T000000Z
+RDATE:20071115T000000Z
+RRULE:FREQ=WEEKLY
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1
+RECURRENCE-ID:20071115T000000Z
+DTSTART:20071115T000000Z
+DTSTAMP:20080601T120000Z
+END:VEVENT
+END:VCALENDAR
+""",
+ 1, 0,
+ ),
)
for title, calendar, result_calendar, result_fixed, result_unfixed in data:
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_memcacheprops.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_memcacheprops.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_memcacheprops.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -28,12 +28,13 @@
from twext.web2.http import HTTPError
+from txdav.xml.base import encodeXMLName
+
from twistedcaldav.memcacheprops import MemcachePropertyCollection
from twistedcaldav.test.util import InMemoryPropertyStore
from twistedcaldav.test.util import TestCase
-
class StubCollection(object):
def __init__(self, path, childNames):
@@ -91,9 +92,8 @@
def qname(self):
return self.ns, self.name
-
def __repr__(self):
- return "{%s}%s = %s" % (self.ns, self.name, self.value)
+ return "%s = %s" % (encodeXMLName(self.ns, self.name), self.value)
class MemcachePropertyCollectionTestCase(TestCase):
Modified: CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_validation.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_validation.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/twistedcaldav/test/test_validation.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -118,10 +118,7 @@
element = XML(err.response.stream.mem)[0]
self.assertEquals(
element.tag,
- "{%s}%s" % (
- MaxAttendeesPerInstance.namespace,
- MaxAttendeesPerInstance.name
- )
+ MaxAttendeesPerInstance.sname()
)
self.assertEquals(int(element.text), config.MaxAttendeesPerInstance)
else:
@@ -165,10 +162,7 @@
element = XML(err.response.stream.mem)[0]
self.assertEquals(
element.tag,
- "{%s}%s" % (
- MaxAttendeesPerInstance.namespace,
- MaxAttendeesPerInstance.name
- )
+ MaxAttendeesPerInstance.sname()
)
self.assertEquals(int(element.text), config.MaxAttendeesPerInstance)
else:
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/base.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/base.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/base.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -25,6 +25,7 @@
from twext.python.log import LoggingMixIn
from txdav.xml import element as davxml
+from txdav.xml.base import encodeXMLName
from twext.web2.dav.resource import TwistedGETContentMD5,\
TwistedQuotaRootProperty
@@ -89,7 +90,7 @@
)
def toString(self):
- return "{%s}%s" % (self.namespace, self.name)
+ return encodeXMLName(self.namespace, self.name)
class AbstractPropertyStore(LoggingMixIn, DictMixin):
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/sql.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/sql.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/sql.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -73,7 +73,11 @@
@classmethod
@inlineCallbacks
- def load(cls, defaultuser, txn, resourceID, created=False):
+ def load(cls, defaultuser, txn, resourceID, created=False, notifyCallback=None):
+ """
+ @param notifyCallback: a callable used to trigger notifications when the
+ property store changes.
+ """
self = cls.__new__(cls)
super(PropertyStore, self).__init__(defaultuser)
self._txn = txn
@@ -81,6 +85,7 @@
self._cached = {}
if not created:
yield self._refresh(txn)
+ self._notifyCallback = notifyCallback
returnValue(self)
@@ -124,9 +129,58 @@
Where=parentColumn == parentID
)
rows = yield query.on(txn)
+ stores = cls._createMultipleStores(defaultUser, txn, rows)
+ returnValue(stores)
+
+ @classmethod
+ @inlineCallbacks
+ def forMultipleResourcesWithResourceIDs(cls, defaultUser, txn, resourceIDs):
+ """
+ Load all property stores for all specified resources. This is used
+ to optimize Depth:1 operations on that collection, by loading all
+ relevant properties in a single query. Note that the caller of this
+ method must make sure that the number of items being queried for is
+ within a reasonable batch size. If the caller is itself batching
+ related queries, that will take care of itself.
+
+ @param defaultUser: the UID of the user who owns / is requesting the
+ property stores; the ones whose per-user properties will be exposed.
+
+ @type defaultUser: C{str}
+
+ @param txn: the transaction within which to fetch the rows.
+
+ @type txn: L{IAsyncTransaction}
+
+ @param resourceIDs: The set of resource ID's to query.
+
+ @return: a L{Deferred} that fires with a C{dict} mapping resource ID (a
+ value taken from C{childColumn}) to a L{PropertyStore} for that ID.
+ """
+ query = Select([
+ prop.RESOURCE_ID, prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID.In(Parameter("resourceIDs", len(resourceIDs)))
+ )
+ rows = yield query.on(txn, resourceIDs=resourceIDs)
+ stores = cls._createMultipleStores(defaultUser, txn, rows)
+ returnValue(stores)
+
+
+ @classmethod
+ def _createMultipleStores(cls, defaultUser, txn, rows):
+ """
+ Create a set of stores for the set of rows passed in.
+ """
+
createdStores = {}
- for object_resource_id, resource_id, name, view_uid, value in rows:
+ for row in rows:
+ if len(row) == 5:
+ object_resource_id, resource_id, name, view_uid, value = row
+ else:
+ object_resource_id = None
+ resource_id, name, view_uid, value = row
if resource_id:
if resource_id not in createdStores:
store = cls.__new__(cls)
@@ -136,7 +190,7 @@
store._cached = {}
createdStores[resource_id] = store
createdStores[resource_id]._cached[(name, view_uid)] = value
- else:
+ elif object_resource_id:
store = cls.__new__(cls)
super(PropertyStore, store).__init__(defaultUser)
store._txn = txn
@@ -144,7 +198,7 @@
store._cached = {}
createdStores[object_resource_id] = store
- returnValue(createdStores)
+ return createdStores
def _getitem_uid(self, key, uid):
@@ -196,6 +250,11 @@
txn, resourceID=self._resourceID, value=value_str,
name=key_str, uid=uid)
self._cacher.delete(str(self._resourceID))
+
+ # Call the registered notification callback
+ if hasattr(self, "_notifyCallback") and self._notifyCallback is not None:
+ self._notifyCallback()
+
self._txn.subtransaction(trySetItem)
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/xattr.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/xattr.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/base/propertystore/xattr.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -34,6 +34,7 @@
from twisted.python.reflect import namedAny
+from txdav.xml.base import encodeXMLName
from txdav.xml.parser import WebDAVDocument
from txdav.base.propertystore.base import AbstractPropertyStore, PropertyName,\
validKey
@@ -116,7 +117,7 @@
qname.namespace)
else:
namespace = qname.namespace
- result = urllib.quote("{%s}%s" % (namespace, qname.name), safe="{}:")
+ result = urllib.quote(encodeXMLName(namespace, qname.name), safe="{}:")
if uid and uid != self._defaultUser:
result = uid + result
r = self.deadPropertyXattrPrefix + result
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/common.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/common.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -717,8 +717,7 @@
yield self.commit()
# Make sure notification fired after commit
- self.assertEquals(self.notifierFactory.history,
- [("update", "CalDAV|home1")])
+ self.assertTrue(("update", "CalDAV|home1") in self.notifierFactory.history)
# Make sure it's available in a new transaction; i.e. test the commit.
home = yield self.homeUnderTest()
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/test_sql.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/caldav/datastore/test/test_sql.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -34,7 +34,8 @@
test_event_text
from txdav.caldav.datastore.test.test_file import setUpCalendarStore
from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
-from txdav.common.datastore.sql import ECALENDARTYPE
+from txdav.common.datastore.sql import ECALENDARTYPE, CommonObjectResource
+from txdav.common.datastore.sql_legacy import PostgresLegacyIndexEmulator
from txdav.common.datastore.sql_tables import schema, _BIND_MODE_DIRECT,\
_BIND_STATUS_ACCEPTED
from txdav.common.datastore.test.util import buildStore, populateCalendarsFrom
@@ -49,6 +50,7 @@
import datetime
from pycalendar.datetime import PyCalendarDateTime
+from pycalendar.timezone import PyCalendarTimezone
class CalendarSQLStorageTests(CalendarCommonTests, unittest.TestCase):
"""
@@ -1190,6 +1192,80 @@
self.assertEqual(rMax, None)
@inlineCallbacks
+ def test_notExpandedWithin(self):
+ """
+ Test PostgresLegacyIndexEmulator.notExpandedWithin to make sure it returns the correct
+ result based on the ranges passed in.
+ """
+
+ self.patch(config, "FreeBusyIndexDelayedExpand", False)
+
+ # Create the index on a new calendar
+ home = yield self.homeUnderTest()
+ newcalendar = yield home.createCalendarWithName("index_testing")
+ index = PostgresLegacyIndexEmulator(newcalendar)
+
+ # Create the calendar object to use for testing
+ nowYear = self.nowYear["now"]
+ caldata = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:instance
+DTSTART:%04d0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=WEEKLY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n") % (nowYear - 3,)
+ component = Component.fromString(caldata)
+ calendarObject = yield newcalendar.createCalendarObjectWithName("indexing.ics", component)
+ rmin, rmax = yield calendarObject.recurrenceMinMax()
+ self.assertEqual(rmin.getYear(), nowYear - 1)
+ self.assertEqual(rmax.getYear(), nowYear + 1)
+
+ # Fully within range
+ testMin = PyCalendarDateTime(nowYear, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ testMax = PyCalendarDateTime(nowYear + 1, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ result = yield index.notExpandedWithin(testMin, testMax)
+ self.assertEqual(result, [])
+
+ # Upper bound exceeded
+ testMin = PyCalendarDateTime(nowYear, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ testMax = PyCalendarDateTime(nowYear + 5, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ result = yield index.notExpandedWithin(testMin, testMax)
+ self.assertEqual(result, ["indexing.ics"])
+
+ # Lower bound exceeded
+ testMin = PyCalendarDateTime(nowYear - 5, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ testMax = PyCalendarDateTime(nowYear + 1, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ result = yield index.notExpandedWithin(testMin, testMax)
+ self.assertEqual(result, ["indexing.ics"])
+
+ # Lower and upper bounds exceeded
+ testMin = PyCalendarDateTime(nowYear - 5, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ testMax = PyCalendarDateTime(nowYear + 5, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ result = yield index.notExpandedWithin(testMin, testMax)
+ self.assertEqual(result, ["indexing.ics"])
+
+ # Lower none within range
+ testMin = None
+ testMax = PyCalendarDateTime(nowYear + 1, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ result = yield index.notExpandedWithin(testMin, testMax)
+ self.assertEqual(result, [])
+
+ # Lower none and upper bounds exceeded
+ testMin = None
+ testMax = PyCalendarDateTime(nowYear + 5, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
+ result = yield index.notExpandedWithin(testMin, testMax)
+ self.assertEqual(result, ["indexing.ics"])
+
+
+ @inlineCallbacks
def test_setComponent_no_instance_indexing(self):
"""
L{ICalendarObject.setComponent} raises L{InvalidCalendarComponentError}
@@ -1249,3 +1325,93 @@
yield calendar.removeCalendarObjectWithName("indexing.ics")
yield self.commit()
+
+ @inlineCallbacks
+ def test_loadObjectResourcesWithName(self):
+ """
+ L{CommonHomeChild.objectResourcesWithNames} returns the correct set of object resources
+ properly configured with a loaded property store. make sure batching works.
+ """
+
+ @inlineCallbacks
+ def _tests(cal):
+ resources = yield cal.objectResourcesWithNames(("1.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics", "3.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics", "3.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("1.ics", "2.ics", "3.ics", "4.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("1.ics", "2.ics", "3.ics", "4.ics",)))
+
+ resources = yield cal.objectResourcesWithNames(("bogus1.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set())
+
+ resources = yield cal.objectResourcesWithNames(("bogus1.ics", "2.ics",))
+ self.assertEqual(set([resource.name() for resource in resources]), set(("2.ics",)))
+
+ # Basic load tests
+ cal = yield self.calendarUnderTest()
+ yield _tests(cal)
+
+ # Adjust batch size and try again
+ self.patch(CommonObjectResource, "BATCH_LOAD_SIZE", 2)
+ yield _tests(cal)
+
+ yield self.commit()
+
+ # Tests on inbox - resources with properties
+ txn = self.transactionUnderTest()
+ yield txn.homeWithUID(ECALENDARTYPE, "byNameTest", create=True)
+ inbox = yield self.calendarUnderTest(txn=txn, name="inbox", home="byNameTest")
+ caldata = """BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+METHOD:REQUEST
+BEGIN:VEVENT
+UID:instance
+DTSTART:%(now)s0102T140000Z
+DURATION:PT1H
+CREATED:20060102T190000Z
+DTSTAMP:20051222T210507Z
+RRULE:FREQ=DAILY
+SUMMARY:instance
+END:VEVENT
+END:VCALENDAR
+""".replace("\n", "\r\n") % self.nowYear
+ component = Component.fromString(caldata)
+
+ @inlineCallbacks
+ def _createInboxItem(rname, pvalue):
+ obj = yield inbox.createCalendarObjectWithName(rname, component)
+ prop = caldavxml.CalendarDescription.fromString(pvalue)
+ obj.properties()[PropertyName.fromElement(prop)] = prop
+
+ yield _createInboxItem("1.ics", "p1")
+ yield _createInboxItem("2.ics", "p2")
+ yield _createInboxItem("3.ics", "p3")
+ yield _createInboxItem("4.ics", "p4")
+ yield self.commit()
+
+ inbox = yield self.calendarUnderTest(name="inbox", home="byNameTest")
+ yield _tests(inbox)
+
+ resources = yield inbox.objectResourcesWithNames(("1.ics",))
+ prop = caldavxml.CalendarDescription.fromString("p1")
+ self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
+
+ resources = yield inbox.objectResourcesWithNames(("1.ics", "2.ics",))
+ resources.sort(key=lambda x:x._name)
+ prop = caldavxml.CalendarDescription.fromString("p1")
+ self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
+ prop = caldavxml.CalendarDescription.fromString("p2")
+ self.assertEqual(resources[1].properties()[PropertyName.fromElement(prop)], prop)
+
+ resources = yield inbox.objectResourcesWithNames(("bogus1.ics", "2.ics",))
+ resources.sort(key=lambda x:x._name)
+ prop = caldavxml.CalendarDescription.fromString("p2")
+ self.assertEqual(resources[0].properties()[PropertyName.fromElement(prop)], prop)
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/file.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/file.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -208,6 +208,7 @@
self._notificationHomes = {}
self._notifierFactory = notifierFactory
self._notifiedAlready = set()
+ self._bumpedAlready = set()
self._migrating = migrating
extraInterfaces = []
@@ -250,12 +251,15 @@
# File-based storage of APN subscriptions not implementated.
- def addAPNSubscription(self, token, key, timestamp, subscriber):
+ def addAPNSubscription(self, token, key, timestamp, subscriber, userAgent, ipAddr):
return NotImplementedError
def removeAPNSubscription(self, token, key):
return NotImplementedError
+ def purgeOldAPNSubscriptions(self, purgeSeconds):
+ return NotImplementedError
+
def apnSubscriptionsByToken(self, token):
return NotImplementedError
@@ -267,11 +271,28 @@
def isNotifiedAlready(self, obj):
return obj in self._notifiedAlready
-
+
def notificationAddedForObject(self, obj):
self._notifiedAlready.add(obj)
+ def isBumpedAlready(self, obj):
+ """
+ Indicates whether or not bumpAddedForObject has already been
+ called for the given object, in order to facilitate calling
+ bumpModified only once per object.
+ """
+ return obj in self._bumpedAlready
+ def bumpAddedForObject(self, obj):
+ """
+ Records the fact that a bumpModified( ) call has already been
+ done, in order to facilitate calling bumpModified only once per
+ object.
+ """
+ self._bumpedAlready.add(obj)
+
+
+
class StubResource(object):
"""
Just enough resource to keep the shared sql DB classes going.
@@ -781,6 +802,18 @@
for name in self.listObjectResources()]
+ def objectResourcesWithNames(self, names):
+ """
+ Return a list of the specified object resource objects.
+ """
+ results = []
+ for name in names:
+ obj = self.objectResourceWithName(name)
+ if obj is not None:
+ results.append(obj)
+ return results
+
+
def listObjectResources(self):
"""
Return a list of object resource names.
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -90,8 +90,6 @@
from cStringIO import StringIO
from sqlparse import parse
-import collections
-import sys
import time
current_sql_schema = getModule(__name__).filePath.sibling("sql_schema").child("current.sql").getContent()
@@ -143,7 +141,7 @@
def __init__(self, sqlTxnFactory, notifierFactory, attachmentsPath,
enableCalendars=True, enableAddressBooks=True,
label="unlabeled", quota=(2 ** 20),
- logLabels=False, logStats=False, logSQL=False,
+ logLabels=False, logStats=False, logStatsLogFile=None, logSQL=False,
logTransactionWaits=0, timeoutTransactions=0,
cacheQueries=True, cachePool="Default",
cacheExpireSeconds=3600):
@@ -158,6 +156,7 @@
self.quota = quota
self.logLabels = logLabels
self.logStats = logStats
+ self.logStatsLogFile = logStatsLogFile
self.logSQL = logSQL
self.logTransactionWaits = logTransactionWaits
self.timeoutTransactions = timeoutTransactions
@@ -220,35 +219,69 @@
class TransactionStatsCollector(object):
+ """
+ Used to log each SQL query and statistics about that query during the course of a single transaction.
+ Results can be printed out where ever needed at the end of the transaction.
+ """
- def __init__(self):
- self.count = collections.defaultdict(int)
- self.times = collections.defaultdict(float)
+ def __init__(self, label, logFileName=None):
+ self.label = label
+ self.logFileName = logFileName
+ self.statements = []
- def startStatement(self, sql):
- self.count[sql] += 1
- return sql, time.time()
+ def startStatement(self, sql, args):
+ """
+ Called prior to an SQL query being run.
- def endStatement(self, context):
- sql, tstamp = context
- self.times[sql] += time.time() - tstamp
+ @param sql: the SQL statement to execute
+ @type sql: C{str}
+ @param args: the arguments (binds) to the SQL statement
+ @type args: C{list}
- def printReport(self, toFile=sys.stdout):
+ @return: C{tuple} containing the index in the statement list for this statement, and the start time
+ """
+ args = ["%s" % (arg,) for arg in args]
+ args = [((arg[:10] + "...") if len(arg) > 40 else arg) for arg in args]
+ self.statements.append(["%s %s" % (sql, args,), 0, 0])
+ return len(self.statements) - 1, time.time()
+
+ def endStatement(self, context, rows):
+ """
+ Called after an SQL query has executed.
+
+ @param context: the tuple returned from startStatement
+ @type context: C{tuple}
+ @param rows: number of rows returned from the query
+ @type rows: C{int}
+ """
+ index, tstamp = context
+ self.statements[index][1] = len(rows) if rows else 0
+ self.statements[index][2] = time.time() - tstamp
+ def printReport(self):
+ """
+ Print a report of all the SQL statements executed to date.
+ """
+
+ toFile = StringIO()
toFile.write("*** SQL Stats ***\n")
toFile.write("\n")
- toFile.write("Unique statements: %d\n" % (len(self.count,),))
- toFile.write("Total statements: %d\n" % (sum(self.count.values()),))
- toFile.write("Total time (ms): %.3f\n" % (sum(self.times.values()) * 1000.0,))
- toFile.write("\n")
- for k, v in self.count.items():
- toFile.write("%s\n" % (k,))
- toFile.write("Count: %s\n" % (v,))
- toFile.write("Total Time (ms): %.3f\n" % (self.times[k] * 1000.0,))
- if v > 1:
- toFile.write("Average Time (ms): %.3f\n" % (self.times[k] * 1000.0 / v,))
+ toFile.write("Label: %s\n" % (self.label,))
+ toFile.write("Unique statements: %d\n" % (len(set([statement[0] for statement in self.statements]),),))
+ toFile.write("Total statements: %d\n" % (len(self.statements),))
+ toFile.write("Total rows: %d\n" % (sum([statement[1] for statement in self.statements]),))
+ toFile.write("Total time (ms): %.3f\n" % (sum([statement[2] for statement in self.statements]) * 1000.0,))
+ for sql, rows, t in self.statements:
toFile.write("\n")
- toFile.write("***\n")
+ toFile.write("SQL: %s\n" % (sql,))
+ toFile.write("Rows: %s\n" % (rows,))
+ toFile.write("Time (ms): %.3f\n" % (t,))
+ toFile.write("***\n\n")
+
+ if self.logFileName:
+ open(self.logFileName, "a").write(toFile.getvalue())
+ else:
+ log.error(toFile.getvalue())
class CommonStoreTransactionMonitor(object):
"""
@@ -321,6 +354,7 @@
self._postAbortOperations = []
self._notifierFactory = notifierFactory
self._notifiedAlready = set()
+ self._bumpedAlready = set()
self._label = label
self._migrating = migrating
self._primaryHomeType = None
@@ -348,7 +382,7 @@
self.paramstyle = sqlTxn.paramstyle
self.dialect = sqlTxn.dialect
- self._stats = TransactionStatsCollector() if self._store.logStats else None
+ self._stats = TransactionStatsCollector(self._label, self._store.logStatsLogFile) if self._store.logStats else None
self.statementCount = 0
self.iudCount = 0
self.currentStatement = None
@@ -454,14 +488,18 @@
return Insert({apn.TOKEN: Parameter("token"),
apn.RESOURCE_KEY: Parameter("resourceKey"),
apn.MODIFIED: Parameter("modified"),
- apn.SUBSCRIBER_GUID: Parameter("subscriber")})
+ apn.SUBSCRIBER_GUID: Parameter("subscriber"),
+ apn.USER_AGENT : Parameter("userAgent"),
+ apn.IP_ADDR : Parameter("ipAddr")})
@classproperty
def _updateAPNSubscriptionQuery(cls): #@NoSelf
apn = schema.APN_SUBSCRIPTIONS
return Update({apn.MODIFIED: Parameter("modified"),
- apn.SUBSCRIBER_GUID: Parameter("subscriber")},
+ apn.SUBSCRIBER_GUID: Parameter("subscriber"),
+ apn.USER_AGENT: Parameter("userAgent"),
+ apn.IP_ADDR : Parameter("ipAddr")},
Where=(apn.TOKEN == Parameter("token")).And(
apn.RESOURCE_KEY == Parameter("resourceKey")))
@@ -478,17 +516,23 @@
@inlineCallbacks
- def addAPNSubscription(self, token, key, timestamp, subscriber):
+ def addAPNSubscription(self, token, key, timestamp, subscriber,
+ userAgent, ipAddr):
if not (token and key and timestamp and subscriber):
raise InvalidSubscriptionValues()
+ # Cap these values at 255 characters
+ userAgent = userAgent[:255]
+ ipAddr = ipAddr[:255]
+
row = yield self._selectAPNSubscriptionQuery.on(self,
token=token, resourceKey=key)
if not row: # Subscription does not yet exist
try:
yield self._insertAPNSubscriptionQuery.on(self,
token=token, resourceKey=key, modified=timestamp,
- subscriber=subscriber)
+ subscriber=subscriber, userAgent=userAgent,
+ ipAddr=ipAddr)
except Exception:
# Subscription may have been added by someone else, which is fine
pass
@@ -497,7 +541,8 @@
try:
yield self._updateAPNSubscriptionQuery.on(self,
token=token, resourceKey=key, modified=timestamp,
- subscriber=subscriber)
+ subscriber=subscriber, userAgent=userAgent,
+ ipAddr=ipAddr)
except Exception:
# Subscription may have been added by someone else, which is fine
pass
@@ -517,6 +562,18 @@
@classproperty
+ def _purgeOldAPNSubscriptionQuery(cls): #@NoSelf
+ apn = schema.APN_SUBSCRIPTIONS
+ return Delete(From=apn,
+ Where=(apn.MODIFIED < Parameter("olderThan")))
+
+
+ def purgeOldAPNSubscriptions(self, olderThan):
+ return self._purgeOldAPNSubscriptionQuery.on(self,
+ olderThan=olderThan)
+
+
+ @classproperty
def _apnSubscriptionsByTokenQuery(cls): #@NoSelf
apn = schema.APN_SUBSCRIPTIONS
return Select([apn.RESOURCE_KEY, apn.MODIFIED, apn.SUBSCRIBER_GUID],
@@ -565,10 +622,26 @@
def isNotifiedAlready(self, obj):
return obj in self._notifiedAlready
-
+
def notificationAddedForObject(self, obj):
self._notifiedAlready.add(obj)
+ def isBumpedAlready(self, obj):
+ """
+ Indicates whether or not bumpAddedForObject has already been
+ called for the given object, in order to facilitate calling
+ bumpModified only once per object.
+ """
+ return obj in self._bumpedAlready
+
+ def bumpAddedForObject(self, obj):
+ """
+ Records the fact that a bumpModified( ) call has already been
+ done, in order to facilitate calling bumpModified only once per
+ object.
+ """
+ self._bumpedAlready.add(obj)
+
_savepointCounter = 0
def _savepoint(self):
@@ -662,7 +735,7 @@
Execute some SQL (delegate to L{IAsyncTransaction}).
"""
if self._stats:
- statsContext = self._stats.startStatement(a[0])
+ statsContext = self._stats.startStatement(a[0], a[1])
self.currentStatement = a[0]
if self._store.logTransactionWaits and a[0].split(" ", 1)[0].lower() in ("insert", "update", "delete",):
self.iudCount += 1
@@ -676,7 +749,7 @@
finally:
self.currentStatement = None
if self._stats:
- self._stats.endStatement(statsContext)
+ self._stats.endStatement(statsContext, results)
returnValue(results)
@inlineCallbacks
@@ -711,9 +784,7 @@
returnValue(ignored)
if self._stats:
- s = StringIO()
- self._stats.printReport(s)
- log.error(s.getvalue())
+ self._stats.printReport()
return self._sqlTxn.commit().addCallback(postCommit)
@@ -1317,7 +1388,8 @@
props = yield PropertyStore.load(
self.uid(),
self._txn,
- self._resourceID
+ self._resourceID,
+ notifyCallback=self.notifyChanged
)
self._propertyStore = props
@@ -1518,6 +1590,10 @@
delay the transaction whilst waiting for deadlock detection to kick in.
"""
+ if self._txn.isBumpedAlready(self):
+ returnValue(None)
+ self._txn.bumpAddedForObject(self)
+
# NB if modified is bumped we know that sync token will have changed too, so invalidate the cached value
self._syncTokenRevision = None
@@ -1536,13 +1612,13 @@
except AllRetriesFailed:
log.debug("CommonHome.bumpModified failed")
-
+
@inlineCallbacks
def notifyChanged(self):
"""
Trigger a notification of a change
"""
-
+
# Update modified if object still exists
if self._resourceID:
yield self.bumpModified()
@@ -2845,6 +2921,19 @@
returnValue(results)
+ @inlineCallbacks
+ def objectResourcesWithNames(self, names):
+ """
+ Load and cache all named children - set of names optimization
+ """
+ results = (yield self._objectResourceClass.loadAllObjectsWithNames(self, names))
+ for result in results:
+ self._objects[result.name()] = result
+ self._objects[result.uid()] = result
+ self._objectNames = sorted([result.name() for result in results])
+ returnValue(results)
+
+
@classproperty
def _objectResourceNamesQuery(cls): #@NoSelf
"""
@@ -3087,7 +3176,8 @@
props = yield PropertyStore.load(
self.ownerHome().uid(),
self._txn,
- self._resourceID
+ self._resourceID,
+ notifyCallback=self.notifyChanged
)
self.initPropertyStore(props)
self._properties = props
@@ -3176,12 +3266,16 @@
delay the transaction whilst waiting for deadlock detection to kick in.
"""
+ if self._txn.isBumpedAlready(self):
+ returnValue(None)
+ self._txn.bumpAddedForObject(self)
+
@inlineCallbacks
def _bumpModified(subtxn):
yield self._lockLastModifiedQuery.on(subtxn, resourceID=self._resourceID)
result = (yield self._changeLastModifiedQuery.on(subtxn, resourceID=self._resourceID))
returnValue(result)
-
+
try:
self._modified = (yield self._txn.subtransaction(_bumpModified, retries=0, failureOK=True))[0][0]
@@ -3191,13 +3285,13 @@
yield queryCacher.invalidateAfterCommit(self._txn, cacheKey)
except AllRetriesFailed:
log.debug("CommonHomeChild.bumpModified failed")
-
+
@inlineCallbacks
def notifyChanged(self):
"""
Trigger a notification of a change
"""
-
+
# Update modified if object still exists
if self._resourceID:
yield self.bumpModified()
@@ -3209,7 +3303,6 @@
self._txn.notificationAddedForObject(self)
-
class CommonObjectResource(LoggingMixIn, FancyEqMixin):
"""
Base class for object resources.
@@ -3224,6 +3317,8 @@
_objectSchema = None
+ BATCH_LOAD_SIZE = 50
+
def __init__(self, parent, name, uid, resourceID=None, metadata=None):
self._parentCollection = parent
self._resourceID = resourceID
@@ -3285,8 +3380,74 @@
returnValue(results)
+ @classmethod
+ def _allColumnsWithParentAndNames(cls, names): #@NoSelf
+ obj = cls._objectSchema
+ return Select(cls._allColumns, From=obj,
+ Where=(obj.PARENT_RESOURCE_ID == Parameter("parentID")).And(
+ obj.RESOURCE_NAME.In(Parameter("names", len(names)))))
+
@classmethod
+ @inlineCallbacks
+ def loadAllObjectsWithNames(cls, parent, names):
+ """
+ Load all child objects with the specified names, doing so in batches.
+ """
+ names = tuple(names)
+ results = []
+ while(len(names)):
+ result_batch = (yield cls._loadAllObjectsWithNames(parent, names[:cls.BATCH_LOAD_SIZE]))
+ results.extend(result_batch)
+ names = names[cls.BATCH_LOAD_SIZE:]
+
+ returnValue(results)
+
+ @classmethod
+ @inlineCallbacks
+ def _loadAllObjectsWithNames(cls, parent, names):
+ """
+ Load all child objects with the specified names. This must create the
+ child classes and initialize them using "batched" SQL operations to keep
+ this constant wrt the number of children. This is an optimization for
+ Depth:1 operations on the collection.
+ """
+
+ # Optimize case of single name to load
+ if len(names) == 1:
+ obj = yield cls.objectWithName(parent, names[0], None)
+ returnValue([obj] if obj else [])
+
+ results = []
+
+ # Load from the main table first
+ dataRows = yield cls._allColumnsWithParentAndNames(names).on(
+ parent._txn, parentID=parent._resourceID, names=names)
+
+ if dataRows:
+ # Get property stores for all these child resources
+ if parent.objectResourcesHaveProperties():
+ propertyStores =(yield PropertyStore.forMultipleResourcesWithResourceIDs(
+ parent._home.uid(),
+ parent._txn,
+ tuple([row[0] for row in dataRows]),
+ ))
+ else:
+ propertyStores = {}
+
+ # Create the actual objects merging in properties
+ for row in dataRows:
+ child = cls(parent, "", None)
+ child._initFromRow(tuple(row))
+ yield child._loadPropertyStore(
+ props=propertyStores.get(child._resourceID, None)
+ )
+ results.append(child)
+
+ returnValue(results)
+
+
+ @classmethod
def objectWithName(cls, parent, name, uid):
objectResource = cls(parent, name, uid, None)
return objectResource.initFromStore()
@@ -3680,7 +3841,8 @@
self._propertyStore = yield PropertyStore.load(
self._uid,
self._txn,
- self._resourceID
+ self._resourceID,
+ notifyCallback=self.notifyChanged
)
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_legacy.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_legacy.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_legacy.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -444,7 +444,7 @@
return Select(
[co.RESOURCE_NAME],
From=co,
- Where=((co.RECURRANCE_MIN < Parameter("minDate"))
+ Where=((co.RECURRANCE_MIN > Parameter("minDate"))
.Or(co.RECURRANCE_MAX < Parameter("maxDate")))
.And(co.CALENDAR_RESOURCE_ID == Parameter("resourceID"))
)
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2012-08-09 19:03:13 UTC (rev 9547)
@@ -250,6 +250,8 @@
"RESOURCE_KEY" nvarchar2(255),
"MODIFIED" integer not null,
"SUBSCRIBER_GUID" nvarchar2(255),
+ "USER_AGENT" nvarchar2(255) default null,
+ "IP_ADDR" nvarchar2(255) default null,
primary key("TOKEN", "RESOURCE_KEY")
);
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current.sql 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/current.sql 2012-08-09 19:03:13 UTC (rev 9547)
@@ -478,6 +478,8 @@
RESOURCE_KEY varchar(255) not null,
MODIFIED integer not null,
SUBSCRIBER_GUID varchar(255) not null,
+ USER_AGENT varchar(255) default null,
+ IP_ADDR varchar(255) default null,
primary key(TOKEN, RESOURCE_KEY) -- implicit index
);
@@ -495,6 +497,6 @@
VALUE varchar(255)
);
-insert into CALENDARSERVER values ('VERSION', '10');
+insert into CALENDARSERVER values ('VERSION', '11');
insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '3');
insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '1');
Copied: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/oracle-dialect/v10.sql (from rev 9546, CalendarServer/trunk/txdav/common/datastore/sql_schema/old/oracle-dialect/v10.sql)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/oracle-dialect/v10.sql (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/oracle-dialect/v10.sql 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,360 @@
+create sequence RESOURCE_ID_SEQ;
+create sequence INSTANCE_ID_SEQ;
+create sequence REVISION_SEQ;
+create table CALENDAR_HOME (
+ "RESOURCE_ID" integer primary key,
+ "OWNER_UID" nvarchar2(255) unique,
+ "DATAVERSION" integer default 0 not null
+);
+
+create table CALENDAR_HOME_METADATA (
+ "RESOURCE_ID" integer primary key references CALENDAR_HOME on delete cascade,
+ "QUOTA_USED_BYTES" integer default 0 not null,
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table CALENDAR (
+ "RESOURCE_ID" integer primary key
+);
+
+create table CALENDAR_METADATA (
+ "RESOURCE_ID" integer primary key references CALENDAR on delete cascade,
+ "SUPPORTED_COMPONENTS" nvarchar2(255) default null,
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table INVITE (
+ "INVITE_UID" nvarchar2(255),
+ "NAME" nvarchar2(255),
+ "RECIPIENT_ADDRESS" nvarchar2(255),
+ "HOME_RESOURCE_ID" integer not null,
+ "RESOURCE_ID" integer not null
+);
+
+create table NOTIFICATION_HOME (
+ "RESOURCE_ID" integer primary key,
+ "OWNER_UID" nvarchar2(255) unique
+);
+
+create table NOTIFICATION (
+ "RESOURCE_ID" integer primary key,
+ "NOTIFICATION_HOME_RESOURCE_ID" integer not null references NOTIFICATION_HOME,
+ "NOTIFICATION_UID" nvarchar2(255),
+ "XML_TYPE" nvarchar2(255),
+ "XML_DATA" nclob,
+ "MD5" nchar(32),
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ unique("NOTIFICATION_UID", "NOTIFICATION_HOME_RESOURCE_ID")
+);
+
+create table CALENDAR_BIND (
+ "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
+ "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
+ "CALENDAR_RESOURCE_NAME" nvarchar2(255),
+ "BIND_MODE" integer not null,
+ "BIND_STATUS" integer not null,
+ "SEEN_BY_OWNER" integer not null,
+ "SEEN_BY_SHAREE" integer not null,
+ "MESSAGE" nclob,
+ primary key("CALENDAR_HOME_RESOURCE_ID", "CALENDAR_RESOURCE_ID"),
+ unique("CALENDAR_HOME_RESOURCE_ID", "CALENDAR_RESOURCE_NAME")
+);
+
+create table CALENDAR_BIND_MODE (
+ "ID" integer primary key,
+ "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('own', 0);
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('read', 1);
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('write', 2);
+insert into CALENDAR_BIND_MODE (DESCRIPTION, ID) values ('direct', 3);
+create table CALENDAR_BIND_STATUS (
+ "ID" integer primary key,
+ "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('invited', 0);
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('accepted', 1);
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('declined', 2);
+insert into CALENDAR_BIND_STATUS (DESCRIPTION, ID) values ('invalid', 3);
+create table CALENDAR_OBJECT (
+ "RESOURCE_ID" integer primary key,
+ "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
+ "RESOURCE_NAME" nvarchar2(255),
+ "ICALENDAR_TEXT" nclob,
+ "ICALENDAR_UID" nvarchar2(255),
+ "ICALENDAR_TYPE" nvarchar2(255),
+ "ATTACHMENTS_MODE" integer default 0 not null,
+ "DROPBOX_ID" nvarchar2(255),
+ "ORGANIZER" nvarchar2(255),
+ "ORGANIZER_OBJECT" integer references CALENDAR_OBJECT,
+ "RECURRANCE_MIN" date,
+ "RECURRANCE_MAX" date,
+ "ACCESS" integer default 0 not null,
+ "SCHEDULE_OBJECT" integer default 0,
+ "SCHEDULE_TAG" nvarchar2(36) default null,
+ "SCHEDULE_ETAGS" nclob default null,
+ "PRIVATE_COMMENTS" integer default 0 not null,
+ "MD5" nchar(32),
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ unique("CALENDAR_RESOURCE_ID", "RESOURCE_NAME")
+);
+
+create table CALENDAR_OBJECT_ATTACHMENTS_MO (
+ "ID" integer primary key,
+ "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into CALENDAR_OBJECT_ATTACHMENTS_MO (DESCRIPTION, ID) values ('none', 0);
+insert into CALENDAR_OBJECT_ATTACHMENTS_MO (DESCRIPTION, ID) values ('read', 1);
+insert into CALENDAR_OBJECT_ATTACHMENTS_MO (DESCRIPTION, ID) values ('write', 2);
+create table CALENDAR_ACCESS_TYPE (
+ "ID" integer primary key,
+ "DESCRIPTION" nvarchar2(32) unique
+);
+
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('', 0);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('public', 1);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('private', 2);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('confidential', 3);
+insert into CALENDAR_ACCESS_TYPE (DESCRIPTION, ID) values ('restricted', 4);
+create table TIME_RANGE (
+ "INSTANCE_ID" integer primary key,
+ "CALENDAR_RESOURCE_ID" integer not null references CALENDAR on delete cascade,
+ "CALENDAR_OBJECT_RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade,
+ "FLOATING" integer not null,
+ "START_DATE" timestamp not null,
+ "END_DATE" timestamp not null,
+ "FBTYPE" integer not null,
+ "TRANSPARENT" integer not null
+);
+
+create table FREE_BUSY_TYPE (
+ "ID" integer primary key,
+ "DESCRIPTION" nvarchar2(16) unique
+);
+
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('unknown', 0);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('free', 1);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('busy', 2);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('busy-unavailable', 3);
+insert into FREE_BUSY_TYPE (DESCRIPTION, ID) values ('busy-tentative', 4);
+create table TRANSPARENCY (
+ "TIME_RANGE_INSTANCE_ID" integer not null references TIME_RANGE on delete cascade,
+ "USER_ID" nvarchar2(255),
+ "TRANSPARENT" integer not null
+);
+
+create table ATTACHMENT (
+ "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
+ "DROPBOX_ID" nvarchar2(255),
+ "CONTENT_TYPE" nvarchar2(255),
+ "SIZE" integer not null,
+ "MD5" nchar(32),
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "PATH" nvarchar2(1024),
+ primary key("DROPBOX_ID", "PATH")
+);
+
+create table RESOURCE_PROPERTY (
+ "RESOURCE_ID" integer not null,
+ "NAME" nvarchar2(255),
+ "VALUE" nclob,
+ "VIEWER_UID" nvarchar2(255),
+ primary key("RESOURCE_ID", "NAME", "VIEWER_UID")
+);
+
+create table ADDRESSBOOK_HOME (
+ "RESOURCE_ID" integer primary key,
+ "OWNER_UID" nvarchar2(255) unique,
+ "DATAVERSION" integer default 0 not null
+);
+
+create table ADDRESSBOOK_HOME_METADATA (
+ "RESOURCE_ID" integer primary key references ADDRESSBOOK_HOME on delete cascade,
+ "QUOTA_USED_BYTES" integer default 0 not null,
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table ADDRESSBOOK (
+ "RESOURCE_ID" integer primary key
+);
+
+create table ADDRESSBOOK_METADATA (
+ "RESOURCE_ID" integer primary key references ADDRESSBOOK on delete cascade,
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
+create table ADDRESSBOOK_BIND (
+ "ADDRESSBOOK_HOME_RESOURCE_ID" integer not null references ADDRESSBOOK_HOME,
+ "ADDRESSBOOK_RESOURCE_ID" integer not null references ADDRESSBOOK on delete cascade,
+ "ADDRESSBOOK_RESOURCE_NAME" nvarchar2(255),
+ "BIND_MODE" integer not null,
+ "BIND_STATUS" integer not null,
+ "SEEN_BY_OWNER" integer not null,
+ "SEEN_BY_SHAREE" integer not null,
+ "MESSAGE" nclob,
+ primary key("ADDRESSBOOK_HOME_RESOURCE_ID", "ADDRESSBOOK_RESOURCE_ID"),
+ unique("ADDRESSBOOK_HOME_RESOURCE_ID", "ADDRESSBOOK_RESOURCE_NAME")
+);
+
+create table ADDRESSBOOK_OBJECT (
+ "RESOURCE_ID" integer primary key,
+ "ADDRESSBOOK_RESOURCE_ID" integer not null references ADDRESSBOOK on delete cascade,
+ "RESOURCE_NAME" nvarchar2(255),
+ "VCARD_TEXT" nclob,
+ "VCARD_UID" nvarchar2(255),
+ "MD5" nchar(32),
+ "CREATED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ "MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC',
+ unique("ADDRESSBOOK_RESOURCE_ID", "RESOURCE_NAME"),
+ unique("ADDRESSBOOK_RESOURCE_ID", "VCARD_UID")
+);
+
+create table CALENDAR_OBJECT_REVISIONS (
+ "CALENDAR_HOME_RESOURCE_ID" integer not null references CALENDAR_HOME,
+ "CALENDAR_RESOURCE_ID" integer references CALENDAR,
+ "CALENDAR_NAME" nvarchar2(255) default null,
+ "RESOURCE_NAME" nvarchar2(255),
+ "REVISION" integer not null,
+ "DELETED" integer not null
+);
+
+create table ADDRESSBOOK_OBJECT_REVISIONS (
+ "ADDRESSBOOK_HOME_RESOURCE_ID" integer not null references ADDRESSBOOK_HOME,
+ "ADDRESSBOOK_RESOURCE_ID" integer references ADDRESSBOOK,
+ "ADDRESSBOOK_NAME" nvarchar2(255) default null,
+ "RESOURCE_NAME" nvarchar2(255),
+ "REVISION" integer not null,
+ "DELETED" integer not null
+);
+
+create table NOTIFICATION_OBJECT_REVISIONS (
+ "NOTIFICATION_HOME_RESOURCE_ID" integer not null references NOTIFICATION_HOME on delete cascade,
+ "RESOURCE_NAME" nvarchar2(255),
+ "REVISION" integer not null,
+ "DELETED" integer not null,
+ unique("NOTIFICATION_HOME_RESOURCE_ID", "RESOURCE_NAME")
+);
+
+create table APN_SUBSCRIPTIONS (
+ "TOKEN" nvarchar2(255),
+ "RESOURCE_KEY" nvarchar2(255),
+ "MODIFIED" integer not null,
+ "SUBSCRIBER_GUID" nvarchar2(255),
+ primary key("TOKEN", "RESOURCE_KEY")
+);
+
+create table CALENDARSERVER (
+ "NAME" nvarchar2(255) primary key,
+ "VALUE" nvarchar2(255)
+);
+
+insert into CALENDARSERVER (NAME, VALUE) values ('VERSION', '10');
+insert into CALENDARSERVER (NAME, VALUE) values ('CALENDAR-DATAVERSION', '2');
+insert into CALENDARSERVER (NAME, VALUE) values ('ADDRESSBOOK-DATAVERSION', '1');
+create index INVITE_INVITE_UID_9b0902ff on INVITE (
+ INVITE_UID
+);
+
+create index INVITE_RESOURCE_ID_b36ddc23 on INVITE (
+ RESOURCE_ID
+);
+
+create index INVITE_HOME_RESOURCE__e9bdf77e on INVITE (
+ HOME_RESOURCE_ID
+);
+
+create index NOTIFICATION_NOTIFICA_f891f5f9 on NOTIFICATION (
+ NOTIFICATION_HOME_RESOURCE_ID
+);
+
+create index CALENDAR_BIND_RESOURC_e57964d4 on CALENDAR_BIND (
+ CALENDAR_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_CALEN_a9a453a9 on CALENDAR_OBJECT (
+ CALENDAR_RESOURCE_ID,
+ ICALENDAR_UID
+);
+
+create index CALENDAR_OBJECT_CALEN_96e83b73 on CALENDAR_OBJECT (
+ CALENDAR_RESOURCE_ID,
+ RECURRANCE_MAX
+);
+
+create index CALENDAR_OBJECT_ORGAN_7ce24750 on CALENDAR_OBJECT (
+ ORGANIZER_OBJECT
+);
+
+create index CALENDAR_OBJECT_DROPB_de041d80 on CALENDAR_OBJECT (
+ DROPBOX_ID
+);
+
+create index TIME_RANGE_CALENDAR_R_beb6e7eb on TIME_RANGE (
+ CALENDAR_RESOURCE_ID
+);
+
+create index TIME_RANGE_CALENDAR_O_acf37bd1 on TIME_RANGE (
+ CALENDAR_OBJECT_RESOURCE_ID
+);
+
+create index TRANSPARENCY_TIME_RAN_5f34467f on TRANSPARENCY (
+ TIME_RANGE_INSTANCE_ID
+);
+
+create index ATTACHMENT_CALENDAR_H_0078845c on ATTACHMENT (
+ CALENDAR_HOME_RESOURCE_ID
+);
+
+create index ADDRESSBOOK_BIND_RESO_205aa75c on ADDRESSBOOK_BIND (
+ ADDRESSBOOK_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_REVIS_3a3956c4 on CALENDAR_OBJECT_REVISIONS (
+ CALENDAR_HOME_RESOURCE_ID,
+ CALENDAR_RESOURCE_ID
+);
+
+create index CALENDAR_OBJECT_REVIS_2643d556 on CALENDAR_OBJECT_REVISIONS (
+ CALENDAR_RESOURCE_ID,
+ RESOURCE_NAME
+);
+
+create index CALENDAR_OBJECT_REVIS_265c8acf on CALENDAR_OBJECT_REVISIONS (
+ CALENDAR_RESOURCE_ID,
+ REVISION
+);
+
+create index ADDRESSBOOK_OBJECT_RE_f460d62d on ADDRESSBOOK_OBJECT_REVISIONS (
+ ADDRESSBOOK_HOME_RESOURCE_ID,
+ ADDRESSBOOK_RESOURCE_ID
+);
+
+create index ADDRESSBOOK_OBJECT_RE_9a848f39 on ADDRESSBOOK_OBJECT_REVISIONS (
+ ADDRESSBOOK_RESOURCE_ID,
+ RESOURCE_NAME
+);
+
+create index ADDRESSBOOK_OBJECT_RE_cb101e6b on ADDRESSBOOK_OBJECT_REVISIONS (
+ ADDRESSBOOK_RESOURCE_ID,
+ REVISION
+);
+
+create index NOTIFICATION_OBJECT_R_036a9cee on NOTIFICATION_OBJECT_REVISIONS (
+ NOTIFICATION_HOME_RESOURCE_ID,
+ REVISION
+);
+
+create index APN_SUBSCRIPTIONS_RES_9610d78e on APN_SUBSCRIPTIONS (
+ RESOURCE_KEY
+);
+
Copied: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/postgres-dialect/v10.sql (from rev 9546, CalendarServer/trunk/txdav/common/datastore/sql_schema/old/postgres-dialect/v10.sql)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/postgres-dialect/v10.sql (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/old/postgres-dialect/v10.sql 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,500 @@
+-- -*- test-case-name: txdav.caldav.datastore.test.test_sql,txdav.carddav.datastore.test.test_sql -*-
+
+----
+-- Copyright (c) 2010-2012 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-----------------
+-- Resource ID --
+-----------------
+
+create sequence RESOURCE_ID_SEQ;
+
+
+-------------------
+-- Calendar Home --
+-------------------
+
+create table CALENDAR_HOME (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ OWNER_UID varchar(255) not null unique, -- implicit index
+ DATAVERSION integer default 0 not null
+);
+
+----------------------------
+-- Calendar Home Metadata --
+----------------------------
+
+create table CALENDAR_HOME_METADATA (
+ RESOURCE_ID integer primary key references CALENDAR_HOME on delete cascade, -- implicit index
+ QUOTA_USED_BYTES integer default 0 not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+--------------
+-- Calendar --
+--------------
+
+create table CALENDAR (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ') -- implicit index
+);
+
+
+-----------------------
+-- Calendar Metadata --
+-----------------------
+
+create table CALENDAR_METADATA (
+ RESOURCE_ID integer primary key references CALENDAR on delete cascade, -- implicit index
+ SUPPORTED_COMPONENTS varchar(255) default null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+
+------------------------
+-- Sharing Invitation --
+------------------------
+
+create table INVITE (
+ INVITE_UID varchar(255) not null,
+ NAME varchar(255) not null,
+ RECIPIENT_ADDRESS varchar(255) not null,
+ HOME_RESOURCE_ID integer not null,
+ RESOURCE_ID integer not null
+
+ -- Need primary key on (INVITE_UID, NAME, RECIPIENT_ADDRESS)?
+);
+
+create index INVITE_INVITE_UID on INVITE(INVITE_UID);
+create index INVITE_RESOURCE_ID on INVITE(RESOURCE_ID);
+create index INVITE_HOME_RESOURCE_ID on INVITE(HOME_RESOURCE_ID);
+
+---------------------------
+-- Sharing Notifications --
+---------------------------
+
+create table NOTIFICATION_HOME (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ OWNER_UID varchar(255) not null unique -- implicit index
+);
+
+create table NOTIFICATION (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ NOTIFICATION_HOME_RESOURCE_ID integer not null references NOTIFICATION_HOME,
+ NOTIFICATION_UID varchar(255) not null,
+ XML_TYPE varchar(255) not null,
+ XML_DATA text not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+ unique(NOTIFICATION_UID, NOTIFICATION_HOME_RESOURCE_ID) -- implicit index
+);
+
+create index NOTIFICATION_NOTIFICATION_HOME_RESOURCE_ID on
+ NOTIFICATION(NOTIFICATION_HOME_RESOURCE_ID);
+
+-------------------
+-- Calendar Bind --
+-------------------
+
+-- Joins CALENDAR_HOME and CALENDAR
+
+create table CALENDAR_BIND (
+ CALENDAR_HOME_RESOURCE_ID integer not null references CALENDAR_HOME,
+ CALENDAR_RESOURCE_ID integer not null references CALENDAR on delete cascade,
+
+ -- An invitation which hasn't been accepted yet will not yet have a resource
+ -- name, so this field may be null.
+
+ CALENDAR_RESOURCE_NAME varchar(255),
+ BIND_MODE integer not null, -- enum CALENDAR_BIND_MODE
+ BIND_STATUS integer not null, -- enum CALENDAR_BIND_STATUS
+ SEEN_BY_OWNER boolean not null,
+ SEEN_BY_SHAREE boolean not null,
+ MESSAGE text,
+
+ primary key(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_ID), -- implicit index
+ unique(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_NAME) -- implicit index
+);
+
+create index CALENDAR_BIND_RESOURCE_ID on CALENDAR_BIND(CALENDAR_RESOURCE_ID);
+
+-- Enumeration of calendar bind modes
+
+create table CALENDAR_BIND_MODE (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_BIND_MODE values (0, 'own' );
+insert into CALENDAR_BIND_MODE values (1, 'read' );
+insert into CALENDAR_BIND_MODE values (2, 'write');
+insert into CALENDAR_BIND_MODE values (3, 'direct');
+
+-- Enumeration of statuses
+
+create table CALENDAR_BIND_STATUS (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_BIND_STATUS values (0, 'invited' );
+insert into CALENDAR_BIND_STATUS values (1, 'accepted');
+insert into CALENDAR_BIND_STATUS values (2, 'declined');
+insert into CALENDAR_BIND_STATUS values (3, 'invalid');
+
+
+---------------------
+-- Calendar Object --
+---------------------
+
+create table CALENDAR_OBJECT (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ CALENDAR_RESOURCE_ID integer not null references CALENDAR on delete cascade,
+ RESOURCE_NAME varchar(255) not null,
+ ICALENDAR_TEXT text not null,
+ ICALENDAR_UID varchar(255) not null,
+ ICALENDAR_TYPE varchar(255) not null,
+ ATTACHMENTS_MODE integer default 0 not null, -- enum CALENDAR_OBJECT_ATTACHMENTS_MODE
+ DROPBOX_ID varchar(255),
+ ORGANIZER varchar(255),
+ ORGANIZER_OBJECT integer references CALENDAR_OBJECT,
+ RECURRANCE_MIN date, -- minimum date that recurrences have been expanded to.
+ RECURRANCE_MAX date, -- maximum date that recurrences have been expanded to.
+ ACCESS integer default 0 not null,
+ SCHEDULE_OBJECT boolean default false,
+ SCHEDULE_TAG varchar(36) default null,
+ SCHEDULE_ETAGS text default null,
+ PRIVATE_COMMENTS boolean default false not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+ unique(CALENDAR_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+
+ -- since the 'inbox' is a 'calendar resource' for the purpose of storing
+ -- calendar objects, this constraint has to be selectively enforced by the
+ -- application layer.
+
+ -- unique(CALENDAR_RESOURCE_ID, ICALENDAR_UID)
+);
+
+create index CALENDAR_OBJECT_CALENDAR_RESOURCE_ID_AND_ICALENDAR_UID on
+ CALENDAR_OBJECT(CALENDAR_RESOURCE_ID, ICALENDAR_UID);
+
+create index CALENDAR_OBJECT_CALENDAR_RESOURCE_ID_RECURRANCE_MAX on
+ CALENDAR_OBJECT(CALENDAR_RESOURCE_ID, RECURRANCE_MAX);
+
+create index CALENDAR_OBJECT_ORGANIZER_OBJECT on
+ CALENDAR_OBJECT(ORGANIZER_OBJECT);
+
+create index CALENDAR_OBJECT_DROPBOX_ID on
+ CALENDAR_OBJECT(DROPBOX_ID);
+
+-- Enumeration of attachment modes
+
+create table CALENDAR_OBJECT_ATTACHMENTS_MODE (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (0, 'none' );
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (1, 'read' );
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (2, 'write');
+
+
+-- Enumeration of calendar access types
+
+create table CALENDAR_ACCESS_TYPE (
+ ID integer primary key,
+ DESCRIPTION varchar(32) not null unique
+);
+
+insert into CALENDAR_ACCESS_TYPE values (0, '' );
+insert into CALENDAR_ACCESS_TYPE values (1, 'public' );
+insert into CALENDAR_ACCESS_TYPE values (2, 'private' );
+insert into CALENDAR_ACCESS_TYPE values (3, 'confidential' );
+insert into CALENDAR_ACCESS_TYPE values (4, 'restricted' );
+
+-----------------
+-- Instance ID --
+-----------------
+
+create sequence INSTANCE_ID_SEQ;
+
+
+----------------
+-- Time Range --
+----------------
+
+create table TIME_RANGE (
+ INSTANCE_ID integer primary key default nextval('INSTANCE_ID_SEQ'), -- implicit index
+ CALENDAR_RESOURCE_ID integer not null references CALENDAR on delete cascade,
+ CALENDAR_OBJECT_RESOURCE_ID integer not null references CALENDAR_OBJECT on delete cascade,
+ FLOATING boolean not null,
+ START_DATE timestamp not null,
+ END_DATE timestamp not null,
+ FBTYPE integer not null,
+ TRANSPARENT boolean not null
+);
+
+create index TIME_RANGE_CALENDAR_RESOURCE_ID on
+ TIME_RANGE(CALENDAR_RESOURCE_ID);
+create index TIME_RANGE_CALENDAR_OBJECT_RESOURCE_ID on
+ TIME_RANGE(CALENDAR_OBJECT_RESOURCE_ID);
+
+
+-- Enumeration of free/busy types
+
+create table FREE_BUSY_TYPE (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into FREE_BUSY_TYPE values (0, 'unknown' );
+insert into FREE_BUSY_TYPE values (1, 'free' );
+insert into FREE_BUSY_TYPE values (2, 'busy' );
+insert into FREE_BUSY_TYPE values (3, 'busy-unavailable');
+insert into FREE_BUSY_TYPE values (4, 'busy-tentative' );
+
+
+------------------
+-- Transparency --
+------------------
+
+create table TRANSPARENCY (
+ TIME_RANGE_INSTANCE_ID integer not null references TIME_RANGE on delete cascade,
+ USER_ID varchar(255) not null,
+ TRANSPARENT boolean not null
+);
+
+create index TRANSPARENCY_TIME_RANGE_INSTANCE_ID on
+ TRANSPARENCY(TIME_RANGE_INSTANCE_ID);
+
+----------------
+-- Attachment --
+----------------
+
+create table ATTACHMENT (
+ CALENDAR_HOME_RESOURCE_ID integer not null references CALENDAR_HOME,
+ DROPBOX_ID varchar(255) not null,
+ CONTENT_TYPE varchar(255) not null,
+ SIZE integer not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ PATH varchar(1024) not null,
+
+ primary key(DROPBOX_ID, PATH) --implicit index
+);
+
+create index ATTACHMENT_CALENDAR_HOME_RESOURCE_ID on
+ ATTACHMENT(CALENDAR_HOME_RESOURCE_ID);
+
+-----------------------
+-- Resource Property --
+-----------------------
+
+create table RESOURCE_PROPERTY (
+ RESOURCE_ID integer not null, -- foreign key: *.RESOURCE_ID
+ NAME varchar(255) not null,
+ VALUE text not null, -- FIXME: xml?
+ VIEWER_UID varchar(255),
+
+ primary key(RESOURCE_ID, NAME, VIEWER_UID) -- implicit index
+);
+
+
+----------------------
+-- AddressBook Home --
+----------------------
+
+create table ADDRESSBOOK_HOME (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ OWNER_UID varchar(255) not null unique, -- implicit index
+ DATAVERSION integer default 0 not null
+);
+
+-------------------------------
+-- AddressBook Home Metadata --
+-------------------------------
+
+create table ADDRESSBOOK_HOME_METADATA (
+ RESOURCE_ID integer primary key references ADDRESSBOOK_HOME on delete cascade, -- implicit index
+ QUOTA_USED_BYTES integer default 0 not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+-----------------
+-- AddressBook --
+-----------------
+
+create table ADDRESSBOOK (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ') -- implicit index
+);
+
+
+--------------------------
+-- AddressBook Metadata --
+--------------------------
+
+create table ADDRESSBOOK_METADATA (
+ RESOURCE_ID integer primary key references ADDRESSBOOK on delete cascade, -- implicit index
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+
+----------------------
+-- AddressBook Bind --
+----------------------
+
+-- Joins ADDRESSBOOK_HOME and ADDRESSBOOK
+
+create table ADDRESSBOOK_BIND (
+ ADDRESSBOOK_HOME_RESOURCE_ID integer not null references ADDRESSBOOK_HOME,
+ ADDRESSBOOK_RESOURCE_ID integer not null references ADDRESSBOOK on delete cascade,
+
+ -- An invitation which hasn't been accepted yet will not yet have a resource
+ -- name, so this field may be null.
+
+ ADDRESSBOOK_RESOURCE_NAME varchar(255),
+ BIND_MODE integer not null, -- enum CALENDAR_BIND_MODE
+ BIND_STATUS integer not null, -- enum CALENDAR_BIND_STATUS
+ SEEN_BY_OWNER boolean not null,
+ SEEN_BY_SHAREE boolean not null,
+ MESSAGE text, -- FIXME: xml?
+
+ primary key(ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_ID), -- implicit index
+ unique(ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_NAME) -- implicit index
+);
+
+create index ADDRESSBOOK_BIND_RESOURCE_ID on
+ ADDRESSBOOK_BIND(ADDRESSBOOK_RESOURCE_ID);
+
+create table ADDRESSBOOK_OBJECT (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ ADDRESSBOOK_RESOURCE_ID integer not null references ADDRESSBOOK on delete cascade,
+ RESOURCE_NAME varchar(255) not null,
+ VCARD_TEXT text not null,
+ VCARD_UID varchar(255) not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+ unique(ADDRESSBOOK_RESOURCE_ID, RESOURCE_NAME), -- implicit index
+ unique(ADDRESSBOOK_RESOURCE_ID, VCARD_UID) -- implicit index
+);
+
+---------------
+-- Revisions --
+---------------
+
+create sequence REVISION_SEQ;
+
+
+---------------
+-- Revisions --
+---------------
+
+create table CALENDAR_OBJECT_REVISIONS (
+ CALENDAR_HOME_RESOURCE_ID integer not null references CALENDAR_HOME,
+ CALENDAR_RESOURCE_ID integer references CALENDAR,
+ CALENDAR_NAME varchar(255) default null,
+ RESOURCE_NAME varchar(255),
+ REVISION integer default nextval('REVISION_SEQ') not null,
+ DELETED boolean not null
+);
+
+create index CALENDAR_OBJECT_REVISIONS_HOME_RESOURCE_ID_CALENDAR_RESOURCE_ID
+ on CALENDAR_OBJECT_REVISIONS(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_ID);
+
+create index CALENDAR_OBJECT_REVISIONS_RESOURCE_ID_RESOURCE_NAME
+ on CALENDAR_OBJECT_REVISIONS(CALENDAR_RESOURCE_ID, RESOURCE_NAME);
+
+create index CALENDAR_OBJECT_REVISIONS_RESOURCE_ID_REVISION
+ on CALENDAR_OBJECT_REVISIONS(CALENDAR_RESOURCE_ID, REVISION);
+
+-------------------------------
+-- AddressBook Object Revisions --
+-------------------------------
+
+create table ADDRESSBOOK_OBJECT_REVISIONS (
+ ADDRESSBOOK_HOME_RESOURCE_ID integer not null references ADDRESSBOOK_HOME,
+ ADDRESSBOOK_RESOURCE_ID integer references ADDRESSBOOK,
+ ADDRESSBOOK_NAME varchar(255) default null,
+ RESOURCE_NAME varchar(255),
+ REVISION integer default nextval('REVISION_SEQ') not null,
+ DELETED boolean not null
+);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_HOME_RESOURCE_ID_ADDRESSBOOK_RESOURCE_ID
+ on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_ID);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_RESOURCE_ID_RESOURCE_NAME
+ on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_RESOURCE_ID, RESOURCE_NAME);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_RESOURCE_ID_REVISION
+ on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_RESOURCE_ID, REVISION);
+
+-----------------------------------
+-- Notification Object Revisions --
+-----------------------------------
+
+create table NOTIFICATION_OBJECT_REVISIONS (
+ NOTIFICATION_HOME_RESOURCE_ID integer not null references NOTIFICATION_HOME on delete cascade,
+ RESOURCE_NAME varchar(255),
+ REVISION integer default nextval('REVISION_SEQ') not null,
+ DELETED boolean not null,
+
+ unique(NOTIFICATION_HOME_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+);
+
+create index NOTIFICATION_OBJECT_REVISIONS_RESOURCE_ID_REVISION
+ on NOTIFICATION_OBJECT_REVISIONS(NOTIFICATION_HOME_RESOURCE_ID, REVISION);
+
+-------------------------------------------
+-- Apple Push Notification Subscriptions --
+-------------------------------------------
+
+create table APN_SUBSCRIPTIONS (
+ TOKEN varchar(255) not null,
+ RESOURCE_KEY varchar(255) not null,
+ MODIFIED integer not null,
+ SUBSCRIBER_GUID varchar(255) not null,
+
+ primary key(TOKEN, RESOURCE_KEY) -- implicit index
+);
+
+create index APN_SUBSCRIPTIONS_RESOURCE_KEY
+ on APN_SUBSCRIPTIONS(RESOURCE_KEY);
+
+
+--------------------
+-- Schema Version --
+--------------------
+
+create table CALENDARSERVER (
+ NAME varchar(255) primary key, -- implicit index
+ VALUE varchar(255)
+);
+
+insert into CALENDARSERVER values ('VERSION', '10');
+insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '3');
+insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '1');
Copied: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_10_to_11.sql (from rev 9546, CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_10_to_11.sql)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_10_to_11.sql (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_10_to_11.sql 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,28 @@
+----
+-- Copyright (c) 2012 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+---------------------------------------------------
+-- Upgrade database schema from VERSION 10 to 11 --
+---------------------------------------------------
+
+alter table APN_SUBSCRIPTIONS
+ add ("USER_AGENT" nvarchar2(255) default null);
+alter table APN_SUBSCRIPTIONS
+ add ("IP_ADDR" nvarchar2(255) default null);
+
+-- Now update the version
+-- No data upgrades
+update CALENDARSERVER set VALUE = '11' where NAME = 'VERSION';
Copied: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_10_to_11.sql (from rev 9546, CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_10_to_11.sql)
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_10_to_11.sql (rev 0)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_10_to_11.sql 2012-08-09 19:03:13 UTC (rev 9547)
@@ -0,0 +1,27 @@
+----
+-- Copyright (c) 2012 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+---------------------------------------------------
+-- Upgrade database schema from VERSION 10 to 11 --
+---------------------------------------------------
+
+alter table APN_SUBSCRIPTIONS
+ add column USER_AGENT varchar(255) default null,
+ add column IP_ADDR varchar(255) default null;
+
+-- Now update the version
+-- No data upgrades
+update CALENDARSERVER set VALUE = '11' where NAME = 'VERSION';
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/test/test_sql.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/datastore/test/test_sql.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -331,7 +331,9 @@
yield _normalizeColumnUUIDs(txn, rp.VIEWER_UID)
self.assertEqual(
(yield Select([rp.RESOURCE_ID, rp.NAME,
- rp.VALUE, rp.VIEWER_UID], From=rp).on(txn)),
+ rp.VALUE, rp.VIEWER_UID], From=rp,
+ OrderBy=rp.RESOURCE_ID, Ascending=True,
+ ).on(txn)),
[[1, "asdf", "property-value", "not-a-uuid"],
[2, "fdsa", "another-value",
"AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"]]
Modified: CalendarServer/branches/users/gaya/inviteclean/txdav/common/icommondatastore.py
===================================================================
--- CalendarServer/branches/users/gaya/inviteclean/txdav/common/icommondatastore.py 2012-08-09 00:35:29 UTC (rev 9546)
+++ CalendarServer/branches/users/gaya/inviteclean/txdav/common/icommondatastore.py 2012-08-09 19:03:13 UTC (rev 9547)
@@ -167,7 +167,7 @@
notification collection exists.
"""
- def addAPNSubscription(token, key, timestamp, subscriber):
+ def addAPNSubscription(token, key, timestamp, subscriber, userAgent, ipAddr):
"""
Add (or update) a subscription entry in the database.
@@ -182,6 +182,12 @@
@param subscriber: The GUID of the subscribing principal
@type subscrbier: C{str}
+
+ @param userAgent: The user agent requesting the subscription
+ @type userAgent: C{str}
+
+ @param ipAddr: The ip address requesting the subscription
+ @type ipAddr: C{str}
"""
def removeAPNSubscription(token, key):
@@ -195,6 +201,15 @@
@type key: C{str}
"""
+ def purgeOldAPNSubscriptions(olderThan):
+ """
+ Remove all subscription entries whose modified timestamp
+ is older than the provided timestamp.
+
+ @param olderThan: The cutoff timestamp in seconds
+ @type token: C{int}
+ """
+
def apnSubscriptionsByToken(token):
"""
Retrieve all subscription entries for the token.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20120809/0ef2ef8b/attachment-0001.html>
More information about the calendarserver-changes
mailing list