[CalendarServer-changes] [10139] CalendarServer/branches/users/cdaboo/managed-attachments
source_changes at macosforge.org
source_changes at macosforge.org
Fri Dec 7 13:06:10 PST 2012
Revision: 10139
http://trac.calendarserver.org//changeset/10139
Author: cdaboo at apple.com
Date: 2012-12-07 13:06:09 -0800 (Fri, 07 Dec 2012)
Log Message:
-----------
Merged from trunk.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/provision/test/test_root.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/amppush.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/applepush.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_amppush.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_applepush.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/util.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tap/util.py
CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tools/ampnotifications.py
CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-apple.plist
CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-test.plist
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/calendarcommonextra.py
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/test/test_commonextra.py
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ampsim.py
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/config.plist
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ical.py
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/sim.py
CalendarServer/branches/users/cdaboo/managed-attachments/contrib/tools/readStats.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/python/memcacheclient.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/channel/http.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/auth.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_acl.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_resource.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/http.py
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/test/test_metafd.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/cachingdirectory.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/accounts.xml
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/augments.xml
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/proxies.xml
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_cachedirectory.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/ical.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/resource.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/test/test_timezonestdservice.py
CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/timezonestdservice.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_auth.py
Removed Paths:
-------------
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-Apache.txt
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-OpenDirectory.txt
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-XML.txt
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryServices.txt
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/ExtendedLogItems.txt
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/LoadSimulation.txt
CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/MultiServerDeployment.txt
Property Changed:
----------------
CalendarServer/branches/users/cdaboo/managed-attachments/
Property changes on: CalendarServer/branches/users/cdaboo/managed-attachments
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:9985-10097
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:9985-10136
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/provision/test/test_root.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/provision/test/test_root.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/provision/test/test_root.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -85,7 +85,8 @@
self.root = auth.AuthenticationWrapper(
root,
portal,
- credentialFactories=(basic.BasicCredentialFactory("Test realm"),),
+ (basic.BasicCredentialFactory("Test realm"),),
+ (basic.BasicCredentialFactory("Test realm"),),
loginInterfaces=(auth.IPrincipal,))
self.site = server.Site(self.root)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/amppush.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/amppush.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/amppush.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -23,6 +23,7 @@
from twisted.internet.protocol import Factory, ServerFactory
from twisted.protocols import amp
from twistedcaldav.notify import getPubSubPath
+import time
import uuid
@@ -44,7 +45,7 @@
# AMP Commands sent to client
class NotificationForID(amp.Command):
- arguments = [('id', amp.String())]
+ arguments = [('id', amp.String()), ('dataChangedTimestamp', amp.Integer())]
response = [('status', amp.String())]
@@ -84,14 +85,13 @@
self.log_debug("Removed subscriber")
self.subscribers.remove(p)
- def enqueue(self, op, id):
+ def enqueue(self, op, id, dataChangedTimestamp=None):
"""
Sends an AMP push notification to any clients subscribing to this id.
@param op: The operation that took place, either "create" or "update"
(ignored in this implementation)
@type op: C{str}
-
@param id: The identifier of the resource that was updated, including
a prefix indicating whether this is CalDAV or CardDAV related.
The prefix is separated from the id with "|", e.g.:
@@ -102,6 +102,9 @@
is used in conjunction with the prefix and the server hostname
to build the actual key value that devices subscribe to.
@type id: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification (Only used for unit tests)
+ @type key: C{int}
"""
try:
@@ -113,29 +116,33 @@
id = getPubSubPath(id, {"host": self.serverHostName})
+ # Unit tests can pass this value in; otherwise it defaults to now
+ if dataChangedTimestamp is None:
+ dataChangedTimestamp = int(time.time())
+
tokens = []
for subscriber in self.subscribers:
token = subscriber.subscribedToID(id)
if token is not None:
tokens.append(token)
if tokens:
- return self.scheduleNotifications(tokens, id)
+ return self.scheduleNotifications(tokens, id, dataChangedTimestamp)
@inlineCallbacks
- def sendNotification(self, token, id):
+ def sendNotification(self, token, id, dataChangedTimestamp):
for subscriber in self.subscribers:
if subscriber.subscribedToID(id):
- yield subscriber.notify(token, id)
+ yield subscriber.notify(token, id, dataChangedTimestamp)
@inlineCallbacks
- def scheduleNotifications(self, tokens, id):
+ def scheduleNotifications(self, tokens, id, dataChangedTimestamp):
if self.scheduler is not None:
- self.scheduler.schedule(tokens, id)
+ self.scheduler.schedule(tokens, id, dataChangedTimestamp)
else:
for token in tokens:
- yield self.sendNotification(token, id)
+ yield self.sendNotification(token, id, dataChangedTimestamp)
class AMPPushNotifierProtocol(amp.AMP, LoggingMixIn):
@@ -162,10 +169,11 @@
return {"status" : "OK"}
UnsubscribeFromID.responder(unsubscribe)
- def notify(self, token, id):
+ def notify(self, token, id, dataChangedTimestamp):
if self.subscribedToID(id) == token:
self.log_debug("Sending notification for %s to %s" % (id, token))
- return self.callRemote(NotificationForID, id=id)
+ return self.callRemote(NotificationForID, id=id,
+ dataChangedTimestamp=dataChangedTimestamp)
def subscribedToID(self, id):
if self.any is not None:
@@ -204,8 +212,8 @@
self.callback = callback
@inlineCallbacks
- def notificationForID(self, id):
- yield self.callback(id)
+ def notificationForID(self, id, dataChangedTimestamp):
+ yield self.callback(id, dataChangedTimestamp)
returnValue( {"status" : "OK"} )
NotificationForID.responder(notificationForID)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/applepush.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/applepush.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/applepush.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -31,6 +31,7 @@
from twisted.internet.task import LoopingCall
from twistedcaldav.extensions import DAVResource, DAVResourceWithoutChildrenMixin
from twistedcaldav.resource import ReadOnlyNoCopyResourceMixIn
+import json
import OpenSSL
import struct
import time
@@ -176,7 +177,7 @@
@inlineCallbacks
- def enqueue(self, op, id):
+ def enqueue(self, op, id, dataChangedTimestamp=None):
"""
Sends an Apple Push Notification to any device token subscribed to
this id.
@@ -184,7 +185,6 @@
@param op: The operation that took place, either "create" or "update"
(ignored in this implementation)
@type op: C{str}
-
@param id: The identifier of the resource that was updated, including
a prefix indicating whether this is CalDAV or CardDAV related.
The prefix is separated from the id with "|", e.g.:
@@ -195,6 +195,9 @@
is used in conjunction with the prefix and the server hostname
to build the actual key value that devices subscribe to.
@type id: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification (Only used for unit tests)
+ @type key: C{int}
"""
try:
@@ -204,6 +207,10 @@
self.log_error("Notification id '%s' is missing protocol" % (id,))
return
+ # Unit tests can pass this value in; otherwise it defaults to now
+ if dataChangedTimestamp is None:
+ dataChangedTimestamp = int(time.time())
+
provider = self.providers.get(protocol, None)
if provider is not None:
key = "/%s/%s/%s/" % (protocol, self.dataHost, id)
@@ -222,7 +229,7 @@
if token and uid:
tokens.append(token)
if tokens:
- provider.scheduleNotifications(tokens, key)
+ provider.scheduleNotifications(tokens, key, dataChangedTimestamp)
@@ -332,19 +339,21 @@
yield txn.commit()
- def sendNotification(self, token, key):
+ def sendNotification(self, token, key, dataChangedTimestamp):
"""
Sends a push notification message for the key to the device associated
with the token.
@param token: The device token subscribed to the key
@type token: C{str}
-
@param key: The key we're sending a notification about
@type key: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification
+ @type key: C{int}
"""
- if not (token and key):
+ if not (token and key and dataChangedTimestamp):
return
try:
@@ -354,7 +363,13 @@
return
identifier = self.history.add(token)
- payload = '{"key" : "%s"}' % (key,)
+ payload = json.dumps(
+ {
+ "key" : key,
+ "dataChangedTimestamp" : dataChangedTimestamp,
+ "pushRequestSubmittedTimestamp" : int(time.time()),
+ }
+ )
payloadLength = len(payload)
self.log_debug("Sending APNS notification to %s: id=%d payload=%s" %
(token, identifier, payload))
@@ -488,12 +503,12 @@
# sent will be put back into the queue.
queued = list(self.queue)
self.queue = []
- for token, key in queued:
- if token and key:
- self.sendNotification(token, key)
+ for (token, key), dataChangedTimestamp in queued:
+ if token and key and dataChangedTimestamp:
+ self.sendNotification(token, key, dataChangedTimestamp)
- def scheduleNotifications(self, tokens, key):
+ def scheduleNotifications(self, tokens, key, dataChangedTimestamp):
"""
The starting point for getting notifications to the APNS server. If there is
a connection to the APNS server, these notifications are scheduled (or directly
@@ -504,59 +519,70 @@
@type tokens: List of strings
@param key: The key to use for this batch of notifications
@type key: String
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification
+ @type key: C{int}
"""
# Service has reference to factory has reference to protocol instance
connection = getattr(self.factory, "connection", None)
if connection is not None:
if self.scheduler is not None:
- self.scheduler.schedule(tokens, key)
+ self.scheduler.schedule(tokens, key, dataChangedTimestamp)
else:
for token in tokens:
- self.sendNotification(token, key)
+ self.sendNotification(token, key, dataChangedTimestamp)
else:
- self._saveForWhenConnected(tokens, key)
+ self._saveForWhenConnected(tokens, key, dataChangedTimestamp)
- def _saveForWhenConnected(self, tokens, key):
+ def _saveForWhenConnected(self, tokens, key, dataChangedTimestamp):
"""
Called in order to save notifications that can't be sent now because there
is no connection to the APNS server. (token, key) tuples are appended to
the queue which is serviced during clientConnectionMade()
@param tokens: The device tokens to schedule notifications for
- @type tokens: List of strings
+ @type tokens: List of C{str}
@param key: The key to use for this batch of notifications
- @type key: String
+ @type key: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification
+ @type key: C{int}
"""
for token in tokens:
tokenKeyPair = (token, key)
- if tokenKeyPair not in self.queue:
+ for existingPair, ignored in self.queue:
+ if tokenKeyPair == existingPair:
+ self.log_debug("APNProviderService has no connection; skipping duplicate: %s %s" % (token, key))
+ break # Already scheduled
+ else:
self.log_debug("APNProviderService has no connection; queuing: %s %s" % (token, key))
- self.queue.append((token, key))
- else:
- self.log_debug("APNProviderService has no connection; skipping duplicate: %s %s" % (token, key))
+ self.queue.append(((token, key), dataChangedTimestamp))
- def sendNotification(self, token, key):
+ def sendNotification(self, token, key, dataChangedTimestamp):
"""
If there is a connection the notification is sent right away, otherwise
the notification is saved for later.
@param token: The device token to send a notifications to
- @type token: Strings
+ @type token: C{str}
@param key: The key to use for this notification
- @type key: String
+ @type key: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification
+ @type key: C{int}
"""
- if not (token and key):
+ if not (token and key and dataChangedTimestamp):
return
# Service has reference to factory has reference to protocol instance
connection = getattr(self.factory, "connection", None)
if connection is None:
- self._saveForWhenConnected([token], key)
+ self._saveForWhenConnected([token], key, dataChangedTimestamp)
else:
- connection.sendNotification(token, key)
+ connection.sendNotification(token, key, dataChangedTimestamp)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_amppush.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_amppush.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_amppush.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -67,27 +67,28 @@
self.assertTrue(client3.subscribedToID("/CalDAV/localhost/user02/"))
self.assertTrue(client3.subscribedToID("/CalDAV/localhost/user03/"))
- service.enqueue("update", "CalDAV|user01")
+ dataChangedTimestamp = 1354815999
+ service.enqueue("update", "CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
self.assertEquals(len(client1.history), 0)
self.assertEquals(len(client2.history), 0)
self.assertEquals(len(client3.history), 0)
clock.advance(1)
- self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/'})])
+ self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp': 1354815999})])
self.assertEquals(len(client2.history), 0)
self.assertEquals(len(client3.history), 0)
clock.advance(3)
- self.assertEquals(client2.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/'})])
+ self.assertEquals(client2.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp': 1354815999})])
self.assertEquals(len(client3.history), 0)
clock.advance(3)
- self.assertEquals(client3.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/'})])
+ self.assertEquals(client3.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp': 1354815999})])
client1.reset()
client2.reset()
client2.unsubscribe("token2", "/CalDAV/localhost/user01/")
- service.enqueue("update", "CalDAV|user01")
+ service.enqueue("update", "CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
self.assertEquals(len(client1.history), 0)
clock.advance(1)
- self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/'})])
+ self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp' : 1354815999})])
self.assertEquals(len(client2.history), 0)
clock.advance(3)
self.assertEquals(len(client2.history), 0)
@@ -97,9 +98,9 @@
client1.reset()
client2.reset()
client2.subscribe("token2", "/CalDAV/localhost/user01/")
- service.enqueue("update", "CalDAV|user01")
- self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/'})])
- self.assertEquals(client2.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/'})])
+ service.enqueue("update", "CalDAV|user01", dataChangedTimestamp=dataChangedTimestamp)
+ self.assertEquals(client1.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp' : 1354815999})])
+ self.assertEquals(client2.history, [(NotificationForID, {'id': '/CalDAV/localhost/user01/', 'dataChangedTimestamp' : 1354815999})])
class TestProtocol(AMPPushNotifierProtocol):
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_applepush.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_applepush.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/test/test_applepush.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -14,6 +14,7 @@
# limitations under the License.
##
+import json
import struct
import time
from calendarserver.push.applepush import (
@@ -125,11 +126,13 @@
# case by doing it prior to startService()
# Notification arrives from calendar server
- yield service.enqueue("update", "CalDAV|user01/calendar")
+ dataChangedTimestamp = 1354815999
+ yield service.enqueue("update", "CalDAV|user01/calendar",
+ dataChangedTimestamp=dataChangedTimestamp)
# The notifications should be in the queue
- self.assertTrue((token, key1) in service.providers["CalDAV"].queue)
- self.assertTrue((token2, key1) in service.providers["CalDAV"].queue)
+ self.assertTrue(((token, key1), dataChangedTimestamp) in service.providers["CalDAV"].queue)
+ self.assertTrue(((token2, key1), dataChangedTimestamp) in service.providers["CalDAV"].queue)
# Start the service, making the connection which should service the
# queue
@@ -141,14 +144,17 @@
# Verify data sent to APN
providerConnector = service.providers["CalDAV"].testConnector
rawData = providerConnector.transport.data
- self.assertEquals(len(rawData), 103)
+ self.assertEquals(len(rawData), 183)
data = struct.unpack("!BIIH32sH", rawData[:45])
self.assertEquals(data[0], 1) # command
self.assertEquals(data[4].encode("hex"), token.replace(" ", "")) # token
payloadLength = data[5]
payload = struct.unpack("%ds" % (payloadLength,),
rawData[45:])
- self.assertEquals(payload[0], '{"key" : "%s"}' % (key1,))
+ payload = json.loads(payload[0])
+ self.assertEquals(payload["key"], u"/CalDAV/calendars.example.com/user01/calendar/")
+ self.assertEquals(payload["dataChangedTimestamp"], dataChangedTimestamp)
+ self.assertTrue(payload.has_key("pushRequestSubmittedTimestamp"))
# Verify token history is updated
self.assertTrue(token in [t for (i, t) in providerConnector.service.protocol.history.history])
self.assertTrue(token2 in [t for (i, t) in providerConnector.service.protocol.history.history])
@@ -163,11 +169,11 @@
# Send notification while service is connected
yield service.enqueue("update", "CalDAV|user01/calendar")
clock.advance(1) # so that first push is sent
- self.assertEquals(len(providerConnector.transport.data), 103)
+ self.assertEquals(len(providerConnector.transport.data), 183)
# Reset sent data
providerConnector.transport.data = None
clock.advance(3) # so that second push is sent
- self.assertEquals(len(providerConnector.transport.data), 103)
+ self.assertEquals(len(providerConnector.transport.data), 183)
def errorTestFunction(status, identifier):
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/util.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/push/util.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -121,7 +121,7 @@
self.callback = callback
self.staggerSeconds = staggerSeconds
- def schedule(self, tokens, key):
+ def schedule(self, tokens, key, dataChangedTimestamp):
"""
Schedules a batch of notifications for the given tokens, staggered
with self.staggerSeconds between each one. Duplicates are ignored,
@@ -129,9 +129,12 @@
one will not be scheduled for that pair.
@param tokens: The device tokens to schedule notifications for
- @type tokens: List of strings
+ @type tokens: List of C{str}
@param key: The key to use for this batch of notifications
- @type key: String
+ @type key: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification
+ @type key: C{int}
"""
scheduleTime = 0.0
for token in tokens:
@@ -141,20 +144,28 @@
(internalKey,))
else:
self.outstanding[internalKey] = self.reactor.callLater(
- scheduleTime, self.send, token, key)
+ scheduleTime, self.send, token, key, dataChangedTimestamp)
self.log_debug("PushScheduler scheduled: %s in %.0f sec" %
(internalKey, scheduleTime))
scheduleTime += self.staggerSeconds
- def send(self, token, key):
+ def send(self, token, key, dataChangedTimestamp):
"""
This method is what actually gets scheduled. Its job is to remove
its corresponding entry from the outstanding dict and call the
callback.
+
+ @param token: The device token to send a notification to
+ @type token: C{str}
+ @param key: The notification key
+ @type key: C{str}
+ @param dataChangedTimestamp: Timestamp (epoch seconds) for the data change
+ which triggered this notification
+ @type key: C{int}
"""
- self.log_debug("PushScheduler fired for %s %s" % (token, key))
+ self.log_debug("PushScheduler fired for %s %s %d" % (token, key, dataChangedTimestamp))
del self.outstanding[(token, key)]
- return self.callback(token, key)
+ return self.callback(token, key, dataChangedTimestamp)
def stop(self):
"""
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tap/util.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tap/util.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -415,7 +415,8 @@
#
# Configure the Site and Wrappers
#
- credentialFactories = []
+ wireEncryptedCredentialFactories = []
+ wireUnencryptedCredentialFactories = []
portal = Portal(auth.DavRealm())
@@ -470,7 +471,9 @@
log.error("Unknown scheme: %s" % (scheme,))
if credFactory:
- credentialFactories.append(credFactory)
+ wireEncryptedCredentialFactories.append(credFactory)
+ if schemeConfig.get("AllowedOverWireUnencrypted", False):
+ wireUnencryptedCredentialFactories.append(credFactory)
#
# Setup Resource hierarchy
@@ -691,9 +694,10 @@
authWrapper = AuthenticationWrapper(
root,
portal,
- credentialFactories,
+ wireEncryptedCredentialFactories,
+ wireUnencryptedCredentialFactories,
(auth.IPrincipal,),
- overrides=overrides,
+ overrides=overrides
)
logWrapper = DirectoryLogWrapperResource(
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tools/ampnotifications.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tools/ampnotifications.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/calendarserver/tools/ampnotifications.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -127,7 +127,7 @@
MonitorAMPNotifications,
)
-def notificationCallback(id):
+def notificationCallback(id, dataChangedTimestamp):
print "Received notification for:", id
return succeed(True)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-apple.plist 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-apple.plist 2012-12-07 21:06:09 UTC (rev 10139)
@@ -242,7 +242,7 @@
<key>Basic</key>
<dict>
<key>Enabled</key>
- <false/>
+ <true/>
</dict>
<!-- Digest challenge/response -->
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-test.plist 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/conf/caldavd-test.plist 2012-12-07 21:06:09 UTC (rev 10139)
@@ -215,14 +215,17 @@
<key>params</key>
<dict>
- <key>restrictEnabledRecords</key>
- <false/>
- <key>restrictToGroup</key>
- <string>odtestgrouptop</string>
+ <key>recordTypes</key>
+ <array>
+ <string>users</string>
+ <string>groups</string>
+ <string>locations</string>
+ <string>resources</string>
+ </array>
<key>cacheTimeout</key>
- <integer>30</integer>
+ <integer>10</integer>
<key>uri</key>
- <string>ldap://example.com/</string>
+ <string>ldap://ldapserver.example.com/</string>
<key>tls</key>
<false/>
<key>tlsCACertFile</key>
@@ -234,96 +237,118 @@
<key>credentials</key>
<dict>
<key>dn</key>
- <string></string>
+ <string>uid=admin,ou=people,o=example.com</string>
<key>password</key>
- <string></string>
+ <string>PASSWORD</string>
</dict>
- <key>authMethod</key>
- <string>LDAP</string>
<key>rdnSchema</key>
<dict>
<key>base</key>
- <string>dc=example,dc=com</string>
+ <string>o=example.com</string>
<key>guidAttr</key>
- <string>entryUUID</string>
+ <string>GUID</string>
<key>users</key>
<dict>
<key>rdn</key>
- <string>cn=users</string>
- <key>attr</key>
- <string>uid</string>
- <key>emailSuffix</key>
- <string></string>
- <key>filter</key>
- <string></string>
- <key>loginEnabledAttr</key>
- <string></string>
- <key>loginEnabledValue</key>
- <string>yes</string>
+ <string>ou=people</string>
<key>mapping</key>
<dict>
- <key>recordName</key>
- <string>uid</string>
- <key>fullName</key>
- <string>cn</string>
- <key>emailAddresses</key>
- <array>
- <string>mail</string>
- </array>
- <key>firstName</key>
- <string>givenName</string>
- <key>lastName</key>
- <string>sn</string>
+ <key>recordName</key>
+ <string>uid</string>
+ <key>fullName</key>
+ <string>cn</string>
+ <key>emailAddresses</key>
+ <array>
+ <string>mail</string>
+ <string>mailAlias</string>
+ </array>
+ <key>firstName</key>
+ <string>givenName</string>
+ <key>lastName</key>
+ <string>sn</string>
</dict>
</dict>
<key>groups</key>
<dict>
<key>rdn</key>
- <string>cn=groups</string>
- <key>attr</key>
- <string>cn</string>
- <key>emailSuffix</key>
- <string></string>
- <key>filter</key>
- <string></string>
+ <string>ou=groups</string>
<key>mapping</key>
<dict>
- <key>recordName</key>
- <string>cn</string>
- <key>fullName</key>
- <string>cn</string>
- <key>emailAddresses</key>
- <array>
- <string>mail</string>
- </array>
- <key>firstName</key>
- <string>givenName</string>
- <key>lastName</key>
- <string>sn</string>
+ <key>recordName</key>
+ <string>cn</string>
+ <key>fullName</key>
+ <string>cn</string>
+ <key>emailAddresses</key>
+ <array>
+ <string>mail</string>
+ <string>mailAlias</string>
+ </array>
+ <key>firstName</key>
+ <string></string>
+ <key>lastName</key>
+ <string></string>
</dict>
</dict>
+ <key>locations</key>
+ <dict>
+ <key>rdn</key>
+ <string>ou=locations</string>
+ <key>mapping</key>
+ <dict>
+ <key>recordName</key>
+ <string>cn</string>
+ <key>fullName</key>
+ <string>cn</string>
+ <key>emailAddresses</key>
+ <array>
+ </array>
+ <key>firstName</key>
+ <string></string>
+ <key>lastName</key>
+ <string></string>
+ </dict>
+ </dict>
+ <key>resources</key>
+ <dict>
+ <key>rdn</key>
+ <string>ou=resources</string>
+ <key>mapping</key>
+ <dict>
+ <key>recordName</key>
+ <string>cn</string>
+ <key>fullName</key>
+ <string>cn</string>
+ <key>emailAddresses</key>
+ <array>
+ </array>
+ <key>firstName</key>
+ <string></string>
+ <key>lastName</key>
+ <string></string>
+ </dict>
+ </dict>
</dict>
<key>groupSchema</key>
<dict>
<key>membersAttr</key>
- <string>apple-group-memberguid</string>
+ <string>uniqueMember</string>
<key>nestedGroupsAttr</key>
- <string>apple-group-nestedgroup</string>
+ <string></string>
<key>memberIdAttr</key>
- <string>apple-generateduid</string>
+ <string></string>
</dict>
<key>resourceSchema</key>
<dict>
- <key>resourceInfoAttr</key>
- <string>apple-resource-info</string>
- <key>autoScheduleAttr</key>
- <string></string>
- <key>autoScheduleEnabledValue</key>
- <string>yes</string>
- <key>proxyAttr</key>
- <string></string>
- <key>readOnlyProxyAttr</key>
- <string></string>
+ <key>resourceInfoAttr</key>
+ <string></string>
+ <key>autoScheduleAttr</key>
+ <string></string>
+ <key>autoScheduleEnabledValue</key>
+ <string></string>
+ <key>proxyAttr</key>
+ <string></string>
+ <key>readOnlyProxyAttr</key>
+ <string></string>
</dict>
</dict>
</dict>
@@ -482,6 +507,8 @@
<dict>
<key>Enabled</key>
<true/>
+ <key>AllowedOverWireUnencrypted</key> <!-- advertised over non SSL? -->
+ <true/>
</dict>
<!-- Digest challenge/response -->
@@ -489,6 +516,8 @@
<dict>
<key>Enabled</key>
<true/>
+ <key>AllowedOverWireUnencrypted</key> <!-- advertised over non SSL? -->
+ <true/>
<key>Algorithm</key>
<string>md5</string>
<key>Qop</key>
@@ -500,6 +529,8 @@
<dict>
<key>Enabled</key>
<true/>
+ <key>AllowedOverWireUnencrypted</key> <!-- advertised over non SSL? -->
+ <true/>
<key>ServicePrincipal</key>
<string></string>
</dict>
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/calendarcommonextra.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/calendarcommonextra.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/calendarcommonextra.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -146,6 +146,7 @@
settings["EnableSSL"] = True
settings["RedirectHTTPToHTTPS"] = True
+ settings.setdefault("Authentication", {}).setdefault("Basic", {})["Enabled"] = True
def setCert(plistPath, otherCert):
"""
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/test/test_commonextra.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/test/test_commonextra.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/migration/test/test_commonextra.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -34,6 +34,7 @@
orig = {
}
expected = {
+ 'Authentication': {'Basic': {'Enabled': True}},
'EnableSSL': True,
'RedirectHTTPToHTTPS': True,
'SSLAuthorityChain': '/test/pchain.pem',
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ampsim.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ampsim.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ampsim.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -63,6 +63,8 @@
Configure this worker process with the text of an XML property list.
"""
arguments = [("plist", String())]
+ # Pass OSError exceptions through, presenting the exception message to the user.
+ errors = {OSError: 'OSError'}
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/config.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/config.plist 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/config.plist 2012-12-07 21:06:09 UTC (rev 10139)
@@ -471,7 +471,7 @@
<key>params</key>
<dict>
<key>enabled</key>
- <false/>
+ <true/>
<!-- Define the interval (in seconds) at which this profile will use
its client to create a new task. -->
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ical.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ical.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/ical.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -772,9 +772,6 @@
for comp in nodes[caldavxml.supported_calendar_component_set].getchildren():
componentTypes.add(comp.get("name").upper())
- if textProps.get(davxml.displayname, None) == "tasks":
- # Until we can fix caldavxml.supported_calendar_component_set
- break
changeTag = davxml.sync_token if self.supportSync else csxml.getctag
calendars.append(Calendar(
nodeType.tag,
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/sim.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/sim.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/performance/loadtest/sim.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -17,7 +17,8 @@
##
from collections import namedtuple
-from os import environ
+from os import environ, mkdir
+from os.path import isdir
from plistlib import readPlist
from random import Random
from sys import argv, stdout
@@ -256,6 +257,13 @@
if 'clientDataSerialization' in config:
if config['clientDataSerialization']['Enabled']:
serializationPath = config['clientDataSerialization']['Path']
+ if not isdir(serializationPath):
+ try:
+ mkdir(serializationPath)
+ except OSError:
+ print "Unable to create client data serialization directory: %s" % (serializationPath)
+ print "Please consult the clientDataSerialization stanza of contrib/performance/loadtest/config.plist"
+ raise
if 'arrival' in config:
arrival = Arrival(
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/contrib/tools/readStats.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/contrib/tools/readStats.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/contrib/tools/readStats.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
##
# Copyright (c) 2012 Apple Inc. All rights reserved.
#
@@ -15,13 +16,14 @@
##
from StringIO import StringIO
+import collections
import datetime
+import getopt
import json
import socket
+import sys
import tables
import time
-import sys
-import getopt
"""
This tool reads data from the server's statistics socket and prints a summary.
@@ -52,23 +54,23 @@
-def printStats(stats):
+def printStats(stats, multimode, showMethods, topUsers):
if len(stats) == 1:
if "Failed" in stats[0]:
printFailedStats(stats[0]["Failed"])
else:
try:
- printStat(stats[0])
+ printStat(stats[0], multimode[0], showMethods, topUsers)
except KeyError, e:
printFailedStats("Unable to find key '%s' in statistics from server socket" % (e,))
sys.exit(1)
else:
- printMultipleStats(stats)
+ printMultipleStats(stats, multimode, showMethods, topUsers)
-def printStat(stats):
+def printStat(stats, index, showMethods, topUsers):
print "- " * 40
print "Server: %s" % (stats["Server"],)
@@ -89,17 +91,19 @@
print "Current Memory Used: Unavailable"
print
printRequestSummary(stats)
- printHistogramSummary(stats["5 Minutes"])
+ printHistogramSummary(stats[index])
+ if showMethods:
+ printMethodCounts(stats[index])
+ if topUsers:
+ printUserCounts(stats[index], topUsers)
-def printMultipleStats(stats):
+def printMultipleStats(stats, multimode, showMethods, topUsers):
labels = serverLabels(stats)
print "- " * 40
- print "Servers: %s" % (", ".join(labels),)
-
print datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
times = []
@@ -109,30 +113,50 @@
except KeyError:
t = "-"
times.append(t)
- print "Service Uptime: %s" % (", ".join(times),)
cpus = []
memories = []
for stat in stats:
if stat["System"]["cpu count"] > 0:
- cpus.append("%.1f%%" % (stat["System"]["cpu use"],))
- memories.append("%.1f%%" % (stat["System"]["memory percent"],))
+ cpus.append(stat["System"]["cpu use"])
+ memories.append(stat["System"]["memory percent"])
else:
- cpus.append("-")
- memories.append("-")
- print "Current CPU: %s" % (", ".join(cpus),)
- print "Current Memory Used: %s" % (", ".join(memories),)
- print
- printMultiRequestSummary(stats, labels, ("5 Minutes", 5 * 60,))
- printMultiHistogramSummary(stats, "5 Minutes")
+ cpus.append(-1)
+ memories.append(-1)
+ printMultiRequestSummary(stats, cpus, memories, times, labels, multimode)
+ printMultiHistogramSummary(stats, multimode[0])
+ if showMethods:
+ printMultiMethodCounts(stats, multimode[0])
+ if topUsers:
+ printMultiUserCounts(stats, multimode[0], topUsers)
+
def serverLabels(stats):
- return [str(stat["Server"]) for stat in stats]
+ servers = [stat["Server"] for stat in stats]
+ if isinstance(servers[0], tuple):
+ hosts = set([item[0] for item in servers])
+ ports = set([item[1] for item in servers])
+ if len(ports) == 1:
+ servers = [item[0] for item in servers]
+ elif len(hosts) == 1:
+ servers = [":%d" % item[1] for item in servers]
+ elif len(hosts) == len(servers):
+ servers = [item[0] for item in servers]
+ else:
+ servers = ["%s:%s" % item for item in servers]
+ servers = [item.split(".") for item in servers]
+ while True:
+ if all([item[-1] == servers[0][-1] for item in servers]):
+ servers = [item[:-1] for item in servers]
+ else:
+ break
+ return [".".join(item) for item in servers]
+
def printFailedStats(message):
print "- " * 40
@@ -185,14 +209,18 @@
-def printMultiRequestSummary(stats, labels, index):
+def printMultiRequestSummary(stats, cpus, memories, times, labels, index):
+
+ key, seconds = index
+
table = tables.Table()
table.addHeader(
- ("Server", "Requests", "Av. Requests", "Av. Response", "Av. Response", "Max. Response", "Slot", "CPU", "500's"),
+ ("Server", "Requests", "Av. Requests", "Av. Response", "Av. Response", "Max. Response", "Slot", "CPU", "CPU", "Memory", "500's", "Uptime",),
)
table.addHeader(
- ("", "", "per second", "(ms)", "no write(ms)", "(ms)", "Average", "Average", ""),
+ (key, "", "per second", "(ms)", "no write(ms)", "(ms)", "Average", "Average", "Current", "Current", "", "",),
)
+ max_column = 5
table.setDefaultColumnFormats(
(
tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
@@ -203,12 +231,14 @@
tables.Table.ColumnFormat("%.1f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.2f", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
)
)
- key, seconds = index
- totals = ["Overall:", 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0]
+ totals = ["Overall:", 0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, "", ]
for ctr, stat in enumerate(stats):
stat = stat[key]
@@ -222,12 +252,18 @@
col.append(stat["T-MAX"])
col.append(safeDivision(float(stat["slots"]), stat["requests"]))
col.append(safeDivision(stat["cpu"], stat["requests"]))
+ col.append(cpus[ctr])
+ col.append(memories[ctr])
col.append(stat["500"])
+ col.append(times[ctr])
table.addRow(col)
- for item in xrange(1, len(col)):
- totals[item] += col[item]
+ for item in xrange(1, len(col) - 1):
+ if item == max_column:
+ totals[item] = max(totals[item], col[item])
+ else:
+ totals[item] += col[item]
- for item in (2, 3, 4, 6, 7):
+ for item in (3, 4, 6, 7, 8, 9):
totals[item] /= len(stats)
table.addFooter(totals)
@@ -247,7 +283,7 @@
)
table.setDefaultColumnFormats(
(
- tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.CENTER_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
@@ -261,7 +297,7 @@
)
for i in ("T", "T-RESP-WR",):
table.addRow((
- "Overall Response" if i == "T" else "Response without Write",
+ "Overall Response" if i == "T" else "Response Write",
(stat[i]["<10ms"], safeDivision(stat[i]["<10ms"], stat["requests"], 100.0)),
(stat[i]["10ms<->100ms"], safeDivision(stat[i]["10ms<->100ms"], stat["requests"], 100.0)),
(stat[i]["100ms<->1s"], safeDivision(stat[i]["100ms<->1s"], stat["requests"], 100.0)),
@@ -293,14 +329,14 @@
for k in keys[1:]:
totals[i][k] += stat[index][i][k]
- print "5 minute average response histogram"
+ print "%s average response histogram" % (index,)
table = tables.Table()
table.addHeader(
("", "<10ms", "10ms<->100ms", "100ms<->1s", "1s<->10s", "10s<->30s", "30s<->60s", ">60s", "Over 1s", "Over 10s"),
)
table.setDefaultColumnFormats(
(
- tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.CENTER_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
tables.Table.ColumnFormat("%d (%.1f%%)", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
@@ -314,7 +350,7 @@
)
for i in ("T", "T-RESP-WR",):
table.addRow((
- "Overall Response" if i == "T" else "Response without Write",
+ "Overall Response" if i == "T" else "Response Write",
(totals[i]["<10ms"], safeDivision(totals[i]["<10ms"], totals[i]["requests"], 100.0)),
(totals[i]["10ms<->100ms"], safeDivision(totals[i]["10ms<->100ms"], totals[i]["requests"], 100.0)),
(totals[i]["100ms<->1s"], safeDivision(totals[i]["100ms<->1s"], totals[i]["requests"], 100.0)),
@@ -331,6 +367,128 @@
+def printMethodCounts(stat):
+
+ print "Method Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("Method", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(stat["method"].values())
+ for method in sorted(stat["method"].keys()):
+ table.addRow((
+ method,
+ stat["method"][method],
+ safeDivision(stat["method"][method], total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
+def printMultiMethodCounts(stats, index):
+
+ methods = collections.defaultdict(int)
+ for stat in stats:
+ for method in stat[index]["method"]:
+ methods[method] += stat[index]["method"][method]
+
+ print "Method Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("Method", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(methods.values())
+ for method in sorted(methods.keys()):
+ table.addRow((
+ method,
+ methods[method],
+ safeDivision(methods[method], total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
+def printUserCounts(stat, topUsers):
+
+ print "User Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("User", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(stat["uid"].values())
+ for uid in sorted(stat["uid"].items(), key=lambda x: x[1], reverse=True)[:topUsers]:
+ table.addRow((
+ uid,
+ stat["uid"][uid],
+ safeDivision(stat["uid"][uid], total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
+def printMultiUserCounts(stats, index, topUsers):
+
+ uids = collections.defaultdict(int)
+ for stat in stats:
+ for uid in stat[index]["uid"]:
+ uids[uid] += stat[index]["uid"][uid]
+
+ print "User Counts"
+ table = tables.Table()
+ table.addHeader(
+ ("User", "Total", "Percentage"),
+ )
+ table.setDefaultColumnFormats(
+ (
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%.1f%%", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ )
+ )
+
+ total = sum(uids.values())
+ for uid, count in sorted(uids.items(), key=lambda x: x[1], reverse=True)[:topUsers]:
+ table.addRow((
+ uid,
+ count,
+ safeDivision(count, total, 100.0),
+ ))
+ os = StringIO()
+ table.printTable(os=os)
+ print os.getvalue()
+
+
+
def usage(error_msg=None):
if error_msg:
print error_msg
@@ -341,6 +499,12 @@
-s Name of local socket to read from
-t Delay in seconds between each sample [10 seconds]
--tcp host:port Use TCP connection with host:port
+ --0 Display multiserver current average
+ --1 Display multiserver 1 minute average
+ --5 Display multiserver 5 minute average (the default)
+ --60 Display multiserver 1 hour average
+ --methods Include details about HTTP method usage
+ --users N Include details about top N users
Description:
This utility will print a summary of statistics read from a
@@ -359,9 +523,14 @@
delay = 10
servers = ("data/Logs/state/caldavd-stats.sock",)
useTCP = False
+ showMethods = False
+ topUsers = 0
- options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", ])
+ multimodes = (("Current", 60,), ("1 Minute", 60,), ("5 Minutes", 5 * 60,), ("1 Hour", 60 * 60,),)
+ multimode = multimodes[2]
+ options, args = getopt.getopt(sys.argv[1:], "hs:t:", ["tcp=", "0", "1", "5", "60", "methods", "users="])
+
for option, value in options:
if option == "-h":
usage()
@@ -372,7 +541,19 @@
elif option == "--tcp":
servers = [(host, int(port),) for host, port in [server.split(":") for server in value.split(",")]]
useTCP = True
+ elif option == "--0":
+ multimode = multimodes[0]
+ elif option == "--1":
+ multimode = multimodes[1]
+ elif option == "--5":
+ multimode = multimodes[2]
+ elif option == "--60":
+ multimode = multimodes[3]
+ elif option == "--methods":
+ showMethods = True
+ elif option == "--users":
+ topUsers = int(value)
while True:
- printStats([readSock(server, useTCP) for server in servers])
+ printStats([readSock(server, useTCP) for server in servers], multimode, showMethods, topUsers)
time.sleep(delay)
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-Apache.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-Apache.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-Apache.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,112 +0,0 @@
-Apache Directory Services
-=========================
-
-**WARNING:** *This directory service implementation is experimental,
-incomplete and not supported.*
-
-The Apache directory services provide principal information that is
-read from configuration files in the `same formats`_ as used by the
-`Apache HTTP server`_, allowing you to easily share user and group
-information with an Apache server.
-
-.. _same formats: http://httpd.apache.org/docs/2.3/howto/auth.html
-.. _Apache HTTP server: http://httpd.apache.org/
-
-The Apache directory services provide principal information for users
-and groups. They do not provide principal information for locations or
-resources.
-
-Configuring the Calendar Server
--------------------------------
-
-The full name of the service is either
-``twistedcaldav.directory.apache.BasicDirectoryService`` or
-``twistedcaldav.directory.apache.DigestDirectoryService``. These
-services implement `basic and digest HTTP authentication`_,
-respectively.
-
-.. _basic and digest HTTP authentication: http://www.ietf.org/rfc/rfc2617.txt
-
-Both services take a ``userFile`` parameter which contains the name of
-the file to read user principal information from and an optional
-``groupFile`` parameter which contains the name of the file to read
-group principal information from.
-
-For example, if you are using digest:
-
-::
-
- <!-- Apache-style Digest Directory Service -->
- <key>DirectoryService</key>
- <dict>
- <key>type</key>
- <string>twistedcaldav.directory.apache.DigestDirectoryService</string>
-
- <key>params</key>
- <dict>
- <key>userFile</key>
- <string>conf/digest</string>
- <key>groupFile</key>
- <string>conf/group</string>
- </dict>
- </dict>
-
-The service re-reads the user and group files if either file's
-timestamp changes, so edits to the files do not require a server
-restart.
-
-Note that basic authentication is highly insecure because it sends
-password information in plain text over the network (where is may be
-intercepted) and should not be enabled on a server unless all
-connections are somehow secured by another means, such as by enabling
-SSL and disabling non-SSL connections.
-
-Configuring Principals
-----------------------
-
-In the case of ``BasicDirectoryService``, the user file must be in the
-form generated by the Apache ``htpasswd`` command; in the case of
-``DigestDirectoryService``, the user file must be in the form
-generated by the Apache ``htdigest`` command.
-
-Both user file formats contain a single entry per line, with fields
-separated by the colon (``:``) character. The basic format has two
-fields, one containing a user identifier and the second containing the
-user's password in the UNIX crypt format. The digest format has three
-fields: a user identifier, a realm name, and the user's hashed
-password.
-
-An example basic user file:
-
-::
-
- wsanchez:Cytm0Bwm7CPJs
- cdaboo:I.Ef5FJl5GVh2
- dreid:LVhqAv4qSrYPs
- lecroy:/7/5VDrkrLxY.
-
-And an example digest user file:
-
-::
-
- wsanchez:Test:decbe233ab3d997cacc2fc058b19db8c
- cdaboo:Test:61164bf3d607d072fe8a7ac420b24aac
- dreid:Test:8ee67801004b2752f72b84e7064889a6
- lecroy:Test:60d4feb424430953be045738041e51be
-
-The group file is in a similar format, with one entry of
-colon-separated field per line. Each line has two fields: a group
-identifier, and a comma- (``,``) separated list of user identifiers
-which identify the members of the group.
-
-And example group file:
-
-::
-
- managers: lecroy
- grunts: wsanchez, cdaboo, dreid
- right_coast: cdaboo
- left_coast: wsanchez, dreid, lecroy
-
-The user files should be edited using the ``htpasswd`` and
-``htdigest`` tools. The group file is typically edited by hand.
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-OpenDirectory.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-OpenDirectory.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-OpenDirectory.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,44 +0,0 @@
-Open Directory Service
-======================
-
-The Open Directory directory service provides principal information
-that is obtained using Apple's Open Directory service.
-
-Open Directory provides principal information for users, groups,
-locations, and resources.
-
-For more information about configuring Open Directory and running Open
-Directory services, see Apple's `Open Directory Administration`_
-document.
-
-.. _Open Directory Administration: http://images.apple.com/server/macosx/docs/Open_Directory_Admin_v10.6.pdf
-
-Configuring the Calendar Server
--------------------------------
-
-The full name of the service is
-``twistedcaldav.directory.appleopendirectory.OpenDirectoryService``
-and the service takes a ``node`` parameter which contains the name of
-the directory node to bind to.
-
-For example:
-
-::
-
- <!-- Open Directory Service -->
- <key>DirectoryService</key>
- <dict>
- <key>type</key>
- <string>twistedcaldav.directory.appleopendirectory.OpenDirectoryService</string>
-
- <key>params</key>
- <dict>
- <key>node</key>
- <string>/Search</string>
- </dict>
- </dict>
-
-The special Open Directory node ``/Search`` causes the server to use
-the default directory search path that the host system the server is
-running on is configured to use. To bind to a specific LDAP service, a
-node in the form ``/LDAPv3/ldapserver.example.com`` may be specified.
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-XML.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-XML.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryService-XML.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,159 +0,0 @@
-XML Directory Service
-=====================
-
-The XML directory service provides principal information that is read
-from an XML file.
-
-The XML file provides principal information for users, groups,
-locations, and resources.
-
-One advantage to this directory service implementation is that it does
-not require a networked directory server to be running somewhere,
-instead simply relying on a file.
-
-Configuring the Calendar Server
--------------------------------
-
-The full name of the service is
-``twistedcaldav.directory.xmlfile.XMLDirectoryService`` and the
-service takes an ``xmlFile`` parameter which contains the name of the
-XML file to read principal information from.
-
-For example:
-
-::
-
- <!-- XML File Directory Service -->
- <key>DirectoryService</key>
- <dict>
- <key>type</key>
- <string>twistedcaldav.directory.xmlfile.XMLDirectoryService</string>
-
- <key>params</key>
- <dict>
- <key>xmlFile</key>
- <string>/etc/caldavd/accounts.xml</string>
- </dict>
- </dict>
-
-The service re-reads the XML file if it's timestamp changes, so edits
-to the XML file do not require a server restart.
-
-Configuring Principals
-----------------------
-
-Principals are expressed in an XML document. The root element
-``accounts`` has an attribute ``realm`` which describes the
-authentication realm. It contains principal elements which in turn
-contain elements describing the principal. The element itself
-(``user``, ``group``, ``location``, ``resource``) denotes the
-principal type.
-
-Principal elements can contain the following elements which provide
-information about the principal:
-
-``uid``
-
- The login identifier for the principal (ie. "user name" or "short
- name").
-
-``guid``
-
- A globally unique identifier for the principal. Must be a UUID
- string that complies with `RFC 4122`_.
-
- .. _RFC 4122: http://tools.ietf.org/html/rfc4122
-
-``password``
-
- The principal's password in plain text.
-
-``name``
-
- The principal's full name (or description).
-
-``members``
-
- A list of uids for the principals which are members of the principal
- being defined. Only group principals may have members. The
- ``members`` element has ``member`` sub-elements used to specify each
- member. The member element has a type attribute that defines the
- principal type of the member (one of ``users``, ``groups``,
- ``locations`` and ``resources``), and the text value inside the
- ``member`` element is the corresponding uid of the principal being
- referenced. Any principal may be a member of a group, including
- other groups. One should avoid creating "loops" by having two groups
- include each other.
-
-``cuaddr``
-
- A "calendar user address" for the principal. Principals may have
- multiple calendar user addresses, but a calendar user addresses must
- be unique to one principal. A calendar user address must be a URI_.
-
- .. _URI: http://tools.ietf.org/html/rfc2396
-
- Note that calendar user addresses here supplement any calendar user
- addresses that are assigned by the server based on other principal
- information.
-
-``disable-calendar``
-
- When present, this element indicates that the principal is able to
- login to the calendar server, but is not provided a calendar home
- and therefore cannot do scheduling. This type of principal is
- typically used to allow access to the calendars of other principals
- or other data on the server. This element may only pecified for user
- principals.
-
-``auto-schedule``
-
- Indicates that the server will automatically process scheduling
- messages for the corresponding principal. For example, when a
- scheduling message arrives, if it does not conflict with an existing
- meeting it will be automatically accepted into the principal's main
- calendar; if it does conflict it will be automatically
- declined. This element can only be defined on location and resource
- principals.
-
-``proxies``
-
- Contains a list of ``member`` elements that define which other
- principals have read-write proxy access to the corresponding
- principal's calendar data.
-
-An example:
-
-::
-
- <?xml version="1.0" encoding="utf-8"?>
- <accounts realm="Test Realm">
- <user>
- <uid>admin</uid>
- <password>admin</password>
- <name>Super User</name>
- </user>
- <user>
- <uid>test</uid>
- <password>test</password>
- <name>Test User</name>
- <cuaddr>mailto:testuser at example.com</cuaddr>
- </user>
- <group>
- <uid>users</uid>
- <password>users</password>
- <name>Users Group</name>
- <members>
- <member type="users">test</member>
- </members>
- </group>
- <location>
- <uid>mercury</uid>
- <password>mercury</password>
- <name>Mecury Conference Room, Building 1, 2nd Floor</name>
- <auto-schedule/>
- <proxies>
- <member type="users">test</member>
- </proxies>
- </location>
- </accounts>
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryServices.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryServices.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/DirectoryServices.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,93 +0,0 @@
-Directory Services
-==================
-
-The Calendar Server needs to be able to obtain information about the
-users, groups and resources ("principals") which access and/or have a
-presence on the server.
-
-About Principals
-----------------
-
-All principals have a "principal resource" on the server which
-represents the principal in the form of an HTTP resource. This is
-useful for obtaining information about a principal, such as the URL of
-the principal's calendar home, the principal's members and/or
-memberships, and so on. This information is exposed via WebDAV
-properties on the principal resource.
-
-Principals can be used to configure access controls for resources on
-the server by granting or denying various privileges to the principal.
-Privileges granted or denied to group principals are also granted or
-denied to all members of the group.
-
-Some principals (often, but not necessarily all) are also given a
-calendar home collection on the server, in which the principal may
-have one or more calendar collections, as well as special collections
-which allow the principals to schedule meetings with others, and so
-on.
-
-The Role of a Directory Service
--------------------------------
-
-A "directory service" is simply an entity which contains information
-about principals.
-
-Directory services are interchangeable, which allows the server to
-obtain this information from a variety of data stores, such as
-configuration files or network directory systems like LDAP.
-
-Most directory services refer to principals as "records" in their
-databases. Internally, Calendar Server will map records from a
-directory service to WebDAV principals.
-
-A given directory service may classify records into "types" such as
-users, groups, resources, and so on. Calendar Server keeps this
-distinction, and some types are treated specially.
-
-Principal types commonly provided by directory services include:
-
-users
-
- Individual (typically human) users of the system.
-
-groups
-
- Principals that contain other principals ("members"). Members can be
- principals of any type, including other group principals.
-
-locations
-
- Locations that can be scheduled.
-
-resources
-
- Other resources (eg. projectors) which can be scheduled.
-
-For example, only user principals are allowed to authenticate with
-(log into) the server. Only group principals have members, and group
-principals do not have calendars.
-
-
-Configuration
-=============
-
-The directory service used by the server is configured in the
-``caldavd.plist`` file by specifying the directory service
-implementation to use, as well as its configuration options. Options
-are specified as a dictionary.
-
-The configuration syntax looks like this:
-
-::
-
- <key>DirectoryService</key>
- <dict>
- <key>type</key>
- <string>ExampleService</string>
-
- <key>params</key>
- <dict>
- <key>option</key>
- <string>value</string>
- </dict>
- </dict>
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/ExtendedLogItems.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/ExtendedLogItems.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/ExtendedLogItems.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,110 +0,0 @@
-Apache-style Access Log Extensions
-==================================
-
-Calendar Server extends the Apache log file format it uses by:
-
- * Adding a "sub-method" to the HTTP method field.
- * Adding key-value pairs at the end of log lines.
-
-The sub-method is adding by appending the name of the sub-method in
-parentheses to the original HTTP request method. This is used to
-clarify what operation is going on for HTTP methods that are commonly
-used to tunnel other operations, such as ``POST`` and ``REPORT``. For
-example, CalDAV uses ``POST`` for free/busy lookups, and ``REPORT``
-can be used to make any sort of query.
-
-Key-value pairs are used to provide additional details about the
-request. They are added to the end of the line as new fields in the
-form ``key=value``.
-
-These keys are always emitted:
-
- ``i``
-
- the port number of the server instance emitting the log
-
- ``t``
-
- the amount of time spent processing the request (in milliseconds).
-
- ``or``
-
- the number of outstanding requests enqueued for the server
- instance emitting the log entry.
-
-Keys that may be emitted depending on the client request and server
-response include:
-
- ``rcount``
-
- the number of resources specified in a ``MULTIGET`` request.
-
- ``responses``
-
- the number of responses included in a ``multi-status`` response to
- a ``PROPFIND`` request or a CalDAV ``calendar-query`` request.
-
- ``recipients``
-
- the number of recipients in a CalDAV scheduling request via
- ``POST`` (which are typically only free/busy requests).
-
- ``itip.requests``
-
- the number of attendees in a scheduling operation triggered by an
- Organizer.
-
- ``itip.refreshes``
-
- the number of attendee refreshes completed after a scheduling
- operation.
-
- ``itip.auto``
-
- the number of auto-accept attendees specified
-
- ``itip.reply``
-
- Either ``reply`` or ``cancel`` depending on...???
-
- ``fwd``
-
- the value of the X-Forwarded-For header, if present
-
-In the following example, we see a ``CalDAV:calendar-multiget``
-``REPORT`` for 32 resources in a user's calendar, which was handled by
-instance ``8459`` in 183.0ms, with one outstanding request (the one
-being logged):
-
-::
-
- 17.108.160.37 - scastillo [15/Sep/2009:20:10:23 +0000] "REPORT(CalDAV:calendar-multiget) /calendars/__uids__/B8CE9430-965B-11DE-B626-EC2E9DB52B69/calendar/ HTTP/1.1" 207 149285 "-" "DAVKit/4.0 (729); CalendarStore/4.0 (965); iCal/4.0 (1362); Mac OS X/10.6.1 (10B504)" i=8459 t=183.0 or=1 rcount=32
-
-
-
-**Fine-grained request time logging**
-
-If the configuration key EnableExtendedTimingAccessLog is set to true, additional key-value pairs will be logged with each request. The overall request time "t" is broken into four phases, and the elapsed time for each phase is logged. The new keys representing the four request phases are:
-
- ``t-req-proc``
-
- time elapsed from when a request object is created up until renderHTTP is about to be called.
- This is the overhead of parsing the request headers and locating the target resource.
-
- ``t-resp-gen``
-
- time elapsed from t-req-proc up until the response is ready to write
-
- ``t-resp-wr``
-
- time elapsed from t-resp-gen up until response is written
-
- ``t-log``
-
- time from t-resp-wr up until log entry is ready to write to master
-
-A sample log line with EnableExtendedTimingAccessLog enabled is shown below:
-
-::
-
- 17.209.103.42 - wsanchez [24/Jul/2012:17:51:29 +0000] "REPORT(CalDAV:calendar-multiget) /calendars/__uids__/F114CA1D-295F-42A5-A5BD-D1A1B19FC049/60E68E32-4C87-4E63-9BF2-12A25E8F2623/ HTTP/1.1" 207 114349 "-" "CalendarStore/5.0.2 (1166); iCal/5.0.2 (1571); Mac OS X/10.7.3 (11D50d)" i=7 or=1 t=764.7 t-req-proc=4.8 t-resp-gen=754.5 t-resp-wr=5.1 t-log=0.2 rcount=2
\ No newline at end of file
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/LoadSimulation.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/LoadSimulation.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/LoadSimulation.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,338 +0,0 @@
-==========================
-Load Simulation
-==========================
-
-Calendar Server includes a flexible, scalable, and easy to use `client simulator <http://trac.calendarserver.org/browser/CalendarServer/trunk/contrib/performance/loadtest>`_ that provides behaviors modeled after Apple's iCal client in Mac OS X 10.6. This sim has a highly modular architecture with the goal of making it as easy as possible to add new behaviors, or entirely new clients. The sim can be used to validate server correctness, and also for performance and scalability testing.
-
-This document contains the following sections:
-
-* `Quick Start`_: Instructions to get the sim up and running from an SVN checkout of Calendar Server
-
-* `Reporting`_: The sim emits both real-time logging during runtime, and also a summary at the end that includes quality of service evaluations
-
-* `Configuration`_: Describes the load sim's configuration options, how to enable / disable certain behaviors, adjust frequency of events, etc
-
-* `Scalability`_: Through the use of an AMP harness, you may easily add additional sim instances on the local host and / or remote hosts
-
----------------------
-Quick Start
----------------------
-
-The sim's default config is pre-configured to work against the Calendar Server's default config (caldavd-test.plist), so we'll use that for this quick start.
-
-- Start Calendar Server using the -test configuration. For further detail on this, see: http://trac.calendarserver.org/wiki/QuickStart
-- Open another shell and cd into the Calendar Server SVN root
-- Start the sim:
-
-::
-
- ./sim
-
-The sim should begin logging its activity, which might look something like:
-
-::
-
- Loaded 99 accounts.
- user01 - - - - - - - - - - - startup BEGIN
- user01 request 207✓[ 0.04 s] PROPFIND /principals/users/user01/
- user01 request 207✓[ 0.02 s] PROPFIND /principals/__uids__/user01/
- user01 request 200✓[ 0.01 s] REPORT /principals/
- user01 - - - - - - - - - - - poll BEGIN
- user01 request 207✓[ 0.19 s] PROPFIND /calendars/__uids__/user01/
- ...
-
-- To stop the sim, simply cntrl-c it.
-
----------------------
-Reporting
----------------------
-
-Runtime Reporting
------------------
-
-While running, the sim will log its activity to stdout. Simulator activity is organized into a two-level hierarchy: client behaviors (e.g. create event, poll, respond to events) which are composed of individual HTTP requests (e.g. GET, PUT, DELETE, etc).
-
-For example, the sim logs the following right at startup as it initializes the users:
-
-::
-
- user01 - - - - - - - - - - - startup BEGIN
- user01 request 207✓[ 0.04 s] PROPFIND /principals/users/user01/
- user01 request 207✓[ 0.02 s] PROPFIND /principals/__uids__/user01/
- user01 request 200✓[ 0.01 s] REPORT /principals/
- user01 - - - - - - - - - - - poll BEGIN
- user01 request 207✓[ 0.29 s] PROPFIND /calendars/__uids__/user01/
- user01 request 207✓[ 0.04 s] PROPFIND /calendars/__uids__/user01/calendar/
- user01 request 207✓[ 0.04 s] PROPFIND /calendars/__uids__/user01/inbox/
- user01 request 207✓[ 0.05 s] PROPFIND /calendars/__uids__/user01/tasks/
- user01 request 207✓[ 0.02 s] PROPFIND /calendars/__uids__/user01/outbox/
- user01 request 207✓[ 0.03 s] PROPFIND /calendars/__uids__/user01/dropbox/
- user01 request 207✓[ 0.03 s] PROPFIND /calendars/__uids__/user01/notification/
- user01 - - - - - - - - - - - poll END [ 0.51 s]
- user01 - - - - - - - - - - - startup END [ 0.59 s]
-
-The first token on each line is the username associated with the activity. Most of the time, client activity is interleaved, so to trace a specific client you may wish to filter for a specific username.
-
-Client operations are groups of requests that are surrounded by lines with many dashes, either BEGIN or END. 'END' lines also include the overall duration of the operation, which is the elapsed wall clock time since the operation started.
-
-Above, we see two behaviors that are nested; the 'startup' operation starts which includes a few requests and also triggers the 'poll' operation.
-
-Individual requests are logged with the word 'request', followed by the HTTP status, the request duration, the HTTP method of the request, and the URI that is being targeted. Successful requests are indicated by a ✓ while failed requests are indicated with —✗.
-
-Most client operations also log a 'lag' time when they start, e.g.
-
-::
-
- user12 - - - - - - - - - - - create BEGIN {lag 0.80 ms}
-
-'lag' in this context specifies the time elapsed between when the sim should have started this operation (based on internal timers / activity distributions) and when it actually did. This is intended to be a measure of client sim health; if the lag values begin to grow too high, that indicates that the client sim is 'slipping' and doesn't have enough CPU to perform its various activities at the rates specified in the config file. This is python, so a single instance of the client sim is limited to one CPU core. See the `Scalability`_ section for information about how to scale the load sim by combining multiple instances.
-
-Summary Reports
----------------
-When the sim stops or is killed, it emits a summary report that displays the total number of users that were active in this run, and overall counts for both individual requests and also the higher-level client operations. Also we display quality of service information used to grade the run as PASS or FAIL. An example report is shown below::
-
- Users : 20
- request count failed >3sec mean median
- DELETE 21 0 0 0.0186 0.0184
- GET 27 0 0 0.0341 0.0223
- POST 17 0 0 0.0709 0.0523
- PROPFIND 265 0 0 0.0593 0.0262
- PUT 44 0 2 0.3544 0.1735
- REPORT 107 0 0 0.0599 0.0280
-
- operation count failed >3sec mean median avglag (ms)
- accept 10 0 0 0.2808 0.2942 0.8490
- create 17 0 0 0.0560 0.0484 1.0024
- invite 17 0 2 0.8713 0.4774 0.9585
- poll 64 0 0 0.3856 0.3234 0.0000
- reply done 12 0 0 0.0177 0.0174 1.9181
- startup 20 0 0 0.7369 0.6023 0.0000
- PASS
-
-The pass / fail criteria are defined in `contrib/performance/loadtest/profiles.py <http://trac.calendarserver.org/browser/CalendarServer/trunk/contrib/performance/loadtest/profiles.py>`_ for operations and `contrib/performance/loadtest/population.py <http://trac.calendarserver.org/browser/CalendarServer/trunk/contrib/performance/loadtest/population.py>`_ for individual requests, and are generally derived from execution time and failure rate.
-
----------------------
-Configuration
----------------------
-
-The client sim's default configuration file is found here::
-
- contrib/performance/loadtest/config.plist
-
-The config file defines
-
-- how to connect to the server
-- which user accounts to use
-- client 'arrival' policy, which specifies how many of the available accounts to use, and how quickly they are initialized
-- which client behaviors are performed, along with optional configuration of each behavior
-
-Server Specification
----------------------
-
-Specify the URI to which the client sim should connect, e.g::
-
- <!-- Identify the server to be load tested. -->
- <key>server</key>
- <string>https://127.0.0.1:8443/</string>
-
-User Accounts
--------------
-
-User accounts are defined in the 'accounts' key of the plist:
-
-::
-
- <!-- Define the credentials of the clients which will be used to load test
- the server. These credentials must already be valid on the server. -->
- <key>accounts</key>
- <dict>
- <!-- The loader is the fully-qualified Python name of a callable which
- returns a list of directory service records defining all of the client accounts
- to use. contrib.performance.loadtest.sim.recordsFromCSVFile reads username,
- password, mailto triples from a CSV file and returns them as a list of faked
- directory service records. -->
- <key>loader</key>
- <string>contrib.performance.loadtest.sim.recordsFromCSVFile</string>
-
- <!-- Keyword arguments may be passed to the loader. -->
- <key>params</key>
- <dict>
- <!-- recordsFromCSVFile interprets the path relative to the config.plist,
- to make it independent of the script's working directory while still allowing
- a relative path. This isn't a great solution. -->
- <key>path</key>
- <string>contrib/performance/loadtest/accounts.csv</string>
- </dict>
- </dict>
-
-The accounts.csv file has lines like shown below::
-
- user01,user01,User 01,user01 at example.com
-
-Client Arrival
-----------------
-
-This section configures the number of accounts to use, and defines how quickly clients are initialized when the sim starts::
-
- <!-- Define how many clients will participate in the load test and how
- they will show up. -->
- <key>arrival</key>
- <dict>
-
- <!-- Specify a class which creates new clients and introduces them into
- the test. contrib.performance.loadtest.population.SmoothRampUp introduces
- groups of new clients at fixed intervals up to a maximum. The size of the
- group, interval, and maximum are configured by the parameters below. The
- total number of clients is groups * groupSize, which needs to be no larger
- than the number of credentials created in the accounts section. -->
- <key>factory</key>
- <string>contrib.performance.loadtest.population.SmoothRampUp</string>
-
- <key>params</key>
- <dict>
- <!-- groups gives the total number of groups of clients to introduce. -->
- <key>groups</key>
- <integer>20</integer>
-
- <!-- groupSize is the number of clients in each group of clients. It's
- really only a "smooth" ramp up if this is pretty small. -->
- <key>groupSize</key>
- <integer>1</integer>
-
- <!-- Number of seconds between the introduction of each group. -->
- <key>interval</key>
- <integer>3</integer>
- </dict>
-
- </dict>
-
-In the default configuration, one client is initialized every 3 seconds, until 20 clients are initialized. As soon as a client is initialized, it begins to perform its specified behaviors at the configured rates (see "Client Behaviors").
-
-To increase the client load, increase the number of groups.
-
-To increase the rate at which clients are initialized, reduce 'interval' and / or increase 'groupSize'
-
-Remember: **The total number of clients is groups * groupSize, which needs to be no larger than the number of credentials created in the accounts section**
-
-Client Behaviors
-----------------
-
-The 'clients' plist key is an array of dictionaries, where each dict has the keys:
-
-- 'software', which specifies the implementation class for this client. For example:
-
-::
-
- <key>software</key>
- <string>contrib.performance.loadtest.ical.SnowLeopard</string>
-
-- 'params', optionally specifying parameters accepted by this software. For example:
-
-::
-
- <!-- Arguments to use to initialize the SnowLeopard instance. -->
- <key>params</key>
- <dict>
- <!-- SnowLeopard can poll the calendar home at some interval. This is
- in seconds. -->
- <key>calendarHomePollInterval</key>
- <integer>30</integer>
-
- <!-- If the server advertises xmpp push, SnowLeopard can wait for notifications
- about calendar home changes instead of polling for them periodically. If
- this option is true, then look for the server advertisement for xmpp push
- and use it if possible. Still fall back to polling if there is no xmpp push
- advertised. -->
- <key>supportPush</key>
- <false />
- </dict>
-
-- 'profiles' is an array of dictionaries specifying individual behaviors of this software. Each dict has a 'class' key which specifies the implementation class for this behavior, and a 'params' dict with options specific to that behavior. For example:
-
-::
-
- <!-- This profile accepts invitations to events, handles cancels, and
- handles replies received. -->
- <dict>
- <key>class</key>
- <string>contrib.performance.loadtest.profiles.Accepter</string>
-
- <key>params</key>
- <dict>
- <key>enabled</key>
- <true/>
-
- <!-- Define how long to wait after seeing a new invitation before
- accepting it. -->
- <key>acceptDelayDistribution</key>
- <dict>
- <key>type</key>
- <string>contrib.performance.stats.NormalDistribution</string>
- <key>params</key>
- <dict>
- <!-- mean -->
- <key>mu</key>
- <integer>60</integer>
- <!-- standard deviation -->
- <key>sigma</key>
- <integer>60</integer>
- </dict>
- </dict>
- </dict>
- </dict>
-
-Some parameters may be safely modified to suit your purposes, for example you might choose to disable certain profiles (by setting 'enabled' to false) in order to simulate only specific types of activity. Also, you can edit the params for the various distributions to configure how often things happen.
-
-Motivated readers may also develop new behaviors for existing clients, or even entirely new clients. An example of adding a new behavior to an existing client can be found here: http://trac.calendarserver.org/changeset/8428. As of this writing, we have only the one Snow Leopard client simulator, and would happily accept patches that implement additional clients!
-
----------------------
-Scalability
----------------------
-
-A good amount of activity can be generated by a single client sim instance, and that should be suitable for most cases. However, if your task is performance or scalability testing, you will likely want to generate more load than can be presented by a single CPU core (which is all you can get from a single Python process). By adding a 'workers' array to the sim's config file you can specify the use of additional sim instances on the local host, and / or remote hosts. In this configuration, the master process will distribute work across all the workers. In general, you shouldn't need additional workers unless you are approaching CPU saturation for your existing sim instance(s). The "lag" statistic is another useful metric for determining whether the client sim is hitting its targets - if it gets too high, consider adding workers.
-
-The specific approach you take when configuring a high load depends on your goals and available resources. If your goal is to beat down a server until it melts into the floor, it is legitimate to use a less accurate simulation by reducing the timers and intervals in the client sim's behavior configuration. If instead you wish to see how many 'realistic' clients your server can service, you will want to stick with reasonable values for timers and intervals, and instead increase load by configuring more user accounts (in the 'arrival' section of the config file, and the separate user accounts file).
-
-To use four instances on the local host::
-
- <key>workers</key>
- <array>
- <string>./python contrib/performance/loadtest/ampsim.py</string>
- <string>./python contrib/performance/loadtest/ampsim.py</string>
- <string>./python contrib/performance/loadtest/ampsim.py</string>
- <string>./python contrib/performance/loadtest/ampsim.py</string>
- </array>
-
-To use four instances each on two different remote hosts, use something like::
-
- <key>workers</key>
- <array>
- <string>exec ssh blade2 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade2 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade2 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade2 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade3 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade3 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade3 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- <string>exec ssh blade3 'cd ~/ccs/CalendarServer ; exec ./python contrib/performance/loadtest/ampsim.py'</string>
- </array>
-
-**When using remote hosts, the ssh commands must work in an unattended fashion, so configure SSH keys as needed**. Also, each remote host needs to have a Calendar Server SVN checkout. In this example, the hosts blade2 and blade3 need to have an SVN checkout of Calendar Server at ~/ccs/CalendarServer.
-
-Configuration of the additional workers is handled by the master, so you need not distribute the sim's config file to the other hosts. Each instance gets an identical copy of the config. The amount of work attempted by the sim is not changed by adding workers; instead, the master distributes work (i.e. user accounts) across the workers. To do more work, add user accounts.
-
-When running the sim using multiple instances, the standard output of each child instance is sent to the master. For example, when starting with four instances::
-
- Loaded 99 accounts.
- Initiating worker configuration
- Initiating worker configuration
- Initiating worker configuration
- Initiating worker configuration
- Worker configuration complete.
- Worker configuration complete.
- Worker configuration complete.
- Worker configuration complete.
- user01 - - - - - - - - - - - startup BEGIN
- ...
-
-
Deleted: CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/MultiServerDeployment.txt
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/MultiServerDeployment.txt 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/doc/Admin/MultiServerDeployment.txt 2012-12-07 21:06:09 UTC (rev 10139)
@@ -1,191 +0,0 @@
-==========================
-Multi-server Deployment
-==========================
-
-Calendar Server version 3 and later uses a database as the primary data store, instead of the filesystem store used by previous versions. This allows for a familiar profile of scalability to multiple servers by offloading the DB to a dedicated server or cluster, and then adding front-end Calendar Server hosts as needed. This document highlights the key elements of a multi-server Calendar Server deployment.
-
-* `Database Connectivity`_: By default, Calendar Server assumes the DB is hosted locally on a unix domain socket, so you must add configuration to connect to an external DB service over the network.
-
-* `Database Setup and Schema Management`_: When connecting to an external DB, the administrator is responsible for applying our database schema to initialize the database for use by Calendar Server, using the calendarserver_bootstrap_database tool.
-
-* `Memcached`_: All Calendar Server hosts need to share access to memcached
-
-* `Proxy Database`_: Normally the Proxy (delegation) database is kept in a local sqlite database, which is not sharable. Create an additional database on the DB server to hold the Proxy DB, then configure all the servers to use it.
-
-* `Directory Services`_: All servers should have access to the same directory services data that defines users, groups, resources, and locations. Calendar Server provides a highly flexible LDAP client to leverage existing directory servers, or you can use local XML files.
-
-* `Client Connectivity`_: Use either a load balancer or round robin dns, and configure all servers to use the same ServerHostName in caldavd.plist
-
-* `Shared Storage for Attachments`_: AttachmentsRoot should point to storage shared across all servers, e.g. an NFS mount. Used for file attachments to calendar events.
-
-* `General Advise`_: *No one wants advice - only corroboration.* --John Steinbeck
-
----------------------
-Database Connectivity
----------------------
-
-First, make sure your Postgres service is connectable over the network ; by default it may use only unix domain sockets. Accepting connections over TCP typically involves changes to `listen_address <http://www.postgresql.org/docs/current/static/runtime-config-connection.html#GUC-LISTEN-ADDRESSES>`_ in the main server config file to bind the network socket, and also edits to `pg_hga.conf <http://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html>`_ to allow connections by IP or netmask. Additional discussion of database server setup and tuning is beyond the scope of this document.
-
-There are a few configuration parameters in caldavd.plist that control Calendar Server's behavior with respect to database use. The relevant caldavd.plist entries and their default values are shown and described below (as defined in `stdconfig.py <https://trac.calendarserver.org/browser/CalendarServer/trunk/twistedcaldav/stdconfig.py>`_)
-
-::
-
- "UseDatabase" : True, # True: database; False: files
-
- "DBType" : "", # 2 possible values: empty, meaning 'spawn postgres
- # yourself', or 'postgres', meaning 'connect to a
- # postgres database as specified by the 'DSN'
- # configuration key. Will support more values in
- # the future.
-
- "DSN" : "", # Data Source Name. Used to connect to an external
- # database if DBType is non-empty. Format varies
- # depending on database type.
-
-All of the above are top-level keys in caldavd.plist.
-
-The DSN is a colon separated string defined in PyGreSQL-4.0/pgdb.py and has the following structure:
-
-::
-
- dbhost = params[0]
- dbbase = params[1]
- dbuser = params[2]
- dbpasswd = params[3]
- dbopt = params[4]
- dbtty = params[5]
-
-When no hostname is specified, Calendar Server assumes the use of a local unix domain socket (found in the directory defined by the RunRoot config key)
-
-Example of a 'remote postgres via TCP' configuration:
-
-::
-
- <key>DBType</key>
- <string>postgres</string>
- <key>DSN</key>
- <string>hostname:dbname:dbuser:dbpass::</string>
-
-
-------------------------------------
-Database Setup and Schema Management
-------------------------------------
-
-Whenever DBType is set, Calendar Server is not responsible for the lifecycle of the database, nor is it responsible for the setup and schema population - these tasks are now the responsibility of the administrator. Once caldavd.plist is configured for your database, use the `calendarserver_bootstrap_database <https://trac.calendarserver.org/browser/CalendarServer/trunk/bin/calendarserver_bootstrap_database>`_ `tool <https://trac.calendarserver.org/browser/CalendarServer/trunk/calendarserver/tools/bootstrapdatabase.py>`_ to populate calendar server `schema <https://trac.calendarserver.org/browser/CalendarServer/trunk/txdav/common/datastore/sql_schema>`_ in your database. Starting and stopping the database should be accomplished using native tools (e.g. pg_ctl). The database should be started before Calendar Server, and stopped after Calendar Server.
-
-It is critically important that your database server keeps updated statistics about your database, which allows the database query planner to select appropriate performance optimizations. Refer to your database server documentation for details.
-
---------------
-Memcached
---------------
-
-The default memcached settings are found in `stdconfig.py <https://trac.calendarserver.org/browser/CalendarServer/trunk/twistedcaldav/stdconfig.py>`_. By default there is one memcached 'pool' that is automatically managed by Calendar Server, and memcache is started and stopped along with Calendar Server. For a multi-server deployment, you should manage the memcached lifecycle externally, to make it independent of your Calendar Server hosts. Also, all Calendar Server hosts must be configured to share this memcached instance. A sample configuration is shown below, which instructs Calendar Server to connect to the 'Default' pool at example.com port 11211.
-
-::
-
- <!-- Memcache Settings -->
- <key>Memcached</key>
- <dict>
- <key>MaxClients</key>
- <integer>5</integer>
- <key>Options</key>
- <array>
- <string>-U</string>
- <string>0</string>
- <string>-m</string>
- <string>6000</string>
- </array>
- <key>Pools</key>
- <dict>
- <key>Default</key>
- <dict>
- <key>ClientEnabled</key>
- <true/>
- <key>ServerEnabled</key>
- <false/>
- <key>BindAddress</key>
- <string>EXAMPLE.COM</string>
- <key>Port</key>
- <integer>11211</integer>
- <key>HandleCacheTypes</key>
- <array>
- <string>Default</string>
- </array>
- </dict>
- </dict>
- <key>memcached</key>
- <string>memcached</string>
- </dict>
-
-In this configuration, the administrator is expected to ensure that there is a memcached instance running on host EXAMPLE.COM listening on port 11211 (Default). All calendar servers need to have the same memcache configuration. Memcache should start first and stop last, relative to calendar server and postgres. Note also that memcached should be as close to the Calendar Server hosts as possible to minimize network latency.
-
-----------------
-Proxy Database
-----------------
-
-The Proxy DB (for delegation) is typically stored on disk in an sqlite DB, which does not allow for concurrent access across multiple hosts. To address this, create an additional DB in the postgres server, then edit caldavd.plist to add something like the following, and disable any other ProxyDB configuration.
-
-::
-
- <!-- PostgreSQL ProxyDB Service -->
- <key>ProxyDBService</key>
- <dict>
- <key>type</key>
- <string>twistedcaldav.directory.calendaruserproxy.ProxyPostgreSQLDB</string>
- <key>params</key>
- <dict>
- <key>dbtype</key>
- <string>ProxyDB</string>
- <key>host</key>
- <string>PARADISE-FALLS</string>
- <key>database</key>
- <string>FOSSILS</string>
- <key>user</key>
- <string>MUNTZ</string>
- </dict>
- </dict>
-
-As with the memcache config, all calendar servers should have the same ProxyDBService config. In the shown example, the server will expect to access a database called FOSSILS as user MUNTZ on the postgres server PARADISE-FALLS. Unlike with the primary calendar data store, calendar server is prepared to initialize the schema of this database at runtime if it does not exist - so nothing is required beyond creating the empty db, creating the db user with appropriate access, and applying some caldavd.plist configuration.
-
--------------------
-Directory Services
--------------------
-
-It is critical that all servers use the same directory services data that defines users (and their passwords), groups, resources, and locations used by Calendar Server. By default, this data is stored in local XML files, which is not ideally suited for a multi-server deployment, although would still work fine if the administrator is willing to manage the workflow around updating and distributing those files to all servers.
-
-In addition, Calendar Server provides a very configurable LDAP client interface for accessing external directory services data. Administrators familiar with LDAP should need little more than to look at `twistedcaldav/stdconfig.py <https://trac.calendarserver.org/browser/CalendarServer/trunk/twistedcaldav/stdconfig.py>`_ for the available options to get started. Calendar Server will perform standard LDAP bind authentication to authenticate clients.
-
-Open Directory is also available when running on Mac OS X or Mac OS X Server.
-
--------------------
-Client Connectivity
--------------------
-
-Use either a load balancer or round robin dns, and configure all servers to use the same ServerHostName in caldavd.plist. A load balancer provides the most optimal distribution of work across available servers, and greater resiliency incase of individual server failure. Round robin DNS is simpler, and should work fine - however be aware that DNS caches may result in a given client 'sticking' to a server for a while. Using the same ServerHostName everywhere allows for all servers to have the exact same caldavd.plist, which is strongly recommended for simplicity.
-
--------------------------------
-Shared Storage for Attachments
--------------------------------
-
-Set the caldavd.plist key AttachmentsRoot to a filesystem directory that is shared and writable by all Calendar Server machines, for example an NFS export. This will be used to store file attachements that users may attach to calendar events.
-
--------------------
-General Advise
--------------------
-
-* Ensure caldavd.plist is identical on all Calendar Server hosts. This is not strictly required, but recommended to keep things as predictable as possible. Since you already have shared storage for AttachmentsRoot, use that to host the 'conf' directory for all servers as well; this way you don't need to push config changes out to the servers.
-
-* Use the various `tools and utilities <https://trac.calendarserver.org/browser/CalendarServer/trunk/contrib/tools>`_ to monitor activity in real time, and also for post-processing access logs.
-
-* Be sure you are getting the most from an individual server before you decide you need additional hosts (other than for redundancy). To optimize the single-server configuration, play with the caldavd.plist keys MultiProcessCount (# of daemons spawned), and MaxRequests (# of requests a daemon will process concurrently). If your Calendar Server isn't above 80% CPU use for sustained periods, you most likely don't need more Calendar Server hosts.
-
-* Ensure that your database's table statistics are updated at a reasonable interval. "Reasonable" depends entirely on how quickly your data changes in shape and size. In particular, be sure to update stats after any bulk changes.
-
-* Tune the database for performance, using the methodologies appropriate for the database you are using. The DB server will need to accept up to MultiProcessCount * MaxRequests connections from each Calendar Server, unless MaxDBConnectionsPerPool is set, in which case the number is MultiProcessCount * MaxDBConnectionsPerPool per server, plus a handful more for other things like the notification sidecar or command line tools.
-
-* Test Scenario: With a well tuned multi-server deployment of identically configured caldav servers behind a load balancer, and a separate Postgres server with a fast RAID 0, in a low-latency lab environment using simulated iCal client load, it takes 5 or 6 caldav servers to saturate the postgres server (which becomes i/o bound at a load of about 55,000 simulated users in this test).
-
-* To eliminate all single points of failure, implement high-availability for memcache, the database, the directory service, the shared storage for AttachmentsRoot, and the network load balancer(s).
-
-* When using an external directory service such as LDAP or Open Directory, overall Calendar Server performance is highly dependent on the responsiveness of the directory service.
-
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/python/memcacheclient.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/python/memcacheclient.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/python/memcacheclient.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -126,13 +126,13 @@
pickler=pickle.Pickler, unpickler=pickle.Unpickler,
pload=None, pid=None):
- if config.Memcached.Pools.Default.ClientEnabled:
- return Client(servers, debug=debug, pickleProtocol=pickleProtocol,
- pickler=pickler, unpickler=unpickler, pload=pload, pid=pid)
- elif cls.allowTestCache:
+ if cls.allowTestCache:
return TestClient(servers, debug=debug,
pickleProtocol=pickleProtocol, pickler=pickler,
unpickler=unpickler, pload=pload, pid=pid)
+ elif config.Memcached.Pools.Default.ClientEnabled:
+ return Client(servers, debug=debug, pickleProtocol=pickleProtocol,
+ pickler=pickler, unpickler=unpickler, pload=pload, pid=pid)
else:
return None
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/channel/http.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/channel/http.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/channel/http.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -637,11 +637,7 @@
self._cleanup()
def getHostInfo(self):
- t=self.channel.transport
- secure = interfaces.ISSLTransport(t, None) is not None
- host = t.getHost()
- host.host = _cachedGetHostByAddr(host.host)
- return host, secure
+ return self.channel._host, self.channel._secure
def getRemoteHost(self):
return self.channel.transport.getPeer()
@@ -783,6 +779,9 @@
self.requests = []
def connectionMade(self):
+ self._secure = interfaces.ISSLTransport(self.transport, None) is not None
+ address = self.transport.getHost()
+ self._host = _cachedGetHostByAddr(address.host)
self.setTimeout(self.inputTimeOut)
self.factory.addConnectedChannel(self)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/auth.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/auth.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/auth.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -37,7 +37,9 @@
class AuthenticationWrapper(WrapperResource):
- def __init__(self, resource, portal, credentialFactories, loginInterfaces):
+ def __init__(self, resource, portal,
+ wireEncryptedCredentialFactories, wireUnencryptedCredentialFactories,
+ loginInterfaces):
"""
Wrap the given resource and use the parameters to set up the request
to allow anyone to challenge and handle authentication.
@@ -45,23 +47,42 @@
@param resource: L{DAVResource} FIXME: This should get promoted to
twext.web2.auth
@param portal: The cred portal
- @param credentialFactories: Sequence of credentialFactories that can
- be used to authenticate by resources in this tree.
+ @param wireEncryptedCredentialFactories: Sequence of credentialFactories
+ that can be used to authenticate by resources in this tree over a
+ wire-encrypted channel (SSL).
+ @param wireUnencryptedCredentialFactories: Sequence of credentialFactories
+ that can be used to authenticate by resources in this tree over a
+ wire-unencrypted channel (non-SSL).
@param loginInterfaces: More cred stuff
"""
super(AuthenticationWrapper, self).__init__(resource)
self.portal = portal
- self.credentialFactories = dict([(factory.scheme, factory)
- for factory in credentialFactories])
+ self.wireEncryptedCredentialFactories = dict([(factory.scheme, factory)
+ for factory in wireEncryptedCredentialFactories])
+ self.wireUnencryptedCredentialFactories = dict([(factory.scheme, factory)
+ for factory in wireUnencryptedCredentialFactories])
self.loginInterfaces = loginInterfaces
+ # FIXME: some unit tests access self.credentialFactories, so assigning here
+ self.credentialFactories = self.wireEncryptedCredentialFactories
+
def hook(self, req):
req.portal = self.portal
- req.credentialFactories = self.credentialFactories
req.loginInterfaces = self.loginInterfaces
+ # If not using SSL, use the factory list which excludes "Basic"
+ if req.chanRequest is None: # This is only None in unit tests
+ secureConnection = True
+ else:
+ ignored, secureConnection = req.chanRequest.getHostInfo()
+ req.credentialFactories = (
+ self.wireEncryptedCredentialFactories
+ if secureConnection
+ else self.wireUnencryptedCredentialFactories
+ )
+
class IPrincipal(Interface):
pass
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_acl.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_acl.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_acl.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -72,6 +72,7 @@
rootResource,
portal,
credentialFactories,
+ credentialFactories,
loginInterfaces
))
Copied: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_auth.py (from rev 10136, CalendarServer/trunk/twext/web2/dav/test/test_auth.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_auth.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_auth.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -0,0 +1,67 @@
+##
+# Copyright (c) 2012 Apple Computer, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# DRI: Wilfredo Sanchez, wsanchez at apple.com
+##
+
+import collections
+from twext.web2.dav.auth import AuthenticationWrapper
+import twext.web2.dav.test.util
+
+class AutoWrapperTestCase(twext.web2.dav.test.util.TestCase):
+
+ def test_basicAuthPrevention(self):
+ """
+ Ensure authentication factories which are not safe to use over an
+ "unencrypted wire" are not advertised when an insecure (i.e. non-SSL
+ connection is made.
+ """
+ FakeFactory = collections.namedtuple("FakeFactory", ("scheme,"))
+ wireEncryptedfactories = [FakeFactory("basic"), FakeFactory("digest"), FakeFactory("xyzzy")]
+ wireUnencryptedfactories = [FakeFactory("digest"), FakeFactory("xyzzy")]
+
+ class FakeChannel(object):
+ def __init__(self, secure):
+ self.secure = secure
+ def getHostInfo(self):
+ return "ignored", self.secure
+
+ class FakeRequest(object):
+ def __init__(self, secure):
+ self.portal = None
+ self.loginInterfaces = None
+ self.credentialFactories = None
+ self.chanRequest = FakeChannel(secure)
+
+ wrapper = AuthenticationWrapper(None, None,
+ wireEncryptedfactories, wireUnencryptedfactories, None)
+ req = FakeRequest(True) # Connection is over SSL
+ wrapper.hook(req)
+ self.assertEquals(
+ set(req.credentialFactories.keys()),
+ set(["basic", "digest", "xyzzy"])
+ )
+ req = FakeRequest(False) # Connection is not over SSL
+ wrapper.hook(req)
+ self.assertEquals(
+ set(req.credentialFactories.keys()),
+ set(["digest", "xyzzy"])
+ )
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_resource.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/dav/test/test_resource.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -240,6 +240,7 @@
self.rootresource,
portal,
credentialFactories,
+ credentialFactories,
loginInterfaces,
))
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/http.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/http.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/http.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -32,6 +32,7 @@
"""
# import traceback; log.msg(''.join(traceback.format_stack()))
+import json
import time
from twisted.internet import interfaces, error
@@ -50,13 +51,13 @@
log = Logger()
-defaultPortForScheme = {'http': 80, 'https':443, 'ftp':21}
+defaultPortForScheme = {'http': 80, 'https': 443, 'ftp': 21}
def splitHostPort(scheme, hostport):
- """Split the host in "host:port" format into host and port fields.
+ """Split the host in "host:port" format into host and port fields.
If port was not specified, use the default for the given scheme, if
known. Returns a tuple of (hostname, portnumber)."""
-
+
# Split hostport into host and port
hostport = hostport.split(':', 1)
try:
@@ -67,6 +68,7 @@
return hostport[0], defaultPortForScheme.get(scheme, 0)
+
def parseVersion(strversion):
"""Parse version strings of the form Protocol '/' Major '.' Minor. E.g. 'HTTP/1.1'.
Returns (protocol, major, minor).
@@ -80,7 +82,9 @@
return (proto.lower(), major, minor)
+
class HTTPError(Exception):
+
def __init__(self, codeOrResponse):
"""An Exception for propagating HTTP Error Responses.
@@ -91,28 +95,30 @@
self.response = iweb.IResponse(codeOrResponse)
Exception.__init__(self, str(self.response))
+
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.response)
+
class Response(object):
"""An object representing an HTTP Response to be sent to the client.
"""
implements(iweb.IResponse)
-
+
code = responsecode.OK
headers = None
stream = None
-
+
def __init__(self, code=None, headers=None, stream=None):
"""
@param code: The HTTP status code for this Response
@type code: C{int}
-
+
@param headers: Headers to be sent to the client.
- @type headers: C{dict}, L{twext.web2.http_headers.Headers}, or
+ @type headers: C{dict}, L{twext.web2.http_headers.Headers}, or
C{None}
-
+
@param stream: Content body to send to the HTTP client
@type stream: L{twext.web2.stream.IByteStream}
"""
@@ -123,13 +129,14 @@
if headers is not None:
if isinstance(headers, dict):
headers = http_headers.Headers(headers)
- self.headers=headers
+ self.headers = headers
else:
self.headers = http_headers.Headers()
if stream is not None:
self.stream = IByteStream(stream)
+
def __repr__(self):
if self.stream is None:
streamlen = None
@@ -139,6 +146,7 @@
return "<%s.%s code=%d, streamlen=%s>" % (self.__module__, self.__class__.__name__, self.code, streamlen)
+
class StatusResponseElement(Element):
"""
Render the HTML for a L{StatusResponse}
@@ -216,10 +224,11 @@
self.headers.setHeader("location", location)
-
+
+
def NotModifiedResponse(oldResponse=None):
if oldResponse is not None:
- headers=http_headers.Headers()
+ headers = http_headers.Headers()
for header in (
# Required from sec 10.3.5:
'date', 'etag', 'content-location', 'expires',
@@ -232,8 +241,9 @@
else:
headers = None
return Response(code=responsecode.NOT_MODIFIED, headers=headers)
-
+
+
def checkPreconditions(request, response=None, entityExists=True, etag=None, lastModified=None):
"""Check to see if this request passes the conditional checks specified
by the client. May raise an HTTPError with result codes L{NOT_MODIFIED}
@@ -246,7 +256,7 @@
However, if you are implementing other request methods, like PUT
for your resource, you will need to call this after determining
the etag and last-modified time of the existing resource but
- before actually doing the requested action. In that case,
+ before actually doing the requested action. In that case,
This examines the appropriate request headers for conditionals,
(If-Modified-Since, If-Unmodified-Since, If-Match, If-None-Match,
@@ -259,15 +269,15 @@
shouldn't be separately specified. Not providing the
response with a GET request may cause the emitted
"Not Modified" responses to be non-conformant.
-
+
@param entityExists: Set to False if the entity in question doesn't
yet exist. Necessary for PUT support with 'If-None-Match: *'.
-
+
@param etag: The etag of the resource to check against, or None.
-
+
@param lastModified: The last modified date of the resource to check
against, or None.
-
+
@raise: HTTPError: Raised when the preconditions fail, in order to
abort processing and emit an error page.
@@ -280,7 +290,8 @@
return False
etag = response.headers.getHeader("etag")
lastModified = response.headers.getHeader("last-modified")
-
+
+
def matchETag(tags, allowWeak):
if entityExists and '*' in tags:
return True
@@ -322,7 +333,7 @@
if inm:
if request.method in ("HEAD", "GET"):
# If it's a range request, don't allow a weak ETag, as that
- # would break.
+ # would break.
canBeWeak = not request.headers.hasHeader('Range')
if notModified != False and matchETag(inm, canBeWeak):
raise HTTPError(NotModifiedResponse(response))
@@ -333,6 +344,8 @@
if notModified == True:
raise HTTPError(NotModifiedResponse(response))
+
+
def checkIfRange(request, response):
"""Checks for the If-Range header, and if it exists, checks if the
test passes. Returns true if the server should return partial data."""
@@ -347,25 +360,29 @@
return ifrange == response.headers.getHeader("last-modified")
+
class _NotifyingProducerStream(stream.ProducerStream):
doStartReading = None
def __init__(self, length=None, doStartReading=None):
stream.ProducerStream.__init__(self, length=length)
self.doStartReading = doStartReading
-
+
+
def read(self):
if self.doStartReading is not None:
doStartReading = self.doStartReading
self.doStartReading = None
doStartReading()
-
+
return stream.ProducerStream.read(self)
+
def write(self, data):
self.doStartReading = None
stream.ProducerStream.write(self, data)
+
def finish(self):
self.doStartReading = None
stream.ProducerStream.finish(self)
@@ -379,18 +396,18 @@
Subclasses should override the process() method to determine how
the request will be processed.
-
+
@ivar method: The HTTP method that was used.
@ivar uri: The full URI that was requested (includes arguments).
@ivar headers: All received headers
@ivar clientproto: client HTTP version
@ivar stream: incoming data stream.
"""
-
+
implements(iweb.IRequest, interfaces.IConsumer)
-
+
known_expects = ('100-continue',)
-
+
def __init__(self, chanRequest, command, path, version, contentLength, headers):
"""
@param chanRequest: the channel request we're associated with.
@@ -399,16 +416,17 @@
self.method = command
self.uri = path
self.clientproto = version
-
+
self.headers = headers
-
+
if '100-continue' in self.headers.getHeader('expect', ()):
doStartReading = self._sendContinue
else:
doStartReading = None
self.stream = _NotifyingProducerStream(contentLength, doStartReading)
self.stream.registerProducer(self.chanRequest, True)
-
+
+
def checkExpect(self):
"""Ensure there are no expectations that cannot be met.
Checks Expect header against self.known_expects."""
@@ -416,50 +434,59 @@
for expect in expects:
if expect not in self.known_expects:
raise HTTPError(responsecode.EXPECTATION_FAILED)
-
+
+
def process(self):
"""Called by channel to let you process the request.
-
+
Can be overridden by a subclass to do something useful."""
pass
-
+
+
def handleContentChunk(self, data):
"""Callback from channel when a piece of data has been received.
Puts the data in .stream"""
self.stream.write(data)
-
+
+
def handleContentComplete(self):
"""Callback from channel when all data has been received. """
self.stream.unregisterProducer()
self.stream.finish()
-
+
+
def connectionLost(self, reason):
"""connection was lost"""
pass
+
def __repr__(self):
- return '<%s %s %s>'% (self.method, self.uri, self.clientproto)
+ return '<%s %s %s>' % (self.method, self.uri, self.clientproto)
+
def _sendContinue(self):
self.chanRequest.writeIntermediateResponse(responsecode.CONTINUE)
+
def _reallyFinished(self, x):
"""We are finished writing data."""
self.chanRequest.finish()
-
+
+
def _finished(self, x):
"""
We are finished writing data.
But we need to check that we have also finished reading all data as we
might have sent a, for example, 401 response before we read any data.
To make sure that the stream/producer sequencing works properly we need
- to discard the remaining data in the request.
+ to discard the remaining data in the request.
"""
if self.stream.length != 0:
return readAndDiscard(self.stream).addCallback(self._reallyFinished).addErrback(self._error)
else:
self._reallyFinished(x)
+
def _error(self, reason):
if reason.check(error.ConnectionLost):
log.msg("Request error: " + reason.getErrorMessage())
@@ -467,7 +494,8 @@
log.err(reason)
# Only bother with cleanup on errors other than lost connection.
self.chanRequest.abortConnection()
-
+
+
def writeResponse(self, response):
"""
Write a response.
@@ -476,10 +504,10 @@
# Expect: 100-continue was requested, but 100 response has not been
# sent, and there's a possibility that data is still waiting to be
# sent.
- #
+ #
# Ideally this means the remote side will not send any data.
# However, because of compatibility requirements, it might timeout,
- # and decide to do so anyways at the same time we're sending back
+ # and decide to do so anyways at the same time we're sending back
# this response. Thus, the read state is unknown after this.
# We must close the connection.
self.chanRequest.channel.setReadPersistent(False)
@@ -493,7 +521,7 @@
elif response.stream.length is not None:
response.headers.setHeader('content-length', response.stream.length)
self.chanRequest.writeHeaders(response.code, response.headers)
-
+
# if this is a "HEAD" request, or a special response code,
# don't return any data.
if self.method == "HEAD" or response.code in NO_BODY_CODES:
@@ -501,10 +529,12 @@
response.stream.close()
self._finished(None)
return
-
+
d = stream.StreamProducer(response.stream).beginProducing(self.chanRequest)
d.addCallback(self._finished).addErrback(self._error)
+
+
class XMLResponse (Response):
"""
XML L{Response} object.
@@ -517,8 +547,21 @@
Response.__init__(self, code, stream=element.toxml())
self.headers.setHeader("content-type", http_headers.MimeType("text", "xml"))
-
+
+
+class JSONResponse (Response):
+ """
+ JSON L{Response} object.
+ Renders itself as an JSON document.
+ """
+ def __init__(self, code, jobj):
+ """
+ @param xml_responses: an iterable of davxml.Response objects.
+ """
+ Response.__init__(self, code, stream=json.dumps(jobj))
+ self.headers.setHeader("content-type", http_headers.MimeType("application", "json"))
+
+
components.registerAdapter(Response, int, iweb.IResponse)
-__all__ = ['HTTPError', 'NotModifiedResponse', 'Request', 'Response', 'StatusResponse', 'RedirectResponse', 'checkIfRange', 'checkPreconditions', 'defaultPortForScheme', 'parseVersion', 'splitHostPort', "XMLResponse"]
-
+__all__ = ['HTTPError', 'NotModifiedResponse', 'Request', 'Response', 'StatusResponse', 'RedirectResponse', 'checkIfRange', 'checkPreconditions', 'defaultPortForScheme', 'parseVersion', 'splitHostPort', "XMLResponse", "JSONResponse"]
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/test/test_metafd.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/test/test_metafd.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twext/web2/test/test_metafd.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -53,7 +53,10 @@
raise SocketError(ENOTCONN, "Transport endpoint not connected")
+ def getsockname(self):
+ return ("4.3.2.1", 4321)
+
class InheritedPortForTesting(sendfdport.InheritedPort):
"""
L{sendfdport.InheritedPort} subclass that prevents certain I/O operations
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/cachingdirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/cachingdirectory.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/cachingdirectory.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -96,9 +96,8 @@
for indexType, indexKey in indexTypes:
self.recordsIndexedBy[indexType][indexKey] = record
if useMemcache:
- key = "dir|%s|%s|%s|%s" % (self.directoryService.baseGUID,
- indexType, indexKey,
- "|".join(self.directoryService.recordTypes()))
+ key = self.directoryService.generateMemcacheKey(indexType, indexKey,
+ record.recordType)
self.log_debug("Memcache: storing %s" % (key,))
try:
self.directoryService.memcacheSet(key, record)
@@ -226,6 +225,29 @@
raise DirectoryMemcacheError("Failed to read from memcache")
return record
+ def generateMemcacheKey(self, indexType, indexKey, recordType):
+ """
+ Return a key that can be used to store/retrieve a record in memcache.
+ if short-name is the indexType the recordType be encoded into the key.
+
+ @param indexType: one of the indexTypes( ) values
+ @type indexType: C{str}
+ @param indexKey: the value being indexed
+ @type indexKey: C{str}
+ @param recordType: the type of record being cached
+ @type recordType: C{str}
+ @return: a memcache key comprised of the passed-in values and the directory
+ service's baseGUID
+ @rtype: C{str}
+ """
+ keyVersion = 2
+ if indexType == CachingDirectoryService.INDEX_TYPE_SHORTNAME:
+ return "dir|v%d|%s|%s|%s|%s" % (keyVersion,self.baseGUID, recordType,
+ indexType, indexKey)
+ else:
+ return "dir|v%d|%s|%s|%s" % (keyVersion, self.baseGUID, indexType,
+ indexKey)
+
def _initCaches(self):
self._recordCaches = dict([
(recordType, self.cacheClass(self, recordType))
@@ -315,8 +337,12 @@
# Check memcache
if config.Memcached.Pools.Default.ClientEnabled:
- key = "dir|%s|%s|%s|%s" % (self.baseGUID, indexType, indexKey,
- "|".join(self.recordTypes()))
+
+ # The only time the recordType arg matters is when indexType is
+ # short-name, and in that case recordTypes will contain exactly
+ # one recordType, so using recordTypes[0] here is always safe:
+ key = self.generateMemcacheKey(indexType, indexKey, recordTypes[0])
+
self.log_debug("Memcache: checking %s" % (key,))
try:
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/calendaruserproxy.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/calendaruserproxy.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -374,7 +374,9 @@
for uid in members:
p = self.pcollection.principalForUID(uid)
if p:
- found.append(p)
+ # Only principals enabledForLogin can be a delegate
+ if p.record.enabledForLogin:
+ found.append(p)
# Make sure any outstanding deletion timer entries for
# existing principals are removed
yield self._index().refreshPrincipal(uid)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/accounts.xml
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/accounts.xml 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/accounts.xml 2012-12-07 21:06:09 UTC (rev 10139)
@@ -110,6 +110,20 @@
<name>佐藤佐藤佐藤</name>
<email-address>nonascii at example.com</email-address>
</user>
+ <user>
+ <uid>delegator</uid>
+ <guid>FC465590-E9E9-4746-ACE8-6C756A49FE4D</guid>
+ <password>a</password>
+ <name>Calendar Delegator</name>
+ <email-address>calendardelegator at example.com</email-address>
+ </user>
+ <user>
+ <uid>occasionaldelegate</uid>
+ <guid>EC465590-E9E9-4746-ACE8-6C756A49FE4D</guid>
+ <password>a</password>
+ <name>Occasional Delegate</name>
+ <email-address>occasional at example.com</email-address>
+ </user>
<user repeat="2">
<uid>user%02d</uid>
<guid>user%02d</guid>
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/augments.xml
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/augments.xml 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/augments.xml 2012-12-07 21:06:09 UTC (rev 10139)
@@ -175,4 +175,15 @@
<enable>true</enable>
<enable-addressbook>true</enable-addressbook>
</record>
+ <record>
+ <uid>FC465590-E9E9-4746-ACE8-6C756A49FE4D</uid>
+ <enable>true</enable>
+ <enable-calendar>true</enable-calendar>
+ </record>
+ <record>
+ <uid>EC465590-E9E9-4746-ACE8-6C756A49FE4D</uid>
+ <enable>true</enable>
+ <enable-calendar>true</enable-calendar>
+ <enable-login>true</enable-login>
+ </record>
</augments>
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/proxies.xml
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/proxies.xml 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/proxies.xml 2012-12-07 21:06:09 UTC (rev 10139)
@@ -59,4 +59,10 @@
<member>9FF60DAD-0BDE-4508-8C77-15F0CA5C8DD2</member>
</proxies>
</record>
+ <record>
+ <guid>FC465590-E9E9-4746-ACE8-6C756A49FE4D</guid>
+ <proxies>
+ <member>EC465590-E9E9-4746-ACE8-6C756A49FE4D</member>
+ </proxies>
+ </record>
</proxies>
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_cachedirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_cachedirectory.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_cachedirectory.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -22,11 +22,13 @@
from twistedcaldav.directory.util import uuidFromName
from twistedcaldav.directory.augment import AugmentRecord
from twistedcaldav.test.util import TestCase
+from twistedcaldav.config import config
class TestDirectoryService (CachingDirectoryService):
realmName = "Dummy Realm"
+ baseGUID = "20CB1593-DE3F-4422-A7D7-BA9C2099B317"
def recordTypes(self):
return (
@@ -37,7 +39,7 @@
)
def queryDirectory(self, recordTypes, indexType, indexKey):
-
+
self.queried = True
for recordType in recordTypes:
@@ -95,6 +97,7 @@
def fakeRecord(
self,
fullName,
+ recordType,
shortNames=None,
guid=None,
emails=None,
@@ -108,7 +111,7 @@
shortNames += (fullName,)
if guid is None:
- guid = self.guidForShortName(shortNames[0])
+ guid = self.guidForShortName(shortNames[0], recordType=recordType)
else:
guid = guid.lower()
@@ -135,26 +138,33 @@
def shortNameForFullName(self, fullName):
return fullName.lower().replace(" ", "")
- def guidForShortName(self, shortName):
- return uuidFromName(self.baseGUID, shortName)
+ def guidForShortName(self, shortName, recordType=""):
+ return uuidFromName(self.baseGUID, "%s%s" % (recordType, shortName))
def dummyRecords(self):
SIZE = 10
- self.loadRecords({
+ records = {
DirectoryService.recordType_users: [
- self.fakeRecord("User %02d" % x, multinames=(x>5)) for x in range(1,SIZE+1)
+ self.fakeRecord("User %02d" % x, DirectoryService.recordType_users, multinames=(x>5)) for x in range(1,SIZE+1)
],
DirectoryService.recordType_groups: [
- self.fakeRecord("Group %02d" % x) for x in range(1,SIZE+1)
+ self.fakeRecord("Group %02d" % x, DirectoryService.recordType_groups) for x in range(1,SIZE+1)
],
DirectoryService.recordType_resources: [
- self.fakeRecord("Resource %02d" % x) for x in range(1,SIZE+1)
+ self.fakeRecord("Resource %02d" % x, DirectoryService.recordType_resources) for x in range(1,SIZE+1)
],
DirectoryService.recordType_locations: [
- self.fakeRecord("Location %02d" % x) for x in range(1,SIZE+1)
+ self.fakeRecord("Location %02d" % x, DirectoryService.recordType_locations) for x in range(1,SIZE+1)
],
- })
+ }
+ # Add duplicate shortnames
+ records[DirectoryService.recordType_users].append(self.fakeRecord("Duplicate", DirectoryService.recordType_users, multinames=True))
+ records[DirectoryService.recordType_groups].append(self.fakeRecord("Duplicate", DirectoryService.recordType_groups, multinames=True))
+ records[DirectoryService.recordType_resources].append(self.fakeRecord("Duplicate", DirectoryService.recordType_resources, multinames=True))
+ records[DirectoryService.recordType_locations].append(self.fakeRecord("Duplicate", DirectoryService.recordType_locations, multinames=True))
+ self.loadRecords(records)
+
def verifyRecords(self, recordType, expectedGUIDs):
records = self.service.listRecords(recordType)
@@ -174,10 +184,10 @@
def test_cacheoneguid(self):
self.dummyRecords()
- self.assertTrue(self.service.recordWithGUID(self.guidForShortName("user01")) is not None)
+ self.assertTrue(self.service.recordWithGUID(self.guidForShortName("user01", recordType=DirectoryService.recordType_users)) is not None)
self.assertTrue(self.service.queried)
self.verifyRecords(DirectoryService.recordType_users, set((
- self.guidForShortName("user01"),
+ self.guidForShortName("user01", recordType=DirectoryService.recordType_users),
)))
self.verifyRecords(DirectoryService.recordType_groups, set())
self.verifyRecords(DirectoryService.recordType_resources, set())
@@ -185,11 +195,11 @@
# Make sure it really is cached and won't cause another query
self.service.queried = False
- self.assertTrue(self.service.recordWithGUID(self.guidForShortName("user01")) is not None)
+ self.assertTrue(self.service.recordWithGUID(self.guidForShortName("user01", recordType=DirectoryService.recordType_users)) is not None)
self.assertFalse(self.service.queried)
# Make sure guid is case-insensitive
- self.assertTrue(self.service.recordWithGUID(self.guidForShortName("user01").lower()) is not None)
+ self.assertTrue(self.service.recordWithGUID(self.guidForShortName("user01", recordType=DirectoryService.recordType_users).lower()) is not None)
def test_cacheoneshortname(self):
self.dummyRecords()
@@ -200,7 +210,7 @@
) is not None)
self.assertTrue(self.service.queried)
self.verifyRecords(DirectoryService.recordType_users, set((
- self.guidForShortName("user02"),
+ self.guidForShortName("user02", recordType=DirectoryService.recordType_users),
)))
self.verifyRecords(DirectoryService.recordType_groups, set())
self.verifyRecords(DirectoryService.recordType_resources, set())
@@ -222,7 +232,7 @@
) is not None)
self.assertTrue(self.service.queried)
self.verifyRecords(DirectoryService.recordType_users, set((
- self.guidForShortName("user03"),
+ self.guidForShortName("user03", recordType=DirectoryService.recordType_users),
)))
self.verifyRecords(DirectoryService.recordType_groups, set())
self.verifyRecords(DirectoryService.recordType_resources, set())
@@ -243,7 +253,7 @@
) is not None)
self.assertTrue(self.service.queried)
self.verifyRecords(DirectoryService.recordType_users, set((
- self.guidForShortName("user03"),
+ self.guidForShortName("user03", recordType=DirectoryService.recordType_users),
)))
self.verifyRecords(DirectoryService.recordType_groups, set())
self.verifyRecords(DirectoryService.recordType_resources, set())
@@ -291,3 +301,42 @@
self.service.queried = False
self.assertEquals(self.service.recordWithGUID(self.guidForShortName("missing")), None)
self.assertTrue(self.service.queried)
+
+ def test_duplicateShortNames(self):
+ """
+ Verify that when looking up records having duplicate short-names, the record of the
+ proper type is returned
+ """
+
+ self.patch(config.Memcached.Pools.Default, "ClientEnabled", True)
+ self.dummyRecords()
+
+ record = self.service.recordWithShortName(DirectoryService.recordType_users,
+ "Duplicate")
+ self.assertEquals(record.recordType, DirectoryService.recordType_users)
+
+ record = self.service.recordWithShortName(DirectoryService.recordType_groups,
+ "Duplicate")
+ self.assertEquals(record.recordType, DirectoryService.recordType_groups)
+
+ record = self.service.recordWithShortName(DirectoryService.recordType_resources,
+ "Duplicate")
+ self.assertEquals(record.recordType, DirectoryService.recordType_resources)
+
+ record = self.service.recordWithShortName(DirectoryService.recordType_locations,
+ "Duplicate")
+ self.assertEquals(record.recordType, DirectoryService.recordType_locations)
+
+ def test_generateMemcacheKey(self):
+ """
+ Verify keys are correctly generated based on the index type -- if index type is
+ short-name, then the recordtype is encoded into the key.
+ """
+ self.assertEquals(
+ self.service.generateMemcacheKey(self.service.INDEX_TYPE_GUID, "foo", "users"),
+ "dir|v2|20CB1593-DE3F-4422-A7D7-BA9C2099B317|guid|foo",
+ )
+ self.assertEquals(
+ self.service.generateMemcacheKey(self.service.INDEX_TYPE_SHORTNAME, "foo", "users"),
+ "dir|v2|20CB1593-DE3F-4422-A7D7-BA9C2099B317|users|shortname|foo",
+ )
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -439,5 +439,34 @@
"""
self.assertEquals(
set((yield calendaruserproxy.ProxyDBService.getAllMembers())),
- set([u'6423F94A-6B76-4A3A-815B-D52CFD77935D', u'8A985493-EE2C-4665-94CF-4DFEA3A89500', u'9FF60DAD-0BDE-4508-8C77-15F0CA5C8DD2', u'both_coasts', u'left_coast', u'non_calendar_group', u'recursive1_coasts', u'recursive2_coasts'])
+ set([u'6423F94A-6B76-4A3A-815B-D52CFD77935D', u'8A985493-EE2C-4665-94CF-4DFEA3A89500', u'9FF60DAD-0BDE-4508-8C77-15F0CA5C8DD2', u'both_coasts', u'left_coast', u'non_calendar_group', u'recursive1_coasts', u'recursive2_coasts', u'EC465590-E9E9-4746-ACE8-6C756A49FE4D'])
)
+
+
+ @inlineCallbacks
+ def test_hideDisabledDelegates(self):
+ """
+ Delegates who are not enabledForLogin are "hidden" from the delegate lists
+ """
+
+ record = self.directoryService.recordWithGUID("EC465590-E9E9-4746-ACE8-6C756A49FE4D")
+
+ record.enabledForLogin = True
+ yield self._groupMembersTest(
+ DirectoryService.recordType_users, "delegator", "calendar-proxy-write",
+ ("Occasional Delegate",),
+ )
+
+ # Login disabled -- no longer shown as a delegate
+ record.enabledForLogin = False
+ yield self._groupMembersTest(
+ DirectoryService.recordType_users, "delegator", "calendar-proxy-write",
+ [],
+ )
+
+ # Login re-enabled -- once again a delegate (it wasn't not removed from proxydb)
+ record.enabledForLogin = True
+ yield self._groupMembersTest(
+ DirectoryService.recordType_users, "delegator", "calendar-proxy-write",
+ ("Occasional Delegate",),
+ )
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/ical.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/ical.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/ical.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -3208,23 +3208,23 @@
if tzexpanded:
if start != tzexpanded[0][0]:
results.append((
- str(start),
- PyCalendarUTCOffsetValue(tzexpanded[0][1]).getText(),
- PyCalendarUTCOffsetValue(tzexpanded[0][1]).getText(),
+ start,
+ tzexpanded[0][1],
+ tzexpanded[0][1],
tzexpanded[0][3],
))
else:
results.append((
- str(start),
- PyCalendarUTCOffsetValue(tzcomp._pycalendar.getTimezoneOffsetSeconds(start)).getText(),
- PyCalendarUTCOffsetValue(tzcomp._pycalendar.getTimezoneOffsetSeconds(start)).getText(),
- tzcomp.getTZName(),
+ start,
+ tzcomp._pycalendar.getTimezoneOffsetSeconds(start),
+ tzcomp._pycalendar.getTimezoneOffsetSeconds(start),
+ tzcomp._pycalendar.getTimezoneDescriptor(start),
))
for tzstart, tzoffsetfrom, tzoffsetto, name in tzexpanded:
results.append((
- tzstart.getText(),
- PyCalendarUTCOffsetValue(tzoffsetfrom).getText(),
- PyCalendarUTCOffsetValue(tzoffsetto).getText(),
+ tzstart,
+ tzoffsetfrom,
+ tzoffsetto,
name,
))
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/resource.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/resource.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -3041,11 +3041,13 @@
""" AuthenticationWrapper implementation which allows overriding
credentialFactories on a per-resource-path basis """
- def __init__(self, resource, portal, credentialFactories, loginInterfaces,
- overrides=None):
+ def __init__(self, resource, portal,
+ wireEncryptedCredentialFactories, wireUnencryptedCredentialFactories,
+ loginInterfaces, overrides=None):
super(AuthenticationWrapper, self).__init__(resource, portal,
- credentialFactories, loginInterfaces)
+ wireEncryptedCredentialFactories, wireUnencryptedCredentialFactories,
+ loginInterfaces)
self.overrides = {}
if overrides:
@@ -3061,7 +3063,7 @@
super(AuthenticationWrapper, self).hook(req)
factories = self.overrides.get(req.path.rstrip("/"),
- self.credentialFactories)
+ req.credentialFactories)
req.credentialFactories = factories
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/stdconfig.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/stdconfig.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -406,15 +406,20 @@
# Authentication
#
"Authentication": {
- "Basic": { "Enabled": False }, # Clear text; best avoided
+ "Basic": { # Clear text; best avoided
+ "Enabled": True,
+ "AllowedOverWireUnencrypted": False, # Advertised over non-SSL?
+ },
"Digest": { # Digest challenge/response
"Enabled": True,
"Algorithm": "md5",
"Qop": "",
+ "AllowedOverWireUnencrypted": True, # Advertised over non-SSL?
},
"Kerberos": { # Kerberos/SPNEGO
"Enabled": False,
- "ServicePrincipal": ""
+ "ServicePrincipal": "",
+ "AllowedOverWireUnencrypted": True, # Advertised over non-SSL?
},
"Wiki": {
"Enabled": False,
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/test/test_timezonestdservice.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/test/test_timezonestdservice.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/test/test_timezonestdservice.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -15,7 +15,7 @@
##
from twistedcaldav.timezones import TimezoneCache
-from twistedcaldav.timezonestdservice import TimezoneInfo,\
+from twistedcaldav.timezonestdservice import TimezoneInfo, \
PrimaryTimezoneDatabase
from xml.etree.ElementTree import Element
import hashlib
@@ -28,13 +28,13 @@
"""
def test_generateXML(self):
-
+
hashed = hashlib.md5("test").hexdigest()
info = TimezoneInfo("America/New_York", ("US/Eastern",), "20110517T120000Z", hashed)
-
+
node = Element("root")
info.generateXML(node)
-
+
timezone = node.find("timezone")
self.assertTrue(timezone is not None)
self.assertEqual(timezone.findtext("tzid"), "America/New_York")
@@ -42,15 +42,16 @@
self.assertEqual(timezone.findtext("alias"), "US/Eastern")
self.assertEqual(timezone.findtext("md5"), hashed)
+
def test_parseXML(self):
-
+
hashed = hashlib.md5("test").hexdigest()
info1 = TimezoneInfo("America/New_York", ("US/Eastern",), "20110517T120000Z", hashed)
-
+
node = Element("root")
info1.generateXML(node)
timezone = node.find("timezone")
-
+
info2 = TimezoneInfo.readXML(timezone)
self.assertEqual(info2.tzid, "America/New_York")
@@ -58,6 +59,8 @@
self.assertEqual(info2.dtstamp, "20110517T120000Z")
self.assertEqual(info2.md5, hashed)
+
+
class TestPrimaryTimezoneDatabase (twistedcaldav.test.util.TestCase):
"""
Timezone support tests
@@ -66,21 +69,23 @@
def setUp(self):
TimezoneCache.create()
+
def testCreate(self):
-
+
xmlfile = self.mktemp()
db = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
-
+
db.createNewDatabase()
self.assertTrue(os.path.exists(xmlfile))
self.assertTrue(db.dtstamp is not None)
self.assertTrue(len(db.timezones) > 0)
+
def testUpdate(self):
-
+
xmlfile = self.mktemp()
db = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
-
+
db.createNewDatabase()
self.assertTrue(os.path.exists(xmlfile))
@@ -88,8 +93,9 @@
self.assertTrue(db.changeCount == 0)
self.assertTrue(len(db.changed) == 0)
+
def testRead(self):
-
+
xmlfile = self.mktemp()
db1 = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
db1.createNewDatabase()
@@ -99,50 +105,54 @@
db2.readDatabase()
self.assertEqual(db1.dtstamp, db2.dtstamp)
self.assertEqual(len(db1.timezones), len(db2.timezones))
-
+
+
def testList(self):
-
+
xmlfile = self.mktemp()
db = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
db.createNewDatabase()
self.assertTrue(os.path.exists(xmlfile))
-
+
tzids = set([tz.tzid for tz in db.listTimezones(None)])
self.assertTrue("America/New_York" in tzids)
self.assertTrue("US/Eastern" not in tzids)
-
+
+
def testListChangedSince(self):
-
+
xmlfile = self.mktemp()
db = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
db.createNewDatabase()
self.assertTrue(os.path.exists(xmlfile))
-
+
tzids = set([tz.tzid for tz in db.listTimezones(db.dtstamp)])
self.assertTrue(len(tzids) == 0)
-
+
+
def testGetNone(self):
-
+
xmlfile = self.mktemp()
db = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
db.createNewDatabase()
self.assertTrue(os.path.exists(xmlfile))
-
+
tz = db.getTimezone("Bogus")
self.assertEqual(tz, None)
-
+
+
def testGetOne(self):
-
+
xmlfile = self.mktemp()
db = PrimaryTimezoneDatabase(TimezoneCache.getDBPath(), xmlfile)
db.createNewDatabase()
self.assertTrue(os.path.exists(xmlfile))
-
+
# Original
tz1 = db.getTimezone("America/New_York")
self.assertTrue(str(tz1).find("VTIMEZONE") != -1)
self.assertTrue(str(tz1).find("TZID:America/New_York") != -1)
-
+
# Alias
tz1 = db.getTimezone("US/Eastern")
self.assertTrue(str(tz1).find("VTIMEZONE") != -1)
Modified: CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/timezonestdservice.py
===================================================================
--- CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/timezonestdservice.py 2012-12-07 21:01:33 UTC (rev 10138)
+++ CalendarServer/branches/users/cdaboo/managed-attachments/twistedcaldav/timezonestdservice.py 2012-12-07 21:06:09 UTC (rev 10139)
@@ -27,30 +27,27 @@
from twext.python.log import Logger
from twext.web2 import responsecode
-from txdav.xml import element as davxml
-from twext.web2.dav.http import ErrorResponse
from twext.web2.dav.method.propfind import http_PROPFIND
from twext.web2.dav.noneprops import NonePropertyStore
-from twext.web2.http import HTTPError, StatusResponse
+from twext.web2.http import HTTPError, JSONResponse
from twext.web2.http import Response
-from twext.web2.http import XMLResponse
from twext.web2.http_headers import MimeType
from twext.web2.stream import MemoryStream
+from txdav.xml import element as davxml
-from twisted.internet.defer import succeed, inlineCallbacks, returnValue,\
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue, \
DeferredList
-from twistedcaldav import timezonexml, xmlutil
+from twistedcaldav import xmlutil
from twistedcaldav.client.geturl import getURL
from twistedcaldav.config import config
-from twistedcaldav.customxml import calendarserver_namespace
-from twistedcaldav.extensions import DAVResource,\
+from twistedcaldav.extensions import DAVResource, \
DAVResourceWithoutChildrenMixin
from twistedcaldav.ical import tzexpandlocal
from twistedcaldav.resource import ReadOnlyNoCopyResourceMixIn
-from twistedcaldav.timezones import TimezoneException, TimezoneCache, readVTZ,\
+from twistedcaldav.timezones import TimezoneException, TimezoneCache, readVTZ, \
addVTZ
-from twistedcaldav.xmlutil import addSubElement, readXMLString
+from twistedcaldav.xmlutil import addSubElement
from pycalendar.calendar import PyCalendar
from pycalendar.datetime import PyCalendarDateTime
@@ -58,6 +55,7 @@
import hashlib
import itertools
+import json
import os
log = Logger()
@@ -69,7 +67,6 @@
Extends L{DAVResource} to provide timezone service functionality.
"""
-
def __init__(self, parent):
"""
@param parent: the parent resource of this one.
@@ -80,6 +77,8 @@
self.parent = parent
self.expandcache = {}
+ self.primary = True
+ self.info_source = None
if config.TimezoneService.Mode == "primary":
log.info("Using primary Timezone Service")
@@ -89,7 +88,8 @@
self._initSecondaryService()
else:
raise ValueError("Invalid TimezoneService mode: %s" % (config.TimezoneService.Mode,))
-
+
+
def _initPrimaryService(self):
tzpath = TimezoneCache.getDBPath()
xmlfile = os.path.join(tzpath, "timezones.xml")
@@ -98,9 +98,11 @@
self.timezones.createNewDatabase()
else:
self.timezones.readDatabase()
+ self.info_source = "Primary"
+
def _initSecondaryService(self):
-
+
# Must have writeable paths
tzpath = TimezoneCache.getDBPath()
xmlfile = config.TimezoneService.XMLInfoPath
@@ -111,24 +113,32 @@
self.timezones.readDatabase()
except:
pass
+ self.info_source = "Secondary"
+ self.primary = False
+
def onStartup(self):
return self.timezones.onStartup()
+
def deadProperties(self):
if not hasattr(self, "_dead_properties"):
self._dead_properties = NonePropertyStore(self)
return self._dead_properties
+
def etag(self):
return succeed(None)
+
def checkPreconditions(self, request):
return None
+
def checkPrivileges(self, request, privileges, recurse=False, principal=None, inherited_aces=None):
return succeed(None)
+
def defaultAccessControlList(self):
return davxml.ACL(
# DAV:Read for all principals (includes anonymous)
@@ -141,21 +151,27 @@
),
)
+
def contentType(self):
- return MimeType.fromString("text/html; charset=utf-8");
+ return MimeType.fromString("text/html; charset=utf-8")
+
def resourceType(self):
return None
+
def isCollection(self):
return False
+
def isCalendarCollection(self):
return False
+
def isPseudoCalendarCollection(self):
return False
+
def render(self, request):
output = """<html>
<head>
@@ -176,10 +192,11 @@
"""
The timezone service POST method.
"""
-
+
# GET and POST do the same thing
return self.http_POST(request)
+
def http_POST(self, request):
"""
The timezone service POST method.
@@ -187,109 +204,99 @@
# Check authentication and access controls
def _gotResult(_):
-
+
if not request.args:
# Do normal GET behavior
return self.render(request)
-
+
action = request.args.get("action", ("",))
if len(action) != 1:
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Invalid action query parameter",
+ {
+ "error": "invalid-action",
+ "description": "Invalid action query parameter",
+ },
))
action = action[0]
-
+
action = {
"capabilities" : self.doCapabilities,
"list" : self.doList,
"get" : self.doGet,
"expand" : self.doExpand,
}.get(action, None)
-
+
if action is None:
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Unknown action query parameter",
+ {
+ "error": "invalid-action",
+ "description": "Unknown action query parameter",
+ },
))
-
+
return action(request)
-
+
d = self.authorize(request, (davxml.Read(),))
d.addCallback(_gotResult)
return d
+
def doCapabilities(self, request):
"""
Return a list of all timezones known to the server.
"""
-
- result = timezonexml.Capabilities(
-
- timezonexml.Operation(
- timezonexml.Action.fromString("capabilities"),
- timezonexml.Description.fromString("Get capabilities of the server"),
- ),
-
- timezonexml.Operation(
- timezonexml.Action.fromString("list"),
- timezonexml.Description.fromString("List timezones on the server"),
- timezonexml.AcceptParameter(
- timezonexml.Name.fromString("changedsince"),
- timezonexml.Required.fromString("false"),
- timezonexml.Multi.fromString("false"),
- ),
- ),
-
- timezonexml.Operation(
- timezonexml.Action.fromString("get"),
- timezonexml.Description.fromString("Get timezones from the server"),
- timezonexml.AcceptParameter(
- timezonexml.Name.fromString("format"),
- timezonexml.Required.fromString("false"),
- timezonexml.Multi.fromString("false"),
- timezonexml.Value.fromString("text/calendar"),
- timezonexml.Value.fromString("text/plain"),
- ),
- timezonexml.AcceptParameter(
- timezonexml.Name.fromString("tzid"),
- timezonexml.Required.fromString("true"),
- timezonexml.Multi.fromString("false"),
- ),
- ),
-
- timezonexml.Operation(
- timezonexml.Action.fromString("expand"),
- timezonexml.Description.fromString("Expand timezones from the server"),
- timezonexml.AcceptParameter(
- timezonexml.Name.fromString("tzid"),
- timezonexml.Required.fromString("true"),
- timezonexml.Multi.fromString("false"),
- ),
- timezonexml.AcceptParameter(
- timezonexml.Name.fromString("start"),
- timezonexml.Required.fromString("false"),
- timezonexml.Multi.fromString("false"),
- ),
- timezonexml.AcceptParameter(
- timezonexml.Name.fromString("end"),
- timezonexml.Required.fromString("false"),
- timezonexml.Multi.fromString("false"),
- ),
- ),
- )
- return XMLResponse(responsecode.OK, result)
+ result = {
+ "info" : {
+ "primary-source" if self.primary else "secondary_source": self.info_source,
+ "contacts" : [],
+ },
+ "actions" : [
+ {
+ "name": "capabilities",
+ "parameters": [],
+ },
+ {
+ "name": "list",
+ "parameters": [
+ {"name": "changedsince", "required": False, "multi": False, },
+ ],
+ },
+ {
+ "name": "get",
+ "parameters": [
+ {"name": "format", "required": False, "multi": False, "values": ["text/calendar", "text/plain", ], },
+ {"name": "tzid", "required": True, "multi": False, },
+ ],
+ },
+ {
+ "name": "expand",
+ "parameters": [
+ {"name": "tzid", "required": True, "multi": False, },
+ {"name": "start", "required": False, "multi": False, },
+ {"name": "end", "required": False, "multi": False, },
+ ],
+ },
+ ]
+ }
+ return JSONResponse(responsecode.OK, result)
+
+
def doList(self, request):
"""
Return a list of all timezones known to the server.
"""
-
+
changedsince = request.args.get("changedsince", ())
if len(changedsince) > 1:
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Invalid changedsince query parameter",
+ {
+ "error": "invalid-changedsince",
+ "description": "Invalid changedsince query parameter",
+ },
))
if len(changedsince) == 1:
# Validate a date-time stamp
@@ -297,58 +304,67 @@
try:
dt = PyCalendarDateTime.parseText(changedsince)
except ValueError:
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Invalid changedsince query parameter value",
+ {
+ "error": "invalid-changedsince",
+ "description": "Invalid changedsince query parameter",
+ },
))
if not dt.utc():
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
"Invalid changedsince query parameter value",
))
-
timezones = []
for tz in self.timezones.listTimezones(changedsince):
- timezones.append(
- timezonexml.Summary(
- timezonexml.Tzid.fromString(tz.tzid),
- timezonexml.LastModified.fromString(tz.dtstamp),
- *tuple([timezonexml.Alias.fromString(alias) for alias in tz.aliases])
- )
- )
- result = timezonexml.TimezoneList(
- timezonexml.Dtstamp.fromString(self.timezones.dtstamp),
- *timezones
- )
- return XMLResponse(responsecode.OK, result)
+ timezones.append({
+ "tzid": tz.tzid,
+ "last-modified": tz.dtstamp,
+ "aliases": tz.aliases,
+ })
+ result = {
+ "dtstamp": self.timezones.dtstamp,
+ "timezones": timezones,
+ }
+ return JSONResponse(responsecode.OK, result)
+
def doGet(self, request):
"""
Return the specified timezone data.
"""
-
+
tzids = request.args.get("tzid", ())
if len(tzids) != 1:
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Invalid tzid query parameter",
+ {
+ "error": "invalid-tzid",
+ "description": "Invalid tzid query parameter",
+ },
))
format = request.args.get("format", ("text/calendar",))
if len(format) != 1 or format[0] not in ("text/calendar", "text/plain",):
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Invalid format query parameter",
+ {
+ "error": "invalid-format",
+ "description": "Invalid format query parameter",
+ },
))
format = format[0]
calendar = self.timezones.getTimezone(tzids[0])
if calendar is None:
- raise HTTPError(ErrorResponse(
+ raise HTTPError(JSONResponse(
responsecode.NOT_FOUND,
- (calendarserver_namespace, "invalid-tzid"),
- "Tzid could not be found",
+ {
+ "error": "missing-tzid",
+ "description": "Tzid could not be found",
+ }
))
tzdata = calendar.getText()
@@ -358,6 +374,7 @@
response.headers.setHeader("content-type", MimeType.fromString("%s; charset=utf-8" % (format,)))
return response
+
def doExpand(self, request):
"""
Expand a timezone within specified start/end dates.
@@ -365,9 +382,12 @@
tzids = request.args.get("tzid", ())
if len(tzids) != 1:
- raise HTTPError(StatusResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- "Invalid tzid query parameter",
+ {
+ "error": "invalid-tzid",
+ "description": "Invalid tzid query parameter",
+ },
))
try:
@@ -381,10 +401,12 @@
start.setDay(1)
start.setMonth(1)
except ValueError:
- raise HTTPError(ErrorResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-start-date"),
- "Invalid start query parameter",
+ {
+ "error": "invalid-start",
+ "description": "Invalid start query parameter",
+ }
))
try:
@@ -401,59 +423,59 @@
if end <= start:
raise ValueError()
except ValueError:
- raise HTTPError(ErrorResponse(
+ raise HTTPError(JSONResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-end-date"),
- "Invalid end query parameter",
+ {
+ "error": "invalid-end",
+ "description": "Invalid end query parameter",
+ }
))
- results = []
-
tzid = tzids[0]
tzdata = self.timezones.getTimezone(tzid)
if tzdata is None:
- raise HTTPError(ErrorResponse(
+ raise HTTPError(JSONResponse(
responsecode.NOT_FOUND,
- (calendarserver_namespace, "invalid-tzid"),
- "Tzid could not be found",
+ {
+ "error": "missing-tzid",
+ "description": "Tzid could not be found",
+ }
))
-
+
# Now do the expansion (but use a cache to avoid re-calculating TZs)
observances = self.expandcache.get((tzid, start, end), None)
if observances is None:
observances = tzexpandlocal(tzdata, start, end)
self.expandcache[(tzid, start, end)] = observances
- # Turn into XML
- results.append(timezonexml.Tzdata(
- timezonexml.Tzid.fromString(tzid),
- *[
- timezonexml.Observance(
- timezonexml.Name(name),
- timezonexml.Onset(onset),
- timezonexml.UTCOffsetFrom(utc_offset_from),
- timezonexml.UTCOffsetTo(utc_offset_to),
- ) for onset, utc_offset_from, utc_offset_to, name in observances
- ]
- ))
-
- result = timezonexml.Timezones(
- timezonexml.Dtstamp.fromString(self.timezones.dtstamp),
- *results
- )
- return XMLResponse(responsecode.OK, result)
+ # Turn into JSON
+ result = {
+ "dtstamp": self.timezones.dtstamp,
+ "observances": [
+ {
+ "name": name,
+ "onset": onset.getXMLText(),
+ "utc-offset-from": utc_offset_from,
+ "utc-offset-to": utc_offset_to,
+ } for onset, utc_offset_from, utc_offset_to, name in observances
+ ],
+ }
+ return JSONResponse(responsecode.OK, result)
+
+
class TimezoneInfo(object):
"""
Maintains information from an on-disk store of timezone files.
"""
-
+
def __init__(self, tzid, aliases, dtstamp, md5):
self.tzid = tzid
self.aliases = aliases
self.dtstamp = dtstamp
self.md5 = md5
-
+
+
@classmethod
def readXML(cls, node):
"""
@@ -466,7 +488,8 @@
aliases = tuple([alias_node.text for alias_node in node.findall("alias")])
md5 = node.findtext("md5")
return cls(tzid, aliases, dtstamp, md5)
-
+
+
def generateXML(self, parent):
"""
Generate the XML element for this timezone info.
@@ -478,11 +501,13 @@
xmlutil.addSubElement(node, "alias", alias)
xmlutil.addSubElement(node, "md5", self.md5)
+
+
class CommonTimezoneDatabase(object):
"""
Maintains the database of timezones read from an XML file.
"""
-
+
def __init__(self, basepath, xmlfile):
self.basepath = basepath
self.xmlfile = xmlfile
@@ -490,9 +515,11 @@
self.timezones = {}
self.aliases = {}
+
def onStartup(self):
return succeed(None)
+
def readDatabase(self):
"""
Read in XML data.
@@ -507,22 +534,24 @@
for alias in tz.aliases:
self.aliases[alias] = tz.tzid
+
def listTimezones(self, changedsince):
"""
List timezones (not aliases) possibly changed since a particular dtstamp.
"""
-
- for tzid, tzinfo in sorted(self.timezones.items(), key=lambda x:x[0]):
+
+ for tzid, tzinfo in sorted(self.timezones.items(), key=lambda x: x[0]):
# Ignore those that are aliases
if tzid in self.aliases:
continue
-
+
# Detect timestamp changes
if changedsince and tzinfo.dtstamp <= changedsince:
continue
-
+
yield tzinfo
+
def getTimezone(self, tzid):
"""
Generate a PyCalendar containing the requested timezone.
@@ -533,7 +562,7 @@
vtz = readVTZ(tzid)
calendar.addComponent(vtz.getComponents()[0].duplicate())
except TimezoneException:
-
+
# Check if an alias exists and create data for that
if tzid in self.aliases:
try:
@@ -552,32 +581,37 @@
return calendar
+
def _dumpTZs(self):
-
+
_ignore, root = xmlutil.newElementTreeWithRoot("timezones")
addSubElement(root, "dtstamp", self.dtstamp)
- for _ignore,v in sorted(self.timezones.items(), key=lambda x:x[0]):
+ for _ignore, v in sorted(self.timezones.items(), key=lambda x: x[0]):
v.generateXML(root)
xmlutil.writeXML(self.xmlfile, root)
-
+
+
def _buildAliases(self):
"""
Rebuild aliases mappings from current tzinfo.
"""
-
+
self.aliases = {}
for tzinfo in self.timezones.values():
for alias in tzinfo.aliases:
self.aliases[alias] = tzinfo.tzid
+
+
class PrimaryTimezoneDatabase(CommonTimezoneDatabase):
"""
Maintains the database of timezones read from an XML file.
"""
-
+
def __init__(self, basepath, xmlfile):
super(PrimaryTimezoneDatabase, self).__init__(basepath, xmlfile)
+
def createNewDatabase(self):
"""
Create a new DB xml file from scratch by scanning zoneinfo.
@@ -587,6 +621,7 @@
self._scanTZs("")
self._dumpTZs()
+
def _scanTZs(self, path, checkIfChanged=False):
# Read in all timezone files first
for item in os.listdir(os.path.join(self.basepath, path)):
@@ -601,7 +636,7 @@
except IOError:
log.error("Unable to read timezone file: %s" % (fullPath,))
continue
-
+
if checkIfChanged:
oldtz = self.timezones.get(tzid)
if oldtz != None and oldtz.md5 == md5:
@@ -609,14 +644,14 @@
self.changeCount += 1
self.changed.add(tzid)
self.timezones[tzid] = TimezoneInfo(tzid, (), self.dtstamp, md5)
-
+
# Try links (aliases) file
try:
aliases = open(os.path.join(self.basepath, "links.txt")).read()
except IOError, e:
log.error("Unable to read links.txt file: %s" % (str(e),))
aliases = ""
-
+
try:
for alias in aliases.splitlines():
alias_from, alias_to = alias.split()
@@ -630,8 +665,8 @@
log.error("Missing alias from '%s' to '%s'" % (alias_from, alias_to,))
except ValueError:
log.error("Unable to parse links.txt file: %s" % (str(e),))
-
-
+
+
def updateDatabase(self):
"""
Update existing DB info by comparing md5's.
@@ -643,11 +678,13 @@
if self.changeCount:
self._dumpTZs()
+
+
class SecondaryTimezoneDatabase(CommonTimezoneDatabase):
"""
Caches a database of timezones from another timezone service.
"""
-
+
def __init__(self, basepath, xmlfile, uri):
super(SecondaryTimezoneDatabase, self).__init__(basepath, xmlfile)
self.uri = uri
@@ -658,22 +695,24 @@
if not os.path.exists(self.basepath):
os.makedirs(self.basepath)
-
+
# Paths need to be writeable
if not os.access(basepath, os.W_OK):
raise ValueError("Secondary Timezone Service needs writeable zoneinfo path at: %s" % (basepath,))
if os.path.exists(xmlfile) and not os.access(xmlfile, os.W_OK):
raise ValueError("Secondary Timezone Service needs writeable xmlfile path at: %s" % (xmlfile,))
+
def onStartup(self):
return self.syncWithServer()
+
@inlineCallbacks
def syncWithServer(self):
"""
Sync local data with that from the server we are replicating.
"""
-
+
log.debug("Sync'ing with secondary server")
result = (yield self._getTimezoneListFromServer())
if result is None:
@@ -681,18 +720,18 @@
log.debug("No changes on secondary server")
returnValue(None)
newdtstamp, newtimezones = result
-
+
# Compare timezone infos
-
+
# New ones on the server
newtzids = set(newtimezones.keys()) - set(self.timezones.keys())
-
+
# Check for changes
changedtzids = set()
for tzid in set(newtimezones.keys()) & set(self.timezones.keys()):
if self.timezones[tzid].dtstamp < newtimezones[tzid].dtstamp:
changedtzids.add(tzid)
-
+
log.debug("Fetching %d new, %d changed timezones on secondary server" % (len(newtzids), len(changedtzids),))
# Now apply changes - do requests in parallel for speedier fetching
@@ -708,15 +747,16 @@
self._buildAliases()
log.debug("Sync with secondary server complete")
-
+
returnValue((len(newtzids), len(changedtzids),))
-
+
+
@inlineCallbacks
def _discoverServer(self):
"""
Make sure we know the timezone service path
"""
-
+
if self.uri is None:
if config.TimezoneService.SecondaryService.Host:
self.uri = "https://%s/.well-known/timezone" % (config.TimezoneService.SecondaryService.Host,)
@@ -724,7 +764,7 @@
self.uri = config.TimezoneService.SecondaryService.URI
elif not self.uri.startswith("https:") and not self.uri.startswith("http:"):
self.uri = "https://%s/.well-known/timezone" % (self.uri,)
-
+
testURI = "%s?action=capabilities" % (self.uri,)
log.debug("Discovering secondary server: %s" % (testURI,))
response = (yield getURL(testURI))
@@ -732,29 +772,30 @@
log.error("Unable to discover secondary server: %s" % (testURI,))
self.discovered = False
returnValue(False)
-
+
# Cache the redirect target
if hasattr(response, "location"):
- self.uri = response.location
+ self.uri = response.location
log.debug("Redirected secondary server to: %s" % (self.uri,))
# TODO: Ignoring the data from capabilities for now
self.discovered = True
returnValue(True)
-
+
+
@inlineCallbacks
def _getTimezoneListFromServer(self):
"""
Retrieve the timezone list from the specified server
"""
-
+
# Make sure we have the server
if not self.discovered:
result = (yield self._discoverServer())
if not result:
returnValue(None)
-
+
# List all from the server
url = "%s?action=list" % (self.uri,)
if self.dtstamp:
@@ -763,26 +804,30 @@
response = (yield getURL(url))
if response is None or response.code / 100 != 2:
returnValue(None)
-
+
ct = response.headers.getRawHeaders("content-type", ("bogus/type",))[0]
ct = ct.split(";", 1)
ct = ct[0]
- if ct not in ("application/xml", "text/xml",):
+ if ct not in ("application/json",):
returnValue(None)
-
- etroot, _ignore = readXMLString(response.data, timezonexml.TimezoneList.sname())
- dtstamp = etroot.findtext(timezonexml.Dtstamp.sname())
- timezones = {}
- for summary in etroot.findall(timezonexml.Summary.sname()):
- tzid = summary.findtext(timezonexml.Tzid.sname())
- lastmod = summary.findtext(timezonexml.LastModified.sname())
- aliases = tuple([alias_node.text for alias_node in summary.findall(timezonexml.Alias.sname())])
- timezones[tzid] = TimezoneInfo(tzid, aliases, lastmod, None)
+ try:
+ jroot = json.loads(response.data)
+ dtstamp = jroot["dtstamp"]
+ timezones = {}
+ for timezone in jroot["timezones"]:
+ tzid = timezone["tzid"]
+ lastmod = timezone["last-modified"]
+ aliases = timezone.get("aliases", ())
+ timezones[tzid] = TimezoneInfo(tzid, aliases, lastmod, None)
+ except ValueError, KeyError:
+ log.debug("Failed to parse JSON timezone list response: %s" % (response.data,))
+ returnValue(None)
log.debug("Got %s timezones from secondary server" % (len(timezones),))
-
+
returnValue((dtstamp, timezones,))
+
@inlineCallbacks
def _getTimezoneFromServer(self, tzinfo):
# List all from the server
@@ -791,14 +836,14 @@
response = (yield getURL(url))
if response is None or response.code / 100 != 2:
returnValue(None)
-
+
ct = response.headers.getRawHeaders("content-type", ("bogus/type",))[0]
ct = ct.split(";", 1)
ct = ct[0]
if ct not in ("text/calendar",):
log.error("Invalid content-type '%s' for tzid : %s" % (ct, tzinfo.tzid,))
returnValue(None)
-
+
ical = response.data
try:
calendar = PyCalendar.parseText(ical)
@@ -808,7 +853,7 @@
ical = calendar.getText()
tzinfo.md5 = hashlib.md5(ical).hexdigest()
-
+
try:
tzpath = os.path.join(self.basepath, tzinfo.tzid) + ".ics"
if not os.path.exists(os.path.dirname(tzpath)):
@@ -821,6 +866,7 @@
else:
self.timezones[tzinfo.tzid] = tzinfo
+
def _removeTimezone(self, tzid):
tzpath = os.path.join(self.basepath, tzid) + ".ics"
try:
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121207/ea270328/attachment-0001.html>
More information about the calendarserver-changes
mailing list