[CalendarServer-changes] [7206] CalendarServer/branches/users/cdaboo/pycalendar
source_changes at macosforge.org
source_changes at macosforge.org
Thu Mar 17 10:54:34 PDT 2011
Revision: 7206
http://trac.macosforge.org/projects/calendarserver/changeset/7206
Author: cdaboo at apple.com
Date: 2011-03-17 10:54:34 -0700 (Thu, 17 Mar 2011)
Log Message:
-----------
Merge from trunk.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/opendirectory.py
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/setup_directory.py
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/test/test_opendirectory.py
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/caldav.py
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/util.py
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/purge.py
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/test_resources.py
CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-apple.plist
CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-test.plist
CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/calendarmigrator.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/test/test_migrator.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_change.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_create.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.sh
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_add_attendee.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_change_summary.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/httpauth.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/loadtest/population.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/nightly.sh
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/sample.sh
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/test_event_change_date.py
CalendarServer/branches/users/cdaboo/pycalendar/doc/calendarserver_purge_events.8
CalendarServer/branches/users/cdaboo/pycalendar/lib-patches/vobject/vobject.icalendar.patch
CalendarServer/branches/users/cdaboo/pycalendar/support/Makefile.Apple
CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh
CalendarServer/branches/users/cdaboo/pycalendar/support/patchapply
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/adbapi2.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/model.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/parseschema.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/syntax.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_parseschema.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_sqlsyntax.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/ienterprise.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/test/test_adbapi2.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/element/base.py
CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/http.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/caldavxml.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/carddavxml.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/customxml.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dateops.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/opendirectorybacker.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/principal.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/accounts.xml
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_principal.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_proxyprincipalmembers.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/wiki.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/xmlfile.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dropbox.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/freebusyurl.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/ical.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/mail.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove_contact.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/get.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcalendar.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcol.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/post.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_addressbook_common.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_common.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_addressbook_query.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_calendar_query.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_freebusy.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_multiget_common.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_sync_collection.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/notify.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarquery.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarqueryfilter.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/sqlgenerator.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/resource.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/schedule.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/caldav.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/imip.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/implicit.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/ischedule.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/scheduler.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/sharing.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/storebridge.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/timezoneservice.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/upgrade.py
CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/vcard.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/datastore/dbapiclient.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/base.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/none.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/sql.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/xattr.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/sql.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_sql.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/util.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_sql.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/file.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_legacy.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_tables.py
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/purge/resources.xml
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence_autoaccept.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_autoaccept.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence_autoaccept.py
CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/some-more-data.sh
CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/util.py
Removed Paths:
-------------
CalendarServer/branches/users/cdaboo/pycalendar/contrib/SBS/
Property Changed:
----------------
CalendarServer/branches/users/cdaboo/pycalendar/
CalendarServer/branches/users/cdaboo/pycalendar/bin/calendarserver_purge_principals
CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh
CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_index_file.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/index_file.py
CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_index_file.py
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:7085-7204
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar/bin/calendarserver_purge_principals
___________________________________________________________________
Added: svn:executable
+ *
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/opendirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/opendirectory.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/opendirectory.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -184,7 +184,7 @@
recordType,
attr,
adjustMatchType(matchType, casei),
- value,
+ value.decode("utf-8"),
attributes,
count,
None)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/setup_directory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/setup_directory.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/setup_directory.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -90,6 +90,30 @@
dsattributes.kDS1AttrPrimaryGroupID : ["20"],
},
),
+ (
+ "odtestat at sign",
+ {
+ dsattributes.kDS1AttrFirstName : ["AtSign"],
+ dsattributes.kDS1AttrLastName : ["Test"],
+ dsattributes.kDS1AttrDistinguishedName : ["At Sign Test"],
+ dsattributes.kDSNAttrEMailAddress : ["attsign at example.com"],
+ dsattributes.kDS1AttrGeneratedUID : ["71646A3A-1CEF-4744-AB1D-0AC855E25DC8"],
+ dsattributes.kDS1AttrUniqueID : ["33305"],
+ dsattributes.kDS1AttrPrimaryGroupID : ["20"],
+ },
+ ),
+ (
+ "odtestsatou",
+ {
+ dsattributes.kDS1AttrFirstName : ["\xe4\xbd\x90\xe8\x97\xa4\xe4\xbd\x90\xe8\x97\xa4\xe4\xbd\x90\xe8\x97\xa4".decode("utf-8")],
+ dsattributes.kDS1AttrLastName : ["Test \xe4\xbd\x90\xe8\x97\xa4".decode("utf-8")],
+ dsattributes.kDS1AttrDistinguishedName : ["\xe4\xbd\x90\xe8\x97\xa4\xe4\xbd\x90\xe8\x97\xa4\xe4\xbd\x90\xe8\x97\xa4 Test \xe4\xbd\x90\xe8\x97\xa4".decode("utf-8")],
+ dsattributes.kDSNAttrEMailAddress : ["satou at example.com"],
+ dsattributes.kDS1AttrGeneratedUID : ["C662F833-75AD-4589-9879-5FF102943CEF"],
+ dsattributes.kDS1AttrUniqueID : ["33306"],
+ dsattributes.kDS1AttrPrimaryGroupID : ["20"],
+ },
+ ),
]
masterGroups = [
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/test/test_opendirectory.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/test/test_opendirectory.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/platform/darwin/od/test/test_opendirectory.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -283,9 +283,11 @@
)
recordNames = [x[0] for x in results]
for recordName, info in setup_directory.masterUsers:
- self.assertTrue(recordName in recordNames)
+ if info[dsattributes.kDS1AttrLastName] == "Test":
+ self.assertTrue(recordName in recordNames)
for recordName, info in setup_directory.localUsers:
- self.assertTrue(recordName in recordNames)
+ if info[dsattributes.kDS1AttrLastName] == "Test":
+ self.assertTrue(recordName in recordNames)
def test_queryRecordsWithAttribute_list_lastname_exact_insensitive_match_multitype(self):
@@ -308,9 +310,11 @@
)
recordNames = [x[0] for x in results]
for recordName, info in setup_directory.masterUsers:
- self.assertTrue(recordName in recordNames)
+ if info[dsattributes.kDS1AttrLastName] == "Test":
+ self.assertTrue(recordName in recordNames)
for recordName, info in setup_directory.localUsers:
- self.assertTrue(recordName in recordNames)
+ if info[dsattributes.kDS1AttrLastName] == "Test":
+ self.assertTrue(recordName in recordNames)
def test_queryRecordsWithAttribute_list_lastname_begins_insensitive_match(self):
@@ -827,7 +831,7 @@
else:
self.assertTrue(type(value) is str)
- def test_nonascii_record(self):
+ def test_nonascii_record_by_guid(self):
directory = opendirectory.odInit("/Search")
@@ -846,3 +850,39 @@
result[dsattributes.kDS1AttrDistinguishedName],
"Unicode Test \xc3\x90"
)
+
+ def test_nonascii_record_by_name(self):
+
+ directory = opendirectory.odInit("/Search")
+
+ results = opendirectory.queryRecordsWithAttribute_list(
+ directory,
+ dsattributes.kDS1AttrDistinguishedName,
+ "Unicode Test \xc3\x90",
+ dsattributes.eDSExact,
+ False,
+ dsattributes.kDSStdRecordTypeUsers,
+ USER_ATTRIBUTES,
+ count=0
+ )
+ result = results[0][1]
+ self.assertEquals(
+ result[dsattributes.kDS1AttrGeneratedUID],
+ "CA795296-D77A-4E09-A72F-869920A3D284"
+ )
+
+ results = opendirectory.queryRecordsWithAttribute_list(
+ directory,
+ dsattributes.kDS1AttrFirstName,
+ "\xe4\xbd\x90\xe8\x97\xa4",
+ dsattributes.eDSStartsWith,
+ False,
+ dsattributes.kDSStdRecordTypeUsers,
+ USER_ATTRIBUTES,
+ count=0
+ )
+ result = results[0][1]
+ self.assertEquals(
+ result[dsattributes.kDS1AttrGeneratedUID],
+ "C662F833-75AD-4589-9879-5FF102943CEF"
+ )
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/caldav.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/caldav.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -69,10 +69,12 @@
from twistedcaldav.mail import IMIPReplyInboxResource
from twistedcaldav import memcachepool
from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
-from twistedcaldav.upgrade import upgradeData
+from twistedcaldav.upgrade import UpgradeFileSystemFormatService
from calendarserver.tap.util import pgServiceFromConfig
+from twext.enterprise.ienterprise import POSTGRES_DIALECT
+from twext.enterprise.ienterprise import ORACLE_DIALECT
from twext.enterprise.adbapi2 import ConnectionPool
from twext.enterprise.adbapi2 import ConnectionPoolConnection
@@ -90,6 +92,7 @@
from calendarserver.tap.util import storeFromConfig
from calendarserver.tap.util import transactionFactoryFromFD
from calendarserver.tap.util import pgConnectorFromConfig
+from calendarserver.tap.util import oracleConnectorFromConfig
from calendarserver.tools.util import checkDirectory
try:
@@ -641,6 +644,8 @@
elif not config.UseDatabase:
txnFactory = None
elif not config.SharedConnectionPool:
+ dialect = POSTGRES_DIALECT
+ paramstyle = 'pyformat'
if config.DBType == '':
# get a PostgresService to tell us what the local connection
# info is, but *don't* start it (that would start one postgres
@@ -649,9 +654,14 @@
config, None).produceConnection
elif config.DBType == 'postgres':
connectionFactory = pgConnectorFromConfig(config)
+ elif config.DBType == 'oracle':
+ dialect = ORACLE_DIALECT
+ paramstyle = 'numeric'
+ connectionFactory = oracleConnectorFromConfig(config)
else:
raise UsageError("unknown DB type: %r" % (config.DBType,))
- pool = ConnectionPool(connectionFactory)
+ pool = ConnectionPool(connectionFactory, dialect=dialect,
+ paramstyle=paramstyle)
txnFactory = pool.connection
else:
raise UsageError(
@@ -850,23 +860,11 @@
return config.BindAddresses
- def scheduleOnDiskUpgrade(self):
- """
- Schedule any on disk upgrades we might need. Note that this will only
- do the filesystem-format upgrades; migration to the database needs to
- be done when the connection and possibly server is already up and
- running.
- """
- addSystemEventTrigger("before", "startup", upgradeData, config)
-
-
def makeService_Single(self, options):
"""
Create a service to be used in a single-process, stand-alone
configuration.
"""
- self.scheduleOnDiskUpgrade()
-
def slaveSvcCreator(pool, store):
return self.requestProcessingService(options, store)
return self.storageService(slaveSvcCreator)
@@ -885,6 +883,7 @@
return self.storageService(toolServiceCreator)
+
def storageService(self, createMainService, uid=None, gid=None):
"""
If necessary, create a service to be started used for storage; for
@@ -939,6 +938,12 @@
return self.subServiceFactoryFactory(createMainService,
uid=overrideUID, gid=overrideGID)(
pgConnectorFromConfig(config))
+ elif config.DBType == 'oracle':
+ # Connect to an Oracle database that is already running.
+ return self.subServiceFactoryFactory(createMainService,
+ uid=overrideUID, gid=overrideGID,
+ dialect=ORACLE_DIALECT, paramstyle='numeric')(
+ oracleConnectorFromConfig(config))
else:
raise UsageError("Unknown database type %r" (config.DBType,))
else:
@@ -946,19 +951,23 @@
return createMainService(None, store)
- def subServiceFactoryFactory(self, createMainService,
- uid=None, gid=None):
+ def subServiceFactoryFactory(self, createMainService, uid=None, gid=None,
+ dialect=POSTGRES_DIALECT,
+ paramstyle='pyformat'):
def subServiceFactory(connectionFactory):
ms = MultiService()
- cp = ConnectionPool(connectionFactory)
+ cp = ConnectionPool(connectionFactory, dialect=dialect,
+ paramstyle=paramstyle)
cp.setServiceParent(ms)
store = storeFromConfig(config, cp.connection)
mainService = createMainService(cp, store)
- maybeUpgradeSvc = UpgradeToDatabaseService.wrapService(
- CachingFilePath(config.DocumentRoot), mainService,
- store, uid=uid, gid=gid
+ upgradeSvc = UpgradeFileSystemFormatService(config,
+ UpgradeToDatabaseService.wrapService(
+ CachingFilePath(config.DocumentRoot), mainService,
+ store, uid=uid, gid=gid
+ )
)
- maybeUpgradeSvc.setServiceParent(ms)
+ upgradeSvc.setServiceParent(ms)
return ms
return subServiceFactory
@@ -970,8 +979,6 @@
"""
s = ErrorLoggingMultiService()
- self.scheduleOnDiskUpgrade()
-
# Make sure no old socket files are lying around.
self.deleteStaleSocketFiles()
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/util.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tap/util.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -69,7 +69,7 @@
NegotiateCredentialFactory = None
from twext.enterprise.adbapi2 import ConnectionPoolClient
-from txdav.base.datastore.dbapiclient import DBAPIConnector
+from txdav.base.datastore.dbapiclient import DBAPIConnector, OracleConnector
from txdav.base.datastore.dbapiclient import postgresPreflight
from txdav.base.datastore.subpostgres import PostgresService
@@ -132,6 +132,14 @@
+def oracleConnectorFromConfig(config):
+ """
+ Create a postgres DB-API connector from the given configuration.
+ """
+ return OracleConnector(config.DSN).connect
+
+
+
class ConnectionWithPeer(Connection):
connected = True
@@ -139,10 +147,12 @@
def getPeer(self):
return "<peer: %r %r>" % (self.socket.fileno(), id(self))
+
def getHost(self):
return "<host: %r %r>" % (self.socket.fileno(), id(self))
+
def transactionFactoryFromFD(dbampfd):
"""
Create a transaction factory from an inherited file descriptor.
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/purge.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/purge.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/purge.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -18,6 +18,7 @@
import os
import sys
+from errno import ENOENT, EACCES
from getopt import getopt, GetoptError
@@ -45,6 +46,9 @@
log = Logger()
+DEFAULT_BATCH_SIZE = 100
+DEFAULT_RETAIN_DAYS = 365
+
def usage_purge_events(e=None):
name = os.path.basename(sys.argv[0])
@@ -53,11 +57,11 @@
print " Remove old events from the calendar server"
print ""
print "options:"
- print " -d --days <number>: specify how many days in the past to retain (default=365)"
- print " -b --batch <number>: number of events to remove in each transaction (default=100)"
+ print " -h --help: print this help and exit"
print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -h --help: print this help and exit"
- print " -n --dry-run: only calculate how many events to purge"
+ print " -d --days <number>: specify how many days in the past to retain (default=%d)" % (DEFAULT_RETAIN_DAYS,)
+ #print " -b --batch <number>: number of events to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ print " -n --dry-run: calculate how many events to purge, but do not purge data"
print " -v --verbose: print progress information"
print ""
@@ -75,10 +79,10 @@
print " Remove orphaned attachments from the calendar server"
print ""
print "options:"
- print " -b --batch <number>: number of attachments to remove in each transaction (default=100)"
+ print " -h --help: print this help and exit"
print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -h --help: print this help and exit"
- print " -n --dry-run: only calculate how many attachments to purge"
+ #print " -b --batch <number>: number of attachments to remove in each transaction (default=%d)" % (DEFAULT_BATCH_SIZE,)
+ print " -n --dry-run: calculate how many attachments to purge, but do not purge data"
print " -v --verbose: print progress information"
print ""
@@ -96,9 +100,9 @@
print " Remove a principal's events and contacts from the calendar server"
print ""
print "options:"
+ print " -h --help: print this help and exit"
print " -f --config <path>: Specify caldavd.plist configuration path"
- print " -h --help: print this help and exit"
- print " -n --dry-run: only calculate how many events and contacts to purge"
+ print " -n --dry-run: calculate how many events and contacts to purge, but do not purge data"
print " -v --verbose: print progress information"
print ""
@@ -109,83 +113,98 @@
sys.exit(0)
-class PurgeOldEventsService(Service):
+class WorkerService(Service):
- cutoff = None
- batchSize = None
- dryrun = False
- verbose = False
-
def __init__(self, store):
self._store = store
+ def rootResource(self):
+ try:
+ rootResource = getRootResource(config, self._store)
+ except OSError, e:
+ if e.errno == ENOENT:
+ # Trying to re-write resources.xml but its parent directory does
+ # not exist. The server's never been started, so we're missing
+ # state required to do any work. (Plus, what would be the point
+ # of purging stuff from a server that's completely empty?)
+ raise ConfigurationError(
+ "It appears that the server has never been started.\n"
+ "Please start it at least once before purging anything.")
+ elif e.errno == EACCES:
+ # Trying to re-write resources.xml but it is not writable by the
+ # current user. This most likely means we're in a system
+ # configuration and the user doesn't have sufficient privileges
+ # to do the other things the tool might need to do either.
+ raise ConfigurationError("You must run this tool as root.")
+ else:
+ raise
+ return rootResource
+
+
@inlineCallbacks
def startService(self):
try:
- rootResource = getRootResource(config, self._store)
- directory = rootResource.getDirectory()
- (yield purgeOldEvents(self._store, directory, rootResource,
- self.cutoff, self.batchSize, verbose=self.verbose,
- dryrun=self.dryrun))
+ yield self.doWork()
+ except ConfigurationError, ce:
+ sys.stderr.write("Error: %s\n" % (str(ce),))
except Exception, e:
sys.stderr.write("Error: %s\n" % (e,))
raise
-
finally:
reactor.stop()
-class PurgeOrphanedAttachmentsService(Service):
+class PurgeOldEventsService(WorkerService):
+
+ cutoff = None
batchSize = None
dryrun = False
verbose = False
- def __init__(self, store):
- self._store = store
+ def doWork(self):
+ rootResource = self.rootResource()
+ directory = rootResource.getDirectory()
+ return purgeOldEvents(self._store, directory, rootResource,
+ self.cutoff, self.batchSize, verbose=self.verbose,
+ dryrun=self.dryrun)
- @inlineCallbacks
- def startService(self):
- try:
- (yield purgeOrphanedAttachments(self._store, self.batchSize,
- verbose=self.verbose, dryrun=self.dryrun))
- except Exception, e:
- sys.stderr.write("Error: %s\n" % (e,))
- raise
- finally:
- reactor.stop()
+class PurgeOrphanedAttachmentsService(WorkerService):
-class PurgePrincipalService(Service):
+ batchSize = None
+ dryrun = False
+ verbose = False
+ def doWork(self):
+ return purgeOrphanedAttachments(
+ self._store, self.batchSize,
+ verbose=self.verbose, dryrun=self.dryrun)
+
+
+
+class PurgePrincipalService(WorkerService):
+
guids = None
dryrun = False
verbose = False
- def __init__(self, store):
- self._store = store
-
@inlineCallbacks
- def startService(self):
- try:
- rootResource = getRootResource(config, self._store)
- directory = rootResource.getDirectory()
- total = (yield purgeGUIDs(directory, rootResource, self.guids,
- verbose=self.verbose, dryrun=self.dryrun))
- if self.verbose:
- amount = "%d event%s" % (total, "s" if total > 1 else "")
- if self.dryrun:
- print "Would have modified or deleted %s" % (amount,)
- else:
- print "Modified or deleted %s" % (amount,)
- except Exception, e:
- sys.stderr.write("Error: %s\n" % (e,))
- raise
- finally:
- reactor.stop()
+ def doWork(self):
+ rootResource = self.rootResource()
+ directory = rootResource.getDirectory()
+ total = (yield purgeGUIDs(directory, rootResource, self.guids,
+ verbose=self.verbose, dryrun=self.dryrun))
+ if self.verbose:
+ amount = "%d event%s" % (total, "s" if total > 1 else "")
+ if self.dryrun:
+ print "Would have modified or deleted %s" % (amount,)
+ else:
+ print "Modified or deleted %s" % (amount,)
+
def shared_main(configFileName, serviceClass):
try:
@@ -227,8 +246,8 @@
# Get configuration
#
configFileName = None
- days = 365
- batchSize = 100
+ days = DEFAULT_RETAIN_DAYS
+ batchSize = DEFAULT_BATCH_SIZE
dryrun = False
verbose = False
@@ -302,7 +321,7 @@
# Get configuration
#
configFileName = None
- batchSize = 100
+ batchSize = DEFAULT_BATCH_SIZE
dryrun = False
verbose = False
@@ -395,20 +414,6 @@
@inlineCallbacks
-def callThenStop(method, *args, **kwds):
- try:
- count = (yield method(*args, **kwds))
- if kwds.get("dryrun", False):
- print "Would have purged %d events" % (count,)
- else:
- print "Purged %d events" % (count,)
- except Exception, e:
- sys.stderr.write("Error: %s\n" % (e,))
- finally:
- reactor.stop()
-
-
- at inlineCallbacks
def purgeOldEvents(store, directory, root, date, batchSize, verbose=False,
dryrun=False):
Copied: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/purge/resources.xml (from rev 7204, CalendarServer/trunk/calendarserver/tools/test/purge/resources.xml)
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/purge/resources.xml (rev 0)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/purge/resources.xml 2011-03-17 17:54:34 UTC (rev 7206)
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<accounts realm="/Search">
+</accounts>
Modified: CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/test_resources.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/test_resources.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/calendarserver/tools/test/test_resources.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -19,13 +19,17 @@
from twistedcaldav.directory import augment
from twistedcaldav.directory.directory import DirectoryService
from twistedcaldav.test.util import TestCase
-import dsattributes
-strGUID = dsattributes.kDS1AttrGeneratedUID
-strName = dsattributes.kDS1AttrDistinguishedName
+try:
+ import dsattributes
+ strGUID = dsattributes.kDS1AttrGeneratedUID
+ strName = dsattributes.kDS1AttrDistinguishedName
+except ImportError:
+ dsattributes = None
+
class StubDirectoryRecord(object):
def __init__(self, recordType, guid=None, shortNames=None, fullName=None):
@@ -83,6 +87,9 @@
class MigrateResourcesTestCase(TestCase):
+ if dsattributes is None:
+ skip = "dsattributes module not available"
+
@inlineCallbacks
def test_migrateResources(self):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-apple.plist 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-apple.plist 2011-03-17 17:54:34 UTC (rev 7206)
@@ -540,13 +540,13 @@
<key>params</key>
<dict>
<key>queryUserRecords</key>
- <true/>
+ <false/>
<key>queryPeopleRecords</key>
- <true/>
+ <false/>
</dict>
</dict>
<key>EnableSearchAddressBook</key>
- <true/>
+ <false/>
<key>OpenDirectoryModule</key>
<string>calendarserver.platform.darwin.od.opendirectory</string>
Modified: CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-test.plist
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-test.plist 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/conf/caldavd-test.plist 2011-03-17 17:54:34 UTC (rev 7206)
@@ -806,6 +806,10 @@
<key>EnableTimezoneService</key>
<true/>
+ <!-- Batch Upload via POST -->
+ <key>EnableBatchUpload</key>
+ <true/>
+
<!-- Shared Calendars & Address Books -->
<key>Sharing</key>
<dict>
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/calendarmigrator.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/calendarmigrator.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/calendarmigrator.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -405,18 +405,24 @@
del combined["DirectoryService"]["params"][key]
# Merge ports
- if not caldav["SSLPort"]:
+ if not caldav.get("HTTPPort", 0):
+ caldav["HTTPPort"] = 8008
+ if not carddav.get("HTTPPort", 0):
+ carddav["HTTPPort"] = 8800
+ if not caldav.get("SSLPort", 0):
caldav["SSLPort"] = 8443
- if not carddav["SSLPort"]:
+ if not carddav.get("SSLPort", 0):
carddav["SSLPort"] = 8843
+
for portType in ["HTTPPort", "SSLPort"]:
- bindPorts = list(set(caldav["Bind%ss" % (portType,)]).union(set(carddav["Bind%ss" % (portType,)])))
- if caldav[portType] and caldav[portType] not in bindPorts:
- bindPorts.append(caldav[portType])
- if carddav[portType] and carddav[portType] not in bindPorts:
- bindPorts.append(carddav[portType])
+ bindPorts = list(set(caldav.get("Bind%ss" % (portType,), [])).union(set(carddav.get("Bind%ss" % (portType,), []))))
+ for prev in (carddav, caldav):
+ port = prev.get(portType, 0)
+ if port and port not in bindPorts:
+ bindPorts.append(port)
bindPorts.sort()
combined["Bind%ss" % (portType,)] = bindPorts
+
combined["HTTPPort"] = caldav["HTTPPort"]
combined["SSLPort"] = caldav["SSLPort"]
@@ -426,10 +432,10 @@
sslPrivateKey = ""
enableSSL = False
for prev in (carddav, caldav):
- if (prev["SSLPort"] and prev["SSLCertificate"]):
- sslAuthorityChain = prev["SSLAuthorityChain"]
- sslCertificate = prev["SSLCertificate"]
- sslPrivateKey = prev["SSLPrivateKey"]
+ if (prev["SSLPort"] and prev.get("SSLCertificate", "")):
+ sslAuthorityChain = prev.get("SSLAuthorityChain", "")
+ sslCertificate = prev.get("SSLCertificate", "")
+ sslPrivateKey = prev.get("SSLPrivateKey", "")
enableSSL = True
combined["SSLAuthorityChain"] = sslAuthorityChain
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/test/test_migrator.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/test/test_migrator.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/migration/test/test_migrator.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -205,4 +205,54 @@
self.assertEquals(newCombined, expected)
+ # Never had SSL enabled, so missing SSLPort
+ oldCalDAV = {
+ "BindHTTPPorts": [],
+ "HTTPPort": 8008,
+ "RedirectHTTPToHTTPS": False,
+ "SSLAuthorityChain": "",
+ "SSLCertificate": "",
+ "SSLPrivateKey": "",
+ }
+ oldCardDAV = {
+ "BindHTTPPorts": [],
+ "HTTPPort": 8800,
+ "RedirectHTTPToHTTPS": False,
+ "SSLAuthorityChain": "",
+ "SSLCertificate": "",
+ "SSLPrivateKey": "",
+ }
+ expected = {
+ "BindHTTPPorts": [8008, 8800],
+ "BindSSLPorts": [8443, 8843],
+ "EnableSSL" : False,
+ "HTTPPort": 8008,
+ "RedirectHTTPToHTTPS": False,
+ "SSLAuthorityChain": "",
+ "SSLCertificate": "",
+ "SSLPort": 8443,
+ "SSLPrivateKey": "",
+ }
+ newCombined = { }
+ mergePlist(oldCalDAV, oldCardDAV, newCombined)
+ self.assertEquals(newCombined, expected)
+
+
+ # All settings missing!
+ oldCalDAV = { }
+ oldCardDAV = { }
+ expected = {
+ "BindHTTPPorts": [8008, 8800],
+ "BindSSLPorts": [8443, 8843],
+ "EnableSSL" : False,
+ "HTTPPort": 8008,
+ "RedirectHTTPToHTTPS": False,
+ "SSLAuthorityChain": "",
+ "SSLCertificate": "",
+ "SSLPort": 8443,
+ "SSLPrivateKey": "",
+ }
+ newCombined = { }
+ mergePlist(oldCalDAV, oldCardDAV, newCombined)
+ self.assertEquals(newCombined, expected)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_change.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_change.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_change.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -32,8 +32,9 @@
from httpclient import StringProducer
from benchlib import initialize, sample
-from event import makeEvent
+from _event_create import makeEvent
+
@inlineCallbacks
def measure(host, port, dtrace, attendeeCount, samples, fieldName,
replacer, eventPerSample=False):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_create.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_create.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/_event_create.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -18,7 +18,9 @@
Various helpers for event-creation benchmarks.
"""
+from uuid import uuid4
from urllib2 import HTTPDigestAuthHandler
+from datetime import datetime, timedelta
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.web.http_headers import Headers
@@ -30,10 +32,107 @@
from benchlib import initialize, sample
from httpclient import StringProducer
+
+# XXX Represent these as vobjects? Would make it easier to add more vevents.
+event = """\
+BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//Apple Inc.//iCal 4.0.3//EN
+BEGIN:VTIMEZONE
+TZID:America/Los_Angeles
+BEGIN:STANDARD
+DTSTART:20071104T020000
+RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
+TZNAME:PST
+TZOFFSETFROM:-0700
+TZOFFSETTO:-0800
+END:STANDARD
+BEGIN:DAYLIGHT
+DTSTART:20070311T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
+TZNAME:PDT
+TZOFFSETFROM:-0800
+TZOFFSETTO:-0700
+END:DAYLIGHT
+END:VTIMEZONE
+%(VEVENTS)s\
+END:VCALENDAR
+"""
+
+SUMMARY = "Some random thing"
+
+vevent = """\
+BEGIN:VEVENT
+UID:%(UID)s
+DTSTART;TZID=America/Los_Angeles:%(START)s
+DTEND;TZID=America/Los_Angeles:%(END)s
+%(RRULE)s\
+CREATED:20100729T193912Z
+DTSTAMP:20100729T195557Z
+%(ORGANIZER)s\
+%(ATTENDEES)s\
+SEQUENCE:0
+SUMMARY:%(SUMMARY)s
+TRANSP:OPAQUE
+END:VEVENT
+"""
+
+attendee = """\
+ATTENDEE;CN=User %(SEQUENCE)02d;CUTYPE=INDIVIDUAL;EMAIL=user%(SEQUENCE)02d at example.com;PARTSTAT=NE
+ EDS-ACTION;ROLE=REQ-PARTICIPANT;RSVP=TRUE:urn:uuid:user%(SEQUENCE)02d
+"""
+
+organizer = """\
+ORGANIZER;CN=User %(SEQUENCE)02d;EMAIL=user%(SEQUENCE)02d at example.com:urn:uuid:user%(SEQUENCE)02d
+ATTENDEE;CN=User %(SEQUENCE)02d;EMAIL=user%(SEQUENCE)02d at example.com;PARTSTAT=ACCEPTE
+ D:urn:uuid:user%(SEQUENCE)02d
+"""
+
def formatDate(d):
return ''.join(filter(str.isalnum, d.isoformat()))
+def makeOrganizer(sequence):
+ return organizer % {'SEQUENCE': sequence}
+
+
+def makeAttendees(count):
+ return [
+ attendee % {'SEQUENCE': n} for n in range(2, count + 2)]
+
+
+def makeVCalendar(uid, start, end, recurrence, organizerSequence, attendees):
+ if recurrence is None:
+ rrule = ""
+ else:
+ rrule = recurrence + "\n"
+ return event % {
+ 'VEVENTS': vevent % {
+ 'UID': uid,
+ 'START': formatDate(start),
+ 'END': formatDate(end),
+ 'SUMMARY': SUMMARY,
+ 'ORGANIZER': makeOrganizer(organizerSequence),
+ 'ATTENDEES': ''.join(attendees),
+ 'RRULE': rrule,
+ },
+ }
+
+
+def makeEvent(i, organizerSequence, attendeeCount):
+ base = datetime(2010, 7, 30, 11, 15, 00)
+ interval = timedelta(0, 5)
+ duration = timedelta(0, 3)
+ return makeVCalendar(
+ uuid4(),
+ base + i * interval,
+ base + i * interval + duration,
+ None,
+ organizerSequence,
+ makeAttendees(attendeeCount))
+
+
@inlineCallbacks
def measure(calendar, organizerSequence, events, host, port, dtrace, samples):
"""
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -19,10 +19,9 @@
from twisted.internet.defer import (
FirstError, DeferredList, inlineCallbacks, returnValue)
-# from twisted.internet.task import deferLater
from twisted.web.http_headers import Headers
-# from twisted.internet import reactor
from twisted.python.log import msg
+from twisted.web.http import NO_CONTENT, NOT_FOUND
from stats import Duration
from httpclient import StringProducer, readBody
@@ -45,7 +44,15 @@
def deleteResource(self, path):
- return self.agent.request('DELETE', self._makeURL(path))
+ url = self._makeURL(path)
+ d = self.agent.request('DELETE', url)
+ def deleted(response):
+ if response.code not in (NO_CONTENT, NOT_FOUND):
+ raise Exception(
+ "Unexpected response to DELETE %s: %d" % (
+ url, response.code))
+ d.addCallback(deleted)
+ return d
def makeCalendar(self, path):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.sh
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.sh 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchlib.sh 2011-03-17 17:54:34 UTC (rev 7206)
@@ -29,12 +29,12 @@
CONF=$SOURCE/conf/caldavd-dev.plist
# Names of benchmarks we can run.
-BENCHMARKS="find_calendars find_events event_move event_delete_attendee event_add_attendee event_change_date event_change_summary event_delete vfreebusy event bounded_recurrence"
+BENCHMARKS="find_calendars find_events event_move event_delete_attendee event_add_attendee event_change_date event_change_summary event_delete vfreebusy event bounded_recurrence unbounded_recurrence event_autoaccept bounded_recurrence_autoaccept unbounded_recurrence_autoaccept"
# Custom scaling parameters for benchmarks that merit it. Be careful
# not to exceed the 99 user limit for benchmarks where the scaling
# parameter represents a number of users!
-SCALE_PARAMETERS="--parameters find_events:1,10,100,1000,10000 --parameters bounded_recurrence:1"
+SCALE_PARAMETERS="--parameters find_events:1,10,100,1000,10000"
# Names of metrics we can collect.
STATISTICS=(HTTP SQL read write pagein pageout)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -22,45 +22,10 @@
from itertools import count
from datetime import datetime, timedelta
-from _event_create import formatDate, measure as _measure
+from _event_create import (
+ makeAttendees, makeVCalendar, formatDate, measure as _measure)
-DAILY_EVENT = """\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VTIMEZONE
-TZID:America/New_York
-BEGIN:DAYLIGHT
-TZOFFSETFROM:-0500
-RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
-DTSTART:20070311T020000
-TZNAME:EDT
-TZOFFSETTO:-0400
-END:DAYLIGHT
-BEGIN:STANDARD
-TZOFFSETFROM:-0400
-RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
-DTSTART:20071104T020000
-TZNAME:EST
-TZOFFSETTO:-0500
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-CREATED:20110223T184640Z
-UID:%(UID)s
-DTEND;TZID=America/New_York:%(DTEND)s
-RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=%(UNTIL)s
-TRANSP:OPAQUE
-SUMMARY:A meeting which occurs daily for several days.
-DTSTART;TZID=America/New_York:%(DTSTART)s
-DTSTAMP:20110223T184658Z
-SEQUENCE:5
-END:VEVENT
-END:VCALENDAR
-"""
-
-def makeEvent(i):
+def makeEvent(i, organizerSequence, attendeeCount):
"""
Create a new half-hour long event that starts soon and recurs
daily for the next five days.
@@ -69,12 +34,10 @@
start = now.replace(minute=15, second=0, microsecond=0) + timedelta(hours=i)
end = start + timedelta(minutes=30)
until = start + timedelta(days=5)
- return DAILY_EVENT % {
- 'DTSTART': formatDate(start),
- 'DTEND': formatDate(end),
- 'UNTIL': formatDate(until),
- 'UID': uuid4(),
- }
+ rrule = "RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=" + formatDate(until)
+ return makeVCalendar(
+ uuid4(), start, end, rrule, organizerSequence,
+ makeAttendees(attendeeCount))
def measure(host, port, dtrace, attendeeCount, samples):
@@ -82,7 +45,7 @@
organizerSequence = 1
# An infinite stream of recurring VEVENTS to PUT to the server.
- events = ((i, makeEvent(i)) for i in count(2))
+ events = ((i, makeEvent(i, organizerSequence, attendeeCount)) for i in count(2))
return _measure(
calendar, organizerSequence, events,
Copied: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence_autoaccept.py (from rev 7204, CalendarServer/trunk/contrib/performance/benchmarks/bounded_recurrence_autoaccept.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence_autoaccept.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/bounded_recurrence_autoaccept.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -0,0 +1,55 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Benchmark a server's handling of events with a bounded recurrence.
+"""
+
+from uuid import uuid4
+from itertools import count
+from datetime import datetime, timedelta
+
+from _event_create import (
+ makeAttendees, makeVCalendar, formatDate, measure as _measure)
+
+def makeEvent(i, organizerSequence, attendeeCount):
+ """
+ Create a new half-hour long event that starts soon and recurs
+ daily for the next five days.
+ """
+ now = datetime.now()
+ start = now.replace(minute=15, second=0, microsecond=0) + timedelta(hours=i)
+ end = start + timedelta(minutes=30)
+ until = start + timedelta(days=5)
+ rrule = "RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=" + formatDate(until)
+ attendees = makeAttendees(attendeeCount)
+ attendees.append(
+ 'ATTENDEE;CN="Resource 01";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=T\n'
+ ' RUE;SCHEDULE-STATUS="1.2":urn:uuid:resource01\n')
+ return makeVCalendar(
+ uuid4(), start, end, rrule, organizerSequence, attendees)
+
+
+def measure(host, port, dtrace, attendeeCount, samples):
+ calendar = "bounded-recurrence-autoaccept"
+ organizerSequence = 1
+
+ # An infinite stream of recurring VEVENTS to PUT to the server.
+ events = ((i, makeEvent(i, organizerSequence, attendeeCount)) for i in count(2))
+
+ return _measure(
+ calendar, organizerSequence, events,
+ host, port, dtrace, samples)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -19,89 +19,9 @@
"""
from itertools import count
-from uuid import uuid4
-from datetime import datetime, timedelta
-from _event_create import formatDate, measure as _measure
+from _event_create import makeEvent, measure as _measure
-# XXX Represent these as vobjects? Would make it easier to add more vevents.
-event = """\
-BEGIN:VCALENDAR
-VERSION:2.0
-CALSCALE:GREGORIAN
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-BEGIN:VTIMEZONE
-TZID:America/Los_Angeles
-BEGIN:STANDARD
-DTSTART:20071104T020000
-RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
-TZNAME:PST
-TZOFFSETFROM:-0700
-TZOFFSETTO:-0800
-END:STANDARD
-BEGIN:DAYLIGHT
-DTSTART:20070311T020000
-RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
-TZNAME:PDT
-TZOFFSETFROM:-0800
-TZOFFSETTO:-0700
-END:DAYLIGHT
-END:VTIMEZONE
-%(VEVENTS)s\
-END:VCALENDAR
-"""
-
-attendee = """\
-ATTENDEE;CN=User %(SEQUENCE)02d;CUTYPE=INDIVIDUAL;EMAIL=user%(SEQUENCE)02d at example.com;PARTSTAT=NE
- EDS-ACTION;ROLE=REQ-PARTICIPANT;RSVP=TRUE:urn:uuid:user%(SEQUENCE)02d
-"""
-
-organizer = """\
-ORGANIZER;CN=User %(SEQUENCE)02d;EMAIL=user%(SEQUENCE)02d at example.com:urn:uuid:user%(SEQUENCE)02d
-ATTENDEE;CN=User %(SEQUENCE)02d;EMAIL=user%(SEQUENCE)02d at example.com;PARTSTAT=ACCEPTE
- D:urn:uuid:user%(SEQUENCE)02d
-"""
-
-def makeOrganizer(sequence):
- return organizer % {'SEQUENCE': sequence}
-
-def makeAttendees(count):
- return ''.join([
- attendee % {'SEQUENCE': n} for n in range(2, count + 2)])
-
-
-SUMMARY = "STUFF IS THINGS"
-
-def makeEvent(i, organizerSequence, attendeeCount):
- s = """\
-BEGIN:VEVENT
-UID:%(UID)s
-DTSTART;TZID=America/Los_Angeles:%(START)s
-DTEND;TZID=America/Los_Angeles:%(END)s
-CREATED:20100729T193912Z
-DTSTAMP:20100729T195557Z
-%(ORGANIZER)s\
-%(ATTENDEES)s\
-SEQUENCE:0
-SUMMARY:%(summary)s
-TRANSP:OPAQUE
-END:VEVENT
-"""
- base = datetime(2010, 7, 30, 11, 15, 00)
- interval = timedelta(0, 5)
- duration = timedelta(0, 3)
- return event % {
- 'VEVENTS': s % {
- 'UID': uuid4(),
- 'START': formatDate(base + i * interval),
- 'END': formatDate(base + i * interval + duration),
- 'ORGANIZER': makeOrganizer(organizerSequence),
- 'ATTENDEES': makeAttendees(attendeeCount),
- 'summary': SUMMARY,
- },
- }
-
-
def measure(host, port, dtrace, attendeeCount, samples):
calendar = "event-creation-benchmark"
organizerSequence = 1
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_add_attendee.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_add_attendee.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_add_attendee.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -14,11 +14,10 @@
# limitations under the License.
##
-import _event_change
+from _event_change import measure as _measure
+from _event_create import makeAttendees
-from event import makeAttendees
-
def measure(host, port, dtrace, attendeeCount, samples):
attendees = makeAttendees(attendeeCount)
@@ -29,8 +28,8 @@
# Find the last CREATED line
created = event.rfind('CREATED')
# Insert the attendees before it.
- return event[:created] + attendees + event[created:]
+ return event[:created] + ''.join(attendees) + event[created:]
- return _event_change.measure(
+ return _measure(
host, port, dtrace, 0, samples, "add-attendee",
addAttendees, eventPerSample=True)
Copied: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_autoaccept.py (from rev 7204, CalendarServer/trunk/contrib/performance/benchmarks/event_autoaccept.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_autoaccept.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_autoaccept.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -0,0 +1,56 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Benchmark a server's handling of event creation involving an
+auto-accept resource.
+"""
+
+from itertools import count
+from uuid import uuid4
+from datetime import datetime, timedelta
+
+from _event_create import (
+ makeAttendees, makeVCalendar, measure as _measure)
+
+
+def makeEvent(i, organizerSequence, attendeeCount):
+ base = datetime(2010, 7, 30, 11, 15, 00)
+ interval = timedelta(0, 5)
+ duration = timedelta(0, 3)
+ attendees = makeAttendees(attendeeCount)
+ attendees.append(
+ 'ATTENDEE;CN="Resource 01";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=T\n'
+ ' RUE;SCHEDULE-STATUS="1.2":urn:uuid:resource01\n')
+ return makeVCalendar(
+ uuid4(),
+ base + i * interval,
+ base + i * interval + duration,
+ None,
+ organizerSequence,
+ attendees)
+
+
+def measure(host, port, dtrace, attendeeCount, samples):
+ calendar = "event-autoaccept-creation-benchmark"
+ organizerSequence = 1
+
+ # An infinite stream of VEVENTs to PUT to the server.
+ events = ((i, makeEvent(i, organizerSequence, attendeeCount)) for i in count(2))
+
+ return _measure(
+ calendar, organizerSequence, events,
+ host, port, dtrace, samples)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_change_summary.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_change_summary.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/event_change_summary.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -14,15 +14,14 @@
# limitations under the License.
##
-from event import SUMMARY
+from _event_create import SUMMARY
+from _event_change import measure as _measure
-import _event_change
-
def replaceSummary(event, i):
return event.replace(SUMMARY, 'Replacement summary %d' % (i,))
def measure(host, port, dtrace, attendeeCount, samples):
- return _event_change.measure(
+ return _measure(
host, port, dtrace, attendeeCount, samples, "change-summary",
replaceSummary)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -22,58 +22,20 @@
from itertools import count
from datetime import datetime, timedelta
-from _event_create import formatDate, measure as _measure
+from _event_create import (
+ makeAttendees, makeVCalendar, measure as _measure)
-DAILY_EVENT = """\
-BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Apple Inc.//iCal 4.0.3//EN
-CALSCALE:GREGORIAN
-BEGIN:VTIMEZONE
-TZID:America/New_York
-BEGIN:DAYLIGHT
-TZOFFSETFROM:-0500
-RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU
-DTSTART:20070311T020000
-TZNAME:EDT
-TZOFFSETTO:-0400
-END:DAYLIGHT
-BEGIN:STANDARD
-TZOFFSETFROM:-0400
-RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU
-DTSTART:20071104T020000
-TZNAME:EST
-TZOFFSETTO:-0500
-END:STANDARD
-END:VTIMEZONE
-BEGIN:VEVENT
-CREATED:20110223T184640Z
-UID:%(UID)s
-DTEND;TZID=America/New_York:%(DTEND)s
-RRULE:FREQ=WEEKLY
-TRANSP:OPAQUE
-SUMMARY:A meeting which occurs daily for several days.
-DTSTART;TZID=America/New_York:%(DTSTART)s
-DTSTAMP:20110223T184658Z
-SEQUENCE:5
-END:VEVENT
-END:VCALENDAR
-"""
-
-def makeEvent(i):
+def makeEvent(i, organizerSequence, attendeeCount):
"""
- Create a new half-hour long event that starts soon and recurs
- daily for the next five days.
+ Create a new half-hour long event that starts soon and weekly for
+ as long the server allows.
"""
now = datetime.now()
start = now.replace(minute=15, second=0, microsecond=0) + timedelta(hours=i)
end = start + timedelta(minutes=30)
- until = start + timedelta(days=5)
- return DAILY_EVENT % {
- 'DTSTART': formatDate(start),
- 'DTEND': formatDate(end),
- 'UID': uuid4(),
- }
+ return makeVCalendar(
+ uuid4(), start, end, "RRULE:FREQ=WEEKLY", organizerSequence,
+ makeAttendees(attendeeCount))
def measure(host, port, dtrace, attendeeCount, samples):
@@ -81,7 +43,7 @@
organizerSequence = 1
# An infinite stream of recurring VEVENTS to PUT to the server.
- events = ((i, makeEvent(i)) for i in count(2))
+ events = ((i, makeEvent(i, organizerSequence, attendeeCount)) for i in count(2))
return _measure(
calendar, organizerSequence, events,
Copied: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence_autoaccept.py (from rev 7204, CalendarServer/trunk/contrib/performance/benchmarks/unbounded_recurrence_autoaccept.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence_autoaccept.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/benchmarks/unbounded_recurrence_autoaccept.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -0,0 +1,54 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Benchmark a server's handling of events with an unbounded recurrence.
+"""
+
+from uuid import uuid4
+from itertools import count
+from datetime import datetime, timedelta
+
+from _event_create import (
+ makeAttendees, makeVCalendar, measure as _measure)
+
+def makeEvent(i, organizerSequence, attendeeCount):
+ """
+ Create a new half-hour long event that starts soon and weekly for
+ as long the server allows.
+ """
+ now = datetime.now()
+ start = now.replace(minute=15, second=0, microsecond=0) + timedelta(hours=i)
+ end = start + timedelta(minutes=30)
+ attendees = makeAttendees(attendeeCount)
+ attendees.append(
+ 'ATTENDEE;CN="Resource 01";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=T\n'
+ ' RUE;SCHEDULE-STATUS="1.2":urn:uuid:resource01\n')
+ return makeVCalendar(
+ uuid4(), start, end, "RRULE:FREQ=WEEKLY", organizerSequence, attendees)
+
+
+def measure(host, port, dtrace, attendeeCount, samples):
+ calendar = "unbounded-recurrence-autoaccept"
+ organizerSequence = 1
+
+ # An infinite stream of recurring VEVENTS to PUT to the server.
+ events = ((i, makeEvent(i, organizerSequence, attendeeCount)) for i in count(2))
+
+ return _measure(
+ calendar, organizerSequence, events,
+ host, port, dtrace, samples)
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/httpauth.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/httpauth.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/httpauth.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -16,7 +16,9 @@
import urlparse, urllib2
+from twisted.python.log import msg
from twisted.web.http_headers import Headers
+from twisted.web.http import UNAUTHORIZED
# CalDAVClientLibrary
from protocol.http.authentication.digest import Digest
@@ -107,11 +109,18 @@
def _authenticate(self, response, method, uri, headers, bodyProducer):
- if response.code == 401:
+ if response.code == UNAUTHORIZED:
+ if headers is None:
+ authorization = None
+ else:
+ authorization = headers.getRawHeaders('authorization')
+ msg("UNAUTHORIZED response to %s %s (Authorization=%r)" % (
+ method, uri, authorization))
# Look for a challenge
authorization = response.headers.getRawHeaders('www-authenticate')
if authorization is None:
- raise Exception("401 response with no WWW-Authenticate header")
+ raise Exception(
+ "UNAUTHORIZED response with no WWW-Authenticate header")
for auth in authorization:
challenge = self._parse(auth)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/loadtest/population.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/loadtest/population.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/loadtest/population.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -185,7 +185,11 @@
self.eventReceived(event)
+ def report(self):
+ pass
+
+
class SimpleStatistics(StatisticsBase):
def __init__(self):
self._times = []
@@ -249,7 +253,7 @@
print format % values
- def summarize(self):
+ def report(self):
print
self._printHeader()
for method, data in self._perMethodTimes.iteritems():
@@ -284,7 +288,7 @@
arrivalPolicy.run(reactor, simulator)
reactor.run()
- report.summarize()
+ report.report()
if __name__ == '__main__':
main()
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/nightly.sh
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/nightly.sh 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/nightly.sh 2011-03-17 17:54:34 UTC (rev 7206)
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
##
# Copyright (c) 2011 Apple Inc. All rights reserved.
@@ -22,7 +22,7 @@
exit
else
svn up
- REV="`./contrib/performance/svn-revno .`"
+ REV="$(./contrib/performance/svn-revno .)"
cd ~codespeed/performance
./sample.sh "$REV" ~codespeed/Projects/CalendarServer/trunk "$1"
fi
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/sample.sh
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/sample.sh 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/sample.sh 2011-03-17 17:54:34 UTC (rev 7206)
@@ -20,11 +20,12 @@
sudo -v # Force up to date sudo token before the user walks away
-REV=$1
-SOURCE_DIR=$2
-RESULTS=$3
+REV_SPEC="$1"
+SOURCE_DIR="$2"
+RESULTS="$3"
-update_and_build $REV
+update_and_build "$REV_SPEC"
+REV="$(./svn-revno "$SOURCE_DIR")"
if [ "$HOSTS_COUNT" != "" ]; then
CONCURRENT="--hosts-count $HOSTS_COUNT --host-index $HOST_INDEX"
Copied: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/some-more-data.sh (from rev 7204, CalendarServer/trunk/contrib/performance/some-more-data.sh)
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/some-more-data.sh (rev 0)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/some-more-data.sh 2011-03-17 17:54:34 UTC (rev 7206)
@@ -0,0 +1,33 @@
+#!/bin/bash -x
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+
+. ./benchlib.sh
+
+sudo -v # Force up to date sudo token before the user walks away
+
+SOURCE_DIR="$1"
+RESULTS="$2"
+
+NOW="$(date +%s)"
+
+WHEN=($((60*60*24*7)) $((60*60*24*31)) $((60*60*24*31*3)))
+for when in ${WHEN[*]}; do
+ THEN=$(($NOW-$when))
+ REV_SPEC="{$(date -r "$THEN" +"%Y-%m-%d")}"
+ ./sample.sh "$REV_SPEC" "$SOURCE_DIR" "$RESULTS"
+done
Modified: CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/test_event_change_date.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/test_event_change_date.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/contrib/performance/test_event_change_date.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -14,11 +14,9 @@
# limitations under the License.
##
-import datetime
-
from twisted.trial.unittest import TestCase
-from event_change_date import replaceTimestamp
+from benchmarks.event_change_date import replaceTimestamp
calendarHead = """\
BEGIN:VCALENDAR
Modified: CalendarServer/branches/users/cdaboo/pycalendar/doc/calendarserver_purge_events.8
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/doc/calendarserver_purge_events.8 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/doc/calendarserver_purge_events.8 2011-03-17 17:54:34 UTC (rev 7206)
@@ -41,12 +41,12 @@
Display usage information
.It Fl f, -config Ar FILE
Use the Calendar Server configuration specified in the given file. Defaults to /etc/caldavd/caldavd.plist.
+.It Fl d, -days Ar NUMBER
+Specify how many days in the past to retain. Defaults to 365 days.
.It Fl n, -dry-run
Calculate and display how many events would be removed, but don't actually remove them.
.It Fl v, -verbose
Print progress information.
-.It Fl d, -days Ar NUMBER
-Specify how many days in the past to retain. Defaults to 365 days.
.El
.Sh FILES
.Bl -tag -width flag
Modified: CalendarServer/branches/users/cdaboo/pycalendar/lib-patches/vobject/vobject.icalendar.patch
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/lib-patches/vobject/vobject.icalendar.patch 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/lib-patches/vobject/vobject.icalendar.patch 2011-03-17 17:54:34 UTC (rev 7206)
@@ -21,16 +21,19 @@
if charList is None:
charList = escapableCharList
-@@ -1656,6 +1660,8 @@
+@@ -1655,7 +1659,10 @@
+
if state == "read normal":
if char == '\\':
- state = "read escaped char"
+- state = "read escaped char"
+ if leaveEscapes:
+ current.append(char)
++ else:
++ state = "read escaped char"
elif char == listSeparator:
state = "read normal"
current = "".join(current)
-@@ -1675,9 +1681,10 @@
+@@ -1675,9 +1682,10 @@
else:
current.append(char)
else:
Modified: CalendarServer/branches/users/cdaboo/pycalendar/support/Makefile.Apple
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/support/Makefile.Apple 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/support/Makefile.Apple 2011-03-17 17:54:34 UTC (rev 7206)
@@ -112,11 +112,6 @@
$(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)/usr/libexec/changeip"
$(_v) $(INSTALL_FILE) "$(Sources)/calendarserver/tools/changeip_calendar.py" "$(DSTROOT)/usr/libexec/changeip/changeip_calendar.py"
$(_v) chmod ugo+x "$(DSTROOT)/usr/libexec/changeip/changeip_calendar.py"
- @echo "Installing backup config..."
- $(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)$(ETCDIR)/server_backup/"
- $(_v) $(INSTALL_DIRECTORY) "$(DSTROOT)$(LIBEXECDIR)/server_backup/"
- $(_v) $(INSTALL_FILE) "$(Sources)/contrib/SBS/conf/85-calendarServer.plist" "$(DSTROOT)$(ETCDIR)/server_backup/"
- $(_v) $(INSTALL_FILE) "$(Sources)/contrib/SBS/bin/calendarServer_restore" "$(DSTROOT)$(LIBEXECDIR)/server_backup/"
install::
@echo "Installing CalDAVTester package..."
Modified: CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh 2011-03-17 17:54:34 UTC (rev 7206)
@@ -210,7 +210,7 @@
#
# Try getting a copy from calendarserver.org.
#
- local tmp="$(mktemp -t "${cache_basename}")";
+ local tmp="$(mktemp "/tmp/${cache_basename}.XXXXX")";
curl -L "http://${pkg_host}${pkg_path}/${cache_basename}" -o "${tmp}";
echo "";
if [ ! -s "${tmp}" ] || grep '<title>404 Not Found</title>' "${tmp}" > /dev/null; then
@@ -633,7 +633,7 @@
"http://svn.osafoundation.org/vobject/trunk";
# XXX actually PyCalendar should be imported in-place.
- py_dependency -fie -r 143 \
+ py_dependency -fie -r 144 \
"pycalendar" "pycalendar" "pycalendar" \
"http://svn.mulberrymail.com/repos/PyCalendar/branches/server";
@@ -662,7 +662,7 @@
if "${do_setup}"; then
cd "${caldav}";
echo "Building our own extension modules...";
- python setup.py build_ext --inplace;
+ "${python}" setup.py build_ext --inplace;
fi;
}
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar/support/build.sh
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/support/build.sh:4379-4443
/CalendarServer/branches/egg-info-351/support/build.sh:4589-4625
/CalendarServer/branches/generic-sqlstore/support/build.sh:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/support/build.sh:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/support/build.sh:5911-5935
/CalendarServer/branches/new-store/support/build.sh:5594-5934
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/support/build.sh:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/support/build.sh:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/support/build.sh:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/support/build.sh:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/support/build.sh:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/support/build.sh:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/support/build.sh:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/support/build.sh:4971-5080
/CalendarServer/branches/users/glyph/dalify/support/build.sh:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect/support/build.sh:6824-6876
/CalendarServer/branches/users/glyph/dont-start-postgres/support/build.sh:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/support/build.sh:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/support/build.sh:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/support/build.sh:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/support/build.sh:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/support/build.sh:6490-6550
/CalendarServer/branches/users/glyph/sql-store/support/build.sh:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/support/build.sh:5084-5149
/CalendarServer/branches/users/sagen/locations-resources-2/support/build.sh:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/support/build.sh:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/support/build.sh:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/support/build.sh:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/support/build.sh:4068-4075
/CalendarServer/branches/users/sagen/resources-2/support/build.sh:5084-5093
/CalendarServer/branches/users/wsanchez/transations/support/build.sh:5515-5593
/CalendarServer/trunk/support/build.sh:4626-7169
+ /CalendarServer/branches/config-separation/support/build.sh:4379-4443
/CalendarServer/branches/egg-info-351/support/build.sh:4589-4625
/CalendarServer/branches/generic-sqlstore/support/build.sh:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/support/build.sh:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/support/build.sh:5911-5935
/CalendarServer/branches/new-store/support/build.sh:5594-5934
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/support/build.sh:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/support/build.sh:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/support/build.sh:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/support/build.sh:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/support/build.sh:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/support/build.sh:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/support/build.sh:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/support/build.sh:4971-5080
/CalendarServer/branches/users/glyph/dalify/support/build.sh:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect/support/build.sh:6824-6876
/CalendarServer/branches/users/glyph/dont-start-postgres/support/build.sh:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/support/build.sh:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/support/build.sh:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/support/build.sh:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/support/build.sh:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/support/build.sh:6490-6550
/CalendarServer/branches/users/glyph/sql-store/support/build.sh:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/support/build.sh:5084-5149
/CalendarServer/branches/users/sagen/locations-resources-2/support/build.sh:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/support/build.sh:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/support/build.sh:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/support/build.sh:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/support/build.sh:4068-4075
/CalendarServer/branches/users/sagen/resources-2/support/build.sh:5084-5093
/CalendarServer/branches/users/wsanchez/transations/support/build.sh:5515-5593
/CalendarServer/trunk/support/build.sh:4626-7204
Modified: CalendarServer/branches/users/cdaboo/pycalendar/support/patchapply
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/support/patchapply 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/support/patchapply 2011-03-17 17:54:34 UTC (rev 7206)
@@ -24,7 +24,7 @@
#
#projects = ("Twisted", "vobject", "dateutil", "xattr")
-projects = ("Twisted", "vobject",)
+projects = ("vobject",)
cwd = os.getcwd()
libpatches = os.path.join(cwd, "lib-patches")
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/adbapi2.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/adbapi2.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/adbapi2.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -1,3 +1,4 @@
+from twext.enterprise.ienterprise import IDerivedParameter
# -*- test-case-name: twext.enterprise.test.test_adbapi2 -*-
##
# Copyright (c) 2010 Apple Inc. All rights reserved.
@@ -53,15 +54,31 @@
from twisted.internet.defer import succeed
from twext.enterprise.ienterprise import ConnectionError
from twisted.internet.defer import fail
-from twext.enterprise.ienterprise import AlreadyFinishedError, IAsyncTransaction
+from twext.enterprise.ienterprise import (
+ AlreadyFinishedError, IAsyncTransaction, POSTGRES_DIALECT
+)
-# FIXME: there should be no default for DEFAULT_PARAM_STYLE, it should be
-# discovered dynamically everywhere. Right now we're only using pgdb so we only
-# support that.
+# FIXME: there should be no defaults for connection metadata, it should be
+# discovered dynamically everywhere. Right now it's specified as an explicit
+# argument to the ConnectionPool but it should probably be determined
+# automatically from the database binding.
DEFAULT_PARAM_STYLE = 'pyformat'
+DEFAULT_DIALECT = POSTGRES_DIALECT
+
+def _forward(thunk):
+ """
+ Forward an attribute to the connection pool.
+ """
+ @property
+ def getter(self):
+ return getattr(self._pool, thunk.func_name)
+ return getter
+
+
+
class _ConnectedTxn(object):
"""
L{IAsyncTransaction} implementation based on a L{ThreadHolder} in the
@@ -69,21 +86,57 @@
"""
implements(IAsyncTransaction)
- # See DEFAULT_PARAM_STYLE FIXME above.
- paramstyle = DEFAULT_PARAM_STYLE
-
def __init__(self, pool, threadHolder, connection, cursor):
self._pool = pool
self._completed = True
self._cursor = cursor
self._connection = connection
self._holder = threadHolder
+ self._first = True
+ @_forward
+ def paramstyle(self):
+ """
+ The paramstyle attribute is mirrored from the connection pool.
+ """
+
+
+ @_forward
+ def dialect(self):
+ """
+ The dialect attribute is mirrored from the connection pool.
+ """
+
+
def _reallyExecSQL(self, sql, args=None, raiseOnZeroRowCount=None):
+ wasFirst = self._first
+ self._first = False
if args is None:
args = []
- self._cursor.execute(sql, args)
+ derived = None
+ for n, arg in enumerate(args):
+ if IDerivedParameter.providedBy(arg):
+ if derived is None:
+ # Be sparing with extra allocations, as this usually isn't
+ # needed, and we're doing a ton of extra work to support it.
+ derived = []
+ derived.append(arg)
+ args[n] = arg.preQuery(self._cursor)
+ try:
+ self._cursor.execute(sql, args)
+ except:
+ if wasFirst:
+ self._connection.close()
+ self._connection = self._pool.connectionFactory()
+ self._cursor = self._connection.cursor()
+ result = self._reallyExecSQL(sql, args, raiseOnZeroRowCount)
+ return result
+ else:
+ raise
+ if derived is not None:
+ for arg in derived:
+ arg.postQuery(self._cursor)
if raiseOnZeroRowCount is not None and self._cursor.rowcount == 0:
raise raiseOnZeroRowCount()
if self._cursor.description:
@@ -113,14 +166,16 @@
def _end(self, really):
"""
- Common logic for commit or abort.
+ Common logic for commit or abort. Executed in the cursor main thread.
"""
if not self._completed:
self._completed = True
def reallySomething():
+ # Executed in the cursor thread.
if self._cursor is None:
return
really()
+ self._first = True
result = self._holder.submit(reallySomething)
self._pool._repoolAfter(self, result)
return result
@@ -133,7 +188,7 @@
def abort(self):
- return self._end(self._connection.rollback)
+ return self._end(self._connection.rollback).addErrback(log.err)
def __del__(self):
@@ -171,6 +226,7 @@
return holder.stop()
+
class _NoTxn(object):
"""
An L{IAsyncTransaction} that indicates a local failure before we could even
@@ -179,12 +235,18 @@
"""
implements(IAsyncTransaction)
+ def __init__(self, pool):
+ self.paramstyle = pool.paramstyle
+ self.dialect = pool.dialect
+
+
def _everything(self, *a, **kw):
"""
Everything fails with a L{ConnectionError}.
"""
return fail(ConnectionError())
+
execSQL = _everything
commit = _everything
abort = _everything
@@ -201,11 +263,10 @@
implements(IAsyncTransaction)
- # See DEFAULT_PARAM_STYLE FIXME above.
- paramstyle = DEFAULT_PARAM_STYLE
-
- def __init__(self):
+ def __init__(self, pool):
self._spool = []
+ self.paramstyle = pool.paramstyle
+ self.dialect = pool.dialect
def _enspool(self, cmd, a=(), kw={}):
@@ -257,7 +318,7 @@
class _SingleTxn(proxyForInterface(iface=IAsyncTransaction,
- originalAttribute='_baseTxn')):
+ originalAttribute='_baseTxn')):
"""
A L{_SingleTxn} is a single-use wrapper for the longer-lived
L{_ConnectedTxn}, so that if a badly-behaved API client accidentally hangs
@@ -320,7 +381,7 @@
Stop waiting for a free transaction and fail.
"""
self._pool._waiting.remove(self)
- self._unspoolOnto(_NoTxn())
+ self._unspoolOnto(_NoTxn(self._pool))
def _checkComplete(self):
@@ -420,11 +481,14 @@
RETRY_TIMEOUT = 10.0
- def __init__(self, connectionFactory, maxConnections=10):
+ def __init__(self, connectionFactory, maxConnections=10,
+ paramstyle=DEFAULT_PARAM_STYLE, dialect=DEFAULT_DIALECT):
super(ConnectionPool, self).__init__()
self.connectionFactory = connectionFactory
self.maxConnections = maxConnections
+ self.paramstyle = paramstyle
+ self.dialect = dialect
self._free = []
self._busy = []
@@ -461,11 +525,7 @@
# Phase 3: All of the busy transactions must be aborted first. As each
# one is aborted, it will remove itself from the list.
while self._busy:
- d = self._busy[0].abort()
- try:
- yield d
- except:
- log.err()
+ yield self._busy[0].abort()
# Phase 4: All transactions should now be in the free list, since
# 'abort()' will have put them there. Shut down all the associated
@@ -495,13 +555,13 @@
@return: an L{IAsyncTransaction}
"""
if self._stopping:
- return _NoTxn()
+ return _NoTxn(self)
if self._free:
basetxn = self._free.pop(0)
self._busy.append(basetxn)
txn = _SingleTxn(self, basetxn)
else:
- txn = _SingleTxn(self, _WaitingTxn())
+ txn = _SingleTxn(self, _WaitingTxn(self))
self._waiting.append(txn)
# FIXME/TESTME: should be len(self._busy) + len(self._finishing)
# (free doesn't need to be considered, as it's tested above)
@@ -562,7 +622,12 @@
self._finishing.remove(finishRecord)
self._repoolNow(txn)
return result
- return d.addBoth(repool)
+ def discard(result):
+ self._finishing.remove(finishRecord)
+ txn._releaseConnection()
+ self._startOneMore()
+ return result
+ return d.addCallbacks(repool, discard)
def _repoolNow(self, txn):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/model.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/model.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/model.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -18,6 +18,7 @@
"""
Model classes for SQL.
"""
+from twisted.python.util import FancyEqMixin
class SQLType(object):
"""
@@ -123,7 +124,7 @@
-class Column(object):
+class Column(FancyEqMixin, object):
"""
A column from a table.
@@ -140,8 +141,15 @@
@ivar references: If this column references a foreign key on another table,
this will be a reference to that table; otherwise (normally) C{None}.
@type references: L{Table} or C{NoneType}
+
+ @ivar cascade: If this column references another table, will this column's
+ row be deleted when the matching row in that other table is deleted?
+ (In other words, the SQL feature 'on delete cascade'.)
+ @type cascade: C{bool}
"""
+ compareAttributes = 'table name'.split()
+
def __init__(self, table, name, type):
_checkstr(name)
self.table = table
@@ -149,6 +157,7 @@
self.type = type
self.default = NO_DEFAULT
self.references = None
+ self.cascade = False
def __repr__(self):
@@ -204,7 +213,7 @@
-class Table(object):
+class Table(FancyEqMixin, object):
"""
A set of columns.
@@ -216,6 +225,8 @@
@ivar schema: a reference to the L{Schema} to which this table belongs.
"""
+ compareAttributes = 'schema name'.split()
+
def __init__(self, schema, name):
_checkstr(name)
self.descriptiveComment = ''
@@ -318,11 +329,13 @@
-class Sequence(object):
+class Sequence(FancyEqMixin, object):
"""
A sequence object.
"""
+ compareAttributes = 'name'.split()
+
def __init__(self, name):
_checkstr(name)
self.name = name
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/parseschema.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/parseschema.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/parseschema.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -215,6 +215,20 @@
return self.parseConstraint(maybeIdent)
+ def namesInParens(self, parens):
+ parens = iterSignificant(parens)
+ expect(parens, ttype=Punctuation, value="(")
+ idorids = parens.next()
+ if isinstance(idorids, Identifier):
+ idnames = [idorids.get_name()]
+ elif isinstance(idorids, IdentifierList):
+ idnames = [x.get_name() for x in idorids.get_identifiers()]
+ else:
+ raise ViolatedExpectation("identifier or list", repr(idorids))
+ expect(parens, ttype=Punctuation, value=")")
+ return idnames
+
+
def parseConstraint(self, constraintType):
"""
Parse a 'free' constraint, described explicitly in the table as opposed
@@ -223,20 +237,12 @@
# only know about PRIMARY KEY and UNIQUE for now
if constraintType.match(Keyword, 'PRIMARY'):
expect(self, ttype=Keyword, value='KEY')
- expect(self, cls=Parenthesis)
- self.primaryKey = 'MULTI-VALUE-KEY'
+ names = self.namesInParens(expect(self, cls=Parenthesis))
+ self.table.primaryKey = tuple(self.table.columnNamed(n)
+ for n in names)
elif constraintType.match(Keyword, 'UNIQUE'):
- parens = iterSignificant(expect(self, cls=Parenthesis))
- expect(parens, ttype=Punctuation, value="(")
- idorids = parens.next()
- if isinstance(idorids, Identifier):
- idnames = [idorids.get_name()]
- elif isinstance(idorids, IdentifierList):
- idnames = [x.get_name() for x in idorids.get_identifiers()]
- else:
- raise ViolatedExpectation("identifier or list", repr(idorids))
- expect(parens, ttype=Punctuation, value=")")
- self.table.tableConstraint(Constraint.UNIQUE, idnames)
+ names = self.namesInParens(expect(self, cls=Parenthesis))
+ self.table.tableConstraint(Constraint.UNIQUE, names)
else:
raise ViolatedExpectation('PRIMARY or UNIQUE', constraintType)
return self.checkEnd(self.next())
@@ -347,6 +353,7 @@
elif val.match(Keyword, 'ON'):
expect(self, ttype=Keyword.DML, value='DELETE')
expect(self, ttype=Keyword, value='CASCADE')
+ theColumn.cascade = True
else:
expected = False
if not expected:
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/syntax.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/syntax.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/syntax.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -19,9 +19,75 @@
Syntax wrappers and generators for SQL.
"""
+import itertools
+
+from zope.interface import implements
+
+from twisted.internet.defer import succeed
+
+from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT
+from twext.enterprise.ienterprise import IDerivedParameter
+
+from twext.enterprise.util import mapOracleOutputType
from twext.enterprise.dal.model import Schema, Table, Column, Sequence
+try:
+ import cx_Oracle
+ cx_Oracle
+except ImportError:
+ cx_Oracle = None
+class ConnectionMetadata(object):
+ """
+ Representation of the metadata about the database connection required to
+ generate some SQL, for a single statement. Contains information necessary
+ to generate placeholder strings and determine the database dialect.
+ """
+
+ def __init__(self, dialect):
+ self.dialect = dialect
+
+
+ def placeholder(self):
+ raise NotImplementedError("See subclasses.")
+
+
+
+class FixedPlaceholder(ConnectionMetadata):
+ """
+ Metadata about a connection which uses a fixed string as its placeholder.
+ """
+
+ def __init__(self, dialect, placeholder):
+ super(FixedPlaceholder, self).__init__(dialect)
+ self._placeholder = placeholder
+
+
+ def placeholder(self):
+ return self._placeholder
+
+
+
+class NumericPlaceholder(ConnectionMetadata):
+
+ def __init__(self, dialect):
+ super(NumericPlaceholder, self).__init__(dialect)
+ self._next = itertools.count(1).next
+
+
+ def placeholder(self):
+ return ':' + str(self._next())
+
+
+
+def defaultMetadata():
+ """
+ Generate a default L{ConnectionMetadata}
+ """
+ return FixedPlaceholder(POSTGRES_DIALECT, '?')
+
+
+
class TableMismatch(Exception):
"""
A table in a statement did not match with a column.
@@ -43,18 +109,37 @@
"""
_paramstyles = {
- 'pyformat': ('%s', lambda s: s.replace("%", "%%"))
+ 'pyformat': lambda dialect: FixedPlaceholder(dialect, "%s"),
+ 'numeric': NumericPlaceholder
}
+
+ def toSQL(self, metadata=None):
+ if metadata is None:
+ metadata = defaultMetadata()
+ return self._toSQL(metadata)
+
+
+ def _extraVars(self, txn, metadata):
+ return {}
+
+
+ def _extraResult(self, result, outvars, metadata):
+ return result
+
+
def on(self, txn, raiseOnZeroRowCount=None, **kw):
"""
Execute this statement on a given L{IAsyncTransaction} and return the
resulting L{Deferred}.
"""
- placeholder, quote = self._paramstyles[txn.paramstyle]
- fragment = self.toSQL(placeholder, quote).bind(**kw)
- return txn.execSQL(fragment.text, fragment.parameters,
- raiseOnZeroRowCount)
+ metadata = self._paramstyles[txn.paramstyle](txn.dialect)
+ outvars = self._extraVars(txn, metadata)
+ kw.update(outvars)
+ fragment = self.toSQL(metadata).bind(**kw)
+ result = txn.execSQL(fragment.text, fragment.parameters,
+ raiseOnZeroRowCount)
+ return self._extraResult(result, outvars, metadata)
@@ -89,6 +174,8 @@
def __(self, other):
if other is None:
return NullComparison(self, comparator)
+ if isinstance(other, Select):
+ return NotImplemented
if isinstance(other, ColumnSyntax):
return ColumnComparison(self, comparator, other)
else:
@@ -123,8 +210,8 @@
class FunctionInvocation(ExpressionSyntax):
- def __init__(self, name, *args):
- self.name = name
+ def __init__(self, function, *args):
+ self.function = function
self.args = args
@@ -139,10 +226,10 @@
return list(ac())
- def subSQL(self, placeholder, quote, allTables):
- result = SQLFragment(self.name)
+ def subSQL(self, metadata, allTables):
+ result = SQLFragment(self.function.nameFor(metadata))
result.append(_inParens(
- _commaJoined(_convert(arg).subSQL(placeholder, quote, allTables)
+ _commaJoined(_convert(arg).subSQL(metadata, allTables)
for arg in self.args)))
return result
@@ -157,8 +244,8 @@
return []
- def subSQL(self, placeholder, quote, allTables):
- return SQLFragment(placeholder, [self.value])
+ def subSQL(self, metadata, allTables):
+ return SQLFragment(metadata.placeholder(), [self.value])
@@ -171,7 +258,7 @@
self.name = name
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
return SQLFragment(self.name)
@@ -181,19 +268,27 @@
An L{Function} is a representation of an SQL Function function.
"""
- def __init__(self, name):
+ def __init__(self, name, oracleName=None):
self.name = name
+ self.oracleName = oracleName
+ def nameFor(self, metadata):
+ if metadata.dialect == ORACLE_DIALECT and self.oracleName is not None:
+ return self.oracleName
+ return self.name
+
+
def __call__(self, *args):
"""
Produce an L{FunctionInvocation}
"""
- return FunctionInvocation(self.name, *args)
+ return FunctionInvocation(self, *args)
+
Max = Function("max")
-Len = Function("character_length")
+Len = Function("character_length", "length")
@@ -234,11 +329,15 @@
modelType = Sequence
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
"""
Convert to an SQL fragment.
"""
- return SQLFragment("nextval('%s')" % (self.model.name,))
+ if metadata.dialect == ORACLE_DIALECT:
+ fmt = "%s.nextval"
+ else:
+ fmt = "nextval('%s')"
+ return SQLFragment(fmt % (self.model.name,))
@@ -255,7 +354,7 @@
return Join(self, type, otherTableSyntax, on)
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
"""
For use in a 'from' clause.
"""
@@ -317,18 +416,18 @@
self.on = on
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
stmt = SQLFragment()
- stmt.append(self.leftSide.subSQL(placeholder, quote, allTables))
+ stmt.append(self.leftSide.subSQL(metadata, allTables))
stmt.text += ' '
if self.type:
stmt.text += self.type
stmt.text += ' '
stmt.text += 'join '
- stmt.append(self.rightSide.subSQL(placeholder, quote, allTables))
+ stmt.append(self.rightSide.subSQL(metadata, allTables))
if self.type != 'cross':
stmt.text += ' on '
- stmt.append(self.on.subSQL(placeholder, quote, allTables))
+ stmt.append(self.on.subSQL(metadata, allTables))
return stmt
@@ -342,6 +441,15 @@
return Join(self, type, otherTable, on)
+_KEYWORDS = ["access",
+ # SQL keyword, but we have a column with this name
+ "path",
+ # Not actually a standard keyword, but a function in oracle, and we
+ # have a column with this name.
+ "size",
+ # not actually sure what this is; only experimentally determined
+ # that not quoting it causes an issue.
+ ]
class ColumnSyntax(ExpressionSyntax):
@@ -356,16 +464,19 @@
return [self]
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
# XXX This, and 'model', could in principle conflict with column names.
# Maybe do something about that.
+ name = self.model.name
+ if metadata.dialect == ORACLE_DIALECT and name.lower() in _KEYWORDS:
+ name = '"%s"' % (name,)
+
for tableSyntax in allTables:
if self.model.table is not tableSyntax.model:
if self.model.name in (c.name for c in
tableSyntax.model.columns):
- return SQLFragment((self.model.table.name + '.' +
- self.model.name))
- return SQLFragment(self.model.name)
+ return SQLFragment((self.model.table.name + '.' + name))
+ return SQLFragment(name)
@@ -377,8 +488,8 @@
self.b = b
- def _subexpression(self, expr, placeholder, quote, allTables):
- result = expr.subSQL(placeholder, quote, allTables)
+ def _subexpression(self, expr, metadata, allTables):
+ result = expr.subSQL(metadata, allTables)
if self.op not in ('and', 'or') and isinstance(expr, Comparison):
result = _inParens(result)
return result
@@ -406,9 +517,9 @@
super(NullComparison, self).__init__(a, op, None)
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
sqls = SQLFragment()
- sqls.append(self.a.subSQL(placeholder, quote, allTables))
+ sqls.append(self.a.subSQL(metadata, allTables))
sqls.text += " is "
if self.op != "=":
sqls.text += "not "
@@ -427,16 +538,16 @@
return self.a.allColumns() + self.b.allColumns()
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
stmt = SQLFragment()
- result = self._subexpression(self.a, placeholder, quote, allTables)
+ result = self._subexpression(self.a, metadata, allTables)
if isinstance(self.a, CompoundComparison) and self.a.op == 'or' and self.op == 'and':
result = _inParens(result)
stmt.append(result)
stmt.text += ' %s ' % (self.op,)
- result = self._subexpression(self.b, placeholder, quote, allTables)
+ result = self._subexpression(self.b, metadata, allTables)
if isinstance(self.b, CompoundComparison) and self.b.op == 'or' and self.op == 'and':
result = _inParens(result)
stmt.append(result)
@@ -454,8 +565,8 @@
class _AllColumns(object):
- def subSQL(self, placeholder, quote, allTables):
- return SQLFragment(quote('*'))
+ def subSQL(self, metadata, allTables):
+ return SQLFragment('*')
ALL_COLUMNS = _AllColumns()
@@ -467,7 +578,7 @@
self.columns = columns
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
first = True
cstatement = SQLFragment()
for column in self.columns:
@@ -475,7 +586,7 @@
first = False
else:
cstatement.append(SQLFragment(", "))
- cstatement.append(column.subSQL(placeholder, quote, allTables))
+ cstatement.append(column.subSQL(metadata, allTables))
return cstatement
@@ -491,7 +602,23 @@
return True
+class Tuple(object):
+ def __init__(self, columns):
+ self.columns = columns
+
+
+ def subSQL(self, metadata, allTables):
+ return _inParens(_commaJoined(c.subSQL(metadata, allTables)
+ for c in self.columns))
+
+
+ def allColumns(self):
+ return self.columns
+
+
+
+
class Select(_Statement):
"""
'select' statement.
@@ -499,9 +626,10 @@
def __init__(self, columns=None, Where=None, From=None, OrderBy=None,
GroupBy=None, Limit=None, ForUpdate=False, Ascending=None,
- Having=None):
+ Having=None, Distinct=False):
self.From = From
self.Where = Where
+ self.Distinct = Distinct
if not isinstance(OrderBy, (list, tuple, type(None))):
OrderBy = [OrderBy]
self.OrderBy = OrderBy
@@ -522,43 +650,54 @@
self.Ascending = Ascending
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def __eq__(self, other):
"""
+ Create a comparison.
+ """
+ if isinstance(other, (list, tuple)):
+ other = Tuple(other)
+ return CompoundComparison(other, '=', self)
+
+
+ def _toSQL(self, metadata):
+ """
@return: a 'select' statement with placeholders and arguments
@rtype: L{SQLFragment}
"""
- stmt = SQLFragment(quote("select "))
+ stmt = SQLFragment("select ")
+ if self.Distinct:
+ stmt.text += "distinct "
allTables = self.From.tables()
- stmt.append(self.columns.subSQL(placeholder, quote, allTables))
- stmt.text += quote(" from ")
- stmt.append(self.From.subSQL(placeholder, quote, allTables))
+ stmt.append(self.columns.subSQL(metadata, allTables))
+ stmt.text += " from "
+ stmt.append(self.From.subSQL(metadata, allTables))
if self.Where is not None:
- wherestmt = self.Where.subSQL(placeholder, quote, allTables)
- stmt.text += quote(" where ")
+ wherestmt = self.Where.subSQL(metadata, allTables)
+ stmt.text += " where "
stmt.append(wherestmt)
if self.GroupBy is not None:
- stmt.text += quote(" group by ")
+ stmt.text += " group by "
fst = True
for subthing in self.GroupBy:
if fst:
fst = False
else:
stmt.text += ', '
- stmt.append(subthing.subSQL(placeholder, quote, allTables))
+ stmt.append(subthing.subSQL(metadata, allTables))
if self.Having is not None:
- havingstmt = self.Having.subSQL(placeholder, quote, allTables)
- stmt.text += quote(" having ")
+ havingstmt = self.Having.subSQL(metadata, allTables)
+ stmt.text += " having "
stmt.append(havingstmt)
if self.OrderBy is not None:
- stmt.text += quote(" order by ")
+ stmt.text += " order by "
fst = True
for subthing in self.OrderBy:
if fst:
fst = False
else:
stmt.text += ', '
- stmt.append(subthing.subSQL(placeholder, quote, allTables))
+ stmt.append(subthing.subSQL(metadata, allTables))
if self.Ascending is not None:
if self.Ascending:
kw = " asc"
@@ -566,17 +705,16 @@
kw = " desc"
stmt.append(SQLFragment(kw))
if self.ForUpdate:
- stmt.text += quote(" for update")
+ stmt.text += " for update"
if self.Limit is not None:
- stmt.text += quote(" limit ")
- stmt.append(Constant(self.Limit).subSQL(placeholder, quote,
- allTables))
+ stmt.text += " limit "
+ stmt.append(Constant(self.Limit).subSQL(metadata, allTables))
return stmt
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
result = SQLFragment("(")
- result.append(self.toSQL(placeholder, quote))
+ result.append(self.toSQL(metadata))
result.append(SQLFragment(")"))
return result
@@ -629,21 +767,109 @@
self.subfragments = subfragments
- def subSQL(self, placeholder, quote, allTables):
- return _commaJoined(f.subSQL(placeholder, quote, allTables)
+ def subSQL(self, metadata, allTables):
+ return _commaJoined(f.subSQL(metadata, allTables)
for f in self.subfragments)
-class Insert(_Statement):
+class _DMLStatement(_Statement):
"""
+ Common functionality of Insert/Update/Delete statements.
+ """
+
+ def _returningClause(self, metadata, stmt, allTables):
+ """
+ Add a dialect-appropriate 'returning' clause to the end of the given SQL
+ statement.
+
+ @param metadata: describes the database we are generating the statement for.
+ @type metadata: L{ConnectionMetadata}
+
+ @param stmt: the SQL fragment generated without the 'returning' clause
+ @type stmt: L{SQLFragment}
+
+ @param allTables: all tables involved in the query; see any C{subSQL}
+ method.
+
+ @return: the C{stmt} parameter.
+ """
+ retclause = self.Return
+ if isinstance(retclause, (tuple, list)):
+ retclause = _CommaList(retclause)
+ if retclause is not None:
+ stmt.text += ' returning '
+ stmt.append(retclause.subSQL(metadata, allTables))
+ if metadata.dialect == ORACLE_DIALECT:
+ stmt.text += ' into '
+ params = []
+ retvals = self._returnAsList()
+ for n, v in enumerate(retvals):
+ params.append(
+ Constant(Parameter("oracle_out_" + str(n)))
+ .subSQL(metadata, allTables)
+ )
+ stmt.append(_commaJoined(params))
+ return stmt
+
+
+ def _returnAsList(self):
+ if not isinstance(self.Return, (tuple, list)):
+ return [self.Return]
+ else:
+ return self.Return
+
+
+ def _extraVars(self, txn, metadata):
+ result = []
+ rvars = self._returnAsList()
+ if metadata.dialect == ORACLE_DIALECT:
+ for n, v in enumerate(rvars):
+ result.append(("oracle_out_" + str(n), _OracleOutParam(v)))
+ return result
+
+
+ def _extraResult(self, result, outvars, metadata):
+ if metadata.dialect == ORACLE_DIALECT and self.Return is not None:
+ def processIt(shouldBeNone):
+ result = [[v.value for k, v in outvars]]
+ return result
+ return result.addCallback(processIt)
+ else:
+ return result
+
+
+
+class _OracleOutParam(object):
+ implements(IDerivedParameter)
+
+ def __init__(self, columnSyntax):
+ self.columnSyntax = columnSyntax
+
+
+ def preQuery(self, cursor):
+ self.columnSyntax
+ typeMap = {'integer': cx_Oracle.NUMBER,
+ 'text': cx_Oracle.CLOB,
+ 'varchar': cx_Oracle.STRING,
+ 'timestamp': cx_Oracle.TIMESTAMP}
+ typeID = self.columnSyntax.model.type.name.lower()
+ self.var = cursor.var(typeMap[typeID])
+ return self.var
+
+
+ def postQuery(self, cursor):
+ self.value = mapOracleOutputType(self.var.getvalue())
+
+
+
+class Insert(_DMLStatement):
+ """
'insert' statement.
"""
def __init__(self, columnMap, Return=None):
self.columnMap = columnMap
- if isinstance(Return, (tuple, list)):
- Return = _CommaList(Return)
self.Return = Return
columns = _modelsFromMap(columnMap)
table = _fromSameTable(columns)
@@ -656,31 +882,38 @@
(', '.join([c.name for c in unspecified])))
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
"""
@return: a 'insert' statement with placeholders and arguments
@rtype: L{SQLFragment}
"""
- sortedColumns = sorted(self.columnMap.items(),
+ columnsAndValues = self.columnMap.items()
+ tableModel = columnsAndValues[0][0].model.table
+ specifiedColumnModels = [x.model for x in self.columnMap.keys()]
+ if metadata.dialect == ORACLE_DIALECT:
+ # See test_nextSequenceDefaultImplicitExplicitOracle.
+ for column in tableModel.columns:
+ if isinstance(column.default, Sequence):
+ columnSyntax = ColumnSyntax(column)
+ if column not in specifiedColumnModels:
+ columnsAndValues.append(
+ (columnSyntax, SequenceSyntax(column.default))
+ )
+ sortedColumns = sorted(columnsAndValues,
key=lambda (c, v): c.model.name)
allTables = []
stmt = SQLFragment('insert into ')
- stmt.append(
- TableSyntax(sortedColumns[0][0].model.table)
- .subSQL(placeholder, quote, allTables))
+ stmt.append(TableSyntax(tableModel).subSQL(metadata, allTables))
stmt.append(SQLFragment(" "))
stmt.append(_inParens(_commaJoined(
- [c.subSQL(placeholder, quote, allTables) for (c, v) in
+ [c.subSQL(metadata, allTables) for (c, v) in
sortedColumns])))
stmt.append(SQLFragment(" values "))
stmt.append(_inParens(_commaJoined(
- [_convert(v).subSQL(placeholder, quote, allTables)
+ [_convert(v).subSQL(metadata, allTables)
for (c, v) in sortedColumns])))
- if self.Return is not None:
- stmt.text += ' returning '
- stmt.append(self.Return.subSQL(placeholder, quote, allTables))
- return stmt
+ return self._returningClause(metadata, stmt, allTables)
@@ -696,7 +929,7 @@
-class Update(_Statement):
+class Update(_DMLStatement):
"""
'update' statement
"""
@@ -706,12 +939,10 @@
_fromSameTable(_modelsFromMap(columnMap))
self.columnMap = columnMap
self.Where = Where
- if isinstance(Return, (tuple, list)):
- Return = _CommaList(Return)
self.Return = Return
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
"""
@return: a 'insert' statement with placeholders and arguments
@@ -723,27 +954,24 @@
result = SQLFragment('update ')
result.append(
TableSyntax(sortedColumns[0][0].model.table).subSQL(
- placeholder, quote, allTables)
+ metadata, allTables)
)
result.text += ' set '
result.append(
_commaJoined(
- [c.subSQL(placeholder, quote, allTables).append(
- SQLFragment(" = ").subSQL(placeholder, quote, allTables)
- ).append(_convert(v).subSQL(placeholder, quote, allTables))
+ [c.subSQL(metadata, allTables).append(
+ SQLFragment(" = ").subSQL(metadata, allTables)
+ ).append(_convert(v).subSQL(metadata, allTables))
for (c, v) in sortedColumns]
)
)
result.append(SQLFragment( ' where '))
- result.append(self.Where.subSQL(placeholder, quote, allTables))
- if self.Return is not None:
- result.append(SQLFragment(' returning '))
- result.append(self.Return.subSQL(placeholder, quote, allTables))
- return result
+ result.append(self.Where.subSQL(metadata, allTables))
+ return self._returningClause(metadata, result, allTables)
-class Delete(_Statement):
+class Delete(_DMLStatement):
"""
'delete' statement.
"""
@@ -754,17 +982,14 @@
self.Return = Return
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
result = SQLFragment()
allTables = self.From.tables()
- result.text += quote('delete from ')
- result.append(self.From.subSQL(placeholder, quote, allTables))
- result.text += quote(' where ')
- result.append(self.Where.subSQL(placeholder, quote, allTables))
- if self.Return is not None:
- result.append(SQLFragment(' returning '))
- result.append(self.Return.subSQL(placeholder, quote, allTables))
- return result
+ result.text += 'delete from '
+ result.append(self.From.subSQL(metadata, allTables))
+ result.text += ' where '
+ result.append(self.Where.subSQL(metadata, allTables))
+ return self._returningClause(metadata, result, allTables)
@@ -783,11 +1008,13 @@
return cls(table, 'exclusive')
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
return SQLFragment('lock table ').append(
- self.table.subSQL(placeholder, quote, [self.table])).append(
+ self.table.subSQL(metadata, [self.table])).append(
SQLFragment(' in %s mode' % (self.mode,)))
+
+
class Savepoint(_Statement):
"""
An SQL 'savepoint' statement.
@@ -797,7 +1024,7 @@
self.name = name
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
return SQLFragment('savepoint %s' % (self.name,))
@@ -810,7 +1037,7 @@
self.name = name
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
return SQLFragment('rollback to savepoint %s' % (self.name,))
@@ -823,24 +1050,42 @@
self.name = name
- def toSQL(self, placeholder="?", quote=lambda x: x):
+ def _toSQL(self, metadata):
return SQLFragment('release savepoint %s' % (self.name,))
+
class SavepointAction(object):
-
+
def __init__(self, name):
self._name = name
-
+
+
def acquire(self, txn):
return Savepoint(self._name).on(txn)
+
def rollback(self, txn):
return RollbackToSavepoint(self._name).on(txn)
+
def release(self, txn):
- return ReleaseSavepoint(self._name).on(txn)
+ if txn.dialect == ORACLE_DIALECT:
+ # There is no 'release savepoint' statement in oracle, but then, we
+ # don't need it because there's no resource to manage. Just don't
+ # do anything.
+ return NoOp()
+ else:
+ return ReleaseSavepoint(self._name).on(txn)
+
+
+class NoOp(object):
+ def on(self, *a, **kw):
+ return succeed(None)
+
+
+
class SQLFragment(object):
"""
Combination of SQL text and arguments; a statement which may be executed
@@ -886,7 +1131,7 @@
return self.__class__.__name__ + repr((self.text, self.parameters))
- def subSQL(self, placeholder, quote, allTables):
+ def subSQL(self, metadata, allTables):
return self
@@ -897,14 +1142,27 @@
self.name = name
+ def __eq__(self, param):
+ if not isinstance(param, Parameter):
+ return NotImplemented
+ return self.name == param.name
+
+
+ def __ne__(self, param):
+ if not isinstance(param, Parameter):
+ return NotImplemented
+ return not self.__eq__(param)
+
+
def __repr__(self):
return 'Parameter(%r)' % (self.name,)
# Common helpers:
-# current timestamp in UTC format.
-utcNowSQL = Function('timezone')('UTC', NamedValue('CURRENT_TIMESTAMP'))
+# current timestamp in UTC format. Hack to support standard syntax for this,
+# rather than the compatibility procedure found in various databases.
+utcNowSQL = NamedValue("CURRENT_TIMESTAMP at time zone 'UTC'")
# You can't insert a column with no rows. In SQL that just isn't valid syntax,
# and in this DAL you need at least one key or we can't tell what table you're
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_parseschema.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_parseschema.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_parseschema.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -215,3 +215,32 @@
[set([b, c]), set([c])])
+ def test_multiPrimaryKey(self):
+ """
+ A table with a multi-column PRIMARY KEY clause will be parsed as a tuple
+ primaryKey attribute on the Table object.
+ """
+ s = Schema()
+ addSQLToSchema(
+ s, "create table a (b integer, c integer, primary key(b, c))")
+ a = s.tableNamed("a")
+ self.assertEquals(a.primaryKey,
+ (a.columnNamed("b"), a.columnNamed("c")))
+
+
+ def test_cascade(self):
+ """
+ A column with an 'on delete cascade' constraint will have its C{cascade}
+ attribute set to True.
+ """
+ s = Schema()
+ addSQLToSchema(
+ s,
+ """
+ create table a (b integer primary key);
+ create table c (d integer references a on delete cascade);
+ """)
+ self.assertEquals(s.tableNamed("a").columnNamed("b").cascade, False)
+ self.assertEquals(s.tableNamed("c").columnNamed("d").cascade, True)
+
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_sqlsyntax.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_sqlsyntax.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/dal/test/test_sqlsyntax.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -20,14 +20,35 @@
from twext.enterprise.dal.model import Schema
from twext.enterprise.dal.parseschema import addSQLToSchema
+from twext.enterprise.dal import syntax
from twext.enterprise.dal.syntax import (
SchemaSyntax, Select, Insert, Update, Delete, Lock, SQLFragment,
TableMismatch, Parameter, Max, Len, NotEnoughValues
, Savepoint, RollbackToSavepoint, ReleaseSavepoint, SavepointAction)
-from twext.enterprise.dal.syntax import FunctionInvocation
+from twext.enterprise.dal.syntax import Function
+
+from twext.enterprise.dal.syntax import FixedPlaceholder, NumericPlaceholder
+from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT
+from twext.enterprise.test.test_adbapi2 import ConnectionFactory
+from twext.enterprise.adbapi2 import ConnectionPool
+from twext.enterprise.test.test_adbapi2 import resultOf
+from twext.enterprise.test.test_adbapi2 import FakeThreadHolder
from twisted.trial.unittest import TestCase
+
+
+class _FakeTransaction(object):
+ """
+ An L{IAsyncTransaction} that provides the relevant metadata for SQL
+ generation.
+ """
+
+ def __init__(self, paramstyle):
+ self.paramstyle = 'qmark'
+
+
+
class GenerationTests(TestCase):
"""
Tests for syntactic helpers to generate SQL queries.
@@ -37,11 +58,12 @@
s = Schema(self.id())
addSQLToSchema(schema=s, schemaData="""
create sequence A_SEQ;
- create table FOO (BAR integer, BAZ integer);
- create table BOZ (QUX integer);
+ create table FOO (BAR integer, BAZ varchar(255));
+ create table BOZ (QUX integer, QUUX integer);
create table OTHER (BAR integer,
FOO_BAR integer not null);
create table TEXTUAL (MYTEXT varchar(255));
+ create table LEVELS (ACCESS integer, USERNAME varchar(255));
""")
self.schema = SchemaSyntax(s)
@@ -65,16 +87,16 @@
SQLFragment("select * from FOO where BAR = ?", [1]))
- def test_quotingAndPlaceholder(self):
+ def test_alternateMetadata(self):
"""
L{Select} generates a 'select' statement with the specified placeholder
- syntax and quoting function.
+ syntax when explicitly given L{ConnectionMetadata} which specifies a
+ placeholder.
"""
self.assertEquals(Select(From=self.schema.FOO,
Where=self.schema.FOO.BAR == 1).toSQL(
- placeholder="*",
- quote=lambda partial: partial.replace("*", "**")),
- SQLFragment("select ** from FOO where BAR = *", [1]))
+ FixedPlaceholder(POSTGRES_DIALECT, "$$")),
+ SQLFragment("select * from FOO where BAR = $$", [1]))
def test_columnComparison(self):
@@ -467,6 +489,60 @@
)
+ def test_insertMultiReturnOracle(self):
+ """
+ In Oracle's SQL dialect, the 'returning' clause requires an 'into'
+ clause indicating where to put the results, as they can't be simply
+ relayed to the cursor. Further, additional bound variables are required
+ to capture the output parameters.
+ """
+ self.assertEquals(
+ Insert({self.schema.FOO.BAR: 40,
+ self.schema.FOO.BAZ: 50},
+ Return=(self.schema.FOO.BAR, self.schema.FOO.BAZ)).toSQL(
+ NumericPlaceholder(ORACLE_DIALECT)
+ ),
+ SQLFragment(
+ "insert into FOO (BAR, BAZ) values (:1, :2) returning BAR, BAZ"
+ " into :3, :4",
+ [40, 50, Parameter("oracle_out_0"), Parameter("oracle_out_1")]
+ )
+ )
+
+
+ def test_insertMultiReturnOnOracleTxn(self):
+ """
+ As described in L{test_insertMultiReturnOracle}, Oracle deals with
+ 'returning' clauses by using out parameters. However, this is not quite
+ enough, as the code needs to actually retrieve the values from the out
+ parameters.
+ """
+ class FakeCXOracleModule(object):
+ NUMBER = 'the NUMBER type'
+ STRING = 'a string type (for varchars)'
+ CLOB = 'the clob type. (for text)'
+ TIMESTAMP = 'for timestamps!'
+ self.patch(syntax, 'cx_Oracle', FakeCXOracleModule)
+ factory = ConnectionFactory()
+ pool = ConnectionPool(factory.connect, maxConnections=2,
+ dialect=ORACLE_DIALECT,
+ paramstyle='numeric')
+ self.paused = False
+ pool._createHolder = lambda : FakeThreadHolder(self)
+ pool.startService()
+ conn = pool.connection()
+ i = Insert({self.schema.FOO.BAR: 40,
+ self.schema.FOO.BAZ: 50},
+ Return=(self.schema.FOO.BAR, self.schema.FOO.BAZ))
+ # See fake result generation in test_adbapi2.py.
+ result = resultOf(i.on(conn))
+ self.assertEquals(result, [[[300, 301]]])
+ curvars = factory.connections[0].cursors[0].variables
+ self.assertEquals(len(curvars), 2)
+ self.assertEquals(curvars[0].type, FakeCXOracleModule.NUMBER)
+ self.assertEquals(curvars[1].type, FakeCXOracleModule.STRING)
+
+
def test_insertMismatch(self):
"""
L{Insert} raises L{TableMismatch} if the columns specified aren't all
@@ -480,6 +556,31 @@
)
+ def test_quotingOnKeywordConflict(self):
+ """
+ 'access' is a keyword, so although our schema parser will leniently
+ accept it, it must be quoted in any outgoing SQL. (This is only done in
+ the Oracle dialect, because it isn't necessary in postgres, and
+ idiosyncratic case-folding rules make it challenging to do it in both.)
+ """
+ self.assertEquals(
+ Insert({self.schema.LEVELS.ACCESS: 1,
+ self.schema.LEVELS.USERNAME:
+ "hi"}).toSQL(FixedPlaceholder(ORACLE_DIALECT, "?")),
+ SQLFragment(
+ 'insert into LEVELS ("ACCESS", USERNAME) values (?, ?)',
+ [1, "hi"])
+ )
+ self.assertEquals(
+ Insert({self.schema.LEVELS.ACCESS: 1,
+ self.schema.LEVELS.USERNAME:
+ "hi"}).toSQL(FixedPlaceholder(POSTGRES_DIALECT, "?")),
+ SQLFragment(
+ 'insert into LEVELS (ACCESS, USERNAME) values (?, ?)',
+ [1, "hi"])
+ )
+
+
def test_updateReturning(self):
"""
L{update}'s C{Return} argument will update an SQL 'returning' clause.
@@ -513,10 +614,11 @@
L{Update} values may be L{FunctionInvocation}s, to update to computed
values in the database.
"""
+ sqlfunc = Function("hello")
self.assertEquals(
Update(
{self.schema.FOO.BAR: 23,
- self.schema.FOO.BAZ: FunctionInvocation("hello")},
+ self.schema.FOO.BAZ: sqlfunc()},
Where=self.schema.FOO.BAZ == 9
).toSQL(),
SQLFragment("update FOO set BAR = ?, BAZ = hello() "
@@ -529,10 +631,11 @@
L{Update} values may be L{FunctionInvocation}s, to update to computed
values in the database.
"""
+ sqlfunc = Function("hello")
self.assertEquals(
Insert(
{self.schema.FOO.BAR: 23,
- self.schema.FOO.BAZ: FunctionInvocation("hello")},
+ self.schema.FOO.BAZ: sqlfunc()},
).toSQL(),
SQLFragment("insert into FOO (BAR, BAZ) "
"values (?, hello())", [23])
@@ -636,6 +739,19 @@
)
+ def test_distinct(self):
+ """
+ A L{Select} object with a 'Disinct' keyword parameter with a value of
+ C{True} will generate a SQL statement with a 'distinct' keyword
+ preceding its list of columns.
+ """
+ self.assertEquals(
+ Select([self.schema.FOO.BAR], From=self.schema.FOO,
+ Distinct=True).toSQL(),
+ SQLFragment("select distinct BAR from FOO")
+ )
+
+
def test_nextSequenceValue(self):
"""
When a sequence is used as a value in an expression, it renders as the
@@ -646,10 +762,77 @@
self.schema.A_SEQ}).toSQL(),
SQLFragment("insert into BOZ (QUX) values (nextval('A_SEQ'))", []))
+
+ def test_nextSequenceValueOracle(self):
+ """
+ When a sequence is used as a value in an expression in the Oracle
+ dialect, it renders as the 'nextval' attribute of the appropriate
+ sequence.
+ """
+ self.assertEquals(
+ Insert({self.schema.BOZ.QUX:
+ self.schema.A_SEQ}).toSQL(
+ FixedPlaceholder(ORACLE_DIALECT, "?")),
+ SQLFragment("insert into BOZ (QUX) values (A_SEQ.nextval)", []))
+
+
+ def test_nextSequenceDefaultImplicitExplicitOracle(self):
+ """
+ In Oracle's dialect, sequence defaults can't be implemented without
+ using triggers, so instead we just explicitly always include the
+ sequence default value.
+ """
+ addSQLToSchema(
+ schema=self.schema.model, schemaData=
+ "create table DFLTR (a varchar(255), "
+ "b integer default nextval('A_SEQ'));"
+ )
+ self.assertEquals(
+ Insert({self.schema.DFLTR.a: 'hello'}).toSQL(
+ FixedPlaceholder(ORACLE_DIALECT, "?")
+ ),
+ SQLFragment("insert into DFLTR (a, b) values "
+ "(?, A_SEQ.nextval)", ['hello']),
+ )
+ # Should be the same if it's explicitly specified.
+ self.assertEquals(
+ Insert({self.schema.DFLTR.a: 'hello',
+ self.schema.DFLTR.b: self.schema.A_SEQ}).toSQL(
+ FixedPlaceholder(ORACLE_DIALECT, "?")
+ ),
+ SQLFragment("insert into DFLTR (a, b) values "
+ "(?, A_SEQ.nextval)", ['hello']),
+ )
+
+
+ def test_numericParams(self):
+ """
+ An L{IAsyncTransaction} with the 'numeric' paramstyle attribute will
+ cause statements to be generated with parameters in the style of :1 :2
+ :3, as per the DB-API.
+ """
+ stmts = []
+ class FakeOracleTxn(object):
+ def execSQL(self, text, params, exc):
+ stmts.append((text, params))
+ dialect = ORACLE_DIALECT
+ paramstyle = 'numeric'
+ Select([self.schema.FOO.BAR],
+ From=self.schema.FOO,
+ Where=(self.schema.FOO.BAR == 7).And(
+ self.schema.FOO.BAZ == 9)
+ ).on(FakeOracleTxn())
+ self.assertEquals(
+ stmts, [("select BAR from FOO where BAR = :1 and BAZ = :2",
+ [7, 9])]
+ )
+
+
def test_nestedLogicalExpressions(self):
"""
- When a sequence is used as a value in an expression, it renders as the
- call to 'nextval' that will produce its next value.
+ Make sure that logical operator precedence inserts proper parenthesis
+ when needed. e.g. 'a.And(b.Or(c))' needs to be 'a and (b or c)' not 'a
+ and b or c'.
"""
self.assertEquals(
Select(
@@ -658,7 +841,8 @@
And(self.schema.FOO.BAZ != 8).
And((self.schema.FOO.BAR == 8).Or(self.schema.FOO.BAZ == 0))
).toSQL(),
- SQLFragment("select * from FOO where BAR != ? and BAZ != ? and (BAR = ? or BAZ = ?)", [7, 8, 8, 0]))
+ SQLFragment("select * from FOO where BAR != ? and BAZ != ? and "
+ "(BAR = ? or BAZ = ?)", [7, 8, 8, 0]))
self.assertEquals(
Select(
@@ -667,7 +851,8 @@
Or(self.schema.FOO.BAZ != 8).
Or((self.schema.FOO.BAR == 8).And(self.schema.FOO.BAZ == 0))
).toSQL(),
- SQLFragment("select * from FOO where BAR != ? or BAZ != ? or BAR = ? and BAZ = ?", [7, 8, 8, 0]))
+ SQLFragment("select * from FOO where BAR != ? or BAZ != ? or "
+ "BAR = ? and BAZ = ?", [7, 8, 8, 0]))
self.assertEquals(
Select(
@@ -676,4 +861,67 @@
Or(self.schema.FOO.BAZ != 8).
And((self.schema.FOO.BAR == 8).Or(self.schema.FOO.BAZ == 0))
).toSQL(),
- SQLFragment("select * from FOO where (BAR != ? or BAZ != ?) and (BAR = ? or BAZ = ?)", [7, 8, 8, 0]))
+ SQLFragment("select * from FOO where (BAR != ? or BAZ != ?) and "
+ "(BAR = ? or BAZ = ?)", [7, 8, 8, 0]))
+
+
+ def test_updateWithNULL(self):
+ """
+ As per the DB-API specification, "SQL NULL values are represented by the
+ Python None singleton on input and output." When a C{None} is provided
+ as a value to an L{Update}, it will be relayed to the database as a
+ parameter.
+ """
+ self.assertEquals(
+ Update({self.schema.BOZ.QUX: None},
+ Where=self.schema.BOZ.QUX == 7).toSQL(),
+ SQLFragment("update BOZ set QUX = ? where QUX = ?", [None, 7])
+ )
+
+
+ def test_subSelectComparison(self):
+ """
+ A comparison of a column to a sub-select in a where clause will result
+ in a parenthetical 'Where' clause.
+ """
+ self.assertEquals(
+ Update(
+ {self.schema.BOZ.QUX: 9},
+ Where=self.schema.BOZ.QUX ==
+ Select([self.schema.FOO.BAR], From=self.schema.FOO,
+ Where=self.schema.FOO.BAZ == 12)).toSQL(),
+ SQLFragment(
+ # NOTE: it's very important that the comparison _always_ go in
+ # this order (column from the UPDATE first, inner SELECT second)
+ # as the other order will be considered a syntax error.
+ "update BOZ set QUX = ? where QUX = ("
+ "select BAR from FOO where BAZ = ?)", [9, 12]
+ )
+ )
+
+
+ def test_tupleComparison(self):
+ """
+ A L{Tuple} allows for simultaneous comparison of multiple values in a
+ C{Where} clause. This feature is particularly useful when issuing an
+ L{Update} or L{Delete}, where the comparison is with values from a
+ subselect. (A L{Tuple} will be automatically generated upon comparison
+ to a C{tuple} or C{list}.)
+ """
+ self.assertEquals(
+ Update(
+ {self.schema.BOZ.QUX: 1},
+ Where=(self.schema.BOZ.QUX, self.schema.BOZ.QUUX) ==
+ Select([self.schema.FOO.BAR, self.schema.FOO.BAZ],
+ From=self.schema.FOO,
+ Where=self.schema.FOO.BAZ == 2)).toSQL(),
+ SQLFragment(
+ # NOTE: it's very important that the comparison _always_ go in
+ # this order (tuple of columns from the UPDATE first, inner
+ # SELECT second) as the other order will be considered a syntax
+ # error.
+ "update BOZ set QUX = ? where (QUX, QUUX) = ("
+ "select BAR, BAZ from FOO where BAZ = ?)", [1, 2]
+ )
+ )
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/ienterprise.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/ienterprise.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/ienterprise.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -40,6 +40,11 @@
+POSTGRES_DIALECT = 'postgres-dialect'
+ORACLE_DIALECT = 'oracle-dialect'
+
+
+
class IAsyncTransaction(Interface):
"""
Asynchronous execution of SQL.
@@ -54,6 +59,13 @@
""")
+ dialect = Attribute(
+ """
+ A copy of the 'dialect' attribute from the connection pool. One of the
+ C{*_DIALECT} constants in this module, such as C{POSTGRES_DIALECT}.
+ """)
+
+
def execSQL(sql, args=(), raiseOnZeroRowCount=None):
"""
Execute some SQL.
@@ -91,3 +103,40 @@
rollback of this transaction.
"""
+
+
+class IDerivedParameter(Interface):
+ """
+ A parameter which needs to be derived from the underlying DB-API cursor;
+ implicitly, meaning that this must also interact with the actual thread
+ manipulating said cursor. If a provider of this interface is passed in the
+ C{args} argument to L{IAsyncTransaction.execSQL}, it will have its
+ C{prequery} and C{postquery} methods invoked on it before and after
+ executing the SQL query in question, respectively.
+ """
+
+ def preQuery(cursor):
+ """
+ Before running a query, invoke this method with the cursor that the
+ query will be run on.
+
+ (This can be used, for example, to allocate a special database-specific
+ variable based on the cursor, like an out parameter.)
+
+ @param cursor: the DB-API cursor.
+
+ @return: the concrete value which should be passed to the DB-API layer.
+ """
+
+
+ def postQuery(cursor):
+ """
+ After running a query, invoke this method in the DB-API thread.
+
+ (This can be used, for example, to manipulate any state created in the
+ preQuery method.)
+
+ @param cursor: the DB-API cursor.
+
+ @return: C{None}
+ """
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/test/test_adbapi2.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/test/test_adbapi2.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/test/test_adbapi2.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -75,8 +75,12 @@
class FakeConnection(Parent, Child):
"""
Fake Stand-in for DB-API 2.0 connection.
+
+ @ivar executions: the number of statements which have been executed.
"""
+ executions = 0
+
def __init__(self, factory):
"""
Initialize list of cursors
@@ -84,8 +88,17 @@
Parent.__init__(self)
Child.__init__(self, factory)
self.id = factory.idcounter.next()
+ self._executeFailQueue = []
+ def executeWillFail(self, thunk):
+ """
+ The next call to L{FakeCursor.execute} will fail with an exception
+ returned from the given callable.
+ """
+ self._executeFailQueue.append(thunk)
+
+
@property
def cursors(self):
"Alias to make tests more readable."
@@ -108,6 +121,7 @@
raise RollbackFail()
+
class RollbackFail(Exception):
"""
Sample rollback-failure exception.
@@ -131,6 +145,7 @@
self.rowcount = 0
# not entirely correct, but all we care about is its truth value.
self.description = False
+ self.variables = []
@property
@@ -140,12 +155,24 @@
def execute(self, sql, args=()):
+ self.connection.executions += 1
+ if self.connection._executeFailQueue:
+ raise self.connection._executeFailQueue.pop(0)()
self.sql = sql
self.description = True
self.rowcount = 1
return
+ def var(self, type, *args):
+ """
+ Return a database variable in the style of the cx_Oracle bindings.
+ """
+ v = FakeVariable(self, type, args)
+ self.variables.append(v)
+ return v
+
+
def fetchall(self):
"""
Just echo the SQL that was executed in the last query.
@@ -154,6 +181,18 @@
+class FakeVariable(object):
+ def __init__(self, cursor, type, args):
+ self.cursor = cursor
+ self.type = type
+ self.args = args
+
+
+ def getvalue(self):
+ return self.cursor.variables.index(self) + 300
+
+
+
class ConnectionFactory(Parent):
rollbackFail = False
@@ -162,7 +201,7 @@
def __init__(self):
Parent.__init__(self)
self.idcounter = count(1)
- self._resultQueue = []
+ self._connectResultQueue = []
self.defaultConnect()
@@ -177,8 +216,8 @@
Implement the C{ConnectionFactory} callable expected by
L{ConnectionPool}.
"""
- if self._resultQueue:
- thunk = self._resultQueue.pop(0)
+ if self._connectResultQueue:
+ thunk = self._connectResultQueue.pop(0)
else:
thunk = self._default
return thunk()
@@ -190,7 +229,7 @@
"""
def thunk():
return FakeConnection(self)
- self._resultQueue.append(thunk)
+ self._connectResultQueue.append(thunk)
def willFail(self):
@@ -199,7 +238,7 @@
"""
def thunk():
raise FakeConnectionError()
- self._resultQueue.append(thunk)
+ self._connectResultQueue.append(thunk)
def defaultConnect(self):
@@ -207,7 +246,7 @@
By default, connection attempts will succeed.
"""
self.willConnect()
- self._default = self._resultQueue.pop()
+ self._default = self._connectResultQueue.pop()
def defaultFail(self):
@@ -215,7 +254,7 @@
By default, connection attempts will fail.
"""
self.willFail()
- self._default = self._resultQueue.pop()
+ self._default = self._connectResultQueue.pop()
@@ -657,4 +696,154 @@
self.assertEquals(len(self.factory.connections), 2)
+ def test_propagateParamstyle(self):
+ """
+ Each different type of L{IAsyncTransaction} relays the C{paramstyle}
+ attribute from the L{ConnectionPool}.
+ """
+ TEST_PARAMSTYLE = "justtesting"
+ self.pool.paramstyle = TEST_PARAMSTYLE
+ normaltxn = self.pool.connection()
+ self.assertEquals(normaltxn.paramstyle, TEST_PARAMSTYLE)
+ self.pauseHolders()
+ extra = []
+ extra.append(self.pool.connection())
+ waitingtxn = self.pool.connection()
+ self.assertEquals(waitingtxn.paramstyle, TEST_PARAMSTYLE)
+ self.flushHolders()
+ self.pool.stopService()
+ notxn = self.pool.connection()
+ self.assertEquals(notxn.paramstyle, TEST_PARAMSTYLE)
+
+ def test_propagateDialect(self):
+ """
+ Each different type of L{IAsyncTransaction} relays the C{dialect}
+ attribute from the L{ConnectionPool}.
+ """
+ TEST_DIALECT = "otherdialect"
+ self.pool.dialect = TEST_DIALECT
+ normaltxn = self.pool.connection()
+ self.assertEquals(normaltxn.dialect, TEST_DIALECT)
+ self.pauseHolders()
+ extra = []
+ extra.append(self.pool.connection())
+ waitingtxn = self.pool.connection()
+ self.assertEquals(waitingtxn.dialect, TEST_DIALECT)
+ self.flushHolders()
+ self.pool.stopService()
+ notxn = self.pool.connection()
+ self.assertEquals(notxn.dialect, TEST_DIALECT)
+
+
+ def test_reConnectWhenFirstExecFails(self):
+ """
+ Generally speaking, DB-API 2.0 adapters do not provide information about
+ the cause of a failed 'execute' method; they definitely don't provide it
+ in a way which can be identified as related to the syntax of the query,
+ the state of the database itself, the state of the connection, etc.
+
+ Therefore the best general heuristic for whether the connection to the
+ database has been lost and needs to be re-established is to catch
+ exceptions which are raised by the I{first} statement executed in a
+ transaction.
+ """
+ # Allow 'connect' to succeed. This should behave basically the same
+ # whether connect() happened to succeed in some previous transaction and
+ # it's recycling the underlying transaction, or connect() just
+ # succeeded. Either way you just have a _SingleTxn wrapping a
+ # _ConnectedTxn.
+ txn = self.pool.connection()
+ self.assertEquals(len(self.factory.connections), 1,
+ "Sanity check failed.")
+ self.factory.connections[0].executeWillFail(RuntimeError)
+ results = resultOf(txn.execSQL("hello, world!"))
+ [[[counter, echo]]] = results
+ self.assertEquals("hello, world!", echo)
+ # Two execution attempts should have been made, one on each connection.
+ # The first failed with a RuntimeError, but that is deliberately
+ # obscured, because then we tried again and it succeeded.
+ self.assertEquals(len(self.factory.connections), 2,
+ "No new connection opened.")
+ self.assertEquals(self.factory.connections[0].executions, 1)
+ self.assertEquals(self.factory.connections[1].executions, 1)
+ self.assertEquals(self.factory.connections[0].closed, True)
+ self.assertEquals(self.factory.connections[1].closed, False)
+
+
+ def test_reConnectWhenSecondExecFailsThenFirstExecFails(self):
+ """
+ Other connection-oriented errors might raise exceptions if they occur in
+ the middle of a transaction, but that should cause the error to be
+ caught, the transaction to be aborted, and the (closed) connection to be
+ recycled, where the next transaction that attempts to do anything with
+ it will encounter the error immediately and discover it needs to be
+ recycled.
+
+ It would be better if this behavior were invisible, but that could only
+ be accomplished with more precise database exceptions. We may come up
+ with support in the future for more precisely identifying exceptions,
+ but I{unknown} exceptions should continue to be treated in this manner,
+ relaying the exception back to application code but attempting a
+ re-connection on the next try.
+ """
+ txn = self.pool.connection()
+ [[[counter, echo]]] = resultOf(txn.execSQL("hello, world!", []))
+ self.factory.connections[0].executeWillFail(ZeroDivisionError)
+ [f] = resultOf(txn.execSQL("divide by zero", []))
+ f.trap(ZeroDivisionError)
+ self.assertEquals(self.factory.connections[0].executions, 2)
+ # Reconnection should work exactly as before.
+ self.assertEquals(self.factory.connections[0].closed, False)
+ # Application code has to roll back its transaction at this point, since
+ # it failed (and we don't necessarily know why it failed: not enough
+ # information).
+ txn.abort()
+ self.factory.connections[0].executions = 0 # re-set for next test
+ self.assertEquals(len(self.factory.connections), 1)
+ self.test_reConnectWhenFirstExecFails()
+
+
+ def test_disconnectOnFailedRollback(self):
+ """
+ When C{rollback} fails for any reason on a connection object, then we
+ don't know what state it's in. Most likely, it's already been
+ disconnected, so the connection should be closed and the transaction
+ de-pooled instead of recycled.
+
+ Also, a new connection will immediately be established to keep the pool
+ size the same.
+ """
+ txn = self.pool.connection()
+ self.factory.rollbackFail = True
+ [x] = resultOf(txn.abort())
+ # Abort does not propagate the error on, the transaction merely gets
+ # disposed of.
+ self.assertIdentical(x, None)
+ self.assertEquals(len(self.pool._free), 1)
+ self.assertNotIn(txn._baseTxn, self.pool._free)
+ self.assertEquals(self.pool._finishing, [])
+ self.assertEquals(len(self.factory.connections), 2)
+ self.assertEquals(self.factory.connections[0].closed, True)
+ self.assertEquals(self.factory.connections[1].closed, False)
+ self.assertEquals(len(self.flushLoggedErrors(RollbackFail)), 1)
+
+
+ def test_exceptionPropagatesFailedCommit(self):
+ """
+ A failed C{rollback} is fine (the premature death of the connection
+ without C{commit} means that the changes are surely gone), but a failed
+ C{commit} has to be relayed to client code, since that actually means
+ some changes didn't hit the database.
+ """
+ txn = self.pool.connection()
+ self.factory.commitFail = True
+ [x] = resultOf(txn.commit())
+ x.trap(CommitFail)
+ self.assertEquals(len(self.pool._free), 1)
+ self.assertNotIn(txn._baseTxn, self.pool._free)
+ self.assertEquals(self.pool._finishing, [])
+ self.assertEquals(len(self.factory.connections), 2)
+ self.assertEquals(self.factory.connections[0].closed, True)
+ self.assertEquals(self.factory.connections[1].closed, False)
+
Copied: CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/util.py (from rev 7204, CalendarServer/trunk/twext/enterprise/util.py)
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/util.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/enterprise/util.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -0,0 +1,61 @@
+
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Utilities for dealing with different databases.
+"""
+
+from datetime import datetime
+from twistedcaldav.dateops import SQL_TIMESTAMP_FORMAT
+
+def mapOracleOutputType(column):
+ """
+ Map a single output value from cx_Oracle based on some rules and
+ expectations that we have based on the pgdb bindings.
+
+ @param column: a single value from a column.
+
+ @return: a converted value based on the type of the input; oracle CLOBs and
+ datetime timestamps will be converted to strings, all other types will
+ be left alone.
+ """
+ if hasattr(column, 'read'):
+ # Try to detect large objects and format convert them to
+ # strings on the fly. We need to do this as we read each
+ # row, due to the issue described here -
+ # http://cx-oracle.sourceforge.net/html/lob.html - in
+ # particular, the part where it says "In particular, do not
+ # use the fetchall() method".
+ return column.read()
+ elif isinstance(column, datetime):
+ # cx_Oracle properly maps the type of timestamps to datetime
+ # objects. However, our code is mostly written against
+ # PyGreSQL, which just emits strings as results and expects
+ # to have to convert them itself.. Since it's easier to
+ # just detect the datetimes and stringify them, for now
+ # we'll do that.
+ return column.strftime(SQL_TIMESTAMP_FORMAT)
+ elif isinstance(column, float):
+ if int(column) == column:
+ return int(column)
+ else:
+ return column
+ else:
+ return column
+
+
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/element/base.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/element/base.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/element/base.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -483,6 +483,13 @@
PCDATAElement: (0, None),
}
+ @classmethod
+ def fromQname(cls, namespace, name):
+ child = cls()
+ child.namespace = namespace
+ child.name = name
+ return child
+
def qname(self):
return (self.namespace, self.name)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/http.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/http.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twext/web2/dav/http.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -79,8 +79,9 @@
error.namespace = xml_namespace
error.name = xml_name
- if description:
- output = davxml.Error(error, davxml.ErrorDescription(description)).toxml()
+ self.description = description
+ if self.description:
+ output = davxml.Error(error, davxml.ErrorDescription(self.description)).toxml()
else:
output = davxml.Error(error).toxml()
@@ -337,7 +338,7 @@
def messageForFailure(failure):
if failure.check(HTTPError):
if isinstance(failure.value.response, ErrorResponse):
- return None
- if isinstance(failure.value.response, StatusResponse):
return failure.value.response.description
+ elif isinstance(failure.value.response, StatusResponse):
+ return failure.value.response.description
return str(failure)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/caldavxml.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/caldavxml.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/caldavxml.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -334,6 +334,8 @@
else:
raise ValueError("Not a calendar: %s" % (calendar,))
+ fromTextData = fromCalendar
+
def __init__(self, *children, **attributes):
super(CalendarData, self).__init__(*children, **attributes)
@@ -427,6 +429,8 @@
else:
return None
+ generateComponent = calendar
+
def calendarData(self):
"""
Returns the calendar data derived from this element.
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/carddavxml.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/carddavxml.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/carddavxml.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -200,6 +200,8 @@
"""
return clazz(davxml.PCDATAElement(addressdata))
+ fromTextData = fromAddressData
+
def __init__(self, *children, **attributes):
super(AddressData, self).__init__(*children, **attributes)
@@ -280,6 +282,8 @@
else:
return None
+ generateComponent = address
+
def addressData(self):
"""
Returns an address component derived from this element.
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/customxml.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/customxml.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/customxml.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -992,7 +992,83 @@
namespace = calendarserver_namespace
name = "link"
+mm_namespace = "http://me.com/_namespace/"
+class Multiput (davxml.WebDAVElement):
+ namespace = mm_namespace
+ name = "multiput"
+
+ allowed_children = {
+ (mm_namespace, "resource") : (1, None),
+ }
+
+class Resource (davxml.WebDAVElement):
+ namespace = mm_namespace
+ name = "resource"
+
+ allowed_children = {
+ (davxml, "href") : (0, 1),
+ (mm_namespace, "if-match") : (0, 1),
+ (davxml, "set") : (0, 1),
+ (davxml, "remove") : (0, 1),
+ (mm_namespace, "delete") : (0, 1),
+ }
+
+class IfMatch (davxml.WebDAVElement):
+ namespace = mm_namespace
+ name = "if-match"
+
+ allowed_children = {
+ (davxml, "getetag") : (1, 1),
+ }
+
+class Delete (davxml.WebDAVEmptyElement):
+ namespace = mm_namespace
+ name = "delete"
+
+
+class BulkRequests (davxml.WebDAVElement):
+ namespace = mm_namespace
+ name = "bulk-requests"
+ hidden = True
+ protected = True
+
+ allowed_children = {
+ (mm_namespace, "simple") : (0, 1),
+ (mm_namespace, "crud") : (0, 1),
+ }
+
+class Simple (davxml.WebDAVElement):
+ namespace = mm_namespace
+ name = "simple"
+ hidden = True
+ protected = True
+
+ allowed_children = {
+ (mm_namespace, "max-resources") : (1, 1),
+ (mm_namespace, "max-bytes") : (1, 1),
+ }
+
+class CRUD (davxml.WebDAVElement):
+ namespace = mm_namespace
+ name = "crud"
+ hidden = True
+ protected = True
+
+ allowed_children = {
+ (mm_namespace, "max-resources") : (1, 1),
+ (mm_namespace, "max-bytes") : (1, 1),
+ }
+
+class MaxBulkResources (davxml.WebDAVTextElement):
+ namespace = mm_namespace
+ name = "max-resources"
+
+class MaxBulkBytes (davxml.WebDAVTextElement):
+ namespace = mm_namespace
+ name = "max-bytes"
+
+
##
# Extensions to davxml.ResourceType
##
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dateops.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dateops.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dateops.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -243,12 +243,13 @@
tzinfo=utc
)
+SQL_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
+
def parseSQLTimestamp(ts):
-
# Handle case where fraction seconds may not be present
if len(ts) < 20:
ts += ".0"
- return datetime.datetime.strptime(ts, "%Y-%m-%d %H:%M:%S.%f")
+ return datetime.datetime.strptime(ts, SQL_TIMESTAMP_FORMAT)
def parseSQLTimestampToPyCalendar(ts):
"""
@@ -265,8 +266,8 @@
def datetimeMktime(dt):
assert isinstance(dt, datetime.date)
-
+
if dt.tzinfo is None:
dt.replace(tzinfo=utc)
return calendar.timegm(dt.utctimetuple())
-
\ No newline at end of file
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/opendirectorybacker.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/opendirectorybacker.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/opendirectorybacker.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -44,7 +44,7 @@
from twext.web2.dav.element.base import twisted_dav_namespace, dav_namespace, parse_date, twisted_private_namespace
from twext.web2.dav.resource import DAVPropertyMixIn
from twext.web2.dav.util import joinURL
-from twext.web2.http_headers import MimeType, generateContentType
+from twext.web2.http_headers import MimeType, generateContentType, ETag
from twistedcaldav import customxml, carddavxml
@@ -1908,7 +1908,7 @@
#print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "getetag":
- result = davxml.GETETag( hashlib.md5(self.vCardText()).hexdigest() )
+ result = davxml.GETETag( ETag(hashlib.md5(self.vCardText()).hexdigest()).generate() )
#print("VCardResource.readProperty: qname = %s, result = %s" % (qname, result))
return result
elif name == "getcontenttype":
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/principal.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/principal.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -992,7 +992,8 @@
return self
if config.EnableProxyPrincipals and name in ("calendar-proxy-read", "calendar-proxy-write"):
- return CalendarUserProxyPrincipalResource(self, name)
+ # name is required to be str
+ return CalendarUserProxyPrincipalResource(self, str(name))
else:
return None
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/accounts.xml
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/accounts.xml 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/accounts.xml 2011-03-17 17:54:34 UTC (rev 7206)
@@ -95,6 +95,14 @@
<name>a adbkonly</name>
<email-address>a-adbkonly at example.com</email-address>
</user>
+ <user>
+ <uid>nonascii</uid>
+ <uid>nonascii佐藤</uid>
+ <guid>320B73A1-46E2-4180-9563-782DFDBE1F63</guid>
+ <password>a</password>
+ <name>佐藤佐藤佐藤</name>
+ <email-address>nonascii at example.com</email-address>
+ </user>
<user repeat="2">
<uid>user%02d</uid>
<guid>user%02d</guid>
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_principal.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_principal.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_principal.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -42,8 +42,10 @@
import twistedcaldav.test.util
from txdav.common.datastore.file import CommonDataStore
+from urllib import quote
+
class ProvisionedPrincipals (twistedcaldav.test.util.TestCase):
"""
Directory service provisioned principals.
@@ -115,14 +117,19 @@
self.assertEquals(set((provisioningURL,)), set(pc.principalCollectionURL() for pc in principalCollections))
shortNames = set((yield typeResource.listChildren()))
- self.assertEquals(shortNames, set(r.shortNames[0] for r in directory.listRecords(recordType)))
+ # Handle records with mulitple shortNames
+ expected = []
+ for r in directory.listRecords(recordType):
+ expected.extend(r.shortNames)
+ self.assertEquals(shortNames, set(expected))
for shortName in shortNames:
#print " -> %s" % (shortName,)
recordResource = typeResource.getChild(shortName)
self.failUnless(isinstance(recordResource, DirectoryPrincipalResource))
- recordURL = typeURL + shortName + "/"
+ # shortName may be non-ascii
+ recordURL = typeURL + quote(shortName) + "/"
self.assertIn(recordURL, (recordResource.principalURL(),) + tuple(recordResource.alternateURIs()))
principalCollections = recordResource.principalCollections()
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_proxyprincipalmembers.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/test/test_proxyprincipalmembers.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -547,3 +547,25 @@
yield self._clearProxy(principal, proxyType)
yield self._clearProxy(fakePrincipal, proxyType)
+
+
+ @inlineCallbacks
+ def test_NonAsciiProxy(self):
+ """
+ Ensure that principalURLs with non-ascii don't cause problems
+ within CalendarUserProxyPrincipalResource
+ """
+
+ recordType = DirectoryService.recordType_users
+ proxyType = "calendar-proxy-read"
+
+ record = self.directoryService.recordWithGUID("320B73A1-46E2-4180-9563-782DFDBE1F63")
+ provisioningResource = self.principalRootResources[self.directoryService.__class__.__name__]
+ principal = provisioningResource.principalForRecord(record)
+ proxyPrincipal = provisioningResource.principalForShortName(recordType,
+ "wsanchez")
+
+ yield self._addProxy(principal, proxyType, proxyPrincipal)
+ memberships = yield proxyPrincipal._calendar_user_proxy_index().getMemberships(proxyPrincipal.principalUID())
+ for uid in memberships:
+ subPrincipal = provisioningResource.principalForUID(uid)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/wiki.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/wiki.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/wiki.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -28,6 +28,7 @@
from twisted.web.xmlrpc import Proxy, Fault
from twext.web2.http import HTTPError, StatusResponse
from twext.web2.auth.wrapper import UnauthorizedResponse
+from twext.web2 import responsecode
from twext.python.log import Logger
@@ -240,7 +241,7 @@
raise HTTPError(
StatusResponse(
- 403,
+ responsecode.FORBIDDEN,
"You are not allowed to access this wiki"
)
)
@@ -252,8 +253,15 @@
wikiID, fault))
if fault.faultCode == 2: # non-existent user
- raise HTTPError(StatusResponse(403, fault.faultString))
+ raise HTTPError(StatusResponse(responsecode.FORBIDDEN, fault.faultString))
else: # fault.faultCode == 12, non-existent wiki
- raise HTTPError(StatusResponse(404, fault.faultString))
+ raise HTTPError(StatusResponse(responsecode.NOT_FOUND, fault.faultString))
+ except HTTPError:
+ # pass through the HTTPError we might have raised above
+ raise
+
+ except Exception, e:
+ log.error("Wiki ACL RPC failed: %s" % (e,))
+ raise HTTPError(StatusResponse(responsecode.SERVICE_UNAVAILABLE, "Wiki ACL RPC failed"))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/xmlfile.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/xmlfile.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/directory/xmlfile.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -358,7 +358,7 @@
element = addSubElement(parent, xmlType)
for value in principal.shortNames:
- addSubElement(element, "uid", text=value)
+ addSubElement(element, "uid", text=value.decode("utf-8"))
addSubElement(element, "guid", text=principal.guid)
if principal.fullName is not None:
addSubElement(element, "name", text=principal.fullName.decode("utf-8"))
@@ -367,11 +367,11 @@
if principal.lastName is not None:
addSubElement(element, "last-name", text=principal.lastName.decode("utf-8"))
for value in principal.emailAddresses:
- addSubElement(element, "email-address", text=value)
+ addSubElement(element, "email-address", text=value.decode("utf-8"))
if principal.extras:
extrasElement = addSubElement(element, "extras")
for key, value in principal.extras.iteritems():
- addSubElement(extrasElement, key, text=value)
+ addSubElement(extrasElement, key, text=value.decode("utf-8"))
return element
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dropbox.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dropbox.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/dropbox.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -119,5 +119,6 @@
def http_PUT(self, request):
return ErrorResponse(
responsecode.FORBIDDEN,
- (calendarserver_namespace, "valid-drop-box")
+ (calendarserver_namespace, "valid-drop-box"),
+ "Cannot store resources in dropbox",
)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/freebusyurl.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/freebusyurl.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/freebusyurl.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -159,7 +159,8 @@
if self.token or self.user:
raise HTTPError(ErrorResponse(
responsecode.NOT_ACCEPTABLE,
- (calendarserver_namespace, "supported-query-parameter")
+ (calendarserver_namespace, "supported-query-parameter"),
+ "Invalid query parameter",
))
# Check format
@@ -168,7 +169,8 @@
if self.format not in ("text/calendar", "text/plain"):
raise HTTPError(ErrorResponse(
responsecode.NOT_ACCEPTABLE,
- (calendarserver_namespace, "supported-format")
+ (calendarserver_namespace, "supported-format"),
+ "Invalid return format requested",
))
else:
self.format = "text/calendar"
@@ -188,7 +190,8 @@
except ValueError:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-query-parameters")
+ (calendarserver_namespace, "valid-query-parameters"),
+ "Invalid query parameters",
))
# Sanity check start/end/duration
@@ -197,14 +200,16 @@
if self.end and self.duration:
raise HTTPError(ErrorResponse(
responsecode.NOT_ACCEPTABLE,
- (calendarserver_namespace, "valid-query-parameters")
+ (calendarserver_namespace, "valid-query-parameters"),
+ "Invalid query parameters",
))
# Duration must be positive
if self.duration and self.duration.getTotalSeconds() < 0:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-query-parameters")
+ (calendarserver_namespace, "valid-query-parameters"),
+ "Invalid query parameters",
))
# Now fill in the missing pieces
@@ -220,7 +225,8 @@
if self.end <= self.start:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-query-parameters")
+ (calendarserver_namespace, "valid-query-parameters"),
+ "Invalid query parameters",
))
# TODO: We should probably verify that the actual time-range is within sensible bounds (e.g. not too far in the past or future and not too long)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/ical.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/ical.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/ical.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -294,6 +294,37 @@
extraRestrictedProperties = ("SUMMARY", "LOCATION",)
@classmethod
+ def allFromString(clazz, string):
+ """
+ Construct a L{Component} from a string.
+ @param string: a string containing iCalendar data.
+ @return: a C{list} of L{Component}s representing the components described by
+ C{string}.
+ """
+ if type(string) is unicode:
+ string = string.encode("utf-8")
+ return clazz.allFromStream(StringIO.StringIO(string))
+
+ @classmethod
+ def allFromStream(clazz, stream):
+ """
+ Construct possibly multiple L{Component}s from a stream.
+ @param stream: a C{read()}able stream containing iCalendar data.
+ @return: a C{list} of L{Component}s representing the components described by
+ C{stream}.
+ """
+
+ results = []
+ try:
+ for vobject in readComponents(stream):
+ results.append(clazz(None, vobject=vobject))
+ return results
+ except vParseError, e:
+ raise InvalidICalendarDataError(e)
+ except StopIteration, e:
+ raise InvalidICalendarDataError(e)
+
+ @classmethod
def fromString(clazz, string):
"""
Construct a L{Component} from a string.
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/mail.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/mail.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/mail.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -1041,7 +1041,7 @@
else:
if inviteState == "new":
details['inviteLabel'] = _("Event Invitation")
- if inviteState == "update":
+ elif inviteState == "update":
details['inviteLabel'] = _("Event Update")
else:
details['inviteLabel'] = _("Event Reply")
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -90,7 +90,8 @@
log.err("Attempt to copy a calendar collection into another calendar collection %s" % destination)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "calendar-collection-location-ok")
+ (caldav_namespace, "calendar-collection-location-ok"),
+ "Cannot copy calendar collection inside another calendar collection",
))
# We also do not allow regular collections in calendar collections
@@ -173,7 +174,8 @@
log.err("Attempt to move a calendar collection into another calendar collection %s" % destination)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "calendar-collection-location-ok")
+ (caldav_namespace, "calendar-collection-location-ok"),
+ "Cannot move calendar collection inside another calendar collection",
))
# We also do not allow regular collections in calendar collections
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove_contact.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove_contact.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/copymove_contact.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -83,7 +83,8 @@
log.err("Attempt to copy an addressbook collection into another addressbook collection %s" % destination)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "addressbook-collection-location-ok")
+ (carddav_namespace, "addressbook-collection-location-ok"),
+ "Cannot copy address book collection inside another address book collection",
))
# We also do not allow regular collections in addressbook collections
@@ -153,7 +154,8 @@
log.err("Attempt to move an addressbook collection into another addressbook collection %s" % destination)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "addressbook-collection-location-ok")
+ (carddav_namespace, "addressbook-collection-location-ok"),
+ "Cannot move address book collection inside another address book collection",
))
# We also do not allow regular collections in addressbook collections
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/get.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/get.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/get.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -48,7 +48,8 @@
if len(action) != 1:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-action")
+ (calendarserver_namespace, "valid-action"),
+ "Invalid action parameter: %s" % (action,),
))
action = action[0]
@@ -59,7 +60,8 @@
if dispatch is None:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "supported-action")
+ (calendarserver_namespace, "supported-action"),
+ "Action not supported: %s" % (action,),
))
response = (yield dispatch(request))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcalendar.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcalendar.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcalendar.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -53,7 +53,8 @@
log.err("Attempt to create collection where resource exists: %s" % (self,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (davxml.dav_namespace, "resource-must-be-null")
+ (davxml.dav_namespace, "resource-must-be-null"),
+ "Resource already exists",
))
if not parent.isCollection():
@@ -61,7 +62,8 @@
% (self,))
raise HTTPError(ErrorResponse(
responsecode.CONFLICT,
- (caldavxml.caldav_namespace, "calendar-collection-location-ok")
+ (caldavxml.caldav_namespace, "calendar-collection-location-ok"),
+ "Cannot create calendar inside another calendar",
))
#
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcol.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcol.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/mkcol.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -58,7 +58,8 @@
% (self,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (davxml.dav_namespace, "resource-must-be-null")
+ (davxml.dav_namespace, "resource-must-be-null"),
+ "Resource already exists",
))
if not parent.isCollection():
@@ -66,7 +67,8 @@
% (self,))
raise HTTPError(ErrorResponse(
responsecode.CONFLICT,
- (davxml.dav_namespace, "collection-location-ok")
+ (davxml.dav_namespace, "collection-location-ok"),
+ "Cannot create calendar inside another calendar",
))
#
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/post.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/post.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/post.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -77,7 +77,8 @@
self.log_error("MIME type %s not allowed in calendar collection" % (content_type,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "supported-calendar-data")
+ (caldav_namespace, "supported-calendar-data"),
+ "Wrong MIME type for calendar collection",
))
# Read the calendar component from the stream
@@ -133,7 +134,8 @@
self.log_error("MIME type %s not allowed in address book collection" % (content_type,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "supported-address-data")
+ (carddav_namespace, "supported-address-data"),
+ "Wrong MIME type for address book collection",
))
# Read the calendar component from the stream
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -54,7 +54,8 @@
log.err("MIME type %s not allowed in calendar collection" % (content_type,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "supported-calendar-data")
+ (caldav_namespace, "supported-calendar-data"),
+ "Invalid MIME type for calendar collection",
))
# Read the calendar component from the stream
@@ -96,7 +97,8 @@
log.err("MIME type %s not allowed in address book collection" % (content_type,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "supported-address-data")
+ (carddav_namespace, "supported-address-data"),
+ "Invalid MIME type for address book collection",
))
# Read the vcard component from the stream
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_addressbook_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_addressbook_common.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_addressbook_common.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -23,7 +23,6 @@
import types
from twisted.internet import reactor
-from twisted.python.failure import Failure
from txdav.common.icommondatastore import ReservationError
@@ -167,7 +166,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- customxml.MaxResources()
+ customxml.MaxResources(),
+ message,
))
if not self.sourceadbk:
@@ -178,7 +178,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "supported-address-data")
+ (carddav_namespace, "supported-address-data"),
+ "Invalid content-type",
))
# At this point we need the calendar data to do more tests
@@ -192,7 +193,8 @@
log.err(str(e))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "valid-address-data")
+ (carddav_namespace, "valid-address-data"),
+ "Could not parse vCard",
))
# Valid vcard data for CalDAV check
@@ -201,7 +203,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "valid-addressbook-object-resource")
+ (carddav_namespace, "valid-addressbook-object-resource"),
+ "Invalid vCard data",
))
# Must have a valid UID at this point
@@ -214,7 +217,8 @@
log.err("Source vcard does not have a UID: %s" % self.source.name())
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "valid-addressbook-object-resource")
+ (carddav_namespace, "valid-addressbook-object-resource"),
+ "Missing UID in vCard",
))
# FIXME: We need this here because we have to re-index the destination. Ideally it
@@ -227,7 +231,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "max-resource-size")
+ (carddav_namespace, "max-resource-size"),
+ "Address data too large",
))
# Check access
@@ -433,7 +438,8 @@
rname.encode("utf-8")
)
)
- )
+ ),
+ "UID already used in another resource",
))
# Do the actual put or copy
@@ -445,20 +451,8 @@
returnValue(response)
except Exception, err:
- # Preserve the real traceback to display later, since the error-
- # handling here yields out of the generator and thereby shreds the
- # stack.
- f = Failure()
if reservation:
yield reservation.unreserve()
-
- # FIXME: transaction needs to be rolled back.
- # Display the traceback. Unfortunately this will usually be
- # duplicated by the higher-level exception handler that captures
- # the thing that raises here, but it's better than losing the
- # information.
- f.printTraceback()
-
raise err
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_common.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/put_common.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -28,7 +28,6 @@
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from twisted.internet.defer import returnValue
-from twisted.python.failure import Failure
from twisted.python import hashlib
from twext.web2.dav.util import joinURL, parentForURL
@@ -215,7 +214,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- customxml.MaxResources()
+ customxml.MaxResources(),
+ "Too many resources in collection",
))
# Valid data sizes - do before parsing the data
@@ -226,7 +226,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "max-resource-size")
+ (caldav_namespace, "max-resource-size"),
+ "Calendar data too large",
))
else:
# Valid calendar data size check
@@ -235,7 +236,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "max-resource-size")
+ (caldav_namespace, "max-resource-size"),
+ "Calendar data too large",
))
if not self.sourcecal:
@@ -246,7 +248,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "supported-calendar-data")
+ (caldav_namespace, "supported-calendar-data"),
+ "Invalid content-type for data",
))
# At this point we need the calendar data to do more tests
@@ -288,7 +291,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-object-resource")
+ (caldav_namespace, "valid-calendar-object-resource"),
+ "Invalid calendar data",
))
# Valid attendee list size check
@@ -298,7 +302,8 @@
raise HTTPError(
ErrorResponse(
responsecode.FORBIDDEN,
- MaxAttendeesPerInstance.fromString(str(config.MaxAttendeesPerInstance))
+ MaxAttendeesPerInstance.fromString(str(config.MaxAttendeesPerInstance)),
+ "Too many attendees in calenbdar data",
)
)
@@ -316,7 +321,8 @@
log.err("Source calendar does not have a UID: %s" % self.source)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-object-resource")
+ (caldav_namespace, "valid-calendar-object-resource"),
+ "Missing UID in calendar data",
))
# FIXME: We need this here because we have to re-index the destination. Ideally it
@@ -526,7 +532,8 @@
if self.access is None:
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (calendarserver_namespace, "valid-access-restriction")
+ (calendarserver_namespace, "valid-access-restriction"),
+ "Private event access level not allowed",
))
# Only DAV:owner is able to set the property to other than PUBLIC
@@ -537,7 +544,8 @@
if davxml.Principal(parent_owner) != authz and self.access != Component.ACCESS_PUBLIC:
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (calendarserver_namespace, "valid-access-restriction-change")
+ (calendarserver_namespace, "valid-access-restriction-change"),
+ "Private event access level change not allowed",
))
return None
@@ -561,11 +569,11 @@
try:
result = self.calendar.truncateRecurrence(config.MaxInstancesForRRULE)
except (ValueError, TypeError), ex:
- msg = "Cannot truncate calendar resource: %s" % (ex,)
- log.err(msg)
+ log.err("Cannot truncate calendar resource: %s" % (ex,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-data"), description=msg
+ (caldav_namespace, "valid-calendar-data"),
+ "Cannot truncate recurrences",
))
if result:
self.calendardata = str(self.calendar)
@@ -795,11 +803,11 @@
try:
self.calendar = PerUserDataFilter(accessUID).merge(self.calendar.duplicate(), oldCal)
except ValueError:
- msg = "Invalid per-user data merge"
- log.err(msg)
+ log.err("Invalid per-user data merge")
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-data"), description=msg
+ (caldav_namespace, "valid-calendar-data"),
+ "Cannot merge per-user data",
))
self.calendardata = None
@@ -807,6 +815,9 @@
@inlineCallbacks
def doStore(self, implicit):
+ # Stash the current calendar data as we may need to return it
+ self.returndata = str(self.calendar)
+
# Always do the per-user data merge right before we store
yield self.mergePerUserData()
@@ -960,7 +971,8 @@
rname.encode("utf-8")
)
)
- )
+ ),
+ "UID already exists",
))
@@ -1029,15 +1041,10 @@
returnValue(response)
except Exception, err:
- # Preserve the real traceback to display later, since the error-
- # handling here yields out of the generator and thereby shreds the
- # stack.
- f = Failure()
+
if reservation:
yield reservation.unreserve()
- # FIXME: transaction needs to be rolled back.
-
if isinstance(err, InvalidOverriddenInstanceError):
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
@@ -1047,12 +1054,8 @@
elif isinstance(err, TooManyInstancesError):
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- NumberOfRecurrencesWithinLimits(PCDATAElement(str(err.max_allowed)))
+ NumberOfRecurrencesWithinLimits(PCDATAElement(str(err.max_allowed))),
+ "Too many recurrence instances",
))
else:
- # Display the traceback. Unfortunately this will usually be
- # duplicated by the higher-level exception handler that captures
- # the thing that raises here, but it's better than losing the
- # information.
- f.printTraceback()
raise err
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -119,7 +119,8 @@
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- davxml.SupportedReport()
+ davxml.SupportedReport(),
+ "Report not supported on this resource",
))
#
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_addressbook_query.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_addressbook_query.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_addressbook_query.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -83,7 +83,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "supported-address-data")
+ (carddav_namespace, "supported-address-data"),
+ "Invalid address-data",
))
else:
@@ -94,7 +95,8 @@
log.err("Invalid filter element: %r" % (filter,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddav_namespace, "valid-filter")
+ (carddav_namespace, "valid-filter"),
+ "Invalid filter element",
))
matchcount = [0,]
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_calendar_query.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_calendar_query.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_calendar_query.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -76,7 +76,8 @@
log.err(msg)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-data"), description=msg
+ (caldav_namespace, "valid-calendar-data"),
+ "Invalid calendar-data",
))
if query_tz:
filter.settimezone(query_tz)
@@ -99,7 +100,8 @@
log.err(message)
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "supported-calendar-data")
+ (caldav_namespace, "supported-calendar-data"),
+ "Invalid calendar-data",
))
else:
@@ -110,7 +112,8 @@
log.err("Invalid filter element: %r" % (xmlfilter,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-filter")
+ (caldav_namespace, "valid-filter"),
+ "Invalid filter element",
))
matchcount = [0]
@@ -241,13 +244,15 @@
log.err("Too many instances need to be computed in calendar-query report")
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- NumberOfRecurrencesWithinLimits(PCDATAElement(str(ex.max_allowed)))
+ NumberOfRecurrencesWithinLimits(PCDATAElement(str(ex.max_allowed))),
+ "Too many instrances",
))
except NumberOfMatchesWithinLimits:
log.err("Too many matching components in calendar-query report")
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- davxml.NumberOfMatchesWithinLimits()
+ davxml.NumberOfMatchesWithinLimits(),
+ "Too many components",
))
if not hasattr(request, "extendedLogItems"):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_freebusy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_freebusy.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_freebusy.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -80,7 +80,11 @@
yield report_common.applyToCalendarCollections(self, request, request.uri, depth, generateFreeBusyInfo, (caldavxml.ReadFreeBusy(),))
except NumberOfMatchesWithinLimits:
log.err("Too many matching components in free-busy report")
- raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, davxml.NumberOfMatchesWithinLimits()))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ davxml.NumberOfMatchesWithinLimits(),
+ "Too many components"
+ ))
# Now build a new calendar object with the free busy info we have
fbcalendar = report_common.buildFreeBusyResult(fbinfo, timerange)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_multiget_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_multiget_common.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_multiget_common.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -94,14 +94,22 @@
result = True
if not result:
log.err(message)
- raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, precondition))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ precondition,
+ "Invalid object data element",
+ ))
else:
raise AssertionError("We shouldn't be here")
# Check size of results is within limit when data property requested
if hasData and len(resources) > config.MaxMultigetWithDataHrefs:
log.err("Too many results in multiget report returning data: %d" % len(resources))
- raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, davxml.NumberOfMatchesWithinLimits()))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ davxml.NumberOfMatchesWithinLimits(),
+ "Too many resources",
+ ))
"""
Three possibilities exist:
@@ -259,14 +267,22 @@
directoryAddressBookLock, limited = (yield self.directory.cacheVCardsForAddressBookQuery(addressBookFilter, propertyreq, limit) )
if limited:
log.err("Too many results in multiget report: %d" % len(resources))
- raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (dav_namespace, "number-of-matches-within-limits")))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (dav_namespace, "number-of-matches-within-limits"),
+ "Too many results",
+ ))
else:
#get vCards and filter
limit = config.DirectoryAddressBook.MaxQueryResults
vCardRecords, limited = (yield self.directory.vCardRecordsForAddressBookQuery( addressBookFilter, propertyreq, limit ))
if limited:
log.err("Too many results in multiget report: %d" % len(resources))
- raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (dav_namespace, "number-of-matches-within-limits")))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (dav_namespace, "number-of-matches-within-limits"),
+ "Too many results",
+ ))
for href in valid_hrefs:
matchingRecord = None
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_sync_collection.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_sync_collection.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/method/report_sync_collection.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -53,7 +53,8 @@
log.err("sync-collection report is only allowed on calendar/inbox/addressbook/notification collection resources %s" % (self,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- davxml.SupportedReport()
+ davxml.SupportedReport(),
+ "Report not supported on this resource",
))
responses = []
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/notify.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/notify.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/notify.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -722,7 +722,13 @@
self.sendDebug("Node publish failed (%s)" % (nodeName,), iq)
# Don't know how to proceed
self.unlockNode(None, nodeName)
+ except AttributeError:
+ # We did not get an XML response; most likely it was a disconnection
+ self.unlockNode(None, nodeName)
+ # Don't re-raise, just unlock and ignore
except:
+ # Note: this block is not a "finally" because in the case of a 404
+ # we don't want to unlock yet
self.unlockNode(None, nodeName)
raise
@@ -908,6 +914,9 @@
self.log_error("PubSub node configuration error: %s" %
(iq.toXml().encode('ascii', 'replace')),)
self.sendError("Failed to configure node (%s)" % (nodeName,), iq)
+ except AttributeError:
+ # We did not get an XML response; most likely it was a disconnection
+ pass
finally:
self.unlockNode(None, nodeName)
@@ -944,6 +953,9 @@
self.log_error("PubSub node delete error: %s" %
(iq.toXml().encode('ascii', 'replace')),)
self.sendDebug("Node delete failed (%s)" % (nodeName,), iq)
+ except AttributeError:
+ # We did not get an XML response; most likely it was a disconnection
+ pass
finally:
self.unlockNode(None, nodeName)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarquery.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarquery.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarquery.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -197,10 +197,10 @@
endfloat = floatoffset(end, tzinfo) if end else None
return (
- str(pyCalendarTodatetime(start)) if start else None,
- str(pyCalendarTodatetime(end)) if end else None,
- str(pyCalendarTodatetime(startfloat)) if startfloat else None,
- str(pyCalendarTodatetime(endfloat)) if endfloat else None,
+ pyCalendarTodatetime(start) if start else None,
+ pyCalendarTodatetime(end) if end else None,
+ pyCalendarTodatetime(startfloat) if startfloat else None,
+ pyCalendarTodatetime(endfloat) if endfloat else None,
)
def sqlcalendarquery(filter, calendarid=None, userid=None, generator=sqlgenerator.sqlgenerator):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarqueryfilter.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarqueryfilter.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/calendarqueryfilter.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -334,7 +334,7 @@
def getmaxtimerange(self, currentMaximum, currentIsStartTime):
"""
- Get the date furthest into the future in any time-range elements
+ Get the date farthest into the future in any time-range elements
@param currentMaximum: current future value to compare with
@type currentMaximum: L{PyCalendarDateTime}
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/sqlgenerator.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/sqlgenerator.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/query/sqlgenerator.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -266,13 +266,13 @@
e3 = expression.notcontainsExpression("SUMMARY", "help", True)
e5 = expression.andExpression([e1, e2, e3])
print e5
- sql = sqlgenerator(e5, None, None)
+ sql = sqlgenerator(e5, 'dummy-cal', 'dummy-user')
print sql.generate()
e6 = expression.inExpression("TYPE", ("VEVENT", "VTODO",), False)
print e6
- sql = sqlgenerator(e6, None, None)
+ sql = sqlgenerator(e6, 'dummy-cal', 'dummy-user')
print sql.generate()
e7 = expression.notinExpression("TYPE", ("VEVENT", "VTODO",), False)
print e7
- sql = sqlgenerator(e7, None, None)
+ sql = sqlgenerator(e7, 'dummy-cal', 'dummy-user')
print sql.generate()
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/resource.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/resource.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -130,7 +130,8 @@
def http_MKCALENDAR(self, request):
return ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "calendar-collection-location-ok")
+ (caldav_namespace, "calendar-collection-location-ok"),
+ "Resource is read-only",
)
class ReadOnlyNoCopyResourceMixIn (ReadOnlyResourceMixIn):
@@ -1346,7 +1347,8 @@
except ValueError:
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (dav_namespace, "valid-sync-token")
+ (dav_namespace, "valid-sync-token"),
+ "Sync token is invalid",
))
else:
revision = 0
@@ -1356,7 +1358,8 @@
except SyncTokenValidException:
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (dav_namespace, "valid-sync-token")
+ (dav_namespace, "valid-sync-token"),
+ "Sync token not recognized",
))
returnValue((changed, removed, notallowed, current_token))
@@ -1454,7 +1457,8 @@
self.log_error("Cannot create a calendar collection within a calendar collection %s" % (parent,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldavxml.caldav_namespace, "calendar-collection-location-ok")
+ (caldavxml.caldav_namespace, "calendar-collection-location-ok"),
+ "Cannot create a calendar collection inside another calendar collection",
))
# Check for any quota limits
@@ -1464,7 +1468,8 @@
self.log_error("Cannot create a calendar collection because there are too many already present in %s" % (parent,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- customxml.MaxCollections()
+ customxml.MaxCollections(),
+ "Too many calendar collections",
))
returnValue((yield self.createCalendarCollection()))
@@ -1547,7 +1552,8 @@
self.log_error("Cannot create an address book collection within an address book collection %s" % (parent,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (carddavxml.carddav_namespace, "addressbook-collection-location-ok")
+ (carddavxml.carddav_namespace, "addressbook-collection-location-ok"),
+ "Cannot create an address book collection inside of an address book collection",
))
# Check for any quota limits
@@ -1557,7 +1563,8 @@
self.log_error("Cannot create a calendar collection because there are too many already present in %s" % (parent,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- customxml.MaxCollections()
+ customxml.MaxCollections(),
+ "Too many address book collections",
))
returnValue((yield self.createAddressBookCollection()))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/schedule.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/schedule.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/schedule.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -210,7 +210,8 @@
# Validate that href's point to a valid calendar.
raise HTTPError(ErrorResponse(
responsecode.CONFLICT,
- (caldav_namespace, "valid-calendar-url")
+ (caldav_namespace, "valid-calendar-url"),
+ "Invalid URI",
))
elif property.qname() == (caldav_namespace, "schedule-default-calendar-URL"):
@@ -226,7 +227,8 @@
# Validate that href's point to a valid calendar.
raise HTTPError(ErrorResponse(
responsecode.CONFLICT,
- (caldav_namespace, "valid-schedule-default-calendar-URL")
+ (caldav_namespace, "valid-schedule-default-calendar-URL"),
+ "Invalid URI",
))
else:
# Canonicalize the URL to __uids__ form
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/caldav.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/caldav.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -114,7 +114,8 @@
log.err("Could not access Inbox for recipient: %s" % (recipient.cuaddr,))
err = HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (caldav_namespace, "recipient-permissions")
+ (caldav_namespace, "recipient-permissions"),
+ "Access to inbox denied",
))
self.responses.add(
recipient.cuaddr,
@@ -159,7 +160,8 @@
log.err("Could not store data in Inbox : %s" % (recipient.inbox,))
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-permissions")
+ (caldav_namespace, "recipient-permissions"),
+ "Could not store data in inbox",
))
responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=e.msg)
returnValue(False)
@@ -190,7 +192,8 @@
log.err("Could not store data in Inbox : %s" % (recipient.inbox,))
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-permissions")
+ (caldav_namespace, "recipient-permissions"),
+ "Could not store data in inbox",
))
responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.NO_AUTHORITY)
returnValue(False)
@@ -231,7 +234,8 @@
log.err("Could not determine free busy information: %s" % (recipient.cuaddr,))
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-permissions")
+ (caldav_namespace, "recipient-permissions"),
+ "Could not determine free busy information",
))
responses.add(
recipient.cuaddr,
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/imip.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/imip.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/imip.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -68,7 +68,8 @@
for recipient in self.recipients:
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-failed")
+ (caldav_namespace, "recipient-failed"),
+ "iMIP method not allowed: %s" % (method,),
))
self.responses.add(
recipient.cuaddr,
@@ -95,7 +96,8 @@
log.err("Could not do server-to-imip request : %s %s" % (self, e))
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-failed")
+ (caldav_namespace, "recipient-failed"),
+ "iMIP request failed",
))
self.responses.add(
recipient.cuaddr,
@@ -116,7 +118,8 @@
for recipient in self.recipients:
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-failed")
+ (caldav_namespace, "recipient-failed"),
+ "iMIP request failed",
))
self.responses.add(
recipient.cuaddr,
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/implicit.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/implicit.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/implicit.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -96,7 +96,8 @@
):
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-attendee-change")
+ (caldav_namespace, "valid-attendee-change"),
+ "Cannot change scheduling object mode",
))
returnValue((self.action != "none", new_type == "schedule",))
@@ -120,7 +121,8 @@
log.debug("Implicit - cannot MOVE with a scheduling object resource")
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "unique-scheduling-object-resource")
+ (caldav_namespace, "unique-scheduling-object-resource"),
+ "Cannot MOVE a scheduling object resource",
))
else:
self.action = "none"
@@ -153,7 +155,8 @@
log.debug("Implicit - cannot COPY with a scheduling object resource")
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "unique-scheduling-object-resource")
+ (caldav_namespace, "unique-scheduling-object-resource"),
+ "Cannot COPY with a scheduling object resource",
))
else:
self.action = "none"
@@ -330,7 +333,8 @@
log.error("Originator '%s' is not enabled for calendaring" % (self.originatorPrincipal,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "invalid-originator")
+ (caldav_namespace, "invalid-originator"),
+ "Originator not enabled",
))
# Pick the first mailto cu address or the first other type
@@ -349,7 +353,8 @@
log.error("Only one ORGANIZER is allowed in an iCalendar object:\n%s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "single-organizer")
+ (caldav_namespace, "single-organizer"),
+ "Only one organizer allowed in scheduling object resource",
))
# Get the ATTENDEEs
@@ -382,7 +387,8 @@
log.debug("Implicit - found component with same UID in a different collection: %s" % (check_uri,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "unique-scheduling-object-resource")
+ (caldav_namespace, "unique-scheduling-object-resource"),
+ "Cannot duplicate scheduling object resource",
))
@inlineCallbacks
@@ -557,7 +563,8 @@
log.error("Cannot change ORGANIZER: UID:%s" % (self.uid,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-organizer-change")
+ (caldav_namespace, "valid-organizer-change"),
+ "Organizer cannot be changed",
))
else:
# Special case of SCHEDULE-FORCE-SEND added to attendees and no other change
@@ -789,7 +796,8 @@
log.error("Cannot change ORGANIZER: UID:%s" % (self.uid,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-attendee-change")
+ (caldav_namespace, "valid-attendee-change"),
+ "Cannot change organizer",
))
else:
self.oldcalendar = None
@@ -803,7 +811,8 @@
log.error("Attendee '%s' is not allowed to change SCHEDULE-AGENT on organizer: UID:%s" % (self.attendeePrincipal, self.uid,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-attendee-change")
+ (caldav_namespace, "valid-attendee-change"),
+ "Cannot alter organizer",
))
# Determine whether the current change is allowed
@@ -820,7 +829,8 @@
log.error("Attendee '%s' is not allowed to make an unauthorized change to an organized event: UID:%s" % (self.attendeePrincipal, self.uid,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-attendee-change")
+ (caldav_namespace, "valid-attendee-change"),
+ "Attendee changes are not allowed",
))
if not doITipReply:
@@ -844,7 +854,8 @@
log.error("Attendee '%s' is not allowed to set SCHEDULE-AGENT=SERVER on organizer: UID:%s" % (self.attendeePrincipal, self.uid,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-attendee-change")
+ (caldav_namespace, "valid-attendee-change"),
+ "Attendee cannot change organizer state",
))
log.debug("Attendee '%s' is not allowed to update UID: '%s' - missing organizer copy - removing entire event" % (self.attendee, self.uid,))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/ischedule.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/ischedule.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/ischedule.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -92,7 +92,8 @@
# Cannot do server-to-server for this recipient.
err = HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (caldav_namespace, "recipient-allowed")
+ (caldav_namespace, "recipient-allowed"),
+ "No server for recipient",
))
self.responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.NO_USER_SUPPORT)
@@ -103,7 +104,8 @@
# Cannot do server-to-server outgoing requests for this server.
err = HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (caldav_namespace, "recipient-allowed")
+ (caldav_namespace, "recipient-allowed"),
+ "Cannot send to recipient's server",
))
self.responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.SERVICE_UNAVAILABLE)
@@ -177,7 +179,8 @@
for recipient in self.recipients:
err = HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-failed")
+ (caldav_namespace, "recipient-failed"),
+ "Server-to-server request failed",
))
self.responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.SERVICE_UNAVAILABLE)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/scheduler.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/scheduler.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/scheduling/scheduler.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -183,7 +183,8 @@
log.err("%s request must have Originator" % (self.method,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-specified")
+ (caldav_namespace, "originator-specified"),
+ "Missing originator",
))
else:
self.originator = originator
@@ -202,7 +203,8 @@
log.err("%s request must have at least one Recipient" % (self.method,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-specified")
+ (caldav_namespace, "recipient-specified"),
+ "Must have recipients",
))
else:
self.recipients = list(attendees)
@@ -221,7 +223,8 @@
log.err("%s request must have Originator header" % (self.method,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-specified")
+ (caldav_namespace, "originator-specified"),
+ "Missing originator",
))
else:
self.originator = originator[0]
@@ -233,7 +236,8 @@
log.err("%s request must have at least one Recipient header" % (self.method,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "recipient-specified")
+ (caldav_namespace, "recipient-specified"),
+ "No recipients",
))
# Recipient header may be comma separated list
@@ -252,7 +256,8 @@
log.err("MIME type %s not allowed in calendar collection" % (contentType,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "supported-calendar-data")
+ (caldav_namespace, "supported-calendar-data"),
+ "Data is not calendar data",
))
# Parse the calendar object from the HTTP request stream
@@ -321,7 +326,8 @@
log.err("X-CALENDARSERVER-ACCESS not allowed in a calendar component %s request: %s" % (self.method, self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (calendarserver_namespace, "no-access-restrictions")
+ (calendarserver_namespace, "no-access-restrictions"),
+ "Private events cannot be scheduled",
))
# Determine iTIP method mode
@@ -339,7 +345,8 @@
log.err("Wrong number of ATTENDEEs in calendar data: %s" % (self.calendardata,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "attendee-allowed")
+ (caldav_namespace, "attendee-allowed"),
+ "Wrong number of attendees",
))
self.attendee = attendees[0]
@@ -361,7 +368,8 @@
log.err("iTIP data is not valid for a VFREEBUSY request: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-data"), description="iTIP data is not valid for a VFREEBUSY request"
+ (caldav_namespace, "valid-calendar-data"),
+ "iTIP data is not valid for a VFREEBUSY request",
))
dtstart = vfreebusies[0].getStartDateUTC()
dtend = vfreebusies[0].getEndDateUTC()
@@ -370,7 +378,7 @@
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
(caldav_namespace, "valid-calendar-data"),
- description="VFREEBUSY start/end not valid"
+ "VFREEBUSY start/end not valid",
))
# Some clients send floating instead of UTC - coerce to UTC
@@ -379,7 +387,7 @@
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
(caldav_namespace, "valid-calendar-data"),
- description="VFREEBUSY start or end not UTC"
+ "VFREEBUSY start or end not UTC",
))
self.timeRange = caldavxml.TimeRange(start=iCalendarString(dtstart), end=iCalendarString(dtend))
@@ -452,7 +460,8 @@
if freebusy and config.Scheduling.Options.LimitFreeBusyAttendees and ctr >= config.Scheduling.Options.LimitFreeBusyAttendees:
err = HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (caldav_namespace, "recipient-limit")
+ (caldav_namespace, "recipient-limit"),
+ "Too many attendees",
))
responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.SERVICE_UNAVAILABLE)
continue
@@ -475,7 +484,8 @@
else:
err = HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (caldav_namespace, "recipient-exists")
+ (caldav_namespace, "recipient-exists"),
+ "Unknown recipient",
))
responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.INVALID_CALENDAR_USER)
@@ -547,7 +557,8 @@
log.err("Unauthenticated originators not allowed: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Invalid originator",
))
def checkOriginator(self):
@@ -562,7 +573,8 @@
log.err("Could not find principal for originator: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "No principal for originator",
))
else:
# Must have a valid Inbox.
@@ -571,7 +583,8 @@
log.err("Could not find inbox for originator: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator cannot be scheduled",
))
self.originator = LocalCalendarUser(self.originator, originatorPrincipal)
@@ -628,7 +641,8 @@
log.err("ORGANIZER not allowed to be an Organizer: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Organizer cannot schedule",
))
self.organizer = LocalCalendarUser(organizer, organizerPrincipal)
@@ -636,7 +650,8 @@
log.err("No outbox for ORGANIZER in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Organizer cannot schedule",
))
else:
localUser = (yield addressmapping.mapper.isCalendarUserInMyDomain(organizer))
@@ -644,7 +659,8 @@
log.err("No principal for ORGANIZER in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "No principal for organizer",
))
else:
self.organizer = RemoteCalendarUser(organizer)
@@ -652,7 +668,8 @@
log.err("ORGANIZER missing in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Missing organizer",
))
def checkOrganizerAsOriginator(self):
@@ -662,7 +679,8 @@
log.err("ORGANIZER is not local to server in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Organizer is not local to server",
))
# Make sure that the ORGANIZER's Outbox is the request URI
@@ -670,7 +688,8 @@
log.err("Wrong outbox for ORGANIZER in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Outbox does not belong to organizer",
))
def checkAttendeeAsOriginator(self):
@@ -686,13 +705,15 @@
log.err("ATTENDEE in calendar data does not match owner of Outbox: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "attendee-allowed")
+ (caldav_namespace, "attendee-allowed"),
+ "Outbox does not belong to attendee",
))
else:
log.err("Unknown ATTENDEE in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "attendee-allowed")
+ (caldav_namespace, "attendee-allowed"),
+ "No principal for attendee",
))
def securityChecks(self):
@@ -793,7 +814,8 @@
log.err("Authenticated originators not allowed: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Authentication not allowed",
))
@inlineCallbacks
@@ -810,7 +832,8 @@
log.err("Cannot use originator that is on this server: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator cannot be local to server",
))
else:
self.originator = PartitionedCalendarUser(self.originator, originatorPrincipal)
@@ -831,7 +854,8 @@
log.err("Originator not on recognized server: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator not recognized by server",
))
else:
# Get the request IP and map to hostname.
@@ -870,7 +894,8 @@
log.err("Originator not on allowed server: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator not allowed to send to this server",
))
def _validPartitionServer(self, principal):
@@ -905,7 +930,8 @@
log.err("Originator not on allowed server: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator not allowed to send to this server",
))
@inlineCallbacks
@@ -923,7 +949,8 @@
log.err("Invalid ORGANIZER in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Organizer is not local to server",
))
else:
# Check that the origin server is the correct partition
@@ -935,7 +962,8 @@
log.err("Unsupported ORGANIZER in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "Organizer not allowed to be originator",
))
else:
self.organizer = RemoteCalendarUser(organizer)
@@ -943,7 +971,8 @@
log.err("ORGANIZER missing in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "organizer-allowed")
+ (caldav_namespace, "organizer-allowed"),
+ "No organizer in calendar data",
))
@inlineCallbacks
@@ -960,7 +989,8 @@
log.err("Invalid ATTENDEE in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "attendee-allowed")
+ (caldav_namespace, "attendee-allowed"),
+ "Local attendee cannot send to this server",
))
else:
self._validPartitionServer(attendeePrincipal)
@@ -970,7 +1000,8 @@
log.err("Unknown ATTENDEE in calendar data: %s" % (self.calendar,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "attendee-allowed")
+ (caldav_namespace, "attendee-allowed"),
+ "Attendee not allowed to schedule",
))
# TODO: in this case we should check that the ORGANIZER is the sole recipient.
@@ -990,11 +1021,11 @@
yield self.checkAttendeeAsOriginator()
else:
- msg = "Unknown iTIP METHOD for security checks: %s" % (self.calendar.propertyValue("METHOD"),)
- log.err(msg)
+ log.err("Unknown iTIP METHOD for security checks: %s" % (self.calendar.propertyValue("METHOD"),))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "valid-calendar-data"), description=msg
+ (caldav_namespace, "valid-calendar-data"),
+ "Unknown iTIP method",
))
@@ -1042,7 +1073,8 @@
log.err("Cannot use originator that is on this server: %s" % (self.originator,))
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator cannot be local to server",
))
else:
self.originator = RemoteCalendarUser(self.originator)
@@ -1069,7 +1101,8 @@
# TODO: verify this is the right response:
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
- (caldav_namespace, "originator-allowed")
+ (caldav_namespace, "originator-allowed"),
+ "Originator server not allowed to send to this server",
))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/sharing.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/sharing.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/sharing.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -114,7 +114,7 @@
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
(customxml.calendarserver_namespace, "valid-request"),
- "invalid share",
+ "Invalid share",
))
record = yield self.invitesDB().recordForInviteUID(inviteUID)
@@ -122,7 +122,7 @@
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
(customxml.calendarserver_namespace, "valid-request"),
- "invalid invitation uid: %s" % (inviteUID,),
+ "Invalid invitation uid: %s" % (inviteUID,),
))
# Only certain states are sharer controlled
@@ -596,179 +596,177 @@
# Add to collections
yield notifications.deleteNotifictionMessageByUID(request, record.inviteuid)
- def xmlPOSTNoAuth(self, encoding, request):
- def _handleErrorResponse(error):
- if isinstance(error.value, HTTPError) and hasattr(error.value, "response"):
- return error.value.response
- return error
+ @inlineCallbacks
+ def _xmlHandleInvite(self, request, docroot):
+ yield self.authorize(request, (davxml.Read(), davxml.Write()))
+ result = (yield self._handleInvite(request, docroot))
+ returnValue(result)
+
+ def _handleInvite(self, request, invitedoc):
+ def _handleInviteSet(inviteset):
+ userid = None
+ cn = None
+ access = None
+ summary = None
+ for item in inviteset.children:
+ if isinstance(item, davxml.HRef):
+ userid = str(item)
+ continue
+ if isinstance(item, customxml.CommonName):
+ cn = str(item)
+ continue
+ if isinstance(item, customxml.InviteSummary):
+ summary = str(item)
+ continue
+ if isinstance(item, customxml.ReadAccess) or isinstance(item, customxml.ReadWriteAccess):
+ access = item
+ continue
+ if userid and access and summary:
+ return (userid, cn, access, summary)
+ else:
+ error_text = []
+ if userid is None:
+ error_text.append("missing href")
+ if access is None:
+ error_text.append("missing access")
+ if summary is None:
+ error_text.append("missing summary")
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (customxml.calendarserver_namespace, "valid-request"),
+ "%s: %s" % (", ".join(error_text), inviteset,),
+ ))
- def _handleInvite(invitedoc):
- def _handleInviteSet(inviteset):
- userid = None
- cn = None
+ def _handleInviteRemove(inviteremove):
+ userid = None
+ access = []
+ for item in inviteremove.children:
+ if isinstance(item, davxml.HRef):
+ userid = str(item)
+ continue
+ if isinstance(item, customxml.ReadAccess) or isinstance(item, customxml.ReadWriteAccess):
+ access.append(item)
+ continue
+ if userid is None:
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (customxml.calendarserver_namespace, "valid-request"),
+ "Missing href: %s" % (inviteremove,),
+ ))
+ if len(access) == 0:
access = None
- summary = None
- for item in inviteset.children:
- if isinstance(item, davxml.HRef):
- userid = str(item)
- continue
- if isinstance(item, customxml.CommonName):
- cn = str(item)
- continue
- if isinstance(item, customxml.InviteSummary):
- summary = str(item)
- continue
- if isinstance(item, customxml.ReadAccess) or isinstance(item, customxml.ReadWriteAccess):
- access = item
- continue
- if userid and access and summary:
- return (userid, cn, access, summary)
- else:
- error_text = []
- if userid is None:
- error_text.append("missing href")
- if access is None:
- error_text.append("missing access")
- if summary is None:
- error_text.append("missing summary")
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace, "valid-request"),
- "%s: %s" % (", ".join(error_text), inviteset,),
- ))
+ else:
+ access = set(access)
+ return (userid, access)
- def _handleInviteRemove(inviteremove):
- userid = None
- access = []
- for item in inviteremove.children:
- if isinstance(item, davxml.HRef):
- userid = str(item)
- continue
- if isinstance(item, customxml.ReadAccess) or isinstance(item, customxml.ReadWriteAccess):
- access.append(item)
- continue
- if userid is None:
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace, "valid-request"),
- "missing href: %s" % (inviteremove,),
- ))
- if len(access) == 0:
- access = None
- else:
- access = set(access)
- return (userid, access)
+ def _autoShare(isShared, request):
+ if not isShared:
+ self.upgradeToShare()
- def _autoShare(isShared, request):
- if not isShared:
- self.upgradeToShare()
-
- @inlineCallbacks
- def _processInviteDoc(_, request):
- setDict, removeDict, updateinviteDict = {}, {}, {}
- okusers = set()
- badusers = set()
- for item in invitedoc.children:
- if isinstance(item, customxml.InviteSet):
- userid, cn, access, summary = _handleInviteSet(item)
- setDict[userid] = (cn, access, summary)
+ @inlineCallbacks
+ def _processInviteDoc(_, request):
+ setDict, removeDict, updateinviteDict = {}, {}, {}
+ okusers = set()
+ badusers = set()
+ for item in invitedoc.children:
+ if isinstance(item, customxml.InviteSet):
+ userid, cn, access, summary = _handleInviteSet(item)
+ setDict[userid] = (cn, access, summary)
+
+ # Validate each userid on add only
+ (okusers if self.validUserIDForShare(userid) else badusers).add(userid)
+ elif isinstance(item, customxml.InviteRemove):
+ userid, access = _handleInviteRemove(item)
+ removeDict[userid] = access
- # Validate each userid on add only
- (okusers if self.validUserIDForShare(userid) else badusers).add(userid)
- elif isinstance(item, customxml.InviteRemove):
- userid, access = _handleInviteRemove(item)
- removeDict[userid] = access
-
- # Treat removed userids as valid as we will fail invalid ones silently
- okusers.add(userid)
+ # Treat removed userids as valid as we will fail invalid ones silently
+ okusers.add(userid)
- # Only make changes if all OK
- if len(badusers) == 0:
- # Special case removing and adding the same user and treat that as an add
- sameUseridInRemoveAndSet = [u for u in removeDict.keys() if u in setDict]
- for u in sameUseridInRemoveAndSet:
- removeACL = removeDict[u]
- cn, newACL, summary = setDict[u]
- updateinviteDict[u] = (cn, removeACL, newACL, summary)
- del removeDict[u]
- del setDict[u]
- for userid, access in removeDict.iteritems():
- result = (yield self.uninviteUserToShare(userid, access, request))
- (okusers if result else badusers).add(userid)
- for userid, (cn, access, summary) in setDict.iteritems():
- result = (yield self.inviteUserToShare(userid, cn, access, summary, request))
- (okusers if result else badusers).add(userid)
- for userid, (cn, removeACL, newACL, summary) in updateinviteDict.iteritems():
- result = (yield self.inviteUserUpdateToShare(userid, cn, removeACL, newACL, summary, request))
- (okusers if result else badusers).add(userid)
+ # Only make changes if all OK
+ if len(badusers) == 0:
+ # Special case removing and adding the same user and treat that as an add
+ sameUseridInRemoveAndSet = [u for u in removeDict.keys() if u in setDict]
+ for u in sameUseridInRemoveAndSet:
+ removeACL = removeDict[u]
+ cn, newACL, summary = setDict[u]
+ updateinviteDict[u] = (cn, removeACL, newACL, summary)
+ del removeDict[u]
+ del setDict[u]
+ for userid, access in removeDict.iteritems():
+ result = (yield self.uninviteUserToShare(userid, access, request))
+ (okusers if result else badusers).add(userid)
+ for userid, (cn, access, summary) in setDict.iteritems():
+ result = (yield self.inviteUserToShare(userid, cn, access, summary, request))
+ (okusers if result else badusers).add(userid)
+ for userid, (cn, removeACL, newACL, summary) in updateinviteDict.iteritems():
+ result = (yield self.inviteUserUpdateToShare(userid, cn, removeACL, newACL, summary, request))
+ (okusers if result else badusers).add(userid)
- # Do a final validation of the entire set of invites
- yield self.validateInvites()
-
- # Create the multistatus response - only needed if some are bad
- if badusers:
- xml_responses = []
- xml_responses.extend([
- davxml.StatusResponse(davxml.HRef(userid), davxml.Status.fromResponseCode(responsecode.FAILED_DEPENDENCY))
- for userid in sorted(okusers)
- ])
- xml_responses.extend([
- davxml.StatusResponse(davxml.HRef(userid), davxml.Status.fromResponseCode(responsecode.FORBIDDEN))
- for userid in sorted(badusers)
- ])
-
- #
- # Return response
- #
- returnValue(MultiStatusResponse(xml_responses))
- else:
- returnValue(responsecode.OK)
-
+ # Do a final validation of the entire set of invites
+ yield self.validateInvites()
+
+ # Create the multistatus response - only needed if some are bad
+ if badusers:
+ xml_responses = []
+ xml_responses.extend([
+ davxml.StatusResponse(davxml.HRef(userid), davxml.Status.fromResponseCode(responsecode.FAILED_DEPENDENCY))
+ for userid in sorted(okusers)
+ ])
+ xml_responses.extend([
+ davxml.StatusResponse(davxml.HRef(userid), davxml.Status.fromResponseCode(responsecode.FORBIDDEN))
+ for userid in sorted(badusers)
+ ])
+
+ #
+ # Return response
+ #
+ returnValue(MultiStatusResponse(xml_responses))
+ else:
+ returnValue(responsecode.OK)
- return self.isShared(request).addCallback(_autoShare, request).addCallback(_processInviteDoc, request)
+ return self.isShared(request).addCallback(_autoShare, request).addCallback(_processInviteDoc, request)
- def _getData(data):
- try:
- doc = davxml.WebDAVDocument.fromString(data)
- except ValueError, e:
- self.log_error("Error parsing doc (%s) Doc:\n %s" % (str(e), data,))
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace, "valid-request")
- ))
+ @inlineCallbacks
+ def _xmlHandleInviteReply(self, request, docroot):
+ yield self.authorize(request, (davxml.Read(), davxml.Write()))
+ result = (yield self._handleInviteReply(request, docroot))
+ returnValue(result)
+
+ def _handleInviteReply(self, request, docroot):
+ raise NotImplementedError
- root = doc.root_element
- xmlDocHanders = {
- customxml.InviteShare: _handleInvite,
- }
- if type(root) in xmlDocHanders:
- return xmlDocHanders[type(root)](root).addErrback(_handleErrorResponse)
- else:
- self.log_error("Unsupported XML (%s)" % (root,))
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace, "valid-request")
- ))
+ @inlineCallbacks
+ def xmlRequestHandler(self, request):
+
+ # Need to read the data and get the root element first
+ xmldata = (yield allDataFromStream(request.stream))
+ try:
+ doc = davxml.WebDAVDocument.fromString(xmldata)
+ except ValueError, e:
+ self.log_error("Error parsing doc (%s) Doc:\n %s" % (str(e), xmldata,))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (customxml.calendarserver_namespace, "valid-request"),
+ "Invalid XML",
+ ))
- return allDataFromStream(request.stream).addCallback(_getData)
+ root = doc.root_element
+ if type(root) in self.xmlDocHanders:
+ result = (yield self.xmlDocHanders[type(root)](self, request, root))
+ returnValue(result)
+ else:
+ self.log_error("Unsupported XML (%s)" % (root,))
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (customxml.calendarserver_namespace, "valid-request"),
+ "Unsupported XML",
+ ))
- def xmlPOSTPreconditions(self, _, request):
- if request.headers.hasHeader("Content-Type"):
- mimetype = request.headers.getHeader("Content-Type")
- if mimetype.mediaType in ("application", "text",) and mimetype.mediaSubtype == "xml":
- encoding = mimetype.params["charset"] if "charset" in mimetype.params else "utf8"
- return succeed(encoding)
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace, "valid-request")
- ))
+ xmlDocHanders = {
+ customxml.InviteShare: _xmlHandleInvite,
+ customxml.InviteReply: _xmlHandleInviteReply,
+ }
- def xmlPOSTAuth(self, request):
- d = self.authorize(request, (davxml.Read(), davxml.Write()))
- d.addCallback(self.xmlPOSTPreconditions, request)
- d.addCallback(self.xmlPOSTNoAuth, request)
- return d
-
def POST_handler_content_type(self, request, contentType):
if self.isCollection():
if contentType:
@@ -781,8 +779,8 @@
return succeed(responsecode.FORBIDDEN)
_postHandlers = {
- ("application", "xml") : xmlPOSTAuth,
- ("text", "xml") : xmlPOSTAuth,
+ ("application", "xml") : xmlRequestHandler,
+ ("text", "xml") : xmlRequestHandler,
}
inviteAccessMapToXML = {
@@ -1105,7 +1103,7 @@
raise HTTPError(ErrorResponse(
responsecode.FORBIDDEN,
(customxml.calendarserver_namespace, "valid-request"),
- "invalid shared collection",
+ "Invalid shared collection",
))
# Change the record
@@ -1142,67 +1140,37 @@
# Add to collections
yield notifications.addNotification(request, notificationUID, xmltype, xmldata)
- def xmlPOSTNoAuth(self, encoding, request):
+ def _handleInviteReply(self, request, invitereplydoc):
+ """ Handle a user accepting or declining a sharing invite """
+ hostUrl = None
+ accepted = None
+ summary = None
+ replytoUID = None
+ for item in invitereplydoc.children:
+ if isinstance(item, customxml.InviteStatusAccepted):
+ accepted = True
+ elif isinstance(item, customxml.InviteStatusDeclined):
+ accepted = False
+ elif isinstance(item, customxml.InviteSummary):
+ summary = str(item)
+ elif isinstance(item, customxml.HostURL):
+ for hosturlItem in item.children:
+ if isinstance(hosturlItem, davxml.HRef):
+ hostUrl = str(hosturlItem)
+ elif isinstance(item, customxml.InReplyTo):
+ replytoUID = str(item)
+
+ if accepted is None or hostUrl is None or replytoUID is None:
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (customxml.calendarserver_namespace, "valid-request"),
+ "Missing required XML elements",
+ ))
+ if accepted:
+ return self.acceptInviteShare(request, hostUrl, replytoUID, displayname=summary)
+ else:
+ return self.declineShare(request, hostUrl, replytoUID)
- def _handleErrorResponse(error):
- if isinstance(error.value, HTTPError) and hasattr(error.value, "response"):
- return error.value.response
- return error
-
- def _handleInviteReply(invitereplydoc):
- """ Handle a user accepting or declining a sharing invite """
- hostUrl = None
- accepted = None
- summary = None
- replytoUID = None
- for item in invitereplydoc.children:
- if isinstance(item, customxml.InviteStatusAccepted):
- accepted = True
- elif isinstance(item, customxml.InviteStatusDeclined):
- accepted = False
- elif isinstance(item, customxml.InviteSummary):
- summary = str(item)
- elif isinstance(item, customxml.HostURL):
- for hosturlItem in item.children:
- if isinstance(hosturlItem, davxml.HRef):
- hostUrl = str(hosturlItem)
- elif isinstance(item, customxml.InReplyTo):
- replytoUID = str(item)
-
- if accepted is None or hostUrl is None or replytoUID is None:
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace, "valid-request"),
- "missing required XML elements",
- ))
- if accepted:
- return self.acceptInviteShare(request, hostUrl, replytoUID, displayname=summary)
- else:
- return self.declineShare(request, hostUrl, replytoUID)
-
- def _getData(data):
- try:
- doc = davxml.WebDAVDocument.fromString(data)
- except ValueError, e:
- print "Error parsing doc (%s) Doc:\n %s" % (str(e), data,)
- raise
-
- root = doc.root_element
- xmlDocHanders = {
- customxml.InviteReply: _handleInviteReply,
- }
- if type(root) in xmlDocHanders:
- return xmlDocHanders[type(root)](root).addErrback(_handleErrorResponse)
- else:
- self.log_error("Unsupported XML (%s)" % (root,))
- raise HTTPError(ErrorResponse(
- responsecode.FORBIDDEN,
- (customxml.calendarserver_namespace,
- "valid-request")
- ))
-
- return allDataFromStream(request.stream).addCallback(_getData)
-
class SharedCollectionRecord(object):
def __init__(self, shareuid, sharetype, hosturl, localname, summary):
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/stdconfig.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/stdconfig.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -356,7 +356,7 @@
},
"Wiki": {
"Enabled": False,
- "Cookie": "_authserver_session",
+ "Cookie": "apple_webauth_token",
"URL": "http://127.0.0.1:8089/RPC2",
"UserMethod": "userForSession",
"WikiMethod": "accessLevelForUserWikiCalendar",
@@ -436,6 +436,10 @@
"EnableDropBox" : False, # Calendar Drop Box
"EnablePrivateEvents" : False, # Private Events
"EnableTimezoneService" : False, # Timezone service
+
+ "EnableBatchUpload" : False, # POST batch uploads
+ "MaxResourcesBatchUpload" : 100, # Maximum number of resources in a batch POST
+ "MaxBytesBatchUpload" : 10485760, # Maximum size of a batch POST (10 MB)
"Sharing": {
"Enabled" : False, # Overall on/off switch
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/storebridge.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/storebridge.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -15,59 +15,52 @@
# limitations under the License.
##
-"""
-Wrappers to translate between the APIs in L{txdav.caldav.icalendarstore} and
-L{txdav.carddav.iaddressbookstore} and those in L{twistedcaldav}.
-"""
-
-from urlparse import urlsplit
-
-from twisted.internet.defer import succeed, inlineCallbacks, returnValue,\
- maybeDeferred
-from twisted.internet.protocol import Protocol
-from twisted.python.log import err as logDefaultException
-from twisted.python.util import FancyEqMixin
-
from twext.python.log import Logger
-
from twext.web2 import responsecode
from twext.web2.dav import davxml
-from twext.web2.dav.element.base import dav_namespace
-from twext.web2.dav.http import ErrorResponse, ResponseQueue
+from twext.web2.dav.element.base import dav_namespace, WebDAVUnknownElement
+from twext.web2.dav.http import ErrorResponse, ResponseQueue, MultiStatusResponse
from twext.web2.dav.noneprops import NonePropertyStore
from twext.web2.dav.resource import TwistedACLInheritable, AccessDeniedError
-from twext.web2.dav.util import parentForURL, allDataFromStream, joinURL, \
- davXMLFromStream
+from twext.web2.dav.util import parentForURL, allDataFromStream, joinURL, davXMLFromStream
from twext.web2.http import HTTPError, StatusResponse, Response
from twext.web2.http_headers import ETag, MimeType
-from twext.web2.responsecode import (
- FORBIDDEN, NO_CONTENT, NOT_FOUND, CREATED, CONFLICT, PRECONDITION_FAILED,
- BAD_REQUEST, OK,
-)
+from twext.web2.responsecode import FORBIDDEN, NO_CONTENT, NOT_FOUND, CREATED, CONFLICT, PRECONDITION_FAILED, BAD_REQUEST, OK
from twext.web2.stream import ProducerStream, readStream, MemoryStream
-
+from twisted.internet.defer import succeed, inlineCallbacks, returnValue, maybeDeferred
+from twisted.internet.protocol import Protocol
+from twisted.python.hashlib import md5
+from twisted.python.log import err as logDefaultException
+from twisted.python.util import FancyEqMixin
+from twistedcaldav import customxml, carddavxml, caldavxml
from twistedcaldav.cache import CacheStoreNotifier, ResponseCacheMixin,\
DisabledCacheNotifier
from twistedcaldav.caldavxml import caldav_namespace
from twistedcaldav.config import config
-from twistedcaldav import customxml
+from twistedcaldav.ical import Component as VCalendar, Property as VProperty,\
+ InvalidICalendarDataError, iCalendarProductID
from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
-from twistedcaldav.notifications import NotificationCollectionResource, \
- NotificationResource
+from twistedcaldav.method.put_addressbook_common import StoreAddressObjectResource
+from twistedcaldav.method.put_common import StoreCalendarObjectResource
+from twistedcaldav.notifications import NotificationCollectionResource, NotificationResource
from twistedcaldav.resource import CalDAVResource, GlobalAddressBookResource
from twistedcaldav.schedule import ScheduleInboxResource
from twistedcaldav.scheduling.implicit import ImplicitScheduler
-from twistedcaldav.ical import Component as VCalendar, iCalendarProductID
-from twistedcaldav.ical import Property as VProperty
-from twistedcaldav.vcard import Component as VCard
+from twistedcaldav.vcard import Component as VCard, InvalidVCardDataError
from txdav.base.propertystore.base import PropertyName
from txdav.common.icommondatastore import NoSuchObjectResourceError
+from urlparse import urlsplit
+import time
from txdav.idav import PropertyChangeNotAllowedError
+"""
+Wrappers to translate between the APIs in L{txdav.caldav.icalendarstore} and
+L{txdav.carddav.iaddressbookstore} and those in L{twistedcaldav}.
+"""
+
log = Logger()
-
class _NewStorePropertiesWrapper(object):
"""
Wrap a new-style property store (a L{txdav.idav.IPropertyStore}) in the old-
@@ -230,6 +223,9 @@
if config.MaxResourcesPerCollection:
props += (customxml.MaxResources.qname(),)
+ if config.EnableBatchUpload:
+ props += (customxml.BulkRequests.qname(),)
+
return props
@inlineCallbacks
@@ -496,6 +492,377 @@
yield self._newStoreObject.rename(basename)
returnValue(NO_CONTENT)
+ @inlineCallbacks
+ def _readGlobalProperty(self, qname, property, request):
+
+ if config.EnableBatchUpload and qname == customxml.BulkRequests.qname():
+ returnValue(customxml.BulkRequests(
+ customxml.Simple(
+ customxml.MaxBulkResources.fromString(str(config.MaxResourcesBatchUpload)),
+ customxml.MaxBulkBytes.fromString(str(config.MaxBytesBatchUpload)),
+ ),
+ customxml.CRUD(
+ customxml.MaxBulkResources.fromString(str(config.MaxResourcesBatchUpload)),
+ customxml.MaxBulkBytes.fromString(str(config.MaxBytesBatchUpload)),
+ ),
+ ))
+ else:
+ result = (yield super(_CommonHomeChildCollectionMixin, self)._readGlobalProperty(qname, property, request))
+ returnValue(result)
+
+ @inlineCallbacks
+ def checkCTagPrecondition(self, request):
+ if request.headers.hasHeader("If"):
+ iffy = request.headers.getRawHeaders("If")[0]
+ prefix = "<%sctag/" % (customxml.mm_namespace,)
+ if prefix in iffy:
+ testctag = iffy[iffy.find(prefix):]
+ testctag = testctag[len(prefix):]
+ testctag = testctag.split(">", 1)[0]
+ ctag = (yield self.getSyncToken())
+ if testctag != ctag:
+ raise HTTPError(StatusResponse(responsecode.PRECONDITION_FAILED, "CTag pre-condition failure"))
+
+ def checkReturnChanged(self, request):
+ if request.headers.hasHeader("X-MobileMe-DAV-Options"):
+ return_changed = request.headers.getRawHeaders("X-MobileMe-DAV-Options")[0]
+ return ("return-changed-data" in return_changed)
+ else:
+ return False
+
+ @requiresPermissions(davxml.Bind())
+ @inlineCallbacks
+ def simpleBatchPOST(self, request):
+
+ # If CTag precondition
+ yield self.checkCTagPrecondition(request)
+
+ # Look for return changed data option
+ return_changed = self.checkReturnChanged(request)
+
+ # Read in all data
+ data = (yield allDataFromStream(request.stream))
+
+ components = self.componentsFromData(data)
+ if components is None:
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Could not parse valid data from request body"))
+
+ # Build response
+ xmlresponses = []
+ for component in components:
+
+ code = None
+ error = None
+ dataChanged = None
+ try:
+ componentdata = str(component)
+
+ # Create a new name if one was not provided
+ name = md5(str(componentdata) + str(time.time()) + request.path).hexdigest() + self.resourceSuffix()
+
+ # Get a resource for the new item
+ newchildURL = joinURL(request.path, name)
+ newchild = (yield request.locateResource(newchildURL))
+ dataChanged = (yield self.storeResourceData(request, newchild, newchildURL, componentdata))
+
+ except HTTPError, e:
+ # Extract the pre-condition
+ code = e.response.code
+ if isinstance(e.response, ErrorResponse):
+ error = e.response.error
+ error = (error.namespace, error.name,)
+ except Exception:
+ code = responsecode.BAD_REQUEST
+
+ if code is None:
+
+ if not return_changed or dataChanged is None:
+ xmlresponses.append(
+ davxml.PropertyStatusResponse(
+ davxml.HRef.fromString(newchildURL),
+ davxml.PropertyStatus(
+ davxml.PropertyContainer(
+ davxml.GETETag.fromString(newchild.etag().generate()),
+ customxml.UID.fromString(component.resourceUID()),
+ ),
+ davxml.Status.fromResponseCode(responsecode.OK),
+ )
+ )
+ )
+ else:
+ xmlresponses.append(
+ davxml.PropertyStatusResponse(
+ davxml.HRef.fromString(newchildURL),
+ davxml.PropertyStatus(
+ davxml.PropertyContainer(
+ davxml.GETETag.fromString(newchild.etag().generate()),
+ self.xmlDataElementType().fromTextData(dataChanged),
+ ),
+ davxml.Status.fromResponseCode(responsecode.OK),
+ )
+ )
+ )
+
+ else:
+ xmlresponses.append(
+ davxml.StatusResponse(
+ davxml.HRef.fromString(""),
+ davxml.Status.fromResponseCode(code),
+ davxml.Error(
+ WebDAVUnknownElement.fromQname(*error),
+ customxml.UID.fromString(component.resourceUID()),
+ ) if error else None,
+ )
+ )
+
+ result = MultiStatusResponse(xmlresponses)
+
+ newctag = (yield self.getSyncToken())
+ result.headers.setRawHeaders("CTag", (newctag,))
+
+ # Setup some useful logging
+ request.submethod = "Simple batch"
+ if not hasattr(request, "extendedLogItems"):
+ request.extendedLogItems = {}
+ request.extendedLogItems["rcount"] = len(xmlresponses)
+
+ returnValue(result)
+
+ @inlineCallbacks
+ def crudBatchPOST(self, request, xmlroot):
+
+ # Need to force some kind of overall authentication on the request
+ yield self.authorize(request, (davxml.Read(), davxml.Write(),))
+
+ # If CTag precondition
+ yield self.checkCTagPrecondition(request)
+
+ # Look for return changed data option
+ return_changed = self.checkReturnChanged(request)
+
+ # Build response
+ xmlresponses = []
+ checkedBindPrivelege = None
+ checkedUnbindPrivelege = None
+ for xmlchild in xmlroot.children:
+
+ # Determine the multiput operation: create, update, delete
+ href = xmlchild.childOfType(davxml.HRef.qname())
+ set = xmlchild.childOfType(davxml.Set.qname())
+ prop = set.childOfType(davxml.PropertyContainer.qname()) if set is not None else None
+ xmldata_root = prop if prop else set
+ xmldata = xmldata_root.childOfType(self.xmlDataElementType().qname()) if xmldata_root is not None else None
+ if href is None:
+
+ if xmldata is None:
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Could not parse valid data from request body without a DAV:Href present"))
+
+ # Do privilege check on collection once
+ if checkedBindPrivelege is None:
+ try:
+ yield self.authorize(request, (davxml.Bind(),))
+ checkedBindPrivelege = True
+ except HTTPError, e:
+ checkedBindPrivelege = e
+
+ # Create operations
+ yield self.crudCreate(request, xmldata.generateComponent(), xmlresponses, return_changed, checkedBindPrivelege)
+ else:
+ delete = xmlchild.childOfType(customxml.Delete.qname())
+ ifmatch = xmlchild.childOfType(customxml.IfMatch.qname())
+ if ifmatch:
+ ifmatch = str(ifmatch.children[0]) if len(ifmatch.children) == 1 else None
+ if delete is None:
+ if set is None:
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Could not parse valid data from request body - no set of delete operation"))
+ if xmldata is None:
+ raise HTTPError(StatusResponse(responsecode.BAD_REQUEST, "Could not parse valid data from request body for set operation"))
+ yield self.crudUpdate(request, str(href), xmldata.generateComponent(), ifmatch, return_changed, xmlresponses)
+ else:
+ # Do privilege check on collection once
+ if checkedUnbindPrivelege is None:
+ try:
+ yield self.authorize(request, (davxml.Unbind(),))
+ checkedUnbindPrivelege = True
+ except HTTPError, e:
+ checkedUnbindPrivelege = e
+
+ yield self.crudDelete(request, str(href), ifmatch, xmlresponses, checkedUnbindPrivelege);
+
+ result = MultiStatusResponse(xmlresponses)
+
+ newctag = (yield self.getSyncToken())
+ result.headers.setRawHeaders("CTag", (newctag,))
+
+ # Setup some useful logging
+ request.submethod = "CRUD batch"
+ if not hasattr(request, "extendedLogItems"):
+ request.extendedLogItems = {}
+ request.extendedLogItems["rcount"] = len(xmlresponses)
+
+ returnValue(result)
+
+ @inlineCallbacks
+ def crudCreate(self, request, component, xmlresponses, return_changed, hasPrivilege):
+
+ code = None
+ error = None
+ try:
+ componentdata = str(component)
+ if isinstance(hasPrivilege, HTTPError):
+ raise hasPrivilege
+
+ # Create a new name if one was not provided
+ name = md5(str(componentdata) + str(time.time()) + request.path).hexdigest() + self.resourceSuffix()
+
+ # Get a resource for the new item
+ newchildURL = joinURL(request.path, name)
+ newchild = (yield request.locateResource(newchildURL))
+ yield self.storeResourceData(request, newchild, newchildURL, componentdata)
+
+ # FIXME: figure out return_changed behavior
+
+ except HTTPError, e:
+ # Extract the pre-condition
+ code = e.response.code
+ if isinstance(e.response, ErrorResponse):
+ error = e.response.error
+ error = (error.namespace, error.name,)
+
+ except Exception:
+ code = responsecode.BAD_REQUEST
+
+ if code is None:
+ xmlresponses.append(
+ davxml.PropertyStatusResponse(
+ davxml.HRef.fromString(newchildURL),
+ davxml.PropertyStatus(
+ davxml.PropertyContainer(
+ davxml.GETETag.fromString(newchild.etag().generate()),
+ customxml.UID.fromString(component.resourceUID()),
+ ),
+ davxml.Status.fromResponseCode(responsecode.OK),
+ )
+ )
+ )
+ else:
+ xmlresponses.append(
+ davxml.StatusResponse(
+ davxml.HRef.fromString(""),
+ davxml.Status.fromResponseCode(code),
+ davxml.Error(
+ WebDAVUnknownElement.fromQname(*error),
+ customxml.UID.fromString(component.resourceUID()),
+ ) if error else None,
+ )
+ )
+
+ @inlineCallbacks
+ def crudUpdate(self, request, href, component, ifmatch, return_changed, xmlresponses):
+ code = None
+ error = None
+ try:
+ componentdata = str(component)
+
+ updateResource = (yield request.locateResource(href))
+ if not updateResource.exists():
+ raise HTTPError(responsecode.NOT_FOUND)
+
+ # Check privilege
+ yield updateResource.authorize(request, (davxml.Write(),))
+
+ # Check if match
+ if ifmatch and ifmatch != updateResource.etag().generate():
+ raise HTTPError(responsecode.PRECONDITION_FAILED)
+
+ yield self.storeResourceData(request, updateResource, href, componentdata)
+
+ # FIXME: figure out return_changed behavior
+
+ except HTTPError, e:
+ # Extract the pre-condition
+ code = e.response.code
+ if isinstance(e.response, ErrorResponse):
+ error = e.response.error
+ error = (error.namespace, error.name,)
+
+ except Exception:
+ code = responsecode.BAD_REQUEST
+
+ if code is None:
+ xmlresponses.append(
+ davxml.PropertyStatusResponse(
+ davxml.HRef.fromString(href),
+ davxml.PropertyStatus(
+ davxml.PropertyContainer(
+ davxml.GETETag.fromString(updateResource.etag().generate()),
+ ),
+ davxml.Status.fromResponseCode(responsecode.OK),
+ )
+ )
+ )
+ else:
+ xmlresponses.append(
+ davxml.StatusResponse(
+ davxml.HRef.fromString(href),
+ davxml.Status.fromResponseCode(code),
+ davxml.Error(
+ WebDAVUnknownElement.fromQname(*error),
+ ) if error else None,
+ )
+ )
+
+ @inlineCallbacks
+ def crudDelete(self, request, href, ifmatch, xmlresponses, hasPrivilege):
+ code = None
+ error = None
+ try:
+ if isinstance(hasPrivilege, HTTPError):
+ raise hasPrivilege
+
+ deleteResource = (yield request.locateResource(href))
+ if not deleteResource.exists():
+ raise HTTPError(responsecode.NOT_FOUND)
+
+ # Check if match
+ if ifmatch and ifmatch != deleteResource.etag().generate():
+ raise HTTPError(responsecode.PRECONDITION_FAILED)
+
+ yield deleteResource.storeRemove(
+ request,
+ True,
+ href,
+ )
+
+ except HTTPError, e:
+ # Extract the pre-condition
+ code = e.response.code
+ if isinstance(e.response, ErrorResponse):
+ error = e.response.error
+ error = (error.namespace, error.name,)
+
+ except Exception:
+ code = responsecode.BAD_REQUEST
+
+ if code is None:
+ xmlresponses.append(
+ davxml.StatusResponse(
+ davxml.HRef.fromString(href),
+ davxml.Status.fromResponseCode(responsecode.OK),
+ )
+ )
+ else:
+ xmlresponses.append(
+ davxml.StatusResponse(
+ davxml.HRef.fromString(href),
+ davxml.Status.fromResponseCode(code),
+ davxml.Error(
+ WebDAVUnknownElement.fromQname(*error),
+ ) if error else None,
+ )
+ )
+
+
def notifierID(self, label="default"):
self._newStoreObject.notifierID(label)
@@ -519,6 +886,9 @@
self._initializeWithHomeChild(calendar, home)
self._name = calendar.name() if calendar else name
+ if config.EnableBatchUpload:
+ self._postHandlers[("text", "calendar")] = _CommonHomeChildCollectionMixin.simpleBatchPOST
+ self.xmlDocHanders[customxml.Multiput] = _CommonHomeChildCollectionMixin.crudBatchPOST
def __repr__(self):
return "<Calendar Collection Resource %r:%r %s>" % (
@@ -598,8 +968,81 @@
createCalendarCollection = _CommonHomeChildCollectionMixin.createCollection
+ @classmethod
+ def componentsFromData(cls, data):
+ """
+ Need to split a single VCALENDAR into separate ones based on UID with the
+ appropriate VTIEMZONES included.
+ """
+
+ results = []
+ # Split into components by UID and TZID
+ try:
+ vcal = VCalendar.fromString(data)
+ except InvalidICalendarDataError:
+ return None
+
+ by_uid = {}
+ by_tzid = {}
+ for subcomponent in vcal.subcomponents():
+ if subcomponent.name() == "VTIMEZONE":
+ by_tzid[subcomponent.propertyValue("TZID")] = subcomponent
+ else:
+ by_uid.setdefault(subcomponent.propertyValue("UID"), []).append(subcomponent)
+
+ # Re-constitute as separate VCALENDAR objects
+ for components in by_uid.values():
+
+ newvcal = VCalendar("VCALENDAR")
+ newvcal.addProperty(VProperty("VERSION", "2.0"))
+ newvcal.addProperty(VProperty("PRODID", vcal.propertyValue("PRODID")))
+
+ # Get the set of TZIDs and include them
+ tzids = set()
+ for component in components:
+ tzids.update(component.timezoneIDs())
+ for tzid in tzids:
+ try:
+ tz = by_tzid[tzid]
+ newvcal.addComponent(tz.duplicate())
+ except KeyError:
+ # We ignore the error and generate invalid ics which someone will
+ # complain about at some point
+ pass
+
+ # Now add each component
+ for component in components:
+ newvcal.addComponent(component.duplicate())
+
+ results.append(newvcal)
+
+ return results
+
+ @classmethod
+ def resourceSuffix(cls):
+ return ".ics"
+
+ @classmethod
+ def xmlDataElementType(cls):
+ return caldavxml.CalendarData
+
@inlineCallbacks
+ def storeResourceData(self, request, newchild, newchildURL, data):
+ storer = StoreCalendarObjectResource(
+ request = request,
+ destination = newchild,
+ destination_uri = newchildURL,
+ destinationcal = True,
+ destinationparent = self,
+ calendar = data,
+ )
+ yield storer.run()
+
+ returnValue(storer.returndata if hasattr(storer, "returndata") else None)
+
+
+ @inlineCallbacks
def storeRemove(self, request, implicitly, where):
"""
Delete this calendar collection resource, first deleting each contained
@@ -634,7 +1077,8 @@
log.err("Cannot DELETE default calendar: %s" % (self,))
raise HTTPError(ErrorResponse(
FORBIDDEN,
- (caldav_namespace, "default-calendar-delete-allowed",)
+ (caldav_namespace, "default-calendar-delete-allowed",),
+ "Cannot delete default calendar",
))
response = (
@@ -1462,6 +1906,9 @@
self._initializeWithHomeChild(addressbook, home)
self._name = addressbook.name() if addressbook else name
+ if config.EnableBatchUpload:
+ self._postHandlers[("text", "vcard")] = _CommonHomeChildCollectionMixin.simpleBatchPOST
+ self.xmlDocHanders[customxml.Multiput] = _CommonHomeChildCollectionMixin.crudBatchPOST
def __repr__(self):
return "<AddressBook Collection Resource %r:%r %s>" % (
@@ -1484,7 +1931,36 @@
createAddressBookCollection = _CommonHomeChildCollectionMixin.createCollection
+ @classmethod
+ def componentsFromData(cls, data):
+ try:
+ return VCard.allFromString(data)
+ except InvalidVCardDataError:
+ return None
+ @classmethod
+ def resourceSuffix(cls):
+ return ".vcf"
+
+ @classmethod
+ def xmlDataElementType(cls):
+ return carddavxml.AddressData
+
+ @inlineCallbacks
+ def storeResourceData(self, request, newchild, newchildURL, data):
+ storer = StoreAddressObjectResource(
+ request = request,
+ sourceadbk = False,
+ destination = newchild,
+ destination_uri = newchildURL,
+ destinationadbk = True,
+ destinationparent = self,
+ vcard = data,
+ )
+ yield storer.run()
+
+ returnValue(storer.returndata if hasattr(storer, "returndata") else None)
+
class GlobalAddressBookCollectionResource(GlobalAddressBookResource, AddressBookCollectionResource):
"""
Wrapper around a L{txdav.carddav.iaddressbook.IAddressBook}.
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/timezoneservice.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/timezoneservice.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/timezoneservice.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -147,7 +147,8 @@
if len(method) != 1:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-method")
+ (calendarserver_namespace, "valid-method"),
+ "Invalid method query parameter",
))
method = method[0]
@@ -160,7 +161,8 @@
if action is None:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "supported-method")
+ (calendarserver_namespace, "supported-method"),
+ "Unknown method query parameter",
))
return action(request)
@@ -188,7 +190,8 @@
if len(tzid) != 1:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-timezone")
+ (calendarserver_namespace, "valid-timezone"),
+ "Invalid tzid query parameter",
))
tzid = tzid[0]
@@ -197,7 +200,8 @@
except TimezoneException:
raise HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (calendarserver_namespace, "timezone-available")
+ (calendarserver_namespace, "timezone-available"),
+ "Timezone not found",
))
response = Response()
@@ -214,7 +218,8 @@
if len(tzid) != 1:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-timezone")
+ (calendarserver_namespace, "valid-timezone"),
+ "Invalid tzid query parameter",
))
tzid = tzid[0]
try:
@@ -222,7 +227,8 @@
except TimezoneException:
raise HTTPError(ErrorResponse(
responsecode.NOT_FOUND,
- (calendarserver_namespace, "timezone-available")
+ (calendarserver_namespace, "timezone-available"),
+ "Timezone not found",
))
try:
@@ -233,7 +239,8 @@
except ValueError:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-start-date")
+ (calendarserver_namespace, "valid-start-date"),
+ "Invalid start query parameter",
))
try:
@@ -246,7 +253,8 @@
except ValueError:
raise HTTPError(ErrorResponse(
responsecode.BAD_REQUEST,
- (calendarserver_namespace, "valid-end-date")
+ (calendarserver_namespace, "valid-end-date"),
+ "Invalid end query parameter",
))
# Now do the expansion (but use a cache to avoid re-calculating TZs)
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/upgrade.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/upgrade.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -23,7 +23,6 @@
from twext.web2.dav.fileop import rmdir
from twext.web2.dav import davxml
-
from twext.python.log import Logger
from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
@@ -35,6 +34,8 @@
from twistedcaldav.ical import Component
from twistedcaldav import caldavxml
+from twisted.application.service import Service
+from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, succeed
from calendarserver.tools.util import getDirectory
@@ -558,11 +559,11 @@
if onDiskVersion < version:
log.warn("Upgrading to version %d" % (version,))
(yield method(config))
+ log.warn("Upgraded to version %d" % (version,))
with open(versionFilePath, "w") as verFile:
verFile.write(str(version))
os.chown(versionFilePath, uid, gid)
-
class UpgradeError(RuntimeError):
"""
Generic upgrade error.
@@ -674,3 +675,41 @@
# Can't rename, must copy/delete
shutil.copy2(srcPath, destPath)
os.remove(srcPath)
+
+
+
+class UpgradeFileSystemFormatService(Service, object):
+ """
+ Upgrade filesystem from previous versions.
+ """
+
+ def __init__(self, config, service):
+ """
+ Initialize the service.
+ """
+ self.wrappedService = service
+ self.config = config
+
+
+ @inlineCallbacks
+ def doUpgrade(self):
+ """
+ Do the upgrade. Called by C{startService}, but a different method
+ because C{startService} should return C{None}, not a L{Deferred}.
+
+ @return: a Deferred which fires when the upgrade is complete.
+ """
+ yield upgradeData(self.config)
+
+ # see http://twistedmatrix.com/trac/ticket/4649
+ reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
+
+
+ def startService(self):
+ """
+ Start the service.
+ """
+ self.doUpgrade()
+
+
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/vcard.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/vcard.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/twistedcaldav/vcard.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -142,6 +142,37 @@
raise InvalidVCardDataError(e)
@classmethod
+ def allFromString(clazz, string):
+ """
+ Construct a L{Component} from a string.
+ @param string: a string containing vCard data.
+ @return: a C{list} of L{Component}s representing the components described by
+ C{string}.
+ """
+ if type(string) is unicode:
+ string = string.encode("utf-8")
+ return clazz.allFromStream(StringIO.StringIO(string))
+
+ @classmethod
+ def allFromStream(clazz, stream):
+ """
+ Construct possibly multiple L{Component}s from a stream.
+ @param stream: a C{read()}able stream containing vCard data.
+ @return: a C{list} of L{Component}s representing the components described by
+ C{stream}.
+ """
+
+ results = []
+ try:
+ for vobject in readComponents(stream):
+ results.append(clazz(None, vobject=vobject))
+ return results
+ except vParseError, e:
+ raise InvalidVCardDataError(e)
+ except StopIteration, e:
+ raise InvalidVCardDataError(e)
+
+ @classmethod
def fromIStream(clazz, stream):
"""
Construct a L{Component} from a stream.
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/datastore/dbapiclient.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/datastore/dbapiclient.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/datastore/dbapiclient.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -17,7 +17,14 @@
"""
General utility client code for interfacing with DB-API 2.0 modules.
"""
+from twext.enterprise.util import mapOracleOutputType
+try:
+ import cx_Oracle
+except ImportError:
+ cx_Oracle = None
+
+
class DiagnosticCursorWrapper(object):
"""
Diagnostic wrapper around a DB-API 2.0 cursor for debugging connection
@@ -41,11 +48,11 @@
def execute(self, sql, args=()):
self.connectionWrapper.state = 'executing %r' % (sql,)
-# Use log.debug
-# sys.stdout.write(
-# "Really executing SQL %r in thread %r\n" %
-# ((sql % tuple(args)), thread.get_ident())
-# )
+ # Use log.debug
+ # sys.stdout.write(
+ # "Really executing SQL %r in thread %r\n" %
+ # ((sql % tuple(args)), thread.get_ident())
+ # )
self.realCursor.execute(sql, args)
@@ -55,21 +62,68 @@
def fetchall(self):
results = self.realCursor.fetchall()
-# Use log.debug
-# sys.stdout.write(
-# "Really fetching results %r thread %r\n" %
-# (results, thread.get_ident())
-# )
+ # Use log.debug
+ # sys.stdout.write(
+ # "Really fetching results %r thread %r\n" %
+ # (results, thread.get_ident())
+ # )
return results
+class OracleCursorWrapper(DiagnosticCursorWrapper):
+ """
+ Wrapper for cx_Oracle DB-API connections which implements fetchall() to read
+ all CLOB objects into strings.
+ """
+ def fetchall(self):
+ accum = []
+ for row in self.realCursor:
+ newRow = []
+ for column in row:
+ newRow.append(mapOracleOutputType(column))
+ accum.append(newRow)
+ return accum
+
+
+ def var(self, *args):
+ """
+ Create a cx_Oracle variable bound to this cursor. (Forwarded in
+ addition to the standard methods so that implementors of
+ L{IDerivedParameter} do not need to be specifically aware of this
+ layer.)
+ """
+ return self.realCursor.var(*args)
+
+
+ def execute(self, sql, args=()):
+ realArgs = []
+ for arg in args:
+ if isinstance(arg, (str, unicode)) and len(arg) > 1024:
+ # This *may* cause a type mismatch, but none of the non-CLOB
+ # strings that we're passing would allow a value this large
+ # anyway. Smaller strings will be automatically converted by
+ # the bindings; larger ones will generate an error. I'm not
+ # sure why cx_Oracle itself doesn't just do the following hack
+ # automatically and internally for larger values too, but, here
+ # it is:
+ v = self.var(cx_Oracle.CLOB, len(arg) + 1)
+ v.setvalue(0, arg)
+ else:
+ v = arg
+ realArgs.append(v)
+ return super(OracleCursorWrapper, self).execute(sql, realArgs)
+
+
+
class DiagnosticConnectionWrapper(object):
"""
Diagnostic wrapper around a DB-API 2.0 connection for debugging connection
status.
"""
+ wrapper = DiagnosticCursorWrapper
+
def __init__(self, realConnection, label):
self.realConnection = realConnection
self.label = label
@@ -77,7 +131,7 @@
def cursor(self):
- return DiagnosticCursorWrapper(self.realConnection.cursor(), self)
+ return self.wrapper(self.realConnection.cursor(), self)
def close(self):
@@ -103,6 +157,8 @@
@ivar dbModule: the DB-API module to use.
"""
+ wrapper = DiagnosticConnectionWrapper
+
def __init__(self, dbModule, preflight, *connectArgs, **connectKw):
self.dbModule = dbModule
self.connectArgs = connectArgs
@@ -112,12 +168,53 @@
def connect(self, label="<unlabeled>"):
connection = self.dbModule.connect(*self.connectArgs, **self.connectKw)
- w = DiagnosticConnectionWrapper(connection, label)
+ w = self.wrapper(connection, label)
self.preflight(w)
return w
+class OracleConnectionWrapper(DiagnosticConnectionWrapper):
+
+ wrapper = OracleCursorWrapper
+
+
+
+class OracleConnector(DBAPIConnector):
+ """
+ A connector for cx_Oracle connections, with some special-cased behavior to
+ make it work more like other DB-API bindings.
+
+ Note: this is currently necessary to make our usage of twext.enterprise.dal
+ work with cx_Oracle, and should be factored somewhere higher-level.
+ """
+
+ wrapper = OracleConnectionWrapper
+
+ def __init__(self, dsn):
+ super(OracleConnector, self).__init__(
+ cx_Oracle, oraclePreflight, dsn, threaded=True)
+
+
+
+def oraclePreflight(connection):
+ """
+ Pre-flight function for Oracle connections: set the timestamp format to be
+ something closely resembling our default assumption from Postgres.
+ """
+ c = connection.cursor()
+ c.execute(
+ "alter session set NLS_TIMESTAMP_FORMAT = "
+ "'YYYY-MM-DD HH24:MI:SS.FF'"
+ )
+ c.execute(
+ "alter session set NLS_TIMESTAMP_TZ_FORMAT = "
+ "'YYYY-MM-DD HH:MI:SS.FF+TZH:TZM'"
+ )
+ connection.commit()
+ c.close()
+
+
def postgresPreflight(connection):
"""
Pre-flight function for PostgreSQL connections: enable standard conforming
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/base.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/base.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/base.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -147,7 +147,10 @@
def _keys_uid(self, uid):
raise NotImplementedError()
-
+
+ def _removeResource(self):
+ raise NotImplementedError()
+
def flush(self):
raise NotImplementedError()
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/none.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/none.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/none.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -50,6 +50,9 @@
def keys(self):
return ()
+ def _removeResource(self):
+ pass
+
#
# I/O
#
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/sql.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/sql.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -204,4 +204,13 @@
if cachedUID == uid:
yield PropertyName.fromString(cachedKey)
+ _deleteResourceQuery = Delete(
+ prop, Where=(prop.RESOURCE_ID == Parameter("resourceID"))
+ )
+ def _removeResource(self):
+
+ self._cached = {}
+ self._deleteResourceQuery.on(self._txn, resourceID=self._resourceID)
+ self._cacher.delete(str(self._resourceID))
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/xattr.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/xattr.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/base/propertystore/xattr.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -257,6 +257,11 @@
if effectivekey[1] == uid and effectivekey not in seen:
yield effectivekey[0]
+ def _removeResource(self):
+ # xattrs are removed when the underlying file is deleted so just clear out cached changes
+ self.removed.clear()
+ self.modified.clear()
+
#
# I/O
#
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -314,6 +314,7 @@
# will have been indexed with an "infinite" value always included.
maxDate, isStartDate = filter.getmaxtimerange()
if maxDate:
+ maxDate = maxDate.duplicate()
maxDate.setDateOnly(True)
if isStartDate:
maxDate += PyCalendarDuration(days=365)
@@ -638,10 +639,9 @@
# Decide how far to expand based on the component
doInstanceIndexing = False
master = calendar.masterComponent()
- if master is None or not calendar.isRecurring() and not calendar.isRecurringUnbounded():
+ if master is None or not calendar.isRecurring():
# When there is no master we have a set of overridden components - index them all.
# When there is one instance - index it.
- # When bounded - index all.
expand = PyCalendarDateTime(2100, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
doInstanceIndexing = True
else:
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/caldav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/index.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/caldav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/caldav/datastore/index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/caldav/datastore/index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/index_file.py:6369
/CalendarServer/branches/users/glyph/oracle/txdav/caldav/datastore/index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/index.py:6322-6394
/CalendarServer/trunk/txdav/caldav/datastore/index_file.py:7085-7204
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/sql.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/sql.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -324,27 +324,6 @@
self.hasPrivateComment = metadata.get("hasPrivateComment", False)
- @classmethod
- def _selectAllColumns(cls):
- return """
- select
- %(column_RESOURCE_ID)s,
- %(column_RESOURCE_NAME)s,
- %(column_UID)s,
- %(column_MD5)s,
- character_length(%(column_TEXT)s),
- %(column_ATTACHMENTS_MODE)s,
- %(column_DROPBOX_ID)s,
- %(column_ACCESS)s,
- %(column_SCHEDULE_OBJECT)s,
- %(column_SCHEDULE_TAG)s,
- %(column_SCHEDULE_ETAGS)s,
- %(column_PRIVATE_COMMENTS)s,
- %(column_CREATED)s,
- %(column_MODIFIED)s
- """ % cls._objectTable
-
-
_allColumns = [
_objectSchema.RESOURCE_ID,
_objectSchema.RESOURCE_NAME,
@@ -420,12 +399,10 @@
# Decide how far to expand based on the component
doInstanceIndexing = False
master = component.masterComponent()
- if ( master is None or not component.isRecurring()
- or not component.isRecurringUnbounded() ):
+ if ( master is None or not component.isRecurring() ):
# When there is no master we have a set of overridden components -
# index them all.
# When there is one instance - index it.
- # When bounded - index all.
expand = PyCalendarDateTime(2100, 1, 1, 0, 0, 0, tzid=PyCalendarTimezone(utc=True))
doInstanceIndexing = True
else:
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/caldav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/caldav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/test/test_index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_index.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/caldav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/caldav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/caldav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/caldav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/caldav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/caldav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/caldav/datastore/test/test_index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/caldav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/caldav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/caldav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/caldav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/caldav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/caldav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/caldav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/caldav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/caldav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/caldav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/caldav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/caldav/datastore/test/test_index_file.py:6369
/CalendarServer/branches/users/glyph/oracle/txdav/caldav/datastore/test/test_index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/caldav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/caldav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/caldav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/caldav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/caldav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/caldav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/caldav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/caldav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/caldav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/caldav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/caldav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_index.py:6322-6394
/CalendarServer/trunk/txdav/caldav/datastore/test/test_index_file.py:7085-7204
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_sql.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/test/test_sql.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -13,34 +13,38 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
-import datetime
-from twistedcaldav.dateops import datetimeMktime
"""
Tests for txdav.caldav.datastore.postgres, mostly based on
L{txdav.caldav.datastore.test.common}.
"""
-from twisted.trial import unittest
+from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import deferLater
-from twisted.internet import reactor
from twisted.python import hashlib
+from twisted.trial import unittest
+from twext.enterprise.dal.syntax import Select, Parameter
from twext.python.vcomponent import VComponent
from twext.web2.dav.element.rfc2518 import GETContentLanguage, ResourceType
-from txdav.caldav.datastore.test.common import CommonTests as CalendarCommonTests
+from txdav.base.propertystore.base import PropertyName
+from txdav.caldav.datastore.test.common import CommonTests as CalendarCommonTests,\
+ event4_text
+from txdav.caldav.datastore.test.test_file import setUpCalendarStore
+from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
from txdav.common.datastore.sql import ECALENDARTYPE
+from txdav.common.datastore.sql_tables import schema
from txdav.common.datastore.test.util import buildStore, populateCalendarsFrom
-from txdav.caldav.datastore.test.test_file import setUpCalendarStore
-from txdav.caldav.datastore.util import _migrateCalendar, migrateHome
-from txdav.base.propertystore.base import PropertyName
-
-from twistedcaldav import memcacher
+from twistedcaldav import memcacher, caldavxml
from twistedcaldav.config import config
+from twistedcaldav.dateops import datetimeMktime
+from twistedcaldav.sharing import SharedCollectionRecord
+import datetime
+
class CalendarSQLStorageTests(CalendarCommonTests, unittest.TestCase):
"""
Calendar SQL storage tests.
@@ -433,5 +437,190 @@
self.assertNotEqual(notification_uid1_1, None)
self.assertNotEqual(notification_uid1_2, None)
+ @inlineCallbacks
+ def test_removeCalendarPropertiesOnDelete(self):
+ """
+ L{ICalendarHome.removeCalendarWithName} removes a calendar that already
+ exists and makes sure properties are also removed.
+ """
-
\ No newline at end of file
+ # Create calendar and add a property
+ home = yield self.homeUnderTest()
+ name = "remove-me"
+ calendar = yield home.createCalendarWithName(name)
+ resourceID = calendar._resourceID
+ calendarProperties = calendar.properties()
+
+ prop = caldavxml.CalendarDescription.fromString("Calendar to be removed")
+ calendarProperties[PropertyName.fromElement(prop)] = prop
+ yield self.commit()
+
+ prop = schema.RESOURCE_PROPERTY
+ _allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID == Parameter("resourceID"))
+
+ # Check that two properties are present
+ home = yield self.homeUnderTest()
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 2)
+ yield self.commit()
+
+ # Remove calendar and check for no properties
+ home = yield self.homeUnderTest()
+ yield home.removeCalendarWithName(name)
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ # Recheck it
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ @inlineCallbacks
+ def test_removeCalendarObjectPropertiesOnDelete(self):
+ """
+ L{ICalendarHome.removeCalendarWithName} removes a calendar object that already
+ exists and makes sure properties are also removed (which is always the case as right
+ now calendar objects never have properties).
+ """
+
+ # Create calendar object
+ calendar1 = yield self.calendarUnderTest()
+ name = "4.ics"
+ component = VComponent.fromString(event4_text)
+ metadata = {
+ "accessMode": "PUBLIC",
+ "isScheduleObject": True,
+ "scheduleTag": "abc",
+ "scheduleEtags": (),
+ "hasPrivateComment": False,
+ }
+ calobject = yield calendar1.createCalendarObjectWithName(name, component, metadata=metadata)
+ resourceID = calobject._resourceID
+
+ prop = schema.RESOURCE_PROPERTY
+ _allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID == Parameter("resourceID"))
+
+ # No properties on existing calendar object
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+
+ yield self.commit()
+
+ # Remove calendar and check for no properties
+ calendar1 = yield self.calendarUnderTest()
+ yield calendar1.removeCalendarObjectWithName(name)
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ # Recheck it
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ @inlineCallbacks
+ def test_removeInboxObjectPropertiesOnDelete(self):
+ """
+ L{ICalendarHome.removeCalendarWithName} removes an inbox calendar object that already
+ exists and makes sure properties are also removed. Inbox calendar objects can have properties.
+ """
+
+ # Create calendar object and add a property
+ home = yield self.homeUnderTest()
+ inbox = yield home.createCalendarWithName("inbox")
+
+ name = "4.ics"
+ component = VComponent.fromString(event4_text)
+ metadata = {
+ "accessMode": "PUBLIC",
+ "isScheduleObject": True,
+ "scheduleTag": "abc",
+ "scheduleEtags": (),
+ "hasPrivateComment": False,
+ }
+ calobject = yield inbox.createCalendarObjectWithName(name, component, metadata=metadata)
+ resourceID = calobject._resourceID
+ calobjectProperties = calobject.properties()
+
+ prop = caldavxml.CalendarDescription.fromString("Calendar object to be removed")
+ calobjectProperties[PropertyName.fromElement(prop)] = prop
+ yield self.commit()
+
+ prop = schema.RESOURCE_PROPERTY
+ _allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID == Parameter("resourceID"))
+
+ # One property exists calendar object
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 1)
+
+ yield self.commit()
+
+ # Remove calendar object and check for no properties
+ home = yield self.homeUnderTest()
+ inbox = yield home.calendarWithName("inbox")
+ yield inbox.removeCalendarObjectWithName(name)
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ # Recheck it
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ @inlineCallbacks
+ def test_directShareCreateConcurrency(self):
+ """
+ Test that two concurrent attempts to create a direct shared calendar
+ work concurrently without an exception.
+ """
+
+ calendarStore = self._sqlCalendarStore
+
+ # Provision the home and calendar now
+ txn = calendarStore.newTransaction()
+ home = yield txn.homeWithUID(ECALENDARTYPE, "uid1", create=True)
+ self.assertNotEqual(home, None)
+ cal = yield home.calendarWithName("calendar")
+ self.assertNotEqual(cal, None)
+ yield txn.commit()
+
+ txn1 = calendarStore.newTransaction()
+ txn2 = calendarStore.newTransaction()
+
+ home1 = yield txn1.homeWithUID(ECALENDARTYPE, "uid1", create=True)
+ home2 = yield txn2.homeWithUID(ECALENDARTYPE, "uid1", create=True)
+
+ shares1 = yield home1.retrieveOldShares()
+ shares2 = yield home2.retrieveOldShares()
+
+ record = SharedCollectionRecord(
+ "abcd",
+ "D",
+ "/calendars/__uids__/uid2/calendar/",
+ "XYZ",
+ "Shared Wiki Calendar",
+ )
+
+ @inlineCallbacks
+ def _defer1():
+ yield shares1.addOrUpdateRecord(record)
+ yield txn1.commit()
+ d1 = _defer1()
+
+ @inlineCallbacks
+ def _defer2():
+ yield shares2.addOrUpdateRecord(record)
+ yield txn2.commit()
+ d2 = _defer2()
+
+ yield d1
+ yield d2
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/util.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/caldav/datastore/util.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -83,7 +83,7 @@
dropboxProperty = (yield calendarObject.component(
)).getFirstPropertyInAnyComponent("X-APPLE-DROPBOX")
if dropboxProperty is not None:
- componentDropboxID = dropboxProperty.value().split("/")[-1]
+ componentDropboxID = dropboxProperty.value().rstrip("/").split("/")[-1]
returnValue(componentDropboxID)
# Now look at each ATTACH property and see if it might be a dropbox item
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/carddav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/vcardindex.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/carddav/datastore/index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/carddav/datastore/index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/carddav/datastore/index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/index_file.py:6369
/CalendarServer/branches/users/glyph/oracle/txdav/carddav/datastore/index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/vcardindex.py:6322-6394
/CalendarServer/trunk/txdav/carddav/datastore/index_file.py:7085-7204
Property changes on: CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_index_file.py
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation/txdav/carddav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store/txdav/carddav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/test/test_index_file.py:6369-6445
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_vcardindex.py:6322-6394
+ /CalendarServer/branches/config-separation/txdav/carddav/datastore/test/test_index_file.py:4379-4443
/CalendarServer/branches/egg-info-351/txdav/carddav/datastore/test/test_index_file.py:4589-4625
/CalendarServer/branches/generic-sqlstore/txdav/carddav/datastore/test/test_index_file.py:6167-6191
/CalendarServer/branches/new-store-no-caldavfile-2/txdav/carddav/datastore/test/test_index_file.py:5936-5981
/CalendarServer/branches/new-store-no-caldavfile/txdav/carddav/datastore/test/test_index_file.py:5911-5935
/CalendarServer/branches/new-store/txdav/carddav/datastore/test/test_index_file.py:5594-5934
/CalendarServer/branches/users/cdaboo/batchupload-6699/txdav/carddav/datastore/test/test_index_file.py:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692/txdav/carddav/datastore/test/test_index_file.py:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627/txdav/carddav/datastore/test/test_index_file.py:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591/txdav/carddav/datastore/test/test_index_file.py:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464/txdav/carddav/datastore/test/test_index_file.py:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070/txdav/carddav/datastore/test/test_index_file.py:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187/txdav/carddav/datastore/test/test_index_file.py:5188-5440
/CalendarServer/branches/users/glyph/conn-limit/txdav/carddav/datastore/test/test_index_file.py:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge/txdav/carddav/datastore/test/test_index_file.py:4971-5080
/CalendarServer/branches/users/glyph/dalify/txdav/carddav/datastore/test/test_index_file.py:6932-7023
/CalendarServer/branches/users/glyph/dont-start-postgres/txdav/carddav/datastore/test/test_index_file.py:6592-6614
/CalendarServer/branches/users/glyph/linux-tests/txdav/carddav/datastore/test/test_index_file.py:6893-6900
/CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py:6322-6334
/CalendarServer/branches/users/glyph/more-deferreds-7/txdav/carddav/datastore/test/test_index_file.py:6369
/CalendarServer/branches/users/glyph/oracle/txdav/carddav/datastore/test/test_index_file.py:7106-7155
/CalendarServer/branches/users/glyph/sendfdport/txdav/carddav/datastore/test/test_index_file.py:5388-5424
/CalendarServer/branches/users/glyph/sharedpool/txdav/carddav/datastore/test/test_index_file.py:6490-6550
/CalendarServer/branches/users/glyph/sql-store/txdav/carddav/datastore/test/test_index_file.py:5929-6073
/CalendarServer/branches/users/glyph/use-system-twisted/txdav/carddav/datastore/test/test_index_file.py:5084-5149
/CalendarServer/branches/users/sagen/locations-resources-2/txdav/carddav/datastore/test/test_index_file.py:5052-5061
/CalendarServer/branches/users/sagen/locations-resources/txdav/carddav/datastore/test/test_index_file.py:5032-5051
/CalendarServer/branches/users/sagen/purge_old_events/txdav/carddav/datastore/test/test_index_file.py:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038/txdav/carddav/datastore/test/test_index_file.py:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066/txdav/carddav/datastore/test/test_index_file.py:4068-4075
/CalendarServer/branches/users/sagen/resources-2/txdav/carddav/datastore/test/test_index_file.py:5084-5093
/CalendarServer/branches/users/wsanchez/transations/txdav/carddav/datastore/test/test_index_file.py:5515-5593
/CalendarServer/trunk/twistedcaldav/test/test_vcardindex.py:6322-6394
/CalendarServer/trunk/txdav/carddav/datastore/test/test_index_file.py:7085-7204
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_sql.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/carddav/datastore/test/test_sql.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -19,23 +19,27 @@
L{txdav.carddav.datastore.test.common}.
"""
-from txdav.carddav.datastore.test.common import CommonTests as AddressBookCommonTests
-
-from txdav.common.datastore.sql import EADDRESSBOOKTYPE
-from txdav.common.datastore.test.util import buildStore
-from txdav.carddav.datastore.test.test_file import setUpAddressBookStore
+from twext.enterprise.dal.syntax import Select, Parameter
from twext.web2.dav.element.rfc2518 import GETContentLanguage, ResourceType
-from txdav.base.propertystore.base import PropertyName
-from txdav.carddav.datastore.util import _migrateAddressbook, migrateHome
+from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.trial import unittest
-from twisted.internet.defer import inlineCallbacks, returnValue
-from twistedcaldav import memcacher
+from twistedcaldav import memcacher, carddavxml
from twistedcaldav.config import config
from twistedcaldav.vcard import Component as VCard
+from twistedcaldav.vcard import Component as VComponent
+from txdav.base.propertystore.base import PropertyName
+from txdav.carddav.datastore.test.common import CommonTests as AddressBookCommonTests,\
+ vcard4_text
+from txdav.carddav.datastore.test.test_file import setUpAddressBookStore
+from txdav.carddav.datastore.util import _migrateAddressbook, migrateHome
+from txdav.common.datastore.sql import EADDRESSBOOKTYPE
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.test.util import buildStore
+
class AddressBookSQLStorageTests(AddressBookCommonTests, unittest.TestCase):
"""
AddressBook SQL storage tests.
@@ -260,3 +264,83 @@
yield d1
yield d2
+
+ @inlineCallbacks
+ def test_removeAddressBookPropertiesOnDelete(self):
+ """
+ L{IAddressBookHome.removeAddressBookWithName} removes an address book that already
+ exists and makes sure properties are also removed.
+ """
+
+ # Create address book and add a property
+ home = yield self.homeUnderTest()
+ name = "remove-me"
+ addressbook = yield home.createAddressBookWithName(name)
+ resourceID = addressbook._resourceID
+ addressbookProperties = addressbook.properties()
+
+ prop = carddavxml.AddressBookDescription.fromString("Address Book to be removed")
+ addressbookProperties[PropertyName.fromElement(prop)] = prop
+ yield self.commit()
+
+ prop = schema.RESOURCE_PROPERTY
+ _allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID == Parameter("resourceID"))
+
+ # Check that two properties are present
+ home = yield self.homeUnderTest()
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 2)
+ yield self.commit()
+
+ # Remove address book and check for no properties
+ home = yield self.homeUnderTest()
+ yield home.removeAddressBookWithName(name)
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ # Recheck it
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ @inlineCallbacks
+ def test_removeAddressBookObjectPropertiesOnDelete(self):
+ """
+ L{IAddressBookHome.removeAddressBookWithName} removes an address book object that already
+ exists and makes sure properties are also removed (which is always the case as right
+ now address book objects never have properties).
+ """
+
+ # Create address book object
+ adbk1 = yield self.addressbookUnderTest()
+ name = "4.vcf"
+ component = VComponent.fromString(vcard4_text)
+ addressobject = yield adbk1.createAddressBookObjectWithName(name, component, metadata={})
+ resourceID = addressobject._resourceID
+
+ prop = schema.RESOURCE_PROPERTY
+ _allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE],
+ From=prop,
+ Where=prop.RESOURCE_ID == Parameter("resourceID"))
+
+ # No properties on existing address book object
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+
+ yield self.commit()
+
+ # Remove address book object and check for no properties
+ adbk1 = yield self.addressbookUnderTest()
+ yield adbk1.removeAddressBookObjectWithName(name)
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
+ # Recheck it
+ rows = yield _allWithID.on(self.transactionUnderTest(), resourceID=resourceID)
+ self.assertEqual(len(tuple(rows)), 0)
+ yield self.commit()
+
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/file.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/file.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -668,6 +668,7 @@
def cleanup():
try:
trash.remove()
+ self.properties()._removeResource()
except Exception, e:
self.log_error("Unable to delete trashed child at %s: %s" % (trash.fp, e))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -62,13 +62,11 @@
from twext.enterprise.dal.syntax import Delete
from twext.enterprise.dal.syntax import Insert
from twext.enterprise.dal.syntax import Len
-from twext.enterprise.dal.syntax import Lock
from twext.enterprise.dal.syntax import Max
from twext.enterprise.dal.syntax import Parameter
from twext.enterprise.dal.syntax import SavepointAction
from twext.enterprise.dal.syntax import Select
from twext.enterprise.dal.syntax import Update
-from twext.enterprise.dal.syntax import default
from txdav.base.propertystore.base import PropertyName
from txdav.base.propertystore.none import PropertyStore as NonePropertyStore
@@ -180,6 +178,7 @@
CommonStoreTransaction._homeClass[EADDRESSBOOKTYPE] = AddressBookHome
self._sqlTxn = sqlTxn
self.paramstyle = sqlTxn.paramstyle
+ self.dialect = sqlTxn.dialect
def store(self):
@@ -485,7 +484,7 @@
{cls._homeMetaDataSchema.RESOURCE_ID: resourceid}).on(txn)
except Exception: # FIXME: Really want to trap the pg.DatabaseError but in a non-DB specific manner
yield savepoint.rollback(txn)
-
+
# Retry the query - row may exist now, if not re-raise
homeObject = cls(txn, uid, notifiers)
homeObject = (yield homeObject.initFromStore())
@@ -1082,8 +1081,7 @@
rev.DELETED: True},
Where=(rev.HOME_RESOURCE_ID == Parameter("homeID")).And(
rev.RESOURCE_ID == Parameter("resourceID")).And(
- rev.RESOURCE_NAME == None),
- #Return=rev.REVISION
+ rev.RESOURCE_NAME == None)
)
@@ -1098,7 +1096,6 @@
rev.DELETED: True},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == None),
- # Return=rev.REVISION,
)
@@ -1502,7 +1499,7 @@
DAL statement to create a home child with all default values.
"""
child = cls._homeChildSchema
- return Insert({child.RESOURCE_ID: default},
+ return Insert({child.RESOURCE_ID: schema.RESOURCE_ID_SEQ},
Return=(child.RESOURCE_ID, child.CREATED, child.MODIFIED))
@@ -1659,6 +1656,8 @@
yield self._deletedSyncToken()
yield self._deleteQuery.on(self._txn, NoSuchHomeChildError,
resourceID=self._resourceID)
+ self.properties()._removeResource()
+
# Set to non-existent state
self._resourceID = None
self._created = None
@@ -1862,54 +1861,34 @@
returnValue(objectResource)
- @classproperty
- def _removeObjectResourceByNameQuery(cls):
- """
- DAL query to remove an object resource from this collection by name,
- returning the UID of the resource it deleted.
- """
- obj = cls._objectSchema
- return Delete(From=obj,
- Where=(obj.RESOURCE_NAME == Parameter("name")).And(
- obj.PARENT_RESOURCE_ID == Parameter("resourceID")),
- Return=obj.UID)
-
-
@inlineCallbacks
def removeObjectResourceWithName(self, name):
- uid = (yield self._removeObjectResourceByNameQuery.on(
- self._txn, NoSuchObjectResourceError,
- name=name, resourceID=self._resourceID))[0][0]
- self._objects.pop(name, None)
- self._objects.pop(uid, None)
- yield self._deleteRevision(name)
- self.notifyChanged()
+
+ child = (yield self.objectResourceWithName(name))
+ if child is None:
+ raise NoSuchObjectResourceError
+ yield self._removeObjectResource(child)
- @classproperty
- def _removeObjectResourceByUIDQuery(cls):
- """
- DAL query to remove an object resource from this collection by UID,
- returning the name of the resource it deleted.
- """
- obj = cls._objectSchema
- return Delete(From=obj,
- Where=(obj.UID == Parameter("uid")).And(
- obj.PARENT_RESOURCE_ID == Parameter("resourceID")),
- Return=obj.RESOURCE_NAME)
-
-
@inlineCallbacks
def removeObjectResourceWithUID(self, uid):
- name = (yield self._removeObjectResourceByUIDQuery.on(
- self._txn, NoSuchObjectResourceError,
- uid=uid, resourceID=self._resourceID
- ))[0][0]
- self._objects.pop(name, None)
- self._objects.pop(uid, None)
- yield self._deleteRevision(name)
+
+ child = (yield self.objectResourceWithUID(uid))
+ if child is None:
+ raise NoSuchObjectResourceError
+ yield self._removeObjectResource(child)
- self.notifyChanged()
+ @inlineCallbacks
+ def _removeObjectResource(self, child):
+ name = child.name()
+ uid = child.uid()
+ try:
+ yield child.remove()
+ finally:
+ self._objects.pop(name, None)
+ self._objects.pop(uid, None)
+ yield self._deleteRevision(name)
+ self.notifyChanged()
def objectResourcesHaveProperties(self):
@@ -2165,25 +2144,6 @@
returnValue(None)
- @classmethod
- def _selectAllColumns(cls):
- """
- Full set of columns in the object table that need to be loaded to
- initialize the object resource state. (XXX: remove me, old string-based
- version, see _allColumns)
- """
- return """
- select
- %(column_RESOURCE_ID)s,
- %(column_RESOURCE_NAME)s,
- %(column_UID)s,
- %(column_MD5)s,
- character_length(%(column_TEXT)s),
- %(column_CREATED)s,
- %(column_MODIFIED)s
- """ % cls._objectTable
-
-
@classproperty
def _allColumns(cls):
"""
@@ -2265,7 +2225,30 @@
def componentType(self):
returnValue((yield self.component()).mainType())
+ @classproperty
+ def _deleteQuery(cls):
+ """
+ DAL statement to delete a L{CommonObjectResource} by its resource ID.
+ """
+ return Delete(cls._objectSchema, Where=cls._objectSchema.RESOURCE_ID == Parameter("resourceID"))
+
+ @inlineCallbacks
+ def remove(self):
+ yield self._deleteQuery.on(self._txn, NoSuchObjectResourceError,
+ resourceID=self._resourceID)
+ self.properties()._removeResource()
+
+ # Set to non-existent state
+ self._resourceID = None
+ self._name = None
+ self._uid = None
+ self._md5 = None
+ self._size = None
+ self._created = None
+ self._modified = None
+ self._objectText = None
+
def uid(self):
return self._uid
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_legacy.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_legacy.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_legacy.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -1,4 +1,4 @@
-# -*- test-case-name: txdav.caldav.datastore.test.test_sql -*-
+# -*- test-case-name: twistedcaldav.test.test_sharing,twistedcaldav.test.test_calendarquery -*-
##
# Copyright (c) 2010 Apple Inc. All rights reserved.
#
@@ -27,6 +27,7 @@
from twisted.python import hashlib
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
+from twext.python.clsprop import classproperty
from twext.python.log import Logger, LoggingMixIn
from twistedcaldav import carddavxml
@@ -34,19 +35,26 @@
from twistedcaldav.dateops import normalizeForIndex, pyCalendarTodatetime
from twistedcaldav.memcachepool import CachePoolUserMixIn
from twistedcaldav.notifications import NotificationRecord
-from twistedcaldav.query import calendarqueryfilter, calendarquery, \
- addressbookquery
+from twistedcaldav.query import (
+ calendarqueryfilter, calendarquery, addressbookquery)
from twistedcaldav.query.sqlgenerator import sqlgenerator
from twistedcaldav.sharing import Invite
-from txdav.common.icommondatastore import IndexedSearchException, \
- ReservationError
-from txdav.common.datastore.sql_tables import \
- _BIND_MODE_OWN, _BIND_MODE_READ, _BIND_MODE_WRITE, _BIND_MODE_DIRECT, \
- _BIND_STATUS_INVITED, _BIND_STATUS_ACCEPTED, _BIND_STATUS_DECLINED, _BIND_STATUS_INVALID, \
- CALENDAR_BIND_TABLE, CALENDAR_HOME_TABLE, ADDRESSBOOK_HOME_TABLE, \
- ADDRESSBOOK_BIND_TABLE
+from txdav.common.icommondatastore import (
+ IndexedSearchException, ReservationError)
+from twext.enterprise.dal.syntax import Update, SavepointAction
+from twext.enterprise.dal.syntax import Insert
+from twext.enterprise.dal.syntax import Select
+from twext.enterprise.dal.syntax import Delete
+from twext.enterprise.dal.syntax import Parameter
+from txdav.common.datastore.sql_tables import (
+ _BIND_MODE_OWN, _BIND_MODE_READ, _BIND_MODE_WRITE, _BIND_MODE_DIRECT,
+ _BIND_STATUS_INVITED, _BIND_STATUS_ACCEPTED, _BIND_STATUS_DECLINED,
+ _BIND_STATUS_INVALID, CALENDAR_BIND_TABLE, CALENDAR_HOME_TABLE,
+ ADDRESSBOOK_HOME_TABLE, ADDRESSBOOK_BIND_TABLE, schema)
+
+
from pycalendar.duration import PyCalendarDuration
log = Logger()
@@ -109,6 +117,9 @@
_homeTable = None
_bindTable = None
+ _homeSchema = None
+ _bindSchema = None
+
def __init__(self, collection):
self._collection = collection
@@ -137,56 +148,63 @@
"No-op, because the index implicitly always exists in the database."
+ @classmethod
+ def _allColumnsQuery(cls, condition):
+ inv = schema.INVITE
+ home = cls._homeSchema
+ bind = cls._bindSchema
+ return Select(
+ [inv.INVITE_UID,
+ inv.NAME,
+ inv.RECIPIENT_ADDRESS,
+ home.OWNER_UID,
+ bind.BIND_MODE,
+ bind.BIND_STATUS,
+ bind.MESSAGE],
+ From=inv.join(home).join(bind),
+ Where=(
+ condition
+ .And(inv.RESOURCE_ID == bind.RESOURCE_ID)
+ .And(inv.HOME_RESOURCE_ID == home.RESOURCE_ID)
+ .And(inv.HOME_RESOURCE_ID == bind.HOME_RESOURCE_ID)),
+ OrderBy=inv.NAME, Ascending=True
+ )
+
+
+ @classproperty
+ def _allRecordsQuery(cls):
+ """
+ DAL query for all invite records with a given resource ID.
+ """
+ inv = schema.INVITE
+ return cls._allColumnsQuery(inv.RESOURCE_ID == Parameter("resourceID"))
+
+
@inlineCallbacks
def allRecords(self):
values = []
- for row in (yield self._txn.execSQL(
- """
- select
- INVITE.INVITE_UID,
- INVITE.NAME,
- INVITE.RECIPIENT_ADDRESS,
- %(HOME:name)s.%(HOME:column_OWNER_UID)s,
- %(BIND:name)s.%(BIND:column_BIND_MODE)s,
- %(BIND:name)s.%(BIND:column_BIND_STATUS)s,
- %(BIND:name)s.%(BIND:column_MESSAGE)s
- from
- INVITE, %(HOME:name)s, %(BIND:name)s
- where
- INVITE.RESOURCE_ID = %%s
- and INVITE.HOME_RESOURCE_ID = %(HOME:name)s.%(HOME:column_RESOURCE_ID)s
- and %(BIND:name)s.%(BIND:column_RESOURCE_ID)s = INVITE.RESOURCE_ID
- and %(BIND:name)s.%(BIND:column_HOME_RESOURCE_ID)s = INVITE.HOME_RESOURCE_ID
- order by
- INVITE.NAME asc
- """ % self._combinedTable,
- [self._collection._resourceID]
- )):
+ rows = yield self._allRecordsQuery.on(
+ self._txn, resourceID=self._collection._resourceID
+ )
+ for row in rows:
values.append(self._makeInvite(row))
returnValue(values)
+ @classproperty
+ def _inviteForRecipientQuery(cls):
+ """
+ DAL query to retrieve an invite record for a given recipient address.
+ """
+ inv = schema.INVITE
+ return cls._allColumnsQuery(
+ inv.RECIPIENT_ADDRESS == Parameter("recipient"))
+
+
@inlineCallbacks
def recordForUserID(self, userid):
- rows = yield self._txn.execSQL(
- """
- select
- INVITE.INVITE_UID,
- INVITE.NAME,
- INVITE.RECIPIENT_ADDRESS,
- %(HOME:name)s.%(HOME:column_OWNER_UID)s,
- %(BIND:name)s.%(BIND:column_BIND_MODE)s,
- %(BIND:name)s.%(BIND:column_BIND_STATUS)s,
- %(BIND:name)s.%(BIND:column_MESSAGE)s
- from
- INVITE, %(HOME:name)s, %(BIND:name)s
- where INVITE.RECIPIENT_ADDRESS = %%s
- and INVITE.HOME_RESOURCE_ID = %(HOME:name)s.%(HOME:column_RESOURCE_ID)s
- and %(BIND:name)s.%(BIND:column_RESOURCE_ID)s = INVITE.RESOURCE_ID
- and %(BIND:name)s.%(BIND:column_HOME_RESOURCE_ID)s = INVITE.HOME_RESOURCE_ID
- """ % self._combinedTable,
- [userid]
- )
+ rows = yield self._inviteForRecipientQuery.on(self._txn,
+ recipient=userid)
returnValue(self._makeInvite(rows[0]) if rows else None)
@@ -197,27 +215,18 @@
returnValue(record)
+ @classproperty
+ def _inviteForUIDQuery(cls):
+ """
+ DAL query to retrieve an invite record for a given recipient address.
+ """
+ inv = schema.INVITE
+ return cls._allColumnsQuery(inv.INVITE_UID == Parameter("uid"))
+
+
@inlineCallbacks
def recordForInviteUID(self, inviteUID):
- rows = yield self._txn.execSQL(
- """
- select
- INVITE.INVITE_UID,
- INVITE.NAME,
- INVITE.RECIPIENT_ADDRESS,
- %(HOME:name)s.%(HOME:column_OWNER_UID)s,
- %(BIND:name)s.%(BIND:column_BIND_MODE)s,
- %(BIND:name)s.%(BIND:column_BIND_STATUS)s,
- %(BIND:name)s.%(BIND:column_MESSAGE)s
- from
- INVITE, %(HOME:name)s, %(BIND:name)s
- where INVITE.INVITE_UID = %%s
- and INVITE.HOME_RESOURCE_ID = %(HOME:name)s.%(HOME:column_RESOURCE_ID)s
- and %(BIND:name)s.%(BIND:column_RESOURCE_ID)s = INVITE.RESOURCE_ID
- and %(BIND:name)s.%(BIND:column_HOME_RESOURCE_ID)s = INVITE.HOME_RESOURCE_ID
- """ % self._combinedTable,
- [inviteUID]
- )
+ rows = yield self._inviteForUIDQuery.on(self._txn, uid=inviteUID)
returnValue(self._makeInvite(rows[0]) if rows else None)
@@ -243,6 +252,71 @@
)
+ @classproperty
+ def _updateBindQuery(cls):
+ bind = cls._bindSchema
+
+ return Update({bind.BIND_MODE: Parameter("mode"),
+ bind.BIND_STATUS: Parameter("status"),
+ bind.MESSAGE: Parameter("message")},
+ Where=
+ (bind.RESOURCE_ID == Parameter("resourceID"))
+ .And(bind.HOME_RESOURCE_ID == Parameter("homeID")))
+
+
+ @classproperty
+ def _idsForRecipient(cls):
+ inv = schema.INVITE
+ return Select([inv.RESOURCE_ID, inv.HOME_RESOURCE_ID],
+ From=inv,
+ Where=inv.RECIPIENT_ADDRESS == Parameter("recipient"))
+
+
+ @classproperty
+ def _updateInviteQuery(cls):
+ """
+ DAL query to update an invitation for a given recipient.
+ """
+ inv = schema.INVITE
+ return Update({inv.NAME: Parameter("name"),
+ inv.INVITE_UID: Parameter("uid")},
+ Where=inv.RECIPIENT_ADDRESS == Parameter("recipient"))
+
+
+ @classproperty
+ def _insertBindQuery(cls):
+ bind = cls._bindSchema
+ return Insert(
+ {
+ bind.HOME_RESOURCE_ID: Parameter("homeID"),
+ bind.RESOURCE_ID: Parameter("resourceID"),
+ bind.BIND_MODE: Parameter("mode"),
+ bind.BIND_STATUS: Parameter("status"),
+ bind.MESSAGE: Parameter("message"),
+
+ # name is NULL because the resource is not bound yet, just
+ # invited; let's be explicit about that.
+ bind.RESOURCE_NAME: None,
+ bind.SEEN_BY_OWNER: False,
+ bind.SEEN_BY_SHAREE: False,
+ }
+ )
+
+
+ @classproperty
+ def _insertInviteQuery(cls):
+ inv = schema.INVITE
+ return Insert(
+ {
+ inv.INVITE_UID: Parameter("uid"),
+ inv.NAME: Parameter("name"),
+ inv.HOME_RESOURCE_ID: Parameter("homeID"),
+ inv.RESOURCE_ID: Parameter("resourceID"),
+ inv.RECIPIENT_ADDRESS: Parameter("recipient")
+ }
+ )
+
+
@inlineCallbacks
def addOrUpdateRecord(self, record):
bindMode = {'read-only': _BIND_MODE_READ,
@@ -258,108 +332,88 @@
# (and may contain a trailing slash).
principalUID = record.principalURL.split("/")[3]
shareeHome = yield self._getHomeWithUID(principalUID)
- rows = yield self._txn.execSQL(
- "select RESOURCE_ID, HOME_RESOURCE_ID from INVITE where RECIPIENT_ADDRESS = %s",
- [record.userid]
- )
+ rows = yield self._idsForRecipient.on(self._txn,
+ recipient=record.userid)
if rows:
[[resourceID, homeResourceID]] = rows
- # Invite(inviteuid, userid, principalURL, common_name, access, state, summary)
- yield self._txn.execSQL(
- """
- update %(BIND:name)s
- set %(BIND:column_BIND_MODE)s = %%s,
- %(BIND:column_BIND_STATUS)s = %%s,
- %(BIND:column_MESSAGE)s = %%s
- where %(BIND:column_RESOURCE_ID)s = %%s
- and %(BIND:column_HOME_RESOURCE_ID)s = %%s
- """ % self._combinedTable,
- [bindMode, bindStatus, record.summary, resourceID, homeResourceID]
+ yield self._updateBindQuery.on(
+ self._txn,
+ mode=bindMode, status=bindStatus, message=record.summary,
+ resourceID=resourceID, homeID=homeResourceID
)
- yield self._txn.execSQL("""
- update INVITE
- set NAME = %s, INVITE_UID = %s
- where RECIPIENT_ADDRESS = %s
- """,
- [record.name, record.inviteuid, record.userid]
+ yield self._updateInviteQuery.on(
+ self._txn, name=record.name, uid=record.inviteuid,
+ recipient=record.userid
)
else:
- yield self._txn.execSQL(
- """
- insert into INVITE
- (
- INVITE_UID, NAME,
- HOME_RESOURCE_ID, RESOURCE_ID,
- RECIPIENT_ADDRESS
- )
- values (%s, %s, %s, %s, %s)
- """,
- [
- record.inviteuid, record.name,
- shareeHome._resourceID, self._collection._resourceID,
- record.userid
- ])
- yield self._txn.execSQL(
- """
- insert into %(BIND:name)s
- (
- %(BIND:column_HOME_RESOURCE_ID)s,
- %(BIND:column_RESOURCE_ID)s,
- %(BIND:column_RESOURCE_NAME)s,
- %(BIND:column_BIND_MODE)s,
- %(BIND:column_BIND_STATUS)s,
- %(BIND:column_SEEN_BY_OWNER)s,
- %(BIND:column_SEEN_BY_SHAREE)s,
- %(BIND:column_MESSAGE)s
- )
- values (%%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s)
- """ % self._combinedTable,
- [
- shareeHome._resourceID,
- self._collection._resourceID,
- None, # this is NULL because it is not bound yet, let's be
- # explicit about that.
- bindMode,
- bindStatus,
- False,
- False,
- record.summary
- ]
+ yield self._insertInviteQuery.on(
+ self._txn, uid=record.inviteuid, name=record.name,
+ homeID=shareeHome._resourceID,
+ resourceID=self._collection._resourceID,
+ recipient=record.userid
)
+ yield self._insertBindQuery.on(
+ self._txn,
+ homeID=shareeHome._resourceID,
+ resourceID=self._collection._resourceID,
+ mode=bindMode,
+ status=bindStatus,
+ message=record.summary
+ )
+ @classmethod
+ def _deleteOneBindQuery(cls, constraint):
+ inv = schema.INVITE
+ bind = cls._bindSchema
+ return Delete(
+ From=bind, Where=(bind.HOME_RESOURCE_ID, bind.RESOURCE_ID) ==
+ Select([inv.HOME_RESOURCE_ID, inv.RESOURCE_ID],
+ From=inv, Where=constraint))
+
+
+ @classmethod
+ def _deleteOneInviteQuery(cls, constraint):
+ inv = schema.INVITE
+ return Delete(From=inv, Where=constraint)
+
+
+ @classproperty
+ def _deleteBindByRecipient(cls):
+ inv = schema.INVITE
+ return cls._deleteOneBindQuery(
+ inv.RECIPIENT_ADDRESS == Parameter("recipient"))
+
+
+ @classproperty
+ def _deleteInviteByRecipient(cls):
+ inv = schema.INVITE
+ return cls._deleteOneInviteQuery(
+ inv.RECIPIENT_ADDRESS == Parameter("recipient"))
+
+
+ @classproperty
+ def _deleteBindByUID(cls):
+ inv = schema.INVITE
+ return cls._deleteOneBindQuery(inv.INVITE_UID == Parameter("uid"))
+
+
+ @classproperty
+ def _deleteInviteByUID(cls):
+ inv = schema.INVITE
+ return cls._deleteOneInviteQuery(inv.INVITE_UID == Parameter("uid"))
+
+
@inlineCallbacks
def removeRecordForUserID(self, userid):
- yield self._txn.execSQL(
- """
- delete from %(BIND:name)s using INVITE
- where INVITE.RECIPIENT_ADDRESS = %%s
- and %(BIND:name)s.%(BIND:column_HOME_RESOURCE_ID)s = INVITE.HOME_RESOURCE_ID
- and %(BIND:name)s.%(BIND:column_RESOURCE_ID)s = INVITE.RESOURCE_ID
- """ % self._combinedTable,
- [userid]
- )
- yield self._txn.execSQL(
- "delete from INVITE where RECIPIENT_ADDRESS = %s",
- [userid]
- )
+ yield self._deleteBindByRecipient.on(self._txn, recipient=userid)
+ yield self._deleteInviteByRecipient.on(self._txn, recipient=userid)
@inlineCallbacks
def removeRecordForInviteUID(self, inviteUID):
- yield self._txn.execSQL(
- """
- delete from %(BIND:name)s using INVITE
- where INVITE.INVITE_UID = %s
- and %(BIND:name)s.%(BIND:column_HOME_RESOURCE_ID)s = INVITE.HOME_RESOURCE_ID
- and %(BIND:name)s.%(BIND:column_RESOURCE_ID)s = INVITE.RESOURCE_ID
- """ % self._combinedTable,
- [inviteUID]
- )
- yield self._txn.execSQL(
- "delete from INVITE where INVITE_UID = %s",
- [inviteUID]
- )
+ yield self._deleteBindByUID.on(self._txn, uid=inviteUID)
+ yield self._deleteInviteByUID.on(self._txn, uid=inviteUID)
@@ -369,11 +423,11 @@
L{twistedcaldav.sharing.InvitesDatabase}.
"""
- def __init__(self, calendar):
- self._homeTable = CALENDAR_HOME_TABLE
- self._bindTable = CALENDAR_BIND_TABLE
- super(SQLLegacyCalendarInvites, self).__init__(calendar)
+ _homeTable = CALENDAR_HOME_TABLE
+ _bindTable = CALENDAR_BIND_TABLE
+ _homeSchema = schema.CALENDAR_HOME
+ _bindSchema = schema.CALENDAR_BIND
def _getHomeWithUID(self, uid):
return self._txn.calendarHomeWithUID(uid, create=True)
@@ -386,11 +440,11 @@
L{twistedcaldav.sharing.InvitesDatabase}.
"""
- def __init__(self, addressbook):
- self._homeTable = ADDRESSBOOK_HOME_TABLE
- self._bindTable = ADDRESSBOOK_BIND_TABLE
- super(SQLLegacyAddressBookInvites, self).__init__(addressbook)
+ _homeTable = ADDRESSBOOK_HOME_TABLE
+ _bindTable = ADDRESSBOOK_BIND_TABLE
+ _homeSchema = schema.ADDRESSBOOK_HOME
+ _bindSchema = schema.ADDRESSBOOK_BIND
def _getHomeWithUID(self, uid):
return self._txn.addressbookHomeWithUID(uid, create=True)
@@ -403,9 +457,13 @@
_bindTable = None
_urlTopSegment = None
+ _homeSchema = None
+ _bindSchema = None
+
def __init__(self, home):
self._home = home
+
@property
def _txn(self):
return self._home._txn
@@ -423,6 +481,50 @@
pass
+ @classproperty
+ def _allSharedToQuery(cls):
+ bind = cls._bindSchema
+ return Select(
+ [bind.RESOURCE_ID, bind.RESOURCE_NAME,
+ bind.BIND_MODE, bind.MESSAGE],
+ From=bind,
+ Where=(bind.HOME_RESOURCE_ID == Parameter("homeID"))
+ .And(bind.BIND_MODE != _BIND_MODE_OWN)
+ .And(bind.RESOURCE_NAME != None)
+ )
+
+
+ @classproperty
+ def _inviteUIDByResourceIDsQuery(cls):
+ inv = schema.INVITE
+ return Select(
+ [inv.INVITE_UID], From=inv, Where=
+ (inv.RESOURCE_ID == Parameter("resourceID"))
+ .And(inv.HOME_RESOURCE_ID == Parameter("homeID"))
+ )
+
+
+ @classproperty
+ def _ownerHomeIDAndName(cls):
+ bind = cls._bindSchema
+ return Select(
+ [bind.HOME_RESOURCE_ID, bind.RESOURCE_NAME], From=bind, Where=
+ (bind.RESOURCE_ID == Parameter("resourceID"))
+ .And(bind.BIND_MODE == _BIND_MODE_OWN)
+ )
+
+
+ @classproperty
+ def _ownerUIDFromHomeID(cls):
+ home = cls._homeSchema
+ return Select(
+ [home.OWNER_UID], From=home,
+ Where=home.RESOURCE_ID == Parameter("homeID")
+ )
+
+
+
+
@inlineCallbacks
def allRecords(self):
# This should have been a smart join that got all these columns at
@@ -430,78 +532,31 @@
# _want_ to do (just look for binds in a particular homes) is
# much simpler anyway; we should just do that.
all = []
- shareRows = yield self._txn.execSQL(
- """
- select %(column_RESOURCE_ID)s, %(column_RESOURCE_NAME)s, %(column_BIND_MODE)s, %(column_MESSAGE)s
- from %(name)s
- where %(column_HOME_RESOURCE_ID)s = %%s
- and %(column_BIND_MODE)s != %%s
- and %(column_RESOURCE_NAME)s is not null
- """ % self._bindTable,
- [self._home._resourceID, _BIND_MODE_OWN]
- )
+ shareRows = yield self._allSharedToQuery.on(
+ self._txn, homeID=self._home._resourceID)
for resourceID, resourceName, bindMode, summary in shareRows:
+ [[ownerHomeID, ownerResourceName]] = yield (
+ self._ownerHomeIDAndName.on(self._txn,
+ resourceID=resourceID))
+ [[ownerUID]] = yield self._ownerUIDFromHomeID.on(
+ self._txn, homeID=ownerHomeID)
+ hosturl = '/%s/__uids__/%s/%s' % (
+ self._urlTopSegment, ownerUID, ownerResourceName
+ )
+ localname = resourceName
if bindMode != _BIND_MODE_DIRECT:
- [[shareuid]] = yield self._txn.execSQL(
- """
- select INVITE_UID
- from INVITE
- where RESOURCE_ID = %s and HOME_RESOURCE_ID = %s
- """,
- [resourceID, self._home._resourceID]
- )
sharetype = 'I'
- [[ownerHomeID, ownerResourceName]] = yield self._txn.execSQL(
- """
- select %(column_HOME_RESOURCE_ID)s, %(column_RESOURCE_NAME)s
- from %(name)s
- where %(column_RESOURCE_ID)s = %%s
- and %(column_BIND_MODE)s = %%s
- """ % self._bindTable,
- [resourceID, _BIND_MODE_OWN]
+ [[shareuid]] = yield self._inviteUIDByResourceIDsQuery.on(
+ self._txn, resourceID=resourceID,
+ homeID=self._home._resourceID
)
- [[ownerUID]] = yield self._txn.execSQL(
- """
- select %(column_OWNER_UID)s from %(name)s
- where %(column_RESOURCE_ID)s = %%s
- """ % self._homeTable,
- [ownerHomeID]
- )
- hosturl = '/%s/__uids__/%s/%s' % (
- self._urlTopSegment, ownerUID, ownerResourceName
- )
- localname = resourceName
- record = SharedCollectionRecord(
- shareuid, sharetype, hosturl, localname, summary
- )
- all.append(record)
else:
sharetype = 'D'
- [[ownerHomeID, ownerResourceName]] = yield self._txn.execSQL(
- """
- select %(column_HOME_RESOURCE_ID)s, %(column_RESOURCE_NAME)s
- from %(name)s
- where %(column_RESOURCE_ID)s = %%s
- and %(column_BIND_MODE)s = %%s
- """ % self._bindTable,
- [resourceID, _BIND_MODE_OWN]
- )
- [[ownerUID]] = yield self._txn.execSQL(
- """
- select %(column_OWNER_UID)s from %(name)s
- where %(column_RESOURCE_ID)s = %%s
- """ % self._homeTable,
- [ownerHomeID]
- )
- hosturl = '/%s/__uids__/%s/%s' % (
- self._urlTopSegment, ownerUID, ownerResourceName
- )
- localname = resourceName
- synthesisedUID = "Direct-%s-%s" % (self._home._resourceID, resourceID,)
- record = SharedCollectionRecord(
- synthesisedUID, sharetype, hosturl, localname, summary
- )
- all.append(record)
+ shareuid = "Direct-%s-%s" % (self._home._resourceID, resourceID,)
+ record = SharedCollectionRecord(
+ shareuid, sharetype, hosturl, localname, summary
+ )
+ all.append(record)
returnValue(all)
@@ -517,15 +572,39 @@
return self._search(shareuid=shareUID)
+ @classproperty
+ def _updateBindName(cls):
+ bind = cls._bindSchema
+ return Update({bind.RESOURCE_NAME: Parameter("localname")},
+ Where=(bind.HOME_RESOURCE_ID == Parameter("homeID"))
+ .And(bind.RESOURCE_ID == Parameter('resourceID')))
+
+
+ @classproperty
+ def _acceptDirectShareQuery(cls):
+ bind = cls._bindSchema
+ return Insert({
+ bind.HOME_RESOURCE_ID: Parameter("homeID"),
+ bind.RESOURCE_ID: Parameter("resourceID"),
+ bind.RESOURCE_NAME: Parameter("name"),
+ bind.MESSAGE: Parameter("message"),
+ bind.BIND_MODE: _BIND_MODE_DIRECT,
+ bind.BIND_STATUS: _BIND_STATUS_ACCEPTED,
+ bind.SEEN_BY_OWNER: True,
+ bind.SEEN_BY_SHAREE: True,
+ })
+
+
@inlineCallbacks
def addOrUpdateRecord(self, record):
# record.hosturl -> /.../__uids__/<uid>/<name>
splithost = record.hosturl.split('/')
-
+
# Double-check the path
if splithost[2] != "__uids__":
- raise ValueError("Sharing URL must be a __uids__ path: %s" % (record.hosturl,))
-
+ raise ValueError(
+ "Sharing URL must be a __uids__ path: %s" % (record.hosturl,))
+
ownerUID = splithost[3]
ownerCollectionName = splithost[4]
ownerHome = yield self._getHomeWithUID(ownerUID)
@@ -533,71 +612,85 @@
collectionResourceID = ownerCollection._resourceID
if record.sharetype == 'I':
-
# There needs to be a bind already, one that corresponds to the
# invitation. The invitation's UID is the same as the share UID. I
# just need to update its 'localname', i.e.
# XXX_BIND.XXX_RESOURCE_NAME.
- yield self._txn.execSQL(
- """
- update %(name)s
- set %(column_RESOURCE_NAME)s = %%s
- where %(column_HOME_RESOURCE_ID)s = %%s
- and %(column_RESOURCE_ID)s = %%s
- """ % self._bindTable,
- [record.localname, self._home._resourceID, collectionResourceID]
+ yield self._updateBindName.on(
+ self._txn, localname=record.localname,
+ homeID=self._home._resourceID, resourceID=collectionResourceID
)
elif record.sharetype == 'D':
- # There is no bind entry already so add one.
+ # There is no bind entry already so add one - but be aware of possible race to create
- yield self._txn.execSQL(
- """
- insert into %(name)s
- (
- %(column_HOME_RESOURCE_ID)s,
- %(column_RESOURCE_ID)s,
- %(column_RESOURCE_NAME)s,
- %(column_BIND_MODE)s,
- %(column_BIND_STATUS)s,
- %(column_SEEN_BY_OWNER)s,
- %(column_SEEN_BY_SHAREE)s,
- %(column_MESSAGE)s
+ # Use savepoint so we can do a partial rollback if there is a race condition
+ # where this row has already been inserted
+ savepoint = SavepointAction("addOrUpdateRecord")
+ yield savepoint.acquire(self._txn)
+
+ try:
+ yield self._acceptDirectShareQuery.on(
+ self._txn, homeID=self._home._resourceID,
+ resourceID=collectionResourceID, name=record.localname,
+ message=record.summary
)
- values (%%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s)
- """ % self._bindTable,
- [
- self._home._resourceID,
- collectionResourceID,
- record.localname,
- _BIND_MODE_DIRECT,
- _BIND_STATUS_ACCEPTED,
- True,
- True,
- record.summary,
- ])
+ except Exception: # FIXME: Really want to trap the pg.DatabaseError but in a non-DB specific manner
+ yield savepoint.rollback(self._txn)
+ # For now we will assume that the insert already done is the winner - so nothing more to do here
+ else:
+ yield savepoint.release(self._txn)
+
shareeCollection = yield self._home.sharedChildWithName(record.localname)
yield shareeCollection._initSyncToken()
+ @classproperty
+ def _unbindShareQuery(cls):
+ bind = cls._bindSchema
+ return Update({
+ bind.RESOURCE_NAME: None
+ }, Where=(bind.RESOURCE_NAME == Parameter("name"))
+ .And(bind.HOME_RESOURCE_ID == Parameter("homeID")))
+
+
@inlineCallbacks
def removeRecordForLocalName(self, localname):
record = yield self.recordForLocalName(localname)
shareeCollection = yield self._home.sharedChildWithName(record.localname)
yield shareeCollection._deletedSyncToken(sharedRemoval=True)
- returnValue((yield self._txn.execSQL(
- """
- update %(name)s
- set %(column_RESOURCE_NAME)s = NULL
- where %(column_RESOURCE_NAME)s = %%s
- and %(column_HOME_RESOURCE_ID)s = %%s
- """ % self._bindTable,
- [localname, self._home._resourceID]
- )))
+ result = yield self._unbindShareQuery.on(self._txn, name=localname,
+ homeID=self._home._resourceID)
+ returnValue(result)
+ @classproperty
+ def _removeInviteShareQuery(cls):
+ """
+ DAL query to remove a non-direct share by invite UID.
+ """
+ bind = cls._bindSchema
+ inv = schema.INVITE
+ return Update(
+ {bind.RESOURCE_NAME: None},
+ Where=(bind.HOME_RESOURCE_ID, bind.RESOURCE_ID) ==
+ Select([inv.HOME_RESOURCE_ID, inv.RESOURCE_ID],
+ From=inv, Where=inv.INVITE_UID == Parameter("uid")))
+
+
+ @classproperty
+ def _removeDirectShareQuery(cls):
+ """
+ DAL query to remove a direct share by its homeID and resourceID.
+ """
+ bind = cls._bindSchema
+ return Delete(From=bind,
+ Where=(bind.HOME_RESOURCE_ID == Parameter("homeID"))
+ .And(bind.RESOURCE_ID == Parameter("resourceID")))
+
+
@inlineCallbacks
def removeRecordForShareUID(self, shareUID):
@@ -606,47 +699,28 @@
yield shareeCollection._deletedSyncToken(sharedRemoval=True)
if not shareUID.startswith("Direct"):
- yield self._txn.execSQL(
- """
- update %(name)s
- set %(column_RESOURCE_NAME)s = NULL
- from INVITE
- where INVITE.INVITE_UID = %%s
- and %(name)s.%(column_HOME_RESOURCE_ID)s = INVITE.HOME_RESOURCE_ID
- and %(name)s.%(column_RESOURCE_ID)s = INVITE.RESOURCE_ID
- """ % self._bindTable,
- [shareUID, ]
- )
+ yield self._removeInviteShareQuery.on(self._txn, uid=shareUID)
else:
# Extract pieces from synthesised UID
homeID, resourceID = shareUID[len("Direct-"):].split("-")
-
# Now remove the binding for the direct share
- yield self._txn.execSQL(
- """
- delete from %(name)s
- where %(column_HOME_RESOURCE_ID)s = %%s
- and %(column_RESOURCE_ID)s = %%s
- """ % self._bindTable,
- [homeID, resourceID, ]
- )
+ yield self._removeDirectShareQuery.on(
+ self._txn, homeID=homeID, resourceID=resourceID)
-
class SQLLegacyCalendarShares(SQLLegacyShares):
"""
Emulator for the implicit interface specified by
L{twistedcaldav.sharing.InvitesDatabase}.
"""
- def __init__(self, home):
- self._homeTable = CALENDAR_HOME_TABLE
- self._bindTable = CALENDAR_BIND_TABLE
- self._urlTopSegment = "calendars"
+ _homeTable = CALENDAR_HOME_TABLE
+ _bindTable = CALENDAR_BIND_TABLE
+ _homeSchema = schema.CALENDAR_HOME
+ _bindSchema = schema.CALENDAR_BIND
+ _urlTopSegment = "calendars"
- super(SQLLegacyCalendarShares, self).__init__(home)
-
def _getHomeWithUID(self, uid):
return self._txn.calendarHomeWithUID(uid, create=True)
@@ -658,17 +732,18 @@
L{twistedcaldav.sharing.InvitesDatabase}.
"""
- def __init__(self, home):
- self._homeTable = ADDRESSBOOK_HOME_TABLE
- self._bindTable = ADDRESSBOOK_BIND_TABLE
- self._urlTopSegment = "addressbooks"
+ _homeTable = ADDRESSBOOK_HOME_TABLE
+ _bindTable = ADDRESSBOOK_BIND_TABLE
+ _homeSchema = schema.ADDRESSBOOK_HOME
+ _bindSchema = schema.ADDRESSBOOK_BIND
+ _urlTopSegment = "addressbooks"
- super(SQLLegacyAddressBookShares, self).__init__(home)
def _getHomeWithUID(self, uid):
return self._txn.addressbookHomeWithUID(uid, create=True)
+
class MemcachedUIDReserver(CachePoolUserMixIn, LoggingMixIn):
def __init__(self, index, cachePool=None):
self.index = index
@@ -780,10 +855,14 @@
key = self._key(uid)
return succeed(key in self.reservations)
-class postgresqlgenerator(sqlgenerator):
+
+
+class RealSQLBehaviorMixin(object):
"""
- Query generator for postgreSQL indexed searches. (Currently unused: work
- in progress.)
+ Class attributes for 'real' SQL behavior; avoid idiosyncracies of SQLite,
+ use standard SQL constructions, and depend on the full schema in
+ sql_schema_vX.sql rather than the partial one in twistedcaldav which depends
+ on the placement of the database in the filesystem for some information.
"""
ISOP = " = "
@@ -793,24 +872,20 @@
"TYPE": "CALENDAR_OBJECT.ICALENDAR_TYPE",
"UID": "CALENDAR_OBJECT.ICALENDAR_UID",
}
+ RESOURCEDB = "CALENDAR_OBJECT"
+ TIMESPANDB = "TIME_RANGE"
- def __init__(self, expr, calendarid, userid):
- self.RESOURCEDB = "CALENDAR_OBJECT"
- self.TIMESPANDB = "TIME_RANGE"
- self.TIMESPANTEST = "((TIME_RANGE.FLOATING = FALSE AND TIME_RANGE.START_DATE < %s AND TIME_RANGE.END_DATE > %s) OR (TIME_RANGE.FLOATING = TRUE AND TIME_RANGE.START_DATE < %s AND TIME_RANGE.END_DATE > %s))"
- self.TIMESPANTEST_NOEND = "((TIME_RANGE.FLOATING = FALSE AND TIME_RANGE.END_DATE > %s) OR (TIME_RANGE.FLOATING = TRUE AND TIME_RANGE.END_DATE > %s))"
- self.TIMESPANTEST_NOSTART = "((TIME_RANGE.FLOATING = FALSE AND TIME_RANGE.START_DATE < %s) OR (TIME_RANGE.FLOATING = TRUE AND TIME_RANGE.START_DATE < %s))"
- self.TIMESPANTEST_TAIL_PIECE = " AND TIME_RANGE.CALENDAR_OBJECT_RESOURCE_ID = CALENDAR_OBJECT.RESOURCE_ID AND TIME_RANGE.CALENDAR_RESOURCE_ID = %s"
- self.TIMESPANTEST_JOIN_ON_PIECE = "TIME_RANGE.INSTANCE_ID = TRANSPARENCY.TIME_RANGE_INSTANCE_ID AND TRANSPARENCY.USER_ID = %s"
+ TIMESPANTEST = "((TIME_RANGE.FLOATING = FALSE AND TIME_RANGE.START_DATE < %s AND TIME_RANGE.END_DATE > %s) OR (TIME_RANGE.FLOATING = TRUE AND TIME_RANGE.START_DATE < %s AND TIME_RANGE.END_DATE > %s))"
+ TIMESPANTEST_NOEND = "((TIME_RANGE.FLOATING = FALSE AND TIME_RANGE.END_DATE > %s) OR (TIME_RANGE.FLOATING = TRUE AND TIME_RANGE.END_DATE > %s))"
+ TIMESPANTEST_NOSTART = "((TIME_RANGE.FLOATING = FALSE AND TIME_RANGE.START_DATE < %s) OR (TIME_RANGE.FLOATING = TRUE AND TIME_RANGE.START_DATE < %s))"
+ TIMESPANTEST_TAIL_PIECE = " AND TIME_RANGE.CALENDAR_OBJECT_RESOURCE_ID = CALENDAR_OBJECT.RESOURCE_ID AND TIME_RANGE.CALENDAR_RESOURCE_ID = %s"
+ TIMESPANTEST_JOIN_ON_PIECE = "TIME_RANGE.INSTANCE_ID = TRANSPARENCY.TIME_RANGE_INSTANCE_ID AND TRANSPARENCY.USER_ID = %s"
- super(postgresqlgenerator, self).__init__(expr, calendarid, userid)
-
-
def generate(self):
"""
Generate the actual SQL 'where ...' expression from the passed in
expression tree.
-
+
@return: a C{tuple} of (C{str}, C{list}), where the C{str} is the
partial SQL statement, and the C{list} is the list of argument
substitutions to use with the SQL API execute method.
@@ -842,23 +917,61 @@
return select, self.arguments
+ def containsArgument(self, arg):
+ return "%%%s%%" % (arg,)
+
+
+
+class FormatParamStyleMixin(object):
+ """
+ Mixin for overriding methods on sqlgenerator that generate arguments
+ according to format/pyformat rules rather than the base class's 'numeric'
+ rules.
+ """
+
def addArgument(self, arg):
self.arguments.append(arg)
self.substitutions.append("%s")
self.sout.write("%s")
+
def setArgument(self, arg):
self.arguments.append(arg)
self.substitutions.append("%s")
+
def frontArgument(self, arg):
self.arguments.insert(0, arg)
self.substitutions.insert(0, "%s")
- def containsArgument(self, arg):
- return "%%%s%%" % (arg,)
+class postgresqlgenerator(FormatParamStyleMixin, RealSQLBehaviorMixin,
+ sqlgenerator):
+ """
+ Query generator for PostgreSQL indexed searches.
+ """
+
+
+def fixbools(sqltext):
+ return sqltext.replace("TRUE", "1").replace("FALSE", "0")
+
+
+
+class oraclesqlgenerator(RealSQLBehaviorMixin, sqlgenerator):
+ """
+ Query generator for Oracle indexed searches.
+ """
+ TIMESPANTEST = fixbools(RealSQLBehaviorMixin.TIMESPANTEST)
+ TIMESPANTEST_NOEND = fixbools(RealSQLBehaviorMixin.TIMESPANTEST_NOEND)
+ TIMESPANTEST_NOSTART = fixbools(RealSQLBehaviorMixin.TIMESPANTEST_NOSTART)
+ TIMESPANTEST_TAIL_PIECE = fixbools(
+ RealSQLBehaviorMixin.TIMESPANTEST_TAIL_PIECE)
+ TIMESPANTEST_JOIN_ON_PIECE = fixbools(
+ RealSQLBehaviorMixin.TIMESPANTEST_JOIN_ON_PIECE)
+
+
+
class LegacyIndexHelper(LoggingMixIn, object):
@inlineCallbacks
@@ -907,6 +1020,8 @@
# This is only used with unit tests
self.reserver = DummyUIDReserver(self)
+ _objectSchema = schema.CALENDAR_OBJECT
+
@property
def _txn(self):
return self.calendar._txn
@@ -940,17 +1055,28 @@
returnValue(name)
+ @classproperty
+ def _notExpandedBeyondQuery(self):
+ """
+ DAL query to satisfy L{PostgresLegacyIndexEmulator.notExpandedBeyond}.
+ """
+ co = schema.CALENDAR_OBJECT
+ return Select([co.RESOURCE_NAME], From=co,
+ Where=(co.RECURRANCE_MAX < Parameter("minDate"))
+ .And(co.CALENDAR_RESOURCE_ID == Parameter("resourceID")))
+
+
@inlineCallbacks
def notExpandedBeyond(self, minDate):
"""
Gives all resources which have not been expanded beyond a given date
in the database. (Unused; see above L{postgresqlgenerator}.
"""
- returnValue([row[0] for row in (yield self._txn.execSQL(
- "select RESOURCE_NAME from CALENDAR_OBJECT "
- "where RECURRANCE_MAX < %s and CALENDAR_RESOURCE_ID = %s",
- [pyCalendarTodatetime(normalizeForIndex(minDate)), self.calendar._resourceID]
- ))])
+ returnValue([row[0] for row in (
+ yield self._notExpandedBeyondQuery.on(
+ self._txn, minDate=pyCalendarTodatetime(normalizeForIndex(minDate)),
+ resourceID=self.calendar._resourceID))]
+ )
@inlineCallbacks
@@ -972,7 +1098,8 @@
# Actually expand recurrence max
for name in names:
- self.log_info("Search falls outside range of index for %s %s" % (name, minDate))
+ self.log_info("Search falls outside range of index for %s %s" %
+ (name, minDate))
yield self.reExpandResource(name, minDate)
@@ -980,20 +1107,29 @@
def indexedSearch(self, filter, useruid='', fbtype=False):
"""
Finds resources matching the given qualifiers.
+
@param filter: the L{Filter} for the calendar-query to execute.
@return: a L{Deferred} which fires with an iterable of tuples for each
resource matching the given C{qualifiers}. The tuples are C{(name,
uid, type)}, where C{name} is the resource name, C{uid} is the
- resource UID, and C{type} is the resource iCalendar component
- type.
+ resource UID, and C{type} is the resource iCalendar component type.
"""
+ # Detect which style of parameter-generation we're using. Naming is a
+ # little off here, because the reason we're using the numeric one is
+ # that it happens to be used by the oracle binding that we're using,
+ # whereas the postgres binding happens to use the 'pyformat' (e.g. %s)
+ # parameter style.
+ if self.calendar._txn.paramstyle == 'numeric':
+ generator = oraclesqlgenerator
+ else:
+ generator = postgresqlgenerator
# Make sure we have a proper Filter element and get the partial SQL
# statement to use.
if isinstance(filter, calendarqueryfilter.Filter):
qualifiers = calendarquery.sqlcalendarquery(
filter, self.calendar._resourceID, useruid,
- generator=postgresqlgenerator
+ generator=generator
)
if qualifiers is not None:
# Determine how far we need to extend the current expansion of
@@ -1003,6 +1139,7 @@
# "infinite" value always included.
maxDate, isStartDate = filter.getmaxtimerange()
if maxDate:
+ maxDate = maxDate.duplicate()
maxDate.setDateOnly(True)
if isStartDate:
maxDate += PyCalendarDuration(days=365)
@@ -1015,13 +1152,7 @@
# Perform the search
if qualifiers is None:
- rowiter = yield self._txn.execSQL(
- """
- select RESOURCE_NAME, ICALENDAR_UID, ICALENDAR_TYPE
- from CALENDAR_OBJECT where CALENDAR_RESOURCE_ID = %s
- """,
- [self.calendar._resourceID],
- )
+ rowiter = yield self.bruteForceSearch()
else:
if fbtype:
# For a free-busy time-range query we return all instances
@@ -1065,28 +1196,49 @@
returnValue(results)
- def bruteForceSearch(self):
- return self._txn.execSQL(
- "select RESOURCE_NAME, ICALENDAR_UID, ICALENDAR_TYPE from "
- "CALENDAR_OBJECT where CALENDAR_RESOURCE_ID = %s",
- [self.calendar._resourceID]
+ @classproperty
+ def _bruteForceQuery(cls):
+ """
+ DAL query for all C{CALENDAR_OBJECT} rows in the calendar represented by
+ this index.
+ """
+ obj = cls._objectSchema
+ return Select(
+ [obj.RESOURCE_NAME, obj.ICALENDAR_UID, obj.ICALENDAR_TYPE],
+ From=obj, Where=obj.PARENT_RESOURCE_ID == Parameter("resourceID")
)
+ def bruteForceSearch(self):
+ return self._bruteForceQuery.on(
+ self._txn, resourceID=self.resource._resourceID)
+
+
@inlineCallbacks
def resourcesExist(self, names):
returnValue(list(set(names).intersection(
set((yield self.calendar.listCalendarObjects())))))
+ @classproperty
+ def _resourceExistsQuery(cls):
+ """
+ DAL query to determine whether a calendar object exists in the
+ collection represented by this index.
+ """
+ obj = cls._objectSchema
+ return Select(
+ [obj.RESOURCE_NAME], From=obj,
+ Where=(obj.RESOURCE_NAME == Parameter("name"))
+ .And(obj.PARENT_RESOURCE_ID == Parameter("resourceID"))
+ )
+
+
@inlineCallbacks
def resourceExists(self, name):
returnValue((bool(
- (yield self._txn.execSQL(
- "select RESOURCE_NAME from CALENDAR_OBJECT where "
- "RESOURCE_NAME = %s and CALENDAR_RESOURCE_ID = %s",
- [name, self.calendar._resourceID]
- ))
+ (yield self._resourceExistsQuery.on(
+ self._txn, name=name, resourceID=self.resource._resourceID))
)))
@@ -1113,10 +1265,9 @@
# CARDDAV
-class postgresqladbkgenerator(sqlgenerator):
+class oraclesqladbkgenerator(sqlgenerator):
"""
- Query generator for postgreSQL indexed searches. (Currently unused: work
- in progress.)
+ Query generator for Oracle indexed searches.
"""
ISOP = " = "
@@ -1125,18 +1276,17 @@
FIELDS = {
"UID": "ADDRESSBOOK_OBJECT.VCARD_UID",
}
+ RESOURCEDB = "ADDRESSBOOK_OBJECT"
- def __init__(self, expr, addressbookid):
- self.RESOURCEDB = "ADDRESSBOOK_OBJECT"
+ def containsArgument(self, arg):
+ return "%%%s%%" % (arg,)
- super(postgresqladbkgenerator, self).__init__(expr, addressbookid)
-
def generate(self):
"""
Generate the actual SQL 'where ...' expression from the passed in
expression tree.
-
+
@return: a C{tuple} of (C{str}, C{list}), where the C{str} is the
partial SQL statement, and the C{list} is the list of argument
substitutions to use with the SQL API execute method.
@@ -1160,29 +1310,24 @@
return select, self.arguments
- def addArgument(self, arg):
- self.arguments.append(arg)
- self.substitutions.append("%s")
- self.sout.write("%s")
- def setArgument(self, arg):
- self.arguments.append(arg)
- self.substitutions.append("%s")
+class postgresqladbkgenerator(FormatParamStyleMixin, oraclesqladbkgenerator):
+ """
+ Query generator for PostgreSQL indexed searches. Inherit 'real' database
+ behavior from L{oracleadbkgenerator}, and %s-style formatting from
+ L{FormatParamStyleMixin}.
+ """
- def frontArgument(self, arg):
- self.arguments.insert(0, arg)
- self.substitutions.insert(0, "%s")
- def containsArgument(self, arg):
- return "%%%s%%" % (arg,)
-
class PostgresLegacyABIndexEmulator(LegacyIndexHelper):
"""
Emulator for L{twistedcaldv.index.Index} and
L{twistedcaldv.index.IndexSchedule}.
"""
+ _objectSchema = schema.ADDRESSBOOK_OBJECT
+
def __init__(self, addressbook):
self.resource = self.addressbook = addressbook
if (
@@ -1235,10 +1380,14 @@
C{name} is the resource name, C{uid} is the resource UID, and
C{type} is the resource iCalendar component type.x
"""
-
+ if self.addressbook._txn.paramstyle == 'numeric':
+ generator = oraclesqladbkgenerator
+ else:
+ generator = postgresqladbkgenerator
# Make sure we have a proper Filter element and get the partial SQL statement to use.
if isinstance(filter, carddavxml.Filter):
- qualifiers = addressbookquery.sqladdressbookquery(filter, self.addressbook._resourceID, generator=postgresqladbkgenerator)
+ qualifiers = addressbookquery.sqladdressbookquery(
+ filter, self.addressbook._resourceID, generator=generator)
else:
qualifiers = None
if qualifiers is not None:
@@ -1248,10 +1397,13 @@
qualifiers[1]
)
else:
- rowiter = yield self._txn.execSQL(
- "select RESOURCE_NAME, VCARD_UID from ADDRESSBOOK_OBJECT where ADDRESSBOOK_RESOURCE_ID = %s",
- [self.addressbook._resourceID, ],
- )
+ rowiter = yield Select(
+ [schema.ADDRESSBOOK_OBJECT.RESOURCE_NAME,
+ schema.ADDRESSBOOK_OBJECT.VCARD_UID],
+ From=schema.ADDRESSBOOK_OBJECT,
+ Where=schema.ADDRESSBOOK_OBJECT.ADDRESSBOOK_RESOURCE_ID ==
+ self.addressbook._resourceID
+ ).on(self.addressbook._txn)
returnValue(list(rowiter))
@@ -1264,26 +1416,8 @@
raise IndexedSearchException()
- def bruteForceSearch(self):
- return self._txn.execSQL(
- "select RESOURCE_NAME, VCARD_UID from "
- "ADDRESSBOOK_OBJECT where ADDRESSBOOK_RESOURCE_ID = %s",
- [self.addressbook._resourceID]
- )
-
-
@inlineCallbacks
def resourcesExist(self, names):
returnValue(list(set(names).intersection(
set((yield self.addressbook.listAddressbookObjects())))))
-
- @inlineCallbacks
- def resourceExists(self, name):
- returnValue(bool(
- (yield self._txn.execSQL(
- "select RESOURCE_NAME from ADDRESSBOOK_OBJECT where "
- "RESOURCE_NAME = %s and ADDRESSBOOK_RESOURCE_ID = %s",
- [name, self.addressbook._resourceID]
- ))
- ))
Modified: CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_tables.py
===================================================================
--- CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_tables.py 2011-03-17 17:51:31 UTC (rev 7205)
+++ CalendarServer/branches/users/cdaboo/pycalendar/txdav/common/datastore/sql_tables.py 2011-03-17 17:54:34 UTC (rev 7206)
@@ -21,6 +21,8 @@
from twisted.python.modules import getModule
from twext.enterprise.dal.syntax import SchemaSyntax
+from twext.enterprise.dal.model import NO_DEFAULT
+from twext.enterprise.dal.model import Sequence, ProcedureCall
from twext.enterprise.dal.parseschema import schemaFromPath
@@ -195,3 +197,72 @@
REV=ADDRESSBOOK_OBJECT_REVISIONS_TABLE,
BIND=ADDRESSBOOK_BIND_TABLE)
+
+
+def _translateSchema(out):
+ """
+ When run as a script, translate the schema to another dialect. Currently
+ only postgres and oracle are supported, and native format is postgres, so
+ emit in oracle format.
+ """
+ for sequence in schema.model.sequences:
+ out.write('drop sequence %s; create sequence %s;\n' % (
+ sequence.name, sequence.name))
+ for table in schema:
+ # The only table name which actually exceeds the length limit right now
+ # is CALENDAR_OBJECT_ATTACHMENTS_MODE, which isn't actually _used_
+ # anywhere, so we can fake it for now.
+ out.write('drop table %s; create table %s (\n' % (
+ table.model.name[:30], table.model.name[:30],))
+ first = True
+ for column in table:
+ if first:
+ first = False
+ else:
+ out.write(",\n")
+ typeName = column.model.type.name
+ if typeName == 'text':
+ typeName = 'clob'
+ if typeName == 'boolean':
+ typeName = 'integer'
+ out.write(' "%s" %s' % (column.model.name, typeName))
+ if column.model.type.length:
+ out.write("(%s)" % (column.model.type.length,))
+ if column.model is table.model.primaryKey:
+ out.write(' primary key')
+ default = column.model.default
+ if default is not NO_DEFAULT:
+ # Can't do default sequence types in Oracle, so don't bother.
+ if not isinstance(default, Sequence):
+ out.write(' default')
+ if default is None:
+ out.write(' null')
+ elif isinstance(default, ProcedureCall):
+ # Cheating, because there are currently no other
+ # functions being used.
+ out.write(" CURRENT_TIMESTAMP at time zone 'UTC'")
+ else:
+ if default is True:
+ default = 1
+ elif default is False:
+ default = 0
+ out.write(" " + repr(default))
+ if not column.model.canBeNull():
+ out.write(' not null')
+ if set([column.model]) in list(table.model.uniques()):
+ out.write(' unique')
+ if column.model.references is not None:
+ out.write(" references %s" % (column.model.references.name,))
+ if column.model.cascade:
+ out.write(" on delete cascade")
+
+ out.write('\n);\n\n')
+
+
+
+if __name__ == '__main__':
+ import sys
+ _translateSchema(sys.stdout)
+
+
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110317/171cf19c/attachment-0001.html>
More information about the calendarserver-changes
mailing list