[CalendarServer-changes] [10074] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Nov 16 14:54:42 PST 2012
Revision: 10074
http://trac.calendarserver.org//changeset/10074
Author: glyph at apple.com
Date: 2012-11-16 14:54:42 -0800 (Fri, 16 Nov 2012)
Log Message:
-----------
Unify all APIs for enumeration of homes in both file and SQL stores into a single one with a scalable interface.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tools/shell/cmd.py
CalendarServer/trunk/calendarserver/tools/shell/terminal.py
CalendarServer/trunk/calendarserver/tools/shell/test/test_vfs.py
CalendarServer/trunk/calendarserver/tools/shell/vfs.py
CalendarServer/trunk/twistedcaldav/directory/idirectory.py
CalendarServer/trunk/twistedcaldav/directory/test/test_xmlfile.py
CalendarServer/trunk/txdav/caldav/datastore/test/common.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_file.py
CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/caldav/icalendarstore.py
CalendarServer/trunk/txdav/carddav/datastore/test/common.py
CalendarServer/trunk/txdav/carddav/datastore/test/test_file.py
CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
CalendarServer/trunk/txdav/carddav/iaddressbookstore.py
CalendarServer/trunk/txdav/common/datastore/file.py
CalendarServer/trunk/txdav/common/datastore/sql.py
CalendarServer/trunk/txdav/common/datastore/test/util.py
CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py
CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py
Property Changed:
----------------
CalendarServer/trunk/
Property changes on: CalendarServer/trunk
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/generic-sqlstore:6167-6191
/CalendarServer/branches/new-store:5594-5934
/CalendarServer/branches/new-store-no-caldavfile:5911-5935
/CalendarServer/branches/new-store-no-caldavfile-2:5936-5981
/CalendarServer/branches/users/cdaboo/batchupload-6699:6700-7198
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/component-set-fixes:8130-8346
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/implicituidrace:8137-8141
/CalendarServer/branches/users/cdaboo/ischedule-dkim:9747-9979
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/pods:7297-7377
/CalendarServer/branches/users/cdaboo/pycalendar:7085-7206
/CalendarServer/branches/users/cdaboo/pycard:7227-7237
/CalendarServer/branches/users/cdaboo/queued-attendee-refreshes:7740-8287
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/cdaboo/timezones:7443-7699
/CalendarServer/branches/users/cdaboo/txn-debugging:8730-8743
/CalendarServer/branches/users/glyph/always-abort-txn-on-error:9958-9969
/CalendarServer/branches/users/glyph/case-insensitive-uid:8772-8805
/CalendarServer/branches/users/glyph/conn-limit:6574-6577
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/dalify:6932-7023
/CalendarServer/branches/users/glyph/db-reconnect:6824-6876
/CalendarServer/branches/users/glyph/deploybuild:7563-7572
/CalendarServer/branches/users/glyph/disable-quota:7718-7727
/CalendarServer/branches/users/glyph/dont-start-postgres:6592-6614
/CalendarServer/branches/users/glyph/imip-and-admin-html:7866-7984
/CalendarServer/branches/users/glyph/ipv6-client:9054-9105
/CalendarServer/branches/users/glyph/linux-tests:6893-6900
/CalendarServer/branches/users/glyph/migrate-merge:8690-8713
/CalendarServer/branches/users/glyph/misc-portability-fixes:7365-7374
/CalendarServer/branches/users/glyph/more-deferreds-6:6322-6368
/CalendarServer/branches/users/glyph/more-deferreds-7:6369-6445
/CalendarServer/branches/users/glyph/multiget-delete:8321-8330
/CalendarServer/branches/users/glyph/new-export:7444-7485
/CalendarServer/branches/users/glyph/one-home-list-api:10048-10073
/CalendarServer/branches/users/glyph/oracle:7106-7155
/CalendarServer/branches/users/glyph/oracle-nulls:7340-7351
/CalendarServer/branches/users/glyph/other-html:8062-8091
/CalendarServer/branches/users/glyph/parallel-sim:8240-8251
/CalendarServer/branches/users/glyph/parallel-upgrade:8376-8400
/CalendarServer/branches/users/glyph/parallel-upgrade_to_1:8571-8583
/CalendarServer/branches/users/glyph/q:9560-9688
/CalendarServer/branches/users/glyph/quota:7604-7637
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/shared-pool-fixes:8436-8443
/CalendarServer/branches/users/glyph/shared-pool-take2:8155-8174
/CalendarServer/branches/users/glyph/sharedpool:6490-6550
/CalendarServer/branches/users/glyph/sharing-api:9192-9205
/CalendarServer/branches/users/glyph/skip-lonely-vtimezones:8524-8535
/CalendarServer/branches/users/glyph/sql-store:5929-6073
/CalendarServer/branches/users/glyph/subtransactions:7248-7258
/CalendarServer/branches/users/glyph/table-alias:8651-8664
/CalendarServer/branches/users/glyph/uidexport:7673-7676
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/glyph/uuid-normalize:9268-9296
/CalendarServer/branches/users/glyph/xattrs-from-files:7757-7769
/CalendarServer/branches/users/sagen/applepush:8126-8184
/CalendarServer/branches/users/sagen/inboxitems:7380-7381
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/purge_old_events:6735-6746
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
Modified: CalendarServer/trunk/calendarserver/tools/shell/cmd.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/shell/cmd.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/calendarserver/tools/shell/cmd.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -67,6 +67,12 @@
class CommandsBase(object):
+ """
+ Base class for commands.
+
+ @ivar protocol: a protocol for parsing the incoming command line.
+ @type protocol: L{calendarserver.tools.shell.terminal.ShellProtocol}
+ """
def __init__(self, protocol):
self.protocol = protocol
Modified: CalendarServer/trunk/calendarserver/tools/shell/terminal.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/shell/terminal.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/calendarserver/tools/shell/terminal.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -89,7 +89,28 @@
super(ShellOptions, self).__init__()
+
class ShellService(Service, object):
+ """
+ A L{ShellService} collects all the information that a shell needs to run;
+ when run, it invokes the shell on stdin/stdout.
+
+ @ivar store: the calendar / addressbook store.
+ @type store: L{txdav.idav.IDataStore}
+
+ @ivar directory: the directory service, to look up principals' names
+ @type directory: L{twistedcaldav.directory.idirectory.IDirectoryService}
+
+ @ivar options: the command-line options used to create this shell service
+ @type options: L{ShellOptions}
+
+ @ivar reactor: the reactor under which this service is running
+ @type reactor: L{IReactorTCP}, L{IReactorTime}, L{IReactorThreads} etc
+
+ @ivar config: the configuration associated with this shell service.
+ @type config: L{twistedcaldav.config.Config}
+ """
+
def __init__(self, store, directory, options, reactor, config):
super(ShellService, self).__init__()
self.store = store
@@ -100,6 +121,7 @@
self.terminalFD = None
self.protocol = None
+
def startService(self):
"""
Start the service.
@@ -114,6 +136,7 @@
self.protocol = ServerProtocol(lambda: ShellProtocol(self))
StandardIO(self.protocol)
+
def stopService(self):
"""
Stop the service.
@@ -123,9 +146,13 @@
os.write(self.terminalFD, "\r\x1bc\r")
+
class ShellProtocol(ReceiveLineProtocol):
"""
Data store shell protocol.
+
+ @ivar service: a service representing the running shell
+ @type service: L{ShellService}
"""
# FIXME:
Modified: CalendarServer/trunk/calendarserver/tools/shell/test/test_vfs.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/shell/test/test_vfs.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/calendarserver/tools/shell/test/test_vfs.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -15,14 +15,18 @@
# limitations under the License.
##
-import twisted.trial.unittest
-from twisted.internet.defer import succeed
+from twisted.trial.unittest import TestCase
+from twisted.internet.defer import succeed, inlineCallbacks
from calendarserver.tools.shell.vfs import ListEntry
from calendarserver.tools.shell.vfs import File, Folder
+from calendarserver.tools.shell.vfs import UIDsFolder
+from calendarserver.tools.shell.terminal import ShellService
+from twistedcaldav.directory.test.test_xmlfile import XMLFileBase
+from txdav.common.datastore.test.util import buildStore
-class TestListEntry(twisted.trial.unittest.TestCase):
+class TestListEntry(TestCase):
def test_toString(self):
self.assertEquals(ListEntry(None, File , "thingo" ).toString(), "thingo" )
self.assertEquals(ListEntry(None, File , "thingo", Foo="foo").toString(), "thingo" )
@@ -100,3 +104,58 @@
def list(self): return succeed(())
list.fieldNames = ()
self.assertEquals(fields(MyFile), ("thingo",))
+
+
+
+class DirectoryStubber(XMLFileBase):
+ """
+ Object which creates a stub L{IDirectoryService}.
+ """
+ def __init__(self, testCase):
+ self.testCase = testCase
+
+ def mktemp(self):
+ return self.testCase.mktemp()
+
+
+
+class UIDsFolderTests(TestCase):
+ """
+ L{UIDsFolder} contains all principals and is keyed by UID.
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+ """
+ Create a L{UIDsFolder}.
+ """
+ self.svc = ShellService(store=(yield buildStore(self, None)),
+ directory=DirectoryStubber(self).service(),
+ options=None, reactor=None, config=None)
+ self.folder = UIDsFolder(self.svc, ())
+
+
+ @inlineCallbacks
+ def test_list(self):
+ """
+ L{UIDsFolder.list} returns a L{Deferred} firing an iterable of
+ L{ListEntry} objects, reflecting the directory information for all
+ calendars and addressbooks created in the store.
+ """
+ txn = self.svc.store.newTransaction()
+ wsanchez = "6423F94A-6B76-4A3A-815B-D52CFD77935D"
+ dreid = "5FF60DAD-0BDE-4508-8C77-15F0CA5C8DD1"
+ yield txn.calendarHomeWithUID(wsanchez, create=True)
+ yield txn.addressbookHomeWithUID(dreid, create=True)
+ yield txn.commit()
+ listing = list((yield self.folder.list()))
+ self.assertEquals(
+ [x.fields for x in listing],
+ [{"Record Type": "users", "Short Name": "wsanchez",
+ "Full Name": "Wilfredo Sanchez", "Name": wsanchez},
+ {"Record Type": "users", "Short Name": "dreid",
+ "Full Name": "David Reid", "Name": dreid}]
+ )
+
+
+
Modified: CalendarServer/trunk/calendarserver/tools/shell/vfs.py
===================================================================
--- CalendarServer/trunk/calendarserver/tools/shell/vfs.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/calendarserver/tools/shell/vfs.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -1,3 +1,4 @@
+# -*- test-case-name: calendarserver.tools.shell.test.test_vfs -*-
##
# Copyright (c) 2011-2012 Apple Inc. All rights reserved.
#
@@ -56,6 +57,7 @@
"""
Information about a C{File} as returned by C{File.list()}.
"""
+
def __init__(self, parent, Class, Name, **fields):
self.parent = parent # The class implementing list()
self.fileClass = Class
@@ -64,9 +66,11 @@
fields["Name"] = Name
+
def __str__(self):
return self.toString()
+
def __repr__(self):
fields = self.fields.copy()
del fields["Name"]
@@ -83,15 +87,18 @@
fields,
)
+
def isFolder(self):
return issubclass(self.fileClass, Folder)
+
def toString(self):
if self.isFolder():
return "%s/" % (self.fileName,)
else:
return self.fileName
+
@property
def fieldNames(self):
if not hasattr(self, "_fieldNames"):
@@ -101,10 +108,12 @@
else:
self._fieldNames = ("Name",) + tuple(self.parent.list.fieldNames)
else:
- self._fieldNames = ["Name"] + sorted(n for n in self.fields if n != "Name")
+ self._fieldNames = ["Name"] + sorted(n for n in self.fields
+ if n != "Name")
return self._fieldNames
+
def toFields(self):
try:
return tuple(self.fields[fieldName] for fieldName in self.fieldNames)
@@ -115,6 +124,7 @@
)
+
class File(object):
"""
Object in virtual data hierarchy.
@@ -217,7 +227,8 @@
"""
Root of virtual data hierarchy.
- Hierarchy:
+ Hierarchy::
+
/ RootFolder
uids/ UIDsFolder
<uid>/ PrincipalHomeFolder
@@ -262,9 +273,8 @@
# FIXME: Merge in directory UIDs also?
# FIXME: Add directory info (eg. name) to list entry
- def addResult(uid):
- if uid in results:
- return
+ def addResult(ignoredTxn, home):
+ uid = home.uid()
record = self.service.directory.recordWithUID(uid)
if record:
@@ -277,22 +287,12 @@
info = {}
results[uid] = ListEntry(self, PrincipalHomeFolder, uid, **info)
-
- txn = self.service.store.newTransaction()
- try:
- for home in (yield txn.calendarHomes()):
- addResult(home.uid())
- for home in (yield txn.addressbookHomes()):
- addResult(home.uid())
- finally:
- (yield txn.abort())
-
+ yield self.service.store.withEachCalendarHomeDo(addResult)
+ yield self.service.store.withEachAddressbookHomeDo(addResult)
returnValue(results.itervalues())
- list.fieldNames = ("Record Name", "Short Name", "Full Name")
-
class RecordFolder(Folder):
def _recordForName(self, name):
recordTypeAttr = "recordType_" + self.recordType
Modified: CalendarServer/trunk/twistedcaldav/directory/idirectory.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/idirectory.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/twistedcaldav/directory/idirectory.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -102,20 +102,21 @@
"""
@param tokens: The tokens to search on
@type tokens: C{list} of C{str} (utf-8 bytes)
- @param context: An indication of what the end user is searching
- for; "attendee", "location", or None
+
+ @param context: An indication of what the end user is searching for;
+ "attendee", "location", or None
@type context: C{str}
- @return: a deferred sequence of L{IDirectoryRecord}s which
- match the given tokens and optional context.
- Each token is searched for within each record's full name and
- email address; if each token is found within a record that
- record is returned in the results.
+ @return: a deferred sequence of L{IDirectoryRecord}s which match the
+ given tokens and optional context.
- If context is None, all record types are considered. If
- context is "location", only locations are considered. If
- context is "attendee", only users, groups, and resources
- are considered.
+ Each token is searched for within each record's full name and email
+ address; if each token is found within a record that record is
+ returned in the results.
+
+ If context is None, all record types are considered. If context is
+ "location", only locations are considered. If context is
+ "attendee", only users, groups, and resources are considered.
"""
Modified: CalendarServer/trunk/twistedcaldav/directory/test/test_xmlfile.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/test/test_xmlfile.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/twistedcaldav/directory/test/test_xmlfile.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -25,6 +25,11 @@
# FIXME: Add tests for GUID hooey, once we figure out what that means here
class XMLFileBase(object):
+ """
+ L{XMLFileBase} is a base/mix-in object for testing L{XMLDirectoryService}
+ (or things that depend on L{IDirectoryService} and need a simple
+ implementation to use).
+ """
recordTypes = set((
DirectoryService.recordType_users,
DirectoryService.recordType_groups,
@@ -44,30 +49,30 @@
}
groups = {
- "admin" : { "password": "admin", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_groups, "managers"),) },
- "managers" : { "password": "managers", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "lecroy"),) },
+ "admin" : { "password": "admin", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_groups, "managers"),)},
+ "managers" : { "password": "managers", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "lecroy"),)},
"grunts" : { "password": "grunts", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "wsanchez"),
(DirectoryService.recordType_users , "cdaboo"),
- (DirectoryService.recordType_users , "dreid")) },
- "right_coast": { "password": "right_coast", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "cdaboo"),) },
+ (DirectoryService.recordType_users , "dreid"))},
+ "right_coast": { "password": "right_coast", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "cdaboo"),)},
"left_coast" : { "password": "left_coast", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "wsanchez"),
(DirectoryService.recordType_users , "dreid"),
- (DirectoryService.recordType_users , "lecroy")) },
+ (DirectoryService.recordType_users , "lecroy"))},
"both_coasts": { "password": "both_coasts", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_groups, "right_coast"),
- (DirectoryService.recordType_groups, "left_coast")) },
+ (DirectoryService.recordType_groups, "left_coast"))},
"recursive1_coasts": { "password": "recursive1_coasts", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_groups, "recursive2_coasts"),
- (DirectoryService.recordType_users, "wsanchez")) },
+ (DirectoryService.recordType_users, "wsanchez"))},
"recursive2_coasts": { "password": "recursive2_coasts", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_groups, "recursive1_coasts"),
- (DirectoryService.recordType_users, "cdaboo")) },
+ (DirectoryService.recordType_users, "cdaboo"))},
"non_calendar_group": { "password": "non_calendar_group", "guid": None, "addresses": (), "members": ((DirectoryService.recordType_users , "cdaboo"),
- (DirectoryService.recordType_users , "lecroy")) },
+ (DirectoryService.recordType_users , "lecroy"))},
}
locations = {
"mercury": { "password": "mercury", "guid": None, "addresses": ("mailto:mercury at example.com",) },
"gemini" : { "password": "gemini", "guid": None, "addresses": ("mailto:gemini at example.com",) },
"apollo" : { "password": "apollo", "guid": None, "addresses": ("mailto:apollo at example.com",) },
- "orion" : { "password": "orion", "guid": None, "addresses": ("mailto:orion at example.com",) },
+ "orion" : { "password": "orion", "guid": None, "addresses": ("mailto:orion at example.com",) },
}
resources = {
@@ -77,17 +82,53 @@
}
def xmlFile(self):
+ """
+ Create a L{FilePath} that points to a temporary file containing a copy
+ of C{twistedcaldav/directory/test/accounts.xml}.
+
+ @see: L{xmlFile}
+
+ @rtype: L{FilePath}
+ """
if not hasattr(self, "_xmlFile"):
self._xmlFile = FilePath(self.mktemp())
xmlFile.copyTo(self._xmlFile)
return self._xmlFile
+
def augmentsFile(self):
+ """
+ Create a L{FilePath} that points to a temporary file containing a copy
+ of C{twistedcaldav/directory/test/augments.xml}.
+
+ @see: L{augmentsFile}
+
+ @rtype: L{FilePath}
+ """
if not hasattr(self, "_augmentsFile"):
self._augmentsFile = FilePath(self.mktemp())
augmentsFile.copyTo(self._augmentsFile)
return self._augmentsFile
+
+ def service(self):
+ """
+ Create an L{XMLDirectoryService} based on the contents of the paths
+ returned by L{XMLFileBase.augmentsFile} and L{XMLFileBase.xmlFile}.
+
+ @rtype: L{XMLDirectoryService}
+ """
+ return XMLDirectoryService(
+ {
+ 'xmlFile': self.xmlFile(),
+ 'augmentService':
+ augment.AugmentXMLDB(xmlFiles=(self.augmentsFile().path,)),
+ },
+ alwaysStat=True
+ )
+
+
+
class XMLFile (
XMLFileBase,
twistedcaldav.directory.test.util.BasicTestCase,
@@ -96,16 +137,6 @@
"""
Test XML file based directory implementation.
"""
- def service(self):
- directory = XMLDirectoryService(
- {
- 'xmlFile' : self.xmlFile(),
- 'augmentService' :
- augment.AugmentXMLDB(xmlFiles=(self.augmentsFile().path,)),
- },
- alwaysStat=True
- )
- return directory
def test_changedXML(self):
service = self.service()
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/common.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/common.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -49,12 +49,12 @@
ICalendarObject, ICalendarHome,
ICalendar, IAttachment, ICalendarTransaction)
-
from twistedcaldav.customxml import InviteNotification, InviteSummary
from txdav.caldav.icalendarstore import IAttachmentStorageTransport
from txdav.caldav.icalendarstore import QuotaExceeded
-from txdav.common.datastore.test.util import deriveQuota
-from txdav.common.datastore.test.util import withSpecialQuota
+from txdav.common.datastore.test.util import (
+ deriveQuota, withSpecialQuota, transactionClean
+)
from txdav.common.icommondatastore import ConcurrentModification
from twistedcaldav.ical import Component
from twistedcaldav.config import config
@@ -593,23 +593,6 @@
@inlineCallbacks
- def test_calendarHomes(self):
- """
- Finding all existing calendar homes.
- """
- calendarHomes = (yield self.transactionUnderTest().calendarHomes())
- self.assertEquals(
- [home.name() for home in calendarHomes],
- [
- "home1",
- "home_no_splits",
- "home_splits",
- "home_splits_shared",
- ]
- )
-
-
- @inlineCallbacks
def test_displayNameNone(self):
"""
L{ICalendarHome.calendarWithName} returns C{None} for calendars which
@@ -2272,31 +2255,64 @@
@inlineCallbacks
- def test_eachCalendarHome(self):
+ def test_withEachCalendarHomeDo(self):
"""
- L{ICalendarTransaction.eachCalendarHome} returns an iterator that
- yields 2-tuples of (transaction, home).
+ L{ICalendarStore.withEachCalendarHomeDo} executes its C{action}
+ argument repeatedly with all homes that have been created.
"""
- # create some additional calendar homes
additionalUIDs = set('alpha-uid home2 home3 beta-uid'.split())
txn = self.transactionUnderTest()
for name in additionalUIDs:
- # maybe it's not actually necessary to yield (i.e. wait) for each
- # one? commit() should wait for all of them.
yield txn.calendarHomeWithUID(name, create=True)
yield self.commit()
- foundUIDs = set([])
- lastTxn = None
- for txn, home in (yield self.storeUnderTest().eachCalendarHome()):
- self.addCleanup(txn.commit)
- foundUIDs.add(home.uid())
- self.assertNotIdentical(lastTxn, txn)
- lastTxn = txn
- requiredUIDs = set([
- uid for uid in self.requirements
- if self.requirements[uid] is not None
- ])
- additionalUIDs.add("home_bad")
- additionalUIDs.add("home_attachments")
- expectedUIDs = additionalUIDs.union(requiredUIDs)
- self.assertEquals(foundUIDs, expectedUIDs)
+ store = yield self.storeUnderTest()
+ def toEachCalendarHome(txn, eachHome):
+ return eachHome.createCalendarWithName("a-new-calendar")
+ result = yield store.withEachCalendarHomeDo(toEachCalendarHome)
+ self.assertEquals(result, None)
+ txn2 = self.transactionUnderTest()
+ for uid in additionalUIDs:
+ home = yield txn2.calendarHomeWithUID(uid)
+ self.assertNotIdentical(
+ None, (yield home.calendarWithName("a-new-calendar"))
+ )
+
+
+ @transactionClean
+ @inlineCallbacks
+ def test_withEachCalendarHomeDont(self):
+ """
+ When the function passed to L{ICalendarStore.withEachCalendarHomeDo}
+ raises an exception, processing is halted and the transaction is
+ aborted. The exception is re-raised.
+ """
+ # create some calendar homes.
+ additionalUIDs = set('home2 home3'.split())
+ txn = self.transactionUnderTest()
+ for uid in additionalUIDs:
+ yield txn.calendarHomeWithUID(uid, create=True)
+ yield self.commit()
+ # try to create a calendar in all of them, then fail.
+ class AnException(Exception): pass
+ caught = []
+ @inlineCallbacks
+ def toEachCalendarHome(txn, eachHome):
+ caught.append(eachHome.uid())
+ yield eachHome.createCalendarWithName("wont-be-created")
+ raise AnException()
+ store = self.storeUnderTest()
+ yield self.failUnlessFailure(
+ store.withEachCalendarHomeDo(toEachCalendarHome), AnException
+ )
+ self.assertEquals(len(caught), 1)
+ @inlineCallbacks
+ def noNewCalendar(x):
+ home = yield txn.calendarHomeWithUID(uid, create=False)
+ self.assertIdentical(
+ (yield home.calendarWithName("wont-be-created")), None
+ )
+ txn = self.transactionUnderTest()
+ yield noNewCalendar(caught[0])
+ yield noNewCalendar('home2')
+ yield noNewCalendar('home3')
+
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_file.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_file.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_file.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -502,25 +502,6 @@
@inlineCallbacks
- def test_calendarHomes(self):
- """
- Finding all existing calendar homes.
- """
- calendarHomes = (yield self.transactionUnderTest().calendarHomes())
- self.assertEquals(
- [home.name() for home in calendarHomes],
- [
- "home1",
- "home_attachments",
- "home_bad",
- "home_no_splits",
- "home_splits",
- "home_splits_shared",
- ]
- )
-
-
- @inlineCallbacks
def test_calendarObjectsWithDotFile(self):
"""
Adding a dotfile to the calendar home should not increase the number of
Modified: CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/caldav/datastore/test/test_sql.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -516,21 +516,8 @@
Where=ch.OWNER_UID == "home_version",
).on(txn)[0][0]
self.assertEqual(int(homeVersion, version))
-
-
- def test_eachCalendarHome(self):
- """
- L{ICalendarStore.eachCalendarHome} is currently stubbed out by
- L{txdav.common.datastore.sql.CommonDataStore}.
- """
- return super(CalendarSQLStorageTests, self).test_eachCalendarHome()
-
- test_eachCalendarHome.todo = (
- "stubbed out, as migration only needs to go from file->sql currently")
-
-
@inlineCallbacks
def test_homeProvisioningConcurrency(self):
"""
Modified: CalendarServer/trunk/txdav/caldav/icalendarstore.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/icalendarstore.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/caldav/icalendarstore.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -88,14 +88,6 @@
Transaction functionality required to be implemented by calendar stores.
"""
- def calendarHomes():
- """
- Retrieve each calendar home in the store.
-
- @return: a L{Deferred} which fires with a list of L{ICalendarHome}.
- """
-
-
def calendarHomeWithUID(uid, create=False):
"""
Retrieve the calendar home for the principal with the given C{uid}.
@@ -114,14 +106,40 @@
API root for calendar data storage.
"""
- def eachCalendarHome(self):
+ def withEachCalendarHomeDo(action, batchSize=None):
"""
- Enumerate all calendar homes in this store, with each one in an
- accompanying transaction.
+ Execute a given action with each calendar home present in this store,
+ in serial, committing after each batch of homes of a given size.
- @return: an iterator of 2-tuples of C{(transaction, calendar home)}
- where C{transaction} is an L{ITransaction} provider and C{calendar
- home} is an L{ICalendarHome} provider.
+ @note: This does not execute an action with each directory principal
+ for which there might be a calendar home; it works only on calendar
+ homes which have already been provisioned. To execute an action on
+ every possible calendar user, you will need to inspect the
+ directory API instead.
+
+ @note: The list of calendar homes is loaded incrementally, so this will
+ not necessarily present a consistent snapshot of the entire
+ database at a particular moment. (If this behavior is desired,
+ pass a C{batchSize} greater than the number of homes in the
+ database.)
+
+ @param action: a 2-argument callable, taking an L{ICalendarTransaction}
+ and an L{ICalendarHome}, and returning a L{Deferred} that fires
+ with C{None} when complete. Note that C{action} should not commit
+ or abort the given L{ICalendarTransaction}. If C{action} completes
+ normally, then it will be called again with the next
+ L{ICalendarHome}. If it raises an exception or returns a
+ L{Deferred} that fails, processing will stop and the L{Deferred}
+ returned from C{withEachCalendarHomeDo} will fail with that same
+ L{Failure}.
+ @type action: L{callable}
+
+ @param batchSize: The maximum count of calendar homes to include in a
+ single transaction.
+ @type batchSize: L{int}
+
+ @return: a L{Deferred} which fires with L{None} when all homes have
+ completed processing, or fails with the traceback.
"""
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/common.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/common.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/common.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -234,20 +234,6 @@
@inlineCallbacks
- def test_addressbookHomes(self):
- """
- Finding all existing addressbook homes.
- """
- addressbookHomes = (yield self.transactionUnderTest().addressbookHomes())
- self.assertEquals(
- [home.name() for home in addressbookHomes],
- [
- "home1",
- ]
- )
-
-
- @inlineCallbacks
def test_addressbookHomeWithUID_exists(self):
"""
Finding an existing addressbook home by UID results in an object that
@@ -967,29 +953,3 @@
(yield addressbook2.addressbookObjectWithUID(obj.uid())), None)
- @inlineCallbacks
- def test_eachAddressbookHome(self):
- """
- L{IAddressbookTransaction.eachAddressbookHome} returns an iterator that
- yields 2-tuples of (transaction, home).
- """
- # create some additional addressbook homes
- additionalUIDs = set('alpha-uid home2 home3 beta-uid'.split())
- txn = self.transactionUnderTest()
- for name in additionalUIDs:
- yield txn.addressbookHomeWithUID(name, create=True)
- yield self.commit()
- foundUIDs = set([])
- lastTxn = None
- for txn, home in (yield self.storeUnderTest().eachAddressbookHome()):
- self.addCleanup(txn.commit)
- foundUIDs.add(home.uid())
- self.assertNotIdentical(lastTxn, txn)
- lastTxn = txn
- requiredUIDs = set([
- uid for uid in self.requirements
- if self.requirements[uid] is not None
- ])
- additionalUIDs.add("home_bad")
- expectedUIDs = additionalUIDs.union(requiredUIDs)
- self.assertEquals(foundUIDs, expectedUIDs)
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/test_file.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/test_file.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/test_file.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -472,21 +472,6 @@
@inlineCallbacks
- def test_addressbookHomes(self):
- """
- Finding all existing addressbook homes.
- """
- addressbookHomes = (yield self.transactionUnderTest().addressbookHomes())
- self.assertEquals(
- [home.name() for home in addressbookHomes],
- [
- "home1",
- "home_bad",
- ]
- )
-
-
- @inlineCallbacks
def test_addressbookObjectsWithDotFile(self):
"""
Adding a dotfile to the addressbook home should not create a new
Modified: CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/carddav/datastore/test/test_sql.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -206,11 +206,11 @@
The DATAVERSION column for new calendar homes must match the
ADDRESSBOOK-DATAVERSION value.
"""
-
+
home = yield self.transactionUnderTest().addressbookHomeWithUID("home_version")
self.assertTrue(home is not None)
yield self.transactionUnderTest().commit
-
+
txn = yield self.transactionUnderTest()
version = yield txn.calendarserverValue("ADDRESSBOOK-DATAVERSION")[0][0]
ch = schema.ADDRESSBOOK_HOME
@@ -220,21 +220,8 @@
Where=ch.OWNER_UID == "home_version",
).on(txn)[0][0]
self.assertEqual(int(homeVersion, version))
-
-
- def test_eachAddressbookHome(self):
- """
- L{IAddressbookStore.eachAddressbookHome} is currently stubbed out by
- L{txdav.common.datastore.sql.CommonDataStore}.
- """
- return super(AddressBookSQLStorageTests, self).test_eachAddressbookHome()
-
- test_eachAddressbookHome.todo = (
- "stubbed out, as migration only needs to go from file->sql currently")
-
-
@inlineCallbacks
def test_putConcurrency(self):
"""
Modified: CalendarServer/trunk/txdav/carddav/iaddressbookstore.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/iaddressbookstore.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/carddav/iaddressbookstore.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -37,14 +37,6 @@
Transaction interface that addressbook stores must provide.
"""
- def addressbookHomes():
- """
- Retrieve each addressbook home in the store.
-
- @return: a L{Deferred} which fires with a list of L{ICalendarHome}.
- """
-
-
def addressbookHomeWithUID(uid, create=False):
"""
Retrieve the addressbook home for the principal with the given C{uid}.
Modified: CalendarServer/trunk/txdav/common/datastore/file.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/file.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/common/datastore/file.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -19,6 +19,7 @@
Common utility functions for a file based datastore.
"""
+import sys
from twext.internet.decorate import memoizedKey
from twext.python.log import LoggingMixIn
from txdav.xml.rfc2518 import ResourceType, GETContentType, HRef
@@ -132,6 +133,38 @@
)
+ @inlineCallbacks
+ def _withEachHomeDo(self, enumerator, action, batchSize):
+ """
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo} and
+ L{IAddressBookStore.withEachAddressbookHomeDo}.
+ """
+ for txn, home in enumerator():
+ try:
+ yield action(txn, home)
+ except:
+ a, b, c = sys.exc_info()
+ yield txn.abort()
+ raise a, b, c
+ else:
+ yield txn.commit()
+
+
+ def withEachCalendarHomeDo(self, action, batchSize=None):
+ """
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo}.
+ """
+ return self._withEachHomeDo(self._eachCalendarHome, action, batchSize)
+
+
+ def withEachAddressbookHomeDo(self, action, batchSize=None):
+ """
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo}.
+ """
+ return self._withEachHomeDo(self._eachAddressbookHome, action,
+ batchSize)
+
+
def setMigrating(self, state):
"""
Set the "migrating" state
@@ -149,9 +182,9 @@
def _homesOfType(self, storeType):
"""
- Common implementation of L{ICalendarStore.eachCalendarHome} and
- L{IAddressBookStore.eachAddressbookHome}; see those for a description
- of the return type.
+ Common implementation of L{_eachCalendarHome} and
+ L{_eachAddressbookHome}; see those for a description of the return
+ type.
@param storeType: one of L{EADDRESSBOOKTYPE} or L{ECALENDARTYPE}.
"""
@@ -172,11 +205,11 @@
yield (txn, home)
- def eachCalendarHome(self):
+ def _eachCalendarHome(self):
return self._homesOfType(ECALENDARTYPE)
- def eachAddressbookHome(self):
+ def _eachAddressbookHome(self):
return self._homesOfType(EADDRESSBOOKTYPE)
@@ -228,18 +261,10 @@
CommonStoreTransaction._homeClass[EADDRESSBOOKTYPE] = AddressBookHome
- def calendarHomes(self):
- return self.homes(ECALENDARTYPE)
-
-
def calendarHomeWithUID(self, uid, create=False):
return self.homeWithUID(ECALENDARTYPE, uid, create=create)
- def addressbookHomes(self):
- return self.homes(EADDRESSBOOKTYPE)
-
-
def addressbookHomeWithUID(self, uid, create=False):
return self.homeWithUID(EADDRESSBOOKTYPE, uid, create=create)
Modified: CalendarServer/trunk/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -25,6 +25,8 @@
"CommonHome",
]
+import sys
+
from uuid import uuid4, UUID
from zope.interface import implements, directlyProvides
@@ -178,20 +180,48 @@
self.queryCacher = None
- def eachCalendarHome(self):
+ @inlineCallbacks
+ def _withEachHomeDo(self, homeTable, homeFromTxn, action, batchSize):
"""
- @see: L{ICalendarStore.eachCalendarHome}
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo} and
+ L{IAddressbookStore.withEachAddressbookHomeDo}.
"""
- return []
+ txn = yield self.newTransaction()
+ try:
+ allUIDs = yield (Select([homeTable.OWNER_UID], From=homeTable)
+ .on(txn))
+ for [uid] in allUIDs:
+ yield action(txn, (yield homeFromTxn(txn, uid)))
+ except:
+ a, b, c = sys.exc_info()
+ yield txn.abort()
+ raise a, b, c
+ else:
+ yield txn.commit()
- def eachAddressbookHome(self):
+ def withEachCalendarHomeDo(self, action, batchSize=None):
"""
- @see: L{IAddressbookStore.eachAddressbookHome}
+ Implementation of L{ICalendarStore.withEachCalendarHomeDo}.
"""
- return []
+ return self._withEachHomeDo(
+ schema.CALENDAR_HOME,
+ lambda txn, uid: txn.calendarHomeWithUID(uid),
+ action, batchSize
+ )
+ def withEachAddressbookHomeDo(self, action, batchSize=None):
+ """
+ Implementation of L{IAddressbookStore.withEachAddressbookHomeDo}.
+ """
+ return self._withEachHomeDo(
+ schema.ADDRESSBOOK_HOME,
+ lambda txn, uid: txn.addressbookHomeWithUID(uid),
+ action, batchSize
+ )
+
+
def newTransaction(self, label="unlabeled", disableCache=False):
"""
@see: L{IDataStore.newTransaction}
@@ -465,18 +495,10 @@
raise RuntimeError("Database key %s cannot be determined." % (key,))
- def calendarHomes(self):
- return self.homes(ECALENDARTYPE)
-
-
def calendarHomeWithUID(self, uid, create=False):
return self.homeWithUID(ECALENDARTYPE, uid, create=create)
- def addressbookHomes(self):
- return self.homes(EADDRESSBOOKTYPE)
-
-
def addressbookHomeWithUID(self, uid, create=False):
return self.homeWithUID(EADDRESSBOOKTYPE, uid, create=create)
Modified: CalendarServer/trunk/txdav/common/datastore/test/util.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/test/util.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/common/datastore/test/util.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -193,7 +193,17 @@
store.label = currentTestID
cp.startService()
def stopIt():
- return cp.stopService()
+ # active transactions should have been shut down.
+ wasBusy = len(cp._busy)
+ busyText = repr(cp._busy)
+ stop = cp.stopService()
+ def checkWasBusy(ignored):
+ if wasBusy:
+ testCase.fail("Outstanding Transactions: " + busyText)
+ return ignored
+ if deriveValue(testCase, _SPECIAL_TXN_CLEAN, lambda tc: False):
+ stop.addBoth(checkWasBusy)
+ return stop
testCase.addCleanup(stopIt)
yield self.cleanStore(testCase, store)
returnValue(store)
@@ -251,13 +261,20 @@
for that test.
@param testCase: the test case instance.
+ @type testCase: L{TestCase}
@param attribute: the name of the attribute (the same name passed to
L{withSpecialValue}).
+ @type attribute: L{str}
@param computeDefault: A 1-argument callable, which will be called with
C{testCase} to compute a default value for the attribute for the given
test if no custom one was specified.
+ @type computeDefault: L{callable}
+
+ @return: the value of the given C{attribute} for the given C{testCase}, as
+ decorated with C{withSpecialValue}.
+ @rtype: same type as the return type of L{computeDefault}
"""
testID = testCase.id()
testMethodName = testID.split(".")[-1]
@@ -296,6 +313,7 @@
_SPECIAL_QUOTA = "__special_quota__"
+_SPECIAL_TXN_CLEAN = "__special_txn_clean__"
@@ -329,12 +347,29 @@
Test method decorator that will cause L{deriveQuota} to return a different
value for test cases that run that test method.
- @see: withSpecialValue
+ @see: L{withSpecialValue}
"""
return withSpecialValue(_SPECIAL_QUOTA, quotaValue)
+def transactionClean(f=None):
+ """
+ Test method decorator that will cause L{buildStore} to check that no
+ transactions were left outstanding at the end of the test, and fail the
+ test if they are outstanding rather than terminating them by shutting down
+ the connection pool service.
+
+ @see: L{withSpecialValue}
+ """
+ decorator = withSpecialValue(_SPECIAL_TXN_CLEAN, True)
+ if f:
+ return decorator(f)
+ else:
+ return decorator
+
+
+
@inlineCallbacks
def populateCalendarsFrom(requirements, store, migrating=False):
"""
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/migrate.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -223,6 +223,9 @@
lambda fileHome:
self.upgrader.migrateOneHome(fileTxn, homeType, fileHome)
)
+ .addCallbacks(lambda ignored: fileTxn.commit(),
+ lambda err: fileTxn.abort()
+ .addCallback(lambda ign: err))
.addCallback(lambda ignored: {})
)
@@ -343,7 +346,6 @@
"%s home %r already existed not migrating" % (
homeType, uid))
yield sqlTxn.abort()
- yield fileTxn.commit()
returnValue(None)
try:
if sqlHome is None:
@@ -351,11 +353,9 @@
yield migrateFunc(fileHome, sqlHome, merge=self.merge)
except:
f = Failure()
- yield fileTxn.abort()
yield sqlTxn.abort()
f.raiseException()
else:
- yield fileTxn.commit()
yield sqlTxn.commit()
# Remove file home after migration. FIXME: instead, this should be a
# public remove...HomeWithUID() API for de-provisioning. (If we had
@@ -402,27 +402,20 @@
)
self.log_warn("Upgrade helpers ready.")
parallelizer = Parallelizer(drivers)
+ else:
+ parallelizer = None
self.log_warn("Beginning filesystem -> database upgrade.")
+
for homeType, eachFunc in [
- ("calendar", self.fileStore.eachCalendarHome),
- ("addressbook", self.fileStore.eachAddressbookHome),
+ ("calendar", self.fileStore.withEachCalendarHomeDo),
+ ("addressbook", self.fileStore.withEachAddressbookHomeDo),
]:
- for fileTxn, fileHome in eachFunc():
- uid = fileHome.uid()
- self.log_warn("Migrating %s UID %r" % (homeType, uid))
- if parallel:
- # No-op transaction here: make sure everything's unlocked
- # before asking the subprocess to handle it.
- yield fileTxn.commit()
- @inlineCallbacks
- def doOneUpgrade(driver, fileUID=uid, homeType=homeType):
- yield driver.oneUpgrade(fileUID, homeType)
- self.log_warn("Completed migration of %s uid %r" %
- (homeType, fileUID))
- yield parallelizer.do(doOneUpgrade)
- else:
- yield self.migrateOneHome(fileTxn, homeType, fileHome)
+ yield eachFunc(
+ lambda txn, home: self._upgradeAction(
+ txn, home, homeType, parallel, parallelizer
+ )
+ )
if parallel:
yield parallelizer.done()
@@ -458,6 +451,23 @@
reactor.callLater(0, wrapped.setServiceParent, self.parent)
+ @inlineCallbacks
+ def _upgradeAction(self, fileTxn, fileHome, homeType, parallel,
+ parallelizer):
+ uid = fileHome.uid()
+ self.log_warn("Migrating %s UID %r" % (homeType, uid))
+ if parallel:
+ @inlineCallbacks
+ def doOneUpgrade(driver, fileUID=uid, homeType=homeType):
+ yield driver.oneUpgrade(fileUID, homeType)
+ self.log_warn("Completed migration of %s uid %r" %
+ (homeType, fileUID))
+ yield parallelizer.do(doOneUpgrade)
+ else:
+ yield self.migrateOneHome(fileTxn, homeType, fileHome)
+
+
+
def startService(self):
"""
Start the service.
Modified: CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py 2012-11-16 22:49:20 UTC (rev 10073)
+++ CalendarServer/trunk/txdav/common/datastore/upgrade/test/test_migrate.py 2012-11-16 22:54:42 UTC (rev 10074)
@@ -156,7 +156,8 @@
class StubService(Service, object):
def startService(self):
super(StubService, self).startService()
- subStarted.callback(None)
+ if not subStarted.called:
+ subStarted.callback(None)
from twisted.python import log
def justOnce(evt):
if evt.get('isError') and not hasattr(subStarted, 'result'):
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20121116/e4b5419b/attachment-0001.html>
More information about the calendarserver-changes
mailing list