[CalendarServer-changes] [8190] CalendarServer/branches/users/cdaboo/component-set-fixes
source_changes at macosforge.org
source_changes at macosforge.org
Thu Oct 13 13:50:21 PDT 2011
Revision: 8190
http://trac.macosforge.org/projects/calendarserver/changeset/8190
Author: cdaboo at apple.com
Date: 2011-10-13 13:50:21 -0700 (Thu, 13 Oct 2011)
Log Message:
-----------
Checkpoint work on supported-calendar-component-set restrictions. Includes proper enforcement of restrictions. Option to
provision single component collections. Auto-detect/provisioning of default scheduling calendar based on component restrictions.
Revised sql schema upgrade process to include a data upgrade step using python.
Modified Paths:
--------------
CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/customxml.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcalendar.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcol.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/put_common.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/resource.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/schedule.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/scheduling/processing.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/stdconfig.py
CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/storebridge.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/file.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/scheduling.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/test/common.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql
Added Paths:
-----------
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/old/v5.sql
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/__init__.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/file/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/file/__init__.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/migrate.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/__init__.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/__init__.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/fake_schema1/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/fake_schema2/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/fake_schema3/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/__init__.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py
Removed Paths:
-------------
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/fake_schema1/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/fake_schema2/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/fake_schema3/
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/test_util.py
CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/util.py
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/calendarserver/tap/caldav.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -57,8 +57,8 @@
from twext.web2.channel.http import LimitingHTTPFactory, SSLRedirectRequest
from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
-from txdav.common.datastore.util import UpgradeToDatabaseService,\
- UpgradeDatabaseSchemaService
+from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService
+from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService
from twistedcaldav.config import ConfigurationError
from twistedcaldav.config import config
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/customxml.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/customxml.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/customxml.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -60,6 +60,17 @@
"calendarserver-sharing-no-scheduling",
)
+class TwistedCalendarSupportedComponents (davxml.WebDAVTextElement):
+ """
+ Contains the calendar supported components list.
+ """
+ namespace = twisted_dav_namespace
+ name = "calendar-supported-components"
+ hidden = True
+
+ def getValue(self):
+ return str(self)
+
class TwistedCalendarAccessProperty (davxml.WebDAVTextElement):
"""
Contains the calendar access level (private events) for the resource.
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcalendar.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcalendar.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcalendar.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -92,7 +92,7 @@
for property in makecalendar.children[0].children[0].children:
try:
if property.qname() == (caldavxml.caldav_namespace, "supported-calendar-component-set"):
- self.writeDeadProperty(property)
+ self.setSupportedComponentSet(property)
else:
yield self.writeProperty(property, request)
except HTTPError:
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcol.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcol.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/mkcol.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -166,7 +166,7 @@
for property in mkcol.children[0].children[0].children:
try:
if rtype == "calendar" and property.qname() == (caldavxml.caldav_namespace, "supported-calendar-component-set"):
- self.writeDeadProperty(property)
+ self.setSupportedComponentSet(property)
else:
yield self.writeProperty(property, request)
except HTTPError:
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/put_common.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/method/put_common.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -298,6 +298,16 @@
"Invalid calendar data",
))
+ # Valid calendar component for check
+ result, message = self.validComponentType()
+ if not result:
+ log.err(message)
+ raise HTTPError(ErrorResponse(
+ responsecode.FORBIDDEN,
+ (caldav_namespace, "supported-component"),
+ "Invalid calendar data",
+ ))
+
# Valid attendee list size check
result, message = self.validAttendeeListSizeCheck()
if not result:
@@ -488,6 +498,19 @@
return result, message
+ def validComponentType(self):
+ """
+ Make sure that any limits on the number of resources in a collection are enforced.
+ """
+ result = True
+ message = ""
+
+ if not self.destinationparent.isSupportedComponent(self.calendar.mainType()):
+ result = False
+ message = "Invalid component type %s for calendar: %s" % (self.calendar.mainType(), self.destinationparent,)
+
+ return result, message
+
def validSizeCheck(self):
"""
Make sure that the content-type of the source resource is text/calendar.
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/resource.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/resource.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -71,7 +71,6 @@
PropertyNotFoundError, DAVResourceWithChildrenMixin
from twistedcaldav.ical import Component
-from twistedcaldav.ical import allowedComponents
from twistedcaldav.icaldav import ICalDAVResource, ICalendarPrincipalResource
from twistedcaldav.linkresource import LinkResource
from twistedcaldav.notify import (
@@ -447,10 +446,6 @@
return super(CalDAVResource, self).liveProperties() + baseProperties
- supportedCalendarComponentSet = caldavxml.SupportedCalendarComponentSet(
- *[caldavxml.CalendarComponent(name=item) for item in allowedComponents]
- )
-
def isShadowableProperty(self, qname):
"""
Shadowable properties are ones on shared resources where a "default" exists until
@@ -596,9 +591,7 @@
returnValue(davxml.AddMember(davxml.HRef.fromString(url + "/;add-member")))
elif qname == caldavxml.SupportedCalendarComponentSet.qname():
- if self.hasDeadProperty(qname):
- returnValue(self.readDeadProperty(qname))
- returnValue(self.supportedCalendarComponentSet)
+ returnValue(self.getSupportedComponentSet())
elif qname == caldavxml.SupportedCalendarData.qname():
returnValue(caldavxml.SupportedCalendarData(
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/schedule.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/schedule.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/schedule.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -284,6 +284,51 @@
davxml.HRef(defaultCalendarURL))
)
+ @inlineCallbacks
+ def defaultCalendar(self, request, componentType):
+ """
+ Find the default calendar for the supplied iCalendar component type. If one does
+ not exist, automatically provision it.
+ """
+
+ # Check any default calendar property first
+ default = (yield self.readProperty((caldav_namespace, "schedule-default-calendar-URL"), request))
+ if len(default.children) == 1:
+ defaultURL = str(default.children[0])
+ default = (yield request.locateResource(defaultURL))
+ else:
+ default = None
+
+ # Check that default handles the component type
+ if default is not None:
+ if not default.isSupportedComponent(componentType):
+ default = None
+
+ # Must have a default - provision one if not
+ if default is None:
+
+ # Try to find a calendar supporting the required component type. If there are multiple, pick
+ # the one with the oldest created timestamp as that will likely be the initial provision.
+ for calendarName in (yield self.parent._newStoreHome.listCalendars()): # These are only unshared children
+ if calendarName == "inbox":
+ continue
+ calendar = (yield self.parent._newStoreHome.calendarWithName(calendarName))
+ if not calendar.isSupportedComponent(componentType):
+ continue
+ if default is None or calendar.created() < default.created():
+ default = calendar
+
+ # If none can be found, provision one
+ if default is None:
+ new_name = "%ss" % (componentType.lower()[1:],)
+ default = yield self.parent._newStoreHome.createCalendarWithName(new_name)
+ default.setSupportedComponents(componentType.upper())
+
+ # Need L{DAVResource} object to return not new store object
+ default = (yield request.locateResource(joinURL(self.parent.url(), default.name())))
+
+ returnValue(default)
+
##
# ACL
##
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/scheduling/processing.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/scheduling/processing.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/scheduling/processing.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -341,14 +341,7 @@
if self.new_resource:
# Check for default calendar
- default = (yield self.recipient.inbox.readProperty((caldav_namespace, "schedule-default-calendar-URL"), self.request))
- if len(default.children) == 1:
- defaultURL = str(default.children[0])
- default = (yield self.request.locateResource(defaultURL))
- else:
- default = None
-
- # Must have a default calendar
+ default = (yield self.recipient.inbox.defaultCalendar(self.request, self.message.mainType()))
if default is None:
log.error("No default calendar for recipient: '%s'." % (self.recipient.cuaddr,))
raise ImplicitProcessorException(iTIPRequestStatus.NO_USER_SUPPORT)
@@ -357,13 +350,13 @@
autoprocessed = self.recipient.principal.getAutoSchedule()
store_inbox = not autoprocessed or self.recipient.principal.getCUType() == "INDIVIDUAL"
new_calendar = iTipProcessing.processNewRequest(self.message, self.recipient.cuaddr, autoprocessing=autoprocessed)
- name = md5(str(new_calendar) + str(time.time()) + defaultURL).hexdigest() + ".ics"
+ name = md5(str(new_calendar) + str(time.time()) + default.url()).hexdigest() + ".ics"
# Handle auto-reply behavior
if autoprocessed:
send_reply, partstat = (yield self.checkAttendeeAutoReply(new_calendar))
- new_resource = (yield self.writeCalendarResource(defaultURL, default, name, new_calendar))
+ new_resource = (yield self.writeCalendarResource(default.url(), default, name, new_calendar))
if autoprocessed and send_reply:
# Track outstanding auto-reply processing
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/stdconfig.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/stdconfig.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -520,6 +520,17 @@
"Enabled" : True, # Address Books on/off switch
}
},
+
+ "CalDAV": {
+ "AccountProvisioning": {
+ "CalendarName": "calendar", # Name to use when provisioning calendar for VEVENTs only, or
+ # VEVENTs and VTODOs if KeepComponentTypesSeparate is False
+ "TasksName": "tasks", # Name to use when provisioning calendar for VTODOs only, not
+ # provisioned if KeepComponentTypesSeparate is False
+ "KeepComponentTypesSeparate" : False, # Provision collections with separate
+ # for separate component types
+ },
+ },
# CardDAV Features
"DirectoryAddressBook": {
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/storebridge.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/twistedcaldav/storebridge.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -43,7 +43,7 @@
from twistedcaldav.carddavxml import carddav_namespace
from twistedcaldav.config import config
from twistedcaldav.ical import Component as VCalendar, Property as VProperty,\
- InvalidICalendarDataError, iCalendarProductID
+ InvalidICalendarDataError, iCalendarProductID, allowedComponents
from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
from twistedcaldav.method.put_addressbook_common import StoreAddressObjectResource
from twistedcaldav.method.put_common import StoreCalendarObjectResource
@@ -929,7 +929,34 @@
"""
return True
+ def setSupportedComponentSet(self, support_components_property):
+ """
+ Parse out XML property into list of components and give to store.
+ """
+ support_components = ",".join(sorted([comp.attributes["name"].upper() for comp in support_components_property.children]))
+ return maybeDeferred(self._newStoreObject.setSupportedComponents, support_components)
+
+ def getSupportedComponentSet(self):
+ comps = self._newStoreObject.getSupportedComponents()
+ if comps:
+ comps = comps.split(",")
+ else:
+ comps = allowedComponents
+ return caldavxml.SupportedCalendarComponentSet(
+ *[caldavxml.CalendarComponent(name=item) for item in comps]
+ )
+ def getSupportedComponents(self):
+ comps = self._newStoreObject.getSupportedComponents()
+ if comps:
+ comps = comps.split(",")
+ else:
+ comps = allowedComponents
+ return comps
+
+ def isSupportedComponent(self, componentType):
+ return self._newStoreObject.isSupportedComponent(componentType)
+
@inlineCallbacks
def iCalendarRolledup(self, request):
# FIXME: uncached: implement cache in the storage layer
@@ -1169,6 +1196,9 @@
def provision(self):
pass
+ def isSupportedComponent(self, componentType):
+ return self._newStoreObject.isSupportedComponent(componentType)
+
def http_DELETE(self, request):
return FORBIDDEN
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/file.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/file.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/file.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
-from twistedcaldav.ical import InvalidICalendarDataError
"""
File calendar store.
@@ -44,6 +43,8 @@
from twistedcaldav import caldavxml, customxml
from twistedcaldav.caldavxml import ScheduleCalendarTransp, Opaque
+from twistedcaldav.config import config
+from twistedcaldav.ical import InvalidICalendarDataError
from twistedcaldav.sharing import InvitesDatabase
from txdav.caldav.icalendarstore import IAttachment
@@ -181,14 +182,24 @@
def createdHome(self):
+
+ # Default calendar
defaultCal = self.createCalendarWithName("calendar")
props = defaultCal.properties()
- props[PropertyName(*ScheduleCalendarTransp.qname())] = ScheduleCalendarTransp(
- Opaque())
+ props[PropertyName(*ScheduleCalendarTransp.qname())] = ScheduleCalendarTransp(Opaque())
+
+ # Check whether components type must be separate
+ if config.CalDAV.AccountProvisioning.KeepComponentTypesSeparate:
+ defaultCal.setSupportedComponents("VEVENT")
+
+ # Default tasks
+ defaultTasks = self.createCalendarWithName(config.CalDAV.AccountProvisioning.TasksName)
+ props = defaultTasks.properties()
+ defaultTasks.setSupportedComponents("VTODO")
+
self.createCalendarWithName("inbox")
-
class Calendar(CommonHomeChild):
"""
File-based implementation of L{ICalendar}.
@@ -241,6 +252,22 @@
raise NotImplementedError()
+ def setSupportedComponents(self, supported_components):
+ """
+ Update the private property with the supported components. Technically this should only happen once
+ on collection creation, but for migration we may need to change after the fact - hence a separate api.
+ """
+
+ pname = PropertyName.fromElement(customxml.TwistedCalendarSupportedComponents)
+ if supported_components:
+ self.properties()[pname] = customxml.TwistedCalendarSupportedComponents.fromString(supported_components)
+ elif pname in self.properties():
+ del self.properties()[pname]
+
+ def getSupportedComponents(self):
+ result = str(self.properties().get(PropertyName.fromElement(customxml.TwistedCalendarSupportedComponents), ""))
+ return result if result else None
+
def initPropertyStore(self, props):
# Setup peruser special properties
props.setSpecialProperties(
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/scheduling.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/scheduling.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/scheduling.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -137,6 +137,7 @@
def __init__(self, parentHome, subCalendar):
self._parentHome = parentHome
self._subCalendar = subCalendar
+ self._supportedComponents = None
# def ownerCalendarHome(self):
# return self._parentHome
@@ -162,7 +163,16 @@
# #FIXME: wrap
# return self._subCalendar.calendarObjectWithName(name)
+ def setSupportedComponents(self, supported_components):
+ """
+ Update the database column with the supported components. Technically this should only happen once
+ on collection creation, but for migration we may need to change after the fact - hence a separate api.
+ """
+ self._supportedComponents = supported_components
+ def getSupportedComponents(self):
+ return self._supportedComponents
+
class ImplicitStore(proxyForInterface(ICalendarStore, "_calendarStore")):
"""
This is a wrapper around an L{ICalendarStore} that implements implicit
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/sql.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -211,10 +211,20 @@
@inlineCallbacks
def createdHome(self):
- defaultCal = yield self.createCalendarWithName("calendar")
+
+ # Default calendar
+ defaultCal = yield self.createCalendarWithName(config.CalDAV.AccountProvisioning.CalendarName)
props = defaultCal.properties()
- props[PropertyName(*ScheduleCalendarTransp.qname())] = ScheduleCalendarTransp(
- Opaque())
+ props[PropertyName(*ScheduleCalendarTransp.qname())] = ScheduleCalendarTransp(Opaque())
+
+ # Check whether components type must be separate
+ if config.CalDAV.AccountProvisioning.KeepComponentTypesSeparate:
+ defaultCal.setSupportedComponents("VEVENT")
+
+ # Default tasks
+ defaultTasks = yield self.createCalendarWithName(config.CalDAV.AccountProvisioning.TasksName)
+ defaultTasks.setSupportedComponents("VTODO")
+
yield self.createCalendarWithName("inbox")
@@ -257,8 +267,42 @@
self._index = PostgresLegacyIndexEmulator(self)
self._invites = SQLLegacyCalendarInvites(self)
self._objectResourceClass = CalendarObject
+
+ self._supportedComponents = None
+ @classmethod
+ def metadataColumns(cls):
+ """
+ Return a list of column name for retrieval of metadata. This allows
+ different child classes to have their own type specific data, but still make use of the
+ common base logic.
+ """
+
+ # Common behavior is to have created and modified
+
+ return (
+ cls._homeChildSchema.CREATED,
+ cls._homeChildSchema.MODIFIED,
+ cls._homeChildSchema.SUPPORTED_COMPONENTS,
+ )
+
+ @classmethod
+ def metadataAttributes(cls):
+ """
+ Return a list of attribute names for retrieval of metadata. This allows
+ different child classes to have their own type specific data, but still make use of the
+ common base logic.
+ """
+
+ # Common behavior is to have created and modified
+
+ return (
+ "_created",
+ "_modified",
+ "_supportedComponents",
+ )
+
@property
def _calendarHome(self):
return self._home
@@ -290,6 +334,31 @@
"""
return self._name == "inbox"
+ @inlineCallbacks
+ def setSupportedComponents(self, supported_components):
+ """
+ Update the database column with the supported components. Technically this should only happen once
+ on collection creation, but for migration we may need to change after the fact - hence a separate api.
+ """
+
+ cal = self._homeChildSchema
+ yield Update(
+ {
+ cal.SUPPORTED_COMPONENTS : supported_components
+ },
+ Where=(cal.RESOURCE_ID == self._resourceID)
+ ).on(self._txn)
+ self._supportedComponents = supported_components
+
+ def getSupportedComponents(self):
+ return self._supportedComponents
+
+ def isSupportedComponent(self, componentType):
+ if self._supportedComponents:
+ return componentType.upper() in self._supportedComponents.split(",")
+ else:
+ return True
+
def initPropertyStore(self, props):
# Setup peruser special properties
props.setSpecialProperties(
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/test/common.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/test/common.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/caldav/datastore/test/common.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -56,6 +56,7 @@
from txdav.common.datastore.test.util import deriveQuota
from txdav.common.datastore.test.util import withSpecialQuota
from twistedcaldav.ical import Component
+from twistedcaldav.config import config
storePath = FilePath(__file__).parent().child("calendar_store")
@@ -535,6 +536,33 @@
@inlineCallbacks
+ def test_calendarTasks_exists(self):
+ """
+ L{ICalendarHome.createdHome} creates a calendar only, or a calendar and tasks
+ collection only, in addition to inbox.
+ """
+ home1 = yield self.transactionUnderTest().calendarHomeWithUID("home_provision1", create=True)
+ for name in ("calendar", "inbox",):
+ calendar = yield home1.calendarWithName(name)
+ if calendar is None:
+ self.fail("calendar %r didn't exist" % (name,))
+ self.assertProvides(ICalendar, calendar)
+ self.assertEquals(calendar.name(), name)
+ for name in ("tasks",):
+ calendar = yield home1.calendarWithName(name)
+ if calendar is not None:
+ self.fail("calendar %r exists" % (name,))
+
+ self.patch(config.CalDAV.AccountProvisioning, "KeepComponentTypesSeparate", True)
+ home2 = yield self.transactionUnderTest().calendarHomeWithUID("home_provision2", create=True)
+ for name in ("calendar", "tasks", "inbox",):
+ calendar = yield home2.calendarWithName(name)
+ if calendar is None:
+ self.fail("calendar %r didn't exist" % (name,))
+ self.assertProvides(ICalendar, calendar)
+ self.assertEquals(calendar.name(), name)
+
+ @inlineCallbacks
def test_calendarWithName_exists(self):
"""
L{ICalendarHome.calendarWithName} returns an L{ICalendar} provider,
@@ -684,6 +712,24 @@
@inlineCallbacks
+ def test_supportedComponentSet(self):
+ """
+ Attempt to remove an non-existing calendar object should raise.
+ """
+ calendar = yield self.calendarUnderTest()
+
+ result = yield maybeDeferred(calendar.getSupportedComponents)
+ self.assertEquals(result, None)
+
+ yield maybeDeferred(calendar.setSupportedComponents, "VEVENT,VTODO")
+ result = yield maybeDeferred(calendar.getSupportedComponents)
+ self.assertEquals(result, "VEVENT,VTODO")
+
+ yield maybeDeferred(calendar.setSupportedComponents, None)
+ result = yield maybeDeferred(calendar.getSupportedComponents)
+ self.assertEquals(result, None)
+
+ @inlineCallbacks
def test_calendarObjects(self):
"""
L{ICalendar.calendarObjects} will enumerate the calendar objects present
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -1483,6 +1483,36 @@
bind.BIND_MODE == _BIND_MODE_OWN))
+ @classmethod
+ def metadataColumns(cls):
+ """
+ Return a list of column name for retrieval of metadata. This allows
+ different child classes to have their own type specific data, but still make use of the
+ common base logic.
+ """
+
+ # Common behavior is to have created and modified
+
+ return (
+ cls._homeChildSchema.CREATED,
+ cls._homeChildSchema.MODIFIED,
+ )
+
+ @classmethod
+ def metadataAttributes(cls):
+ """
+ Return a list of attribute names for retrieval of metadata. This allows
+ different child classes to have their own type specific data, but still make use of the
+ common base logic.
+ """
+
+ # Common behavior is to have created and modified
+
+ return (
+ "_created",
+ "_modified",
+ )
+
@classproperty
def _sharedChildListQuery(cls): #@NoSelf
bind = cls._bindSchema
@@ -1492,7 +1522,6 @@
bind.BIND_MODE != _BIND_MODE_OWN).And(
bind.RESOURCE_NAME != None))
-
@classmethod
@inlineCallbacks
def listObjects(cls, home, owned):
@@ -1521,10 +1550,10 @@
else:
ownedPiece = (bind.BIND_MODE != _BIND_MODE_OWN).And(
bind.RESOURCE_NAME != None)
- return Select([child.RESOURCE_ID,
- bind.RESOURCE_NAME,
- child.CREATED,
- child.MODIFIED],
+
+ columns = [child.RESOURCE_ID, bind.RESOURCE_NAME,]
+ columns.extend(cls.metadataColumns())
+ return Select(columns,
From=child.join(
bind, child.RESOURCE_ID == bind.RESOURCE_ID,
'left outer'),
@@ -1585,10 +1614,12 @@
revisions = dict(revisions)
# Create the actual objects merging in properties
- for resourceID, resource_name, created, modified in dataRows:
+ for items in dataRows:
+ resourceID, resource_name = items[:2]
+ metadata = items[2:]
child = cls(home, resource_name, resourceID, owned)
- child._created = created
- child._modified = modified
+ for attr, value in zip(cls.metadataAttributes(), metadata):
+ setattr(child, attr, value)
child._syncTokenRevision = revisions[resourceID]
propstore = propertyStores.get(resourceID, None)
yield child._loadPropertyStore(propstore)
@@ -1759,12 +1790,12 @@
@classproperty
- def _datesByIDQuery(cls): #@NoSelf
+ def _metadataByIDQuery(cls): #@NoSelf
"""
DAL query to retrieve created/modified dates based on a resource ID.
"""
child = cls._homeChildSchema
- return Select([child.CREATED, child.MODIFIED],
+ return Select(cls.metadataColumns(),
From=child,
Where=child.RESOURCE_ID == Parameter("resourceID"))
@@ -1776,9 +1807,11 @@
resource ID. We read in and cache all the extra metadata from the DB to
avoid having to do DB queries for those individually later.
"""
- self._created, self._modified = (
- yield self._datesByIDQuery.on(self._txn,
+ dataRows = (
+ yield self._metadataByIDQuery.on(self._txn,
resourceID=self._resourceID))[0]
+ for attr, value in zip(self.metadataAttributes(), dataRows):
+ setattr(self, attr, value)
yield self._loadPropertyStore()
Modified: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/current.sql 2011-10-13 20:50:21 UTC (rev 8190)
@@ -47,8 +47,9 @@
create table CALENDAR (
RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
- CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
- MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+ SUPPORTED_COMPONENTS varchar(255) default null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
);
@@ -444,5 +445,5 @@
VALUE varchar(255)
);
-insert into CALENDARSERVER values ('VERSION', '5');
+insert into CALENDARSERVER values ('VERSION', '6');
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/old/v5.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/old/v5.sql (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/old/v5.sql 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,448 @@
+-- -*- test-case-name: txdav.caldav.datastore.test.test_sql,txdav.carddav.datastore.test.test_sql -*-
+
+----
+-- Copyright (c) 2010-2011 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-----------------
+-- Resource ID --
+-----------------
+
+create sequence RESOURCE_ID_SEQ;
+
+
+-------------------
+-- Calendar Home --
+-------------------
+
+create table CALENDAR_HOME (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ OWNER_UID varchar(255) not null unique -- implicit index
+);
+
+----------------------------
+-- Calendar Home Metadata --
+----------------------------
+
+create table CALENDAR_HOME_METADATA (
+ RESOURCE_ID integer primary key references CALENDAR_HOME on delete cascade, -- implicit index
+ QUOTA_USED_BYTES integer default 0 not null
+);
+
+--------------
+-- Calendar --
+--------------
+
+create table CALENDAR (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+
+------------------------
+-- Sharing Invitation --
+------------------------
+
+create table INVITE (
+ INVITE_UID varchar(255) not null,
+ NAME varchar(255) not null,
+ RECIPIENT_ADDRESS varchar(255) not null,
+ HOME_RESOURCE_ID integer not null,
+ RESOURCE_ID integer not null
+
+ -- Need primary key on (INVITE_UID, NAME, RECIPIENT_ADDRESS)?
+);
+
+create index INVITE_INVITE_UID on INVITE(INVITE_UID);
+create index INVITE_RESOURCE_ID on INVITE(RESOURCE_ID);
+create index INVITE_HOME_RESOURCE_ID on INVITE(HOME_RESOURCE_ID);
+
+---------------------------
+-- Sharing Notifications --
+---------------------------
+
+create table NOTIFICATION_HOME (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ OWNER_UID varchar(255) not null unique -- implicit index
+);
+
+create table NOTIFICATION (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ NOTIFICATION_HOME_RESOURCE_ID integer not null references NOTIFICATION_HOME,
+ NOTIFICATION_UID varchar(255) not null,
+ XML_TYPE varchar(255) not null,
+ XML_DATA text not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+ unique(NOTIFICATION_UID, NOTIFICATION_HOME_RESOURCE_ID) -- implicit index
+);
+
+create index NOTIFICATION_NOTIFICATION_HOME_RESOURCE_ID on
+ NOTIFICATION(NOTIFICATION_HOME_RESOURCE_ID);
+
+-------------------
+-- Calendar Bind --
+-------------------
+
+-- Joins CALENDAR_HOME and CALENDAR
+
+create table CALENDAR_BIND (
+ CALENDAR_HOME_RESOURCE_ID integer not null references CALENDAR_HOME,
+ CALENDAR_RESOURCE_ID integer not null references CALENDAR on delete cascade,
+
+ -- An invitation which hasn't been accepted yet will not yet have a resource
+ -- name, so this field may be null.
+
+ CALENDAR_RESOURCE_NAME varchar(255),
+ BIND_MODE integer not null, -- enum CALENDAR_BIND_MODE
+ BIND_STATUS integer not null, -- enum CALENDAR_BIND_STATUS
+ SEEN_BY_OWNER boolean not null,
+ SEEN_BY_SHAREE boolean not null,
+ MESSAGE text,
+
+ primary key(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_ID), -- implicit index
+ unique(CALENDAR_HOME_RESOURCE_ID, CALENDAR_RESOURCE_NAME) -- implicit index
+);
+
+create index CALENDAR_BIND_RESOURCE_ID on CALENDAR_BIND(CALENDAR_RESOURCE_ID);
+
+-- Enumeration of calendar bind modes
+
+create table CALENDAR_BIND_MODE (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_BIND_MODE values (0, 'own' );
+insert into CALENDAR_BIND_MODE values (1, 'read' );
+insert into CALENDAR_BIND_MODE values (2, 'write');
+insert into CALENDAR_BIND_MODE values (3, 'direct');
+
+-- Enumeration of statuses
+
+create table CALENDAR_BIND_STATUS (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_BIND_STATUS values (0, 'invited' );
+insert into CALENDAR_BIND_STATUS values (1, 'accepted');
+insert into CALENDAR_BIND_STATUS values (2, 'declined');
+insert into CALENDAR_BIND_STATUS values (3, 'invalid');
+
+
+---------------------
+-- Calendar Object --
+---------------------
+
+create table CALENDAR_OBJECT (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ CALENDAR_RESOURCE_ID integer not null references CALENDAR on delete cascade,
+ RESOURCE_NAME varchar(255) not null,
+ ICALENDAR_TEXT text not null,
+ ICALENDAR_UID varchar(255) not null,
+ ICALENDAR_TYPE varchar(255) not null,
+ ATTACHMENTS_MODE integer default 0 not null, -- enum CALENDAR_OBJECT_ATTACHMENTS_MODE
+ DROPBOX_ID varchar(255),
+ ORGANIZER varchar(255),
+ ORGANIZER_OBJECT integer references CALENDAR_OBJECT,
+ RECURRANCE_MAX date, -- maximum date that recurrences have been expanded to.
+ ACCESS integer default 0 not null,
+ SCHEDULE_OBJECT boolean default false,
+ SCHEDULE_TAG varchar(36) default null,
+ SCHEDULE_ETAGS text default null,
+ PRIVATE_COMMENTS boolean default false not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+ unique(CALENDAR_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+
+ -- since the 'inbox' is a 'calendar resource' for the purpose of storing
+ -- calendar objects, this constraint has to be selectively enforced by the
+ -- application layer.
+
+ -- unique(CALENDAR_RESOURCE_ID, ICALENDAR_UID)
+);
+
+create index CALENDAR_OBJECT_CALENDAR_RESOURCE_ID_AND_ICALENDAR_UID on
+ CALENDAR_OBJECT(CALENDAR_RESOURCE_ID, ICALENDAR_UID);
+
+create index CALENDAR_OBJECT_CALENDAR_RESOURCE_ID_RECURRANCE_MAX on
+ CALENDAR_OBJECT(CALENDAR_RESOURCE_ID, RECURRANCE_MAX);
+
+create index CALENDAR_OBJECT_ORGANIZER_OBJECT on
+ CALENDAR_OBJECT(ORGANIZER_OBJECT);
+
+create index CALENDAR_OBJECT_DROPBOX_ID on
+ CALENDAR_OBJECT(DROPBOX_ID);
+
+-- Enumeration of attachment modes
+
+create table CALENDAR_OBJECT_ATTACHMENTS_MODE (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (0, 'none' );
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (1, 'read' );
+insert into CALENDAR_OBJECT_ATTACHMENTS_MODE values (2, 'write');
+
+
+-- Enumeration of calendar access types
+
+create table CALENDAR_ACCESS_TYPE (
+ ID integer primary key,
+ DESCRIPTION varchar(32) not null unique
+);
+
+insert into CALENDAR_ACCESS_TYPE values (0, '' );
+insert into CALENDAR_ACCESS_TYPE values (1, 'public' );
+insert into CALENDAR_ACCESS_TYPE values (2, 'private' );
+insert into CALENDAR_ACCESS_TYPE values (3, 'confidential' );
+insert into CALENDAR_ACCESS_TYPE values (4, 'restricted' );
+
+-----------------
+-- Instance ID --
+-----------------
+
+create sequence INSTANCE_ID_SEQ;
+
+
+----------------
+-- Time Range --
+----------------
+
+create table TIME_RANGE (
+ INSTANCE_ID integer primary key default nextval('INSTANCE_ID_SEQ'), -- implicit index
+ CALENDAR_RESOURCE_ID integer not null references CALENDAR on delete cascade,
+ CALENDAR_OBJECT_RESOURCE_ID integer not null references CALENDAR_OBJECT on delete cascade,
+ FLOATING boolean not null,
+ START_DATE timestamp not null,
+ END_DATE timestamp not null,
+ FBTYPE integer not null,
+ TRANSPARENT boolean not null
+);
+
+create index TIME_RANGE_CALENDAR_RESOURCE_ID on
+ TIME_RANGE(CALENDAR_RESOURCE_ID);
+create index TIME_RANGE_CALENDAR_OBJECT_RESOURCE_ID on
+ TIME_RANGE(CALENDAR_OBJECT_RESOURCE_ID);
+
+
+-- Enumeration of free/busy types
+
+create table FREE_BUSY_TYPE (
+ ID integer primary key,
+ DESCRIPTION varchar(16) not null unique
+);
+
+insert into FREE_BUSY_TYPE values (0, 'unknown' );
+insert into FREE_BUSY_TYPE values (1, 'free' );
+insert into FREE_BUSY_TYPE values (2, 'busy' );
+insert into FREE_BUSY_TYPE values (3, 'busy-unavailable');
+insert into FREE_BUSY_TYPE values (4, 'busy-tentative' );
+
+
+------------------
+-- Transparency --
+------------------
+
+create table TRANSPARENCY (
+ TIME_RANGE_INSTANCE_ID integer not null references TIME_RANGE on delete cascade,
+ USER_ID varchar(255) not null,
+ TRANSPARENT boolean not null
+);
+
+create index TRANSPARENCY_TIME_RANGE_INSTANCE_ID on
+ TRANSPARENCY(TIME_RANGE_INSTANCE_ID);
+
+----------------
+-- Attachment --
+----------------
+
+create table ATTACHMENT (
+ CALENDAR_HOME_RESOURCE_ID integer not null references CALENDAR_HOME,
+ DROPBOX_ID varchar(255) not null,
+ CONTENT_TYPE varchar(255) not null,
+ SIZE integer not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ PATH varchar(1024) not null,
+
+ primary key(DROPBOX_ID, PATH) --implicit index
+);
+
+create index ATTACHMENT_CALENDAR_HOME_RESOURCE_ID on
+ ATTACHMENT(CALENDAR_HOME_RESOURCE_ID);
+
+-----------------------
+-- Resource Property --
+-----------------------
+
+create table RESOURCE_PROPERTY (
+ RESOURCE_ID integer not null, -- foreign key: *.RESOURCE_ID
+ NAME varchar(255) not null,
+ VALUE text not null, -- FIXME: xml?
+ VIEWER_UID varchar(255),
+
+ primary key(RESOURCE_ID, NAME, VIEWER_UID) -- implicit index
+);
+
+
+----------------------
+-- AddressBook Home --
+----------------------
+
+create table ADDRESSBOOK_HOME (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ OWNER_UID varchar(255) not null unique -- implicit index
+);
+
+--------------------------------
+-- AddressBook Home Meta-data --
+--------------------------------
+
+create table ADDRESSBOOK_HOME_METADATA (
+ RESOURCE_ID integer primary key references ADDRESSBOOK_HOME on delete cascade, -- implicit index
+ QUOTA_USED_BYTES integer default 0 not null
+);
+
+-----------------
+-- AddressBook --
+-----------------
+
+create table ADDRESSBOOK (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
+
+----------------------
+-- AddressBook Bind --
+----------------------
+
+-- Joins ADDRESSBOOK_HOME and ADDRESSBOOK
+
+create table ADDRESSBOOK_BIND (
+ ADDRESSBOOK_HOME_RESOURCE_ID integer not null references ADDRESSBOOK_HOME,
+ ADDRESSBOOK_RESOURCE_ID integer not null references ADDRESSBOOK on delete cascade,
+
+ -- An invitation which hasn't been accepted yet will not yet have a resource
+ -- name, so this field may be null.
+
+ ADDRESSBOOK_RESOURCE_NAME varchar(255),
+ BIND_MODE integer not null, -- enum CALENDAR_BIND_MODE
+ BIND_STATUS integer not null, -- enum CALENDAR_BIND_STATUS
+ SEEN_BY_OWNER boolean not null,
+ SEEN_BY_SHAREE boolean not null,
+ MESSAGE text, -- FIXME: xml?
+
+ primary key(ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_ID), -- implicit index
+ unique(ADDRESSBOOK_HOME_RESOURCE_ID, ADDRESSBOOK_RESOURCE_NAME) -- implicit index
+);
+
+create index ADDRESSBOOK_BIND_RESOURCE_ID on
+ ADDRESSBOOK_BIND(ADDRESSBOOK_RESOURCE_ID);
+
+create table ADDRESSBOOK_OBJECT (
+ RESOURCE_ID integer primary key default nextval('RESOURCE_ID_SEQ'), -- implicit index
+ ADDRESSBOOK_RESOURCE_ID integer not null references ADDRESSBOOK on delete cascade,
+ RESOURCE_NAME varchar(255) not null,
+ VCARD_TEXT text not null,
+ VCARD_UID varchar(255) not null,
+ MD5 char(32) not null,
+ CREATED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+ MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP),
+
+ unique(ADDRESSBOOK_RESOURCE_ID, RESOURCE_NAME), -- implicit index
+ unique(ADDRESSBOOK_RESOURCE_ID, VCARD_UID) -- implicit index
+);
+
+---------------
+-- Revisions --
+---------------
+
+create sequence REVISION_SEQ;
+
+
+---------------
+-- Revisions --
+---------------
+
+create table CALENDAR_OBJECT_REVISIONS (
+ CALENDAR_HOME_RESOURCE_ID integer not null references CALENDAR_HOME,
+ CALENDAR_RESOURCE_ID integer references CALENDAR,
+ CALENDAR_NAME varchar(255) default null,
+ RESOURCE_NAME varchar(255),
+ REVISION integer default nextval('REVISION_SEQ') not null,
+ DELETED boolean not null,
+
+ unique(CALENDAR_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+);
+
+
+create index CALENDAR_OBJECT_REVISIONS_HOME_RESOURCE_ID
+ on CALENDAR_OBJECT_REVISIONS(CALENDAR_HOME_RESOURCE_ID);
+
+
+-------------------------------
+-- AddressBook Object Revisions --
+-------------------------------
+
+create table ADDRESSBOOK_OBJECT_REVISIONS (
+ ADDRESSBOOK_HOME_RESOURCE_ID integer not null references ADDRESSBOOK_HOME,
+ ADDRESSBOOK_RESOURCE_ID integer references ADDRESSBOOK,
+ ADDRESSBOOK_NAME varchar(255) default null,
+ RESOURCE_NAME varchar(255),
+ REVISION integer default nextval('REVISION_SEQ') not null,
+ DELETED boolean not null,
+
+ unique(ADDRESSBOOK_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+);
+
+create index ADDRESSBOOK_OBJECT_REVISIONS_HOME_RESOURCE_ID
+ on ADDRESSBOOK_OBJECT_REVISIONS(ADDRESSBOOK_HOME_RESOURCE_ID);
+
+-----------------------------------
+-- Notification Object Revisions --
+-----------------------------------
+
+create table NOTIFICATION_OBJECT_REVISIONS (
+ NOTIFICATION_HOME_RESOURCE_ID integer not null references NOTIFICATION_HOME on delete cascade,
+ RESOURCE_NAME varchar(255),
+ REVISION integer default nextval('REVISION_SEQ') not null,
+ DELETED boolean not null,
+
+ unique(NOTIFICATION_HOME_RESOURCE_ID, RESOURCE_NAME) -- implicit index
+);
+
+
+--------------------
+-- Schema Version --
+--------------------
+
+create table CALENDARSERVER (
+ NAME varchar(255) primary key, -- implicit index
+ VALUE varchar(255)
+);
+
+insert into CALENDARSERVER values ('VERSION', '5');
+
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_5_to_6.sql 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,27 @@
+----
+-- Copyright (c) 2011 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-------------------------------------------------
+-- Upgrade database schema from VERSION 5 to 6 --
+-------------------------------------------------
+
+-- Just need to modify one column
+alter table CALENDAR_OBJECT
+ add ("SUPPORTED_COMPONENTS" nvarchar2(255) default null);
+
+-- Now update the version
+update CALENDARSERVER set VALUE = '6' where NAME = 'VERSION';
+
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_5_to_6.sql 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,27 @@
+----
+-- Copyright (c) 2011 Apple Inc. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+----
+
+-------------------------------------------------
+-- Upgrade database schema from VERSION 5 to 6 --
+-------------------------------------------------
+
+-- Just need to add one column
+alter table CALENDAR
+ add column SUPPORTED_COMPONENTS varchar(255) default null;
+
+-- Now update the version
+update CALENDARSERVER set VALUE = '6' where NAME = 'VERSION';
+
Deleted: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/test_util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/test_util.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/test/test_util.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -1,361 +0,0 @@
-##
-# Copyright (c) 2010-2011 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT
-
-"""
-Tests for L{txdav.common.datastore.util}.
-"""
-
-from twext.python.filepath import CachingFilePath
-from twext.web2.http_headers import MimeType
-from twisted.application.service import Service, MultiService
-from twisted.internet.defer import inlineCallbacks, Deferred, returnValue
-from twisted.internet.protocol import Protocol
-from twisted.python.modules import getModule
-from twisted.trial.unittest import TestCase
-from txdav.caldav.datastore.test.common import CommonTests
-from txdav.carddav.datastore.test.common import CommonTests as ABCommonTests
-from txdav.common.datastore.file import CommonDataStore
-from txdav.common.datastore.test.util import theStoreBuilder, \
- populateCalendarsFrom, StubNotifierFactory, resetCalendarMD5s,\
- populateAddressBooksFrom, resetAddressBookMD5s
-from txdav.common.datastore.util import UpgradeToDatabaseService,\
- UpgradeDatabaseSchemaService
-import re
-
-class HomeMigrationTests(TestCase):
- """
- Tests for L{UpgradeToDatabaseService}.
- """
-
- @inlineCallbacks
- def setUp(self):
- """
- Set up two stores to migrate between.
- """
- # Add some files to the file store.
-
- self.filesPath = CachingFilePath(self.mktemp())
- self.filesPath.createDirectory()
- fileStore = self.fileStore = CommonDataStore(
- self.filesPath, StubNotifierFactory(), True, True
- )
- self.sqlStore = yield theStoreBuilder.buildStore(
- self, StubNotifierFactory()
- )
- subStarted = self.subStarted = Deferred()
- class StubService(Service, object):
- def startService(self):
- super(StubService, self).startService()
- subStarted.callback(None)
- self.stubService = StubService()
- self.topService = MultiService()
- self.upgrader = UpgradeToDatabaseService(
- fileStore, self.sqlStore, self.stubService
- )
- self.upgrader.setServiceParent(self.topService)
-
- requirements = CommonTests.requirements
- yield populateCalendarsFrom(requirements, fileStore)
- md5s = CommonTests.md5s
- yield resetCalendarMD5s(md5s, fileStore)
- self.filesPath.child("calendars").child(
- "__uids__").child("ho").child("me").child("home1").child(
- ".some-extra-data").setContent("some extra data")
-
- requirements = ABCommonTests.requirements
- yield populateAddressBooksFrom(requirements, fileStore)
- md5s = ABCommonTests.md5s
- yield resetAddressBookMD5s(md5s, fileStore)
- self.filesPath.child("addressbooks").child(
- "__uids__").child("ho").child("me").child("home1").child(
- ".some-extra-data").setContent("some extra data")
-
-
- @inlineCallbacks
- def test_upgradeCalendarHomes(self):
- """
- L{UpgradeToDatabaseService.startService} will do the upgrade, then
- start its dependent service by adding it to its service hierarchy.
- """
- self.topService.startService()
- yield self.subStarted
- self.assertEquals(self.stubService.running, True)
- txn = self.sqlStore.newTransaction()
- self.addCleanup(txn.commit)
- for uid in CommonTests.requirements:
- if CommonTests.requirements[uid] is not None:
- self.assertNotIdentical(
- None, (yield txn.calendarHomeWithUID(uid))
- )
- # Successfully migrated calendar homes are deleted
- self.assertFalse(self.filesPath.child("calendars").child(
- "__uids__").child("ho").child("me").child("home1").exists())
-
- # Want metadata preserved
- home = (yield txn.calendarHomeWithUID("home1"))
- calendar = (yield home.calendarWithName("calendar_1"))
- for name, metadata, md5 in (
- ("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
- ("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
- ("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
- ):
- object = (yield calendar.calendarObjectWithName(name))
- self.assertEquals(object.getMetadata(), metadata)
- self.assertEquals(object.md5(), md5)
-
-
- @inlineCallbacks
- def test_upgradeExistingHome(self):
- """
- L{UpgradeToDatabaseService.startService} will skip migrating existing
- homes.
- """
- startTxn = self.sqlStore.newTransaction("populate empty sample")
- yield startTxn.calendarHomeWithUID("home1", create=True)
- yield startTxn.commit()
- self.topService.startService()
- yield self.subStarted
- vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
- self.addCleanup(vrfyTxn.commit)
- home = yield vrfyTxn.calendarHomeWithUID("home1")
- # The default calendar is still there.
- self.assertNotIdentical(None, (yield home.calendarWithName("calendar")))
- # The migrated calendar isn't.
- self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
-
-
- @inlineCallbacks
- def test_upgradeAttachments(self):
- """
- L{UpgradeToDatabaseService.startService} upgrades calendar attachments
- as well.
- """
-
- txn = self.fileStore.newTransaction()
- committed = []
- def maybeCommit():
- if not committed:
- committed.append(True)
- return txn.commit()
- self.addCleanup(maybeCommit)
-
- @inlineCallbacks
- def getSampleObj():
- home = (yield txn.calendarHomeWithUID("home1"))
- calendar = (yield home.calendarWithName("calendar_1"))
- object = (yield calendar.calendarObjectWithName("1.ics"))
- returnValue(object)
-
- inObject = yield getSampleObj()
- someAttachmentName = "some-attachment"
- someAttachmentType = MimeType.fromString("application/x-custom-type")
- attachment = yield inObject.createAttachmentWithName(
- someAttachmentName,
- )
- transport = attachment.store(someAttachmentType)
- someAttachmentData = "Here is some data for your attachment, enjoy."
- transport.write(someAttachmentData)
- yield transport.loseConnection()
- yield maybeCommit()
- self.topService.startService()
- yield self.subStarted
- committed = []
- txn = self.sqlStore.newTransaction()
- outObject = yield getSampleObj()
- outAttachment = yield outObject.attachmentWithName(someAttachmentName)
- allDone = Deferred()
- class SimpleProto(Protocol):
- data = ''
- def dataReceived(self, data):
- self.data += data
- def connectionLost(self, reason):
- allDone.callback(self.data)
- self.assertEquals(outAttachment.contentType(), someAttachmentType)
- outAttachment.retrieve(SimpleProto())
- allData = yield allDone
- self.assertEquals(allData, someAttachmentData)
-
-
- @inlineCallbacks
- def test_upgradeAddressBookHomes(self):
- """
- L{UpgradeToDatabaseService.startService} will do the upgrade, then
- start its dependent service by adding it to its service hierarchy.
- """
- self.topService.startService()
- yield self.subStarted
- self.assertEquals(self.stubService.running, True)
- txn = self.sqlStore.newTransaction()
- self.addCleanup(txn.commit)
- for uid in ABCommonTests.requirements:
- if ABCommonTests.requirements[uid] is not None:
- self.assertNotIdentical(
- None, (yield txn.addressbookHomeWithUID(uid))
- )
- # Successfully migrated addressbook homes are deleted
- self.assertFalse(self.filesPath.child("addressbooks").child(
- "__uids__").child("ho").child("me").child("home1").exists())
-
- # Want metadata preserved
- home = (yield txn.addressbookHomeWithUID("home1"))
- adbk = (yield home.addressbookWithName("addressbook_1"))
- for name, md5 in (
- ("1.vcf", ABCommonTests.md5Values[0]),
- ("2.vcf", ABCommonTests.md5Values[1]),
- ("3.vcf", ABCommonTests.md5Values[2]),
- ):
- object = (yield adbk.addressbookObjectWithName(name))
- self.assertEquals(object.md5(), md5)
-
-class SchemaUpgradeTests(TestCase):
- """
- Tests for L{UpgradeDatabaseSchemaService}.
- """
-
- def _getSchemaVersion(self, fp):
- schema = fp.getContent()
- found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", schema)
- if found is None:
- self.fail("Could not determine schema version for: %s" % (fp,))
- return int(found.group(1))
-
- def test_scanUpgradeFiles(self):
-
- upgrader = UpgradeDatabaseSchemaService(None, None)
-
- upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
- files = upgrader.scanForUpgradeFiles("fake_dialect")
- self.assertEqual(files,
- [(3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"))],
- )
-
- upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
- files = upgrader.scanForUpgradeFiles("fake_dialect")
- self.assertEqual(files,
- [
- (3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")),
- (3, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")),
- (4, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")),
- ]
- )
-
- def test_determineUpgradeSequence(self):
-
- upgrader = UpgradeDatabaseSchemaService(None, None)
-
- upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
- files = upgrader.scanForUpgradeFiles("fake_dialect")
- upgrades = upgrader.determineUpgradeSequence(3, 4, files, "fake_dialect")
- self.assertEqual(upgrades,
- [upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")],
- )
- self.assertRaises(RuntimeError, upgrader.determineUpgradeSequence, 3, 5, files, "fake_dialect")
-
- upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
- files = upgrader.scanForUpgradeFiles("fake_dialect")
- upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
- self.assertEqual(upgrades,
- [upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")]
- )
- upgrades = upgrader.determineUpgradeSequence(4, 5, files, "fake_dialect")
- self.assertEqual(upgrades,
- [upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")]
- )
-
- upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema3")
- files = upgrader.scanForUpgradeFiles("fake_dialect")
- upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
- self.assertEqual(upgrades,
- [
- upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"),
- upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql"),
- ]
- )
-
- def test_upgradeAvailability(self):
- """
- Make sure that each old schema has a valid upgrade path to the current one.
- """
-
- for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
- upgrader = UpgradeDatabaseSchemaService(None, None)
- files = upgrader.scanForUpgradeFiles(dialect)
-
- current_version = self._getSchemaVersion(upgrader.schemaLocation.child("current.sql"))
-
- for child in upgrader.schemaLocation.child("old").globChildren("*.sql"):
- old_version = self._getSchemaVersion(child)
- upgrades = upgrader.determineUpgradeSequence(old_version, current_version, files, dialect)
- self.assertNotEqual(len(upgrades), 0)
-
- @inlineCallbacks
- def test_dbUpgrades(self):
- """
- This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
- then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
- postgres.
- """
-
- store = yield theStoreBuilder.buildStore(
- self, StubNotifierFactory()
- )
-
- @inlineCallbacks
- def _loadOldSchema(path):
- """
- Use the postgres schema mechanism to do tests under a separate "namespace"
- in postgres that we can quickly wipe clean afterwards.
- """
- startTxn = store.newTransaction("test_dbUpgrades")
- yield startTxn.execSQL("create schema test_dbUpgrades;")
- yield startTxn.execSQL("set search_path to test_dbUpgrades;")
- yield startTxn.execSQL(path.getContent())
- yield startTxn.commit()
-
- @inlineCallbacks
- def _loadVersion():
- startTxn = store.newTransaction("test_dbUpgrades")
- new_version = yield startTxn.execSQL("select value from calendarserver where name = 'VERSION';")
- yield startTxn.commit()
- returnValue(int(new_version[0][0]))
-
- @inlineCallbacks
- def _unloadOldSchema():
- startTxn = store.newTransaction("test_dbUpgrades")
- yield startTxn.execSQL("set search_path to public;")
- yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
- yield startTxn.commit()
-
- @inlineCallbacks
- def _cleanupOldSchema():
- startTxn = store.newTransaction("test_dbUpgrades")
- yield startTxn.execSQL("set search_path to public;")
- yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade;")
- yield startTxn.commit()
-
- self.addCleanup(_cleanupOldSchema)
-
- test_upgrader = UpgradeDatabaseSchemaService(None, None)
- expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"))
- for child in test_upgrader.schemaLocation.child("old").globChildren("*.sql"):
- upgrader = UpgradeDatabaseSchemaService(store, None)
- yield _loadOldSchema(child)
- yield upgrader.doUpgrade()
- new_version = yield _loadVersion()
- yield _unloadOldSchema()
-
- self.assertEqual(new_version, expected_version)
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/__init__.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/__init__.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/__init__.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,16 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/file/__init__.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/file/__init__.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/file/__init__.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,16 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/migrate.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/migrate.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/migrate.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,191 @@
+# -*- test-case-name: txdav.common.datastore.upgrade.test.test_migrate -*-
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Migrating from file store to sql store.
+"""
+
+import os
+import errno
+import xattr
+
+from twext.python.log import LoggingMixIn
+from twisted.application.service import Service
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.runtime import platform
+
+from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
+from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
+from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
+from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
+from txdav.base.propertystore.appledouble_xattr import (
+ PropertyStore as AppleDoubleStore)
+
+class UpgradeToDatabaseService(Service, LoggingMixIn, object):
+ """
+ Upgrade resources from a filesystem store to a database store.
+ """
+
+ @classmethod
+ def wrapService(cls, path, service, store, uid=None, gid=None):
+ """
+ Create an L{UpgradeToDatabaseService} if there are still file-based
+ calendar or addressbook homes remaining in the given path.
+
+ @param path: a path pointing at the document root, where the file-based
+ data-store is located.
+ @type path: L{CachingFilePath}
+
+ @param service: the service to wrap. This service should be started
+ when the upgrade is complete. (This is accomplished by returning
+ it directly when no upgrade needs to be done, and by adding it to
+ the service hierarchy when the upgrade completes; assuming that the
+ service parent of the resulting service will be set to a
+ L{MultiService} or similar.)
+
+ @param store: the SQL storage service.
+
+ @type service: L{IService}
+
+ @return: a service
+ @rtype: L{IService}
+ """
+ # TODO: TOPPATHS should be computed based on enabled flags in 'store',
+ # not hard coded.
+ for homeType in TOPPATHS:
+ if path.child(homeType).exists():
+ if platform.isMacOSX():
+ appropriateStoreClass = XattrPropertyStore
+ else:
+ attrs = xattr.xattr(path.path)
+ try:
+ attrs.get('user.should-not-be-set')
+ except IOError, ioe:
+ if ioe.errno == errno.ENODATA:
+ # xattrs are supported and enabled on the filesystem
+ # where the calendar data lives. this takes some
+ # doing (you have to edit fstab), so this means
+ # we're trying to migrate some 2.x data from a
+ # previous linux installation.
+ appropriateStoreClass = XattrPropertyStore
+ elif ioe.errno == errno.EOPNOTSUPP:
+ # The operation wasn't supported. This is what will
+ # usually happen on a naively configured filesystem,
+ # so this means we're most likely trying to migrate
+ # some data from an untarred archive created on an
+ # OS X installation using xattrs.
+ appropriateStoreClass = AppleDoubleStore
+ else:
+ # No need to check for ENOENT and the like; we just
+ # checked above to make sure the parent exists.
+ # Other errors are not anticipated here, so fail
+ # fast.
+ raise
+
+ appropriateStoreClass = AppleDoubleStore
+
+ self = cls(
+ FileStore(path, None, True, True,
+ propertyStoreClass=appropriateStoreClass),
+ store, service, uid=uid, gid=gid,
+ )
+ return self
+ return service
+
+
+ def __init__(self, fileStore, sqlStore, service, uid=None, gid=None):
+ """
+ Initialize the service.
+ """
+ self.wrappedService = service
+ self.fileStore = fileStore
+ self.sqlStore = sqlStore
+ self.uid = uid
+ self.gid = gid
+
+
+ @inlineCallbacks
+ def doMigration(self):
+ """
+ Do the migration. Called by C{startService}, but a different method
+ because C{startService} should return C{None}, not a L{Deferred}.
+
+ @return: a Deferred which fires when the migration is complete.
+ """
+ self.log_warn("Beginning filesystem -> database upgrade.")
+ for homeType, migrateFunc, eachFunc, destFunc, _ignore_topPathName in [
+ ("calendar", migrateCalendarHome,
+ self.fileStore.eachCalendarHome,
+ lambda txn: txn.calendarHomeWithUID,
+ "calendars"),
+ ("addressbook", migrateAddressbookHome,
+ self.fileStore.eachAddressbookHome,
+ lambda txn: txn.addressbookHomeWithUID,
+ "addressbooks")
+ ]:
+ for fileTxn, fileHome in eachFunc():
+ uid = fileHome.uid()
+ self.log_warn("Migrating %s UID %r" % (homeType, uid))
+ sqlTxn = self.sqlStore.newTransaction(migrating=True)
+ homeGetter = destFunc(sqlTxn)
+ if (yield homeGetter(uid, create=False)) is not None:
+ self.log_warn(
+ "%s home %r already existed not migrating" % (
+ homeType, uid))
+ yield sqlTxn.abort()
+ yield fileTxn.commit()
+ continue
+ sqlHome = yield homeGetter(uid, create=True)
+ if sqlHome is None:
+ raise RuntimeError("THIS SHOULD NOT BE POSSIBLE.")
+ yield migrateFunc(fileHome, sqlHome)
+ yield fileTxn.commit()
+ yield sqlTxn.commit()
+ # FIXME: need a public remove...HomeWithUID() for de-
+ # provisioning
+
+ # Remove file home after migration
+ fileHome._path.remove()
+ for homeType in TOPPATHS:
+ homesPath = self.fileStore._path.child(homeType)
+ if homesPath.isdir():
+ homesPath.remove()
+
+ # Set attachment directory ownership. FIXME: is this still necessary
+ # since attachments started living outside the database directory
+ # created by initdb? default permissions might be correct now.
+ sqlAttachmentsPath = self.sqlStore.attachmentsPath
+ if (sqlAttachmentsPath and sqlAttachmentsPath.exists() and
+ (self.uid or self.gid)):
+ uid = self.uid or -1
+ gid = self.gid or -1
+ for fp in sqlAttachmentsPath.walk():
+ os.chown(fp.path, uid, gid)
+
+ self.log_warn(
+ "Filesystem upgrade complete, launching database service."
+ )
+ # see http://twistedmatrix.com/trac/ticket/4649
+ reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
+
+
+ def startService(self):
+ """
+ Start the service.
+ """
+ self.doMigration()
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/__init__.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/__init__.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/__init__.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,16 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/__init__.py
===================================================================
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/test/test_upgrade.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,180 @@
+##
+# Copyright (c) 2010-2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+import types
+
+"""
+Tests for L{txdav.common.datastore.upgrade.sql.upgrade}.
+"""
+
+from twext.enterprise.ienterprise import ORACLE_DIALECT, POSTGRES_DIALECT
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python.modules import getModule
+from twisted.trial.unittest import TestCase
+from txdav.common.datastore.test.util import theStoreBuilder, StubNotifierFactory
+from txdav.common.datastore.upgrade.sql.upgrade import UpgradeDatabaseSchemaService
+import re
+
+class SchemaUpgradeTests(TestCase):
+ """
+ Tests for L{UpgradeDatabaseSchemaService}.
+ """
+
+ def _getSchemaVersion(self, fp):
+ schema = fp.getContent()
+ found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", schema)
+ if found is None:
+ self.fail("Could not determine schema version for: %s" % (fp,))
+ return int(found.group(1))
+
+ def test_scanUpgradeFiles(self):
+
+ upgrader = UpgradeDatabaseSchemaService(None, None)
+
+ upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
+ files = upgrader.scanForUpgradeFiles("fake_dialect")
+ self.assertEqual(files,
+ [(3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"))],
+ )
+
+ upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
+ files = upgrader.scanForUpgradeFiles("fake_dialect")
+ self.assertEqual(files,
+ [
+ (3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")),
+ (3, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")),
+ (4, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")),
+ ]
+ )
+
+ def test_determineUpgradeSequence(self):
+
+ upgrader = UpgradeDatabaseSchemaService(None, None)
+
+ upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
+ files = upgrader.scanForUpgradeFiles("fake_dialect")
+ upgrades = upgrader.determineUpgradeSequence(3, 4, files, "fake_dialect")
+ self.assertEqual(upgrades,
+ [upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")],
+ )
+ self.assertRaises(RuntimeError, upgrader.determineUpgradeSequence, 3, 5, files, "fake_dialect")
+
+ upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
+ files = upgrader.scanForUpgradeFiles("fake_dialect")
+ upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
+ self.assertEqual(upgrades,
+ [upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")]
+ )
+ upgrades = upgrader.determineUpgradeSequence(4, 5, files, "fake_dialect")
+ self.assertEqual(upgrades,
+ [upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")]
+ )
+
+ upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema3")
+ files = upgrader.scanForUpgradeFiles("fake_dialect")
+ upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
+ self.assertEqual(upgrades,
+ [
+ upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"),
+ upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql"),
+ ]
+ )
+
+ def test_upgradeAvailability(self):
+ """
+ Make sure that each old schema has a valid upgrade path to the current one.
+ """
+
+ for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
+ upgrader = UpgradeDatabaseSchemaService(None, None)
+ files = upgrader.scanForUpgradeFiles(dialect)
+
+ current_version = self._getSchemaVersion(upgrader.schemaLocation.child("current.sql"))
+
+ for child in upgrader.schemaLocation.child("old").globChildren("*.sql"):
+ old_version = self._getSchemaVersion(child)
+ upgrades = upgrader.determineUpgradeSequence(old_version, current_version, files, dialect)
+ self.assertNotEqual(len(upgrades), 0)
+
+ def test_upgradeDataAvailability(self):
+ """
+ Make sure that each upgrade file has a valid data upgrade file or None.
+ """
+
+ for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
+ upgrader = UpgradeDatabaseSchemaService(None, None)
+ files = upgrader.scanForUpgradeFiles(dialect)
+ for _ignore_from, _ignore_to, fp in files:
+ result = upgrader.getDataUpgrade(fp)
+ if result is not None:
+ self.assertIsInstance(result, types.FunctionType)
+
+ @inlineCallbacks
+ def test_dbUpgrades(self):
+ """
+ This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
+ then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
+ postgres.
+ """
+
+ store = yield theStoreBuilder.buildStore(
+ self, StubNotifierFactory()
+ )
+
+ @inlineCallbacks
+ def _loadOldSchema(path):
+ """
+ Use the postgres schema mechanism to do tests under a separate "namespace"
+ in postgres that we can quickly wipe clean afterwards.
+ """
+ startTxn = store.newTransaction("test_dbUpgrades")
+ yield startTxn.execSQL("create schema test_dbUpgrades;")
+ yield startTxn.execSQL("set search_path to test_dbUpgrades;")
+ yield startTxn.execSQL(path.getContent())
+ yield startTxn.commit()
+
+ @inlineCallbacks
+ def _loadVersion():
+ startTxn = store.newTransaction("test_dbUpgrades")
+ new_version = yield startTxn.execSQL("select value from calendarserver where name = 'VERSION';")
+ yield startTxn.commit()
+ returnValue(int(new_version[0][0]))
+
+ @inlineCallbacks
+ def _unloadOldSchema():
+ startTxn = store.newTransaction("test_dbUpgrades")
+ yield startTxn.execSQL("set search_path to public;")
+ yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
+ yield startTxn.commit()
+
+ @inlineCallbacks
+ def _cleanupOldSchema():
+ startTxn = store.newTransaction("test_dbUpgrades")
+ yield startTxn.execSQL("set search_path to public;")
+ yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade;")
+ yield startTxn.commit()
+
+ self.addCleanup(_cleanupOldSchema)
+
+ test_upgrader = UpgradeDatabaseSchemaService(None, None)
+ expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"))
+ for child in test_upgrader.schemaLocation.child("old").globChildren("*.sql"):
+ upgrader = UpgradeDatabaseSchemaService(store, None)
+ yield _loadOldSchema(child)
+ yield upgrader.doUpgrade()
+ new_version = yield _loadVersion()
+ yield _unloadOldSchema()
+
+ self.assertEqual(new_version, expected_version)
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,236 @@
+# -*- test-case-name: txdav.common.datastore.upgrade.sql.test -*-
+##
+# Copyright (c) 2010-2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+from twisted.python.reflect import namedObject
+
+"""
+Utilities, mostly related to upgrading, common to calendar and addresbook
+data stores.
+"""
+
+import re
+
+from twext.python.log import LoggingMixIn
+from twisted.application.service import Service
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
+from twisted.python.modules import getModule
+
+class UpgradeDatabaseSchemaService(Service, LoggingMixIn, object):
+ """
+ Checks and upgrades the database schema. This assumes there are a bunch of
+ upgrade files in sql syntax that we can execute against the database to accomplish
+ the upgrade.
+ """
+
+ @classmethod
+ def wrapService(cls, service, store, uid=None, gid=None):
+ """
+ Create an L{UpgradeDatabaseSchemaService} when starting the database
+ so we can check the schema version and do any upgrades.
+
+ @param service: the service to wrap. This service should be started
+ when the upgrade is complete. (This is accomplished by returning
+ it directly when no upgrade needs to be done, and by adding it to
+ the service hierarchy when the upgrade completes; assuming that the
+ service parent of the resulting service will be set to a
+ L{MultiService} or similar.)
+
+ @param store: the SQL storage service.
+
+ @type service: L{IService}
+
+ @return: a service
+ @rtype: L{IService}
+ """
+ return cls(store, service, uid=uid, gid=gid,)
+
+
+ def __init__(self, sqlStore, service, uid=None, gid=None):
+ """
+ Initialize the service.
+
+ @param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
+ @param service: Wrapped service. Can be C{None} when doing unit tests.
+ """
+ self.wrappedService = service
+ self.sqlStore = sqlStore
+ self.uid = uid
+ self.gid = gid
+ self.schemaLocation = getModule(__name__).filePath.parent().parent().sibling("sql_schema")
+ self.pyLocation = getModule(__name__).filePath.parent()
+
+ @inlineCallbacks
+ def doUpgrade(self):
+ """
+ Do the schema check and upgrade if needed. Called by C{startService}, but a different method
+ because C{startService} should return C{None}, not a L{Deferred}.
+
+ @return: a Deferred which fires when the migration is complete.
+ """
+ self.log_warn("Beginning database schema check.")
+
+ # Retrieve the version number from the schema file
+ current_schema = self.schemaLocation.child("current.sql").getContent()
+ found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", current_schema)
+ if found is None:
+ msg = "Schema is missing required schema VERSION insert statement: %s" % (current_schema,)
+ self.log_error(msg)
+ raise RuntimeError(msg)
+ else:
+ required_version = int(found.group(1))
+ self.log_warn("Required schema version: %s." % (required_version,))
+
+ # Get the schema version in the current database
+ sqlTxn = self.sqlStore.newTransaction()
+ dialect = sqlTxn.dialect
+ try:
+ actual_version = yield sqlTxn.schemaVersion()
+ yield sqlTxn.commit()
+ except RuntimeError:
+ self.log_error("Database schema version cannot be determined.")
+ yield sqlTxn.abort()
+ raise
+
+ self.log_warn("Actual schema version: %s." % (actual_version,))
+
+ if required_version == actual_version:
+ self.log_warn("Schema version check complete: no upgrade needed.")
+ elif required_version < actual_version:
+ msg = "Actual schema version %s is more recent than the expected version %s. The service cannot be started" % (actual_version, required_version,)
+ self.log_error(msg)
+ raise RuntimeError(msg)
+ else:
+ yield self.upgradeVersion(actual_version, required_version, dialect)
+
+ self.log_warn(
+ "Database schema check complete, launching database service."
+ )
+ # see http://twistedmatrix.com/trac/ticket/4649
+ if self.wrappedService is not None:
+ reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
+
+ @inlineCallbacks
+
+ def upgradeVersion(self, fromVersion, toVersion, dialect):
+ """
+ Update the database from one version to another (the current one). Do this by
+ looking for upgrade_from_X_to_Y.sql files that cover the full range of upgrades.
+ """
+
+ self.log_warn("Starting schema upgrade from version %d to %d." % (fromVersion, toVersion,))
+
+ # Scan for all possible upgrade files - returned sorted
+ files = self.scanForUpgradeFiles(dialect)
+
+ # Determine upgrade sequence and run each upgrade
+ upgrades = self.determineUpgradeSequence(fromVersion, toVersion, files, dialect)
+
+ # Use one transaction for the entire set of upgrades
+ sqlTxn = self.sqlStore.newTransaction()
+ try:
+ for fp in upgrades:
+ yield self.applyUpgrade(sqlTxn, fp)
+ yield sqlTxn.commit()
+ except RuntimeError:
+ self.log_error("Database upgrade failed:" % (fp.basename(),))
+ yield sqlTxn.abort()
+ raise
+
+ self.log_warn("Schema upgraded from version %d to %d." % (fromVersion, toVersion,))
+
+ def scanForUpgradeFiles(self, dialect):
+ """
+ Scan the module path for upgrade files with the require name.
+ """
+
+ fp = self.schemaLocation.child("upgrades").child(dialect)
+ upgrades = []
+ regex = re.compile("upgrade_from_(\d)+_to_(\d)+.sql")
+ for child in fp.globChildren("upgrade_*.sql"):
+ matched = regex.match(child.basename())
+ if matched is not None:
+ fromV = int(matched.group(1))
+ toV = int(matched.group(2))
+ upgrades.append((fromV, toV, child))
+
+ upgrades.sort(key=lambda x:(x[0], x[1]))
+ return upgrades
+
+ def determineUpgradeSequence(self, fromVersion, toVersion, files, dialect):
+ """
+ Determine the upgrade_from_X_to_Y.sql files that cover the full range of upgrades.
+ Note that X and Y may not be consecutive, e.g., we might have an upgrade from 3 to 4,
+ 4 to 5, and 3 to 5 - the later because it is more efficient to jump over the intermediate
+ step. As a result we will always try and pick the upgrade file that gives the biggest
+ jump from one version to another at each step.
+ """
+
+ # Now find the path from the old version to the current one
+ filesByFromVersion = {}
+ for fromV, toV, fp in files:
+ if fromV not in filesByFromVersion or filesByFromVersion[fromV][1] < toV:
+ filesByFromVersion[fromV] = fromV, toV, fp
+
+ upgrades = []
+ nextVersion = fromVersion
+ while nextVersion != toVersion:
+ if nextVersion not in filesByFromVersion:
+ msg = "Missing upgrade file from version %d with dialect %s" % (nextVersion, dialect,)
+ self.log_error(msg)
+ raise RuntimeError(msg)
+ else:
+ upgrades.append(filesByFromVersion[nextVersion][2])
+ nextVersion = filesByFromVersion[nextVersion][1]
+
+ return upgrades
+
+ @inlineCallbacks
+ def applyUpgrade(self, sqlTxn, fp):
+ """
+ Apply the schema upgrade .sql file to the database.
+ """
+ self.log_warn("Applying schema upgrade: %s" % (fp.basename(),))
+ sql = fp.getContent()
+ yield sqlTxn.execSQLBlock(sql)
+
+ doDataUpgrade = self.getDataUpgrade(fp)
+ if doDataUpgrade is not None:
+ yield doDataUpgrade(sqlTxn)
+
+ def getDataUpgrade(self, fp):
+ # Also look for python module to execute
+ check_name = self.pyLocation.child(fp.basename()[:-4] + ".py")
+ if check_name.exists():
+ try:
+ module = getModule(__name__)
+ module = ".".join(module.name.split(".")[:-1]) + "." + fp.basename()[:-4] + ".doUpgrade"
+ doUpgrade = namedObject(module)
+ self.log_warn("Applying data upgrade: %s" % (module,))
+ return doUpgrade
+ except ImportError:
+ msg = "Failed data upgrade: %s" % (fp.basename()[:-4],)
+ self.log_error(msg)
+ raise RuntimeError(msg)
+ else:
+ self.log_warn("No data upgrade: %s" % (fp.basename()[:-4],))
+ return None
+
+ def startService(self):
+ """
+ Start the service.
+ """
+ self.doUpgrade()
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/upgrade_from_5_to_6.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,50 @@
+# -*- test-case-name: txdav.common.datastore.upgrade.sql.test -*-
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.syntax import Update
+from twext.web2.dav.element.parser import WebDAVDocument
+from twisted.internet.defer import inlineCallbacks
+from twistedcaldav import caldavxml
+from txdav.common.datastore.sql_tables import schema
+from txdav.common.datastore.upgrade.sql.util import rowsForProperty,\
+ removeProperty
+
+"""
+Data upgrade from database version 5 to 6
+"""
+
+ at inlineCallbacks
+def doUpgrade(sqlTxn):
+ """
+ Need to move all the CalDAV:supported-component-set properties in the RESOURCE_PROPERTY
+ table to the new CALENDAR table column, extracting the new format value from the XML property.
+ """
+
+ rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet))
+ for calendar_rid, value in rows:
+ prop = WebDAVDocument.fromString(value).root_element
+ supported_components = ",".join(sorted([comp.attributes["name"].upper() for comp in prop.children]))
+
+ cal = schema.CALENDAR
+ yield Update(
+ {
+ cal.SUPPORTED_COMPONENTS : supported_components
+ },
+ Where=(cal.RESOURCE_ID == calendar_rid)
+ ).on(sqlTxn)
+
+ yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/sql/util.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,43 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twext.enterprise.dal.syntax import Select, Delete
+from twisted.internet.defer import inlineCallbacks, returnValue
+from txdav.base.propertystore.base import PropertyName
+from txdav.common.datastore.sql_tables import schema
+
+ at inlineCallbacks
+def rowsForProperty(txn, propelement):
+ pname = PropertyName.fromElement(propelement)
+
+ rp = schema.RESOURCE_PROPERTY
+ rows = yield Select(
+ [rp.RESOURCE_ID, rp.VALUE,],
+ From=rp,
+ Where=rp.NAME == pname.toString(),
+ ).on(txn)
+
+ returnValue(rows)
+
+ at inlineCallbacks
+def removeProperty(txn, propelement):
+ pname = PropertyName.fromElement(propelement)
+
+ rp = schema.RESOURCE_PROPERTY
+ yield Delete(
+ From=rp,
+ Where=rp.NAME == pname.toString(),
+ ).on(txn)
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/__init__.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/__init__.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/__init__.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,16 @@
+##
+# Copyright (c) 2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
Added: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py (rev 0)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/upgrade/test/test_migrate.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -0,0 +1,218 @@
+##
+# Copyright (c) 2010-2011 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+Tests for L{txdav.common.datastore.upgrade.migrate}.
+"""
+
+from twext.python.filepath import CachingFilePath
+from twext.web2.http_headers import MimeType
+from twisted.application.service import Service, MultiService
+from twisted.internet.defer import inlineCallbacks, Deferred, returnValue
+from twisted.internet.protocol import Protocol
+from twisted.trial.unittest import TestCase
+from txdav.caldav.datastore.test.common import CommonTests
+from txdav.carddav.datastore.test.common import CommonTests as ABCommonTests
+from txdav.common.datastore.file import CommonDataStore
+from txdav.common.datastore.test.util import theStoreBuilder, \
+ populateCalendarsFrom, StubNotifierFactory, resetCalendarMD5s,\
+ populateAddressBooksFrom, resetAddressBookMD5s
+from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService
+
+class HomeMigrationTests(TestCase):
+ """
+ Tests for L{UpgradeToDatabaseService}.
+ """
+
+ @inlineCallbacks
+ def setUp(self):
+ """
+ Set up two stores to migrate between.
+ """
+ # Add some files to the file store.
+
+ self.filesPath = CachingFilePath(self.mktemp())
+ self.filesPath.createDirectory()
+ fileStore = self.fileStore = CommonDataStore(
+ self.filesPath, StubNotifierFactory(), True, True
+ )
+ self.sqlStore = yield theStoreBuilder.buildStore(
+ self, StubNotifierFactory()
+ )
+ subStarted = self.subStarted = Deferred()
+ class StubService(Service, object):
+ def startService(self):
+ super(StubService, self).startService()
+ subStarted.callback(None)
+ self.stubService = StubService()
+ self.topService = MultiService()
+ self.upgrader = UpgradeToDatabaseService(
+ fileStore, self.sqlStore, self.stubService
+ )
+ self.upgrader.setServiceParent(self.topService)
+
+ requirements = CommonTests.requirements
+ yield populateCalendarsFrom(requirements, fileStore)
+ md5s = CommonTests.md5s
+ yield resetCalendarMD5s(md5s, fileStore)
+ self.filesPath.child("calendars").child(
+ "__uids__").child("ho").child("me").child("home1").child(
+ ".some-extra-data").setContent("some extra data")
+
+ requirements = ABCommonTests.requirements
+ yield populateAddressBooksFrom(requirements, fileStore)
+ md5s = ABCommonTests.md5s
+ yield resetAddressBookMD5s(md5s, fileStore)
+ self.filesPath.child("addressbooks").child(
+ "__uids__").child("ho").child("me").child("home1").child(
+ ".some-extra-data").setContent("some extra data")
+
+
+ @inlineCallbacks
+ def test_upgradeCalendarHomes(self):
+ """
+ L{UpgradeToDatabaseService.startService} will do the upgrade, then
+ start its dependent service by adding it to its service hierarchy.
+ """
+ self.topService.startService()
+ yield self.subStarted
+ self.assertEquals(self.stubService.running, True)
+ txn = self.sqlStore.newTransaction()
+ self.addCleanup(txn.commit)
+ for uid in CommonTests.requirements:
+ if CommonTests.requirements[uid] is not None:
+ self.assertNotIdentical(
+ None, (yield txn.calendarHomeWithUID(uid))
+ )
+ # Successfully migrated calendar homes are deleted
+ self.assertFalse(self.filesPath.child("calendars").child(
+ "__uids__").child("ho").child("me").child("home1").exists())
+
+ # Want metadata preserved
+ home = (yield txn.calendarHomeWithUID("home1"))
+ calendar = (yield home.calendarWithName("calendar_1"))
+ for name, metadata, md5 in (
+ ("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
+ ("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
+ ("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
+ ):
+ object = (yield calendar.calendarObjectWithName(name))
+ self.assertEquals(object.getMetadata(), metadata)
+ self.assertEquals(object.md5(), md5)
+
+
+ @inlineCallbacks
+ def test_upgradeExistingHome(self):
+ """
+ L{UpgradeToDatabaseService.startService} will skip migrating existing
+ homes.
+ """
+ startTxn = self.sqlStore.newTransaction("populate empty sample")
+ yield startTxn.calendarHomeWithUID("home1", create=True)
+ yield startTxn.commit()
+ self.topService.startService()
+ yield self.subStarted
+ vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
+ self.addCleanup(vrfyTxn.commit)
+ home = yield vrfyTxn.calendarHomeWithUID("home1")
+ # The default calendar is still there.
+ self.assertNotIdentical(None, (yield home.calendarWithName("calendar")))
+ # The migrated calendar isn't.
+ self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
+
+
+ @inlineCallbacks
+ def test_upgradeAttachments(self):
+ """
+ L{UpgradeToDatabaseService.startService} upgrades calendar attachments
+ as well.
+ """
+
+ txn = self.fileStore.newTransaction()
+ committed = []
+ def maybeCommit():
+ if not committed:
+ committed.append(True)
+ return txn.commit()
+ self.addCleanup(maybeCommit)
+
+ @inlineCallbacks
+ def getSampleObj():
+ home = (yield txn.calendarHomeWithUID("home1"))
+ calendar = (yield home.calendarWithName("calendar_1"))
+ object = (yield calendar.calendarObjectWithName("1.ics"))
+ returnValue(object)
+
+ inObject = yield getSampleObj()
+ someAttachmentName = "some-attachment"
+ someAttachmentType = MimeType.fromString("application/x-custom-type")
+ attachment = yield inObject.createAttachmentWithName(
+ someAttachmentName,
+ )
+ transport = attachment.store(someAttachmentType)
+ someAttachmentData = "Here is some data for your attachment, enjoy."
+ transport.write(someAttachmentData)
+ yield transport.loseConnection()
+ yield maybeCommit()
+ self.topService.startService()
+ yield self.subStarted
+ committed = []
+ txn = self.sqlStore.newTransaction()
+ outObject = yield getSampleObj()
+ outAttachment = yield outObject.attachmentWithName(someAttachmentName)
+ allDone = Deferred()
+ class SimpleProto(Protocol):
+ data = ''
+ def dataReceived(self, data):
+ self.data += data
+ def connectionLost(self, reason):
+ allDone.callback(self.data)
+ self.assertEquals(outAttachment.contentType(), someAttachmentType)
+ outAttachment.retrieve(SimpleProto())
+ allData = yield allDone
+ self.assertEquals(allData, someAttachmentData)
+
+
+ @inlineCallbacks
+ def test_upgradeAddressBookHomes(self):
+ """
+ L{UpgradeToDatabaseService.startService} will do the upgrade, then
+ start its dependent service by adding it to its service hierarchy.
+ """
+ self.topService.startService()
+ yield self.subStarted
+ self.assertEquals(self.stubService.running, True)
+ txn = self.sqlStore.newTransaction()
+ self.addCleanup(txn.commit)
+ for uid in ABCommonTests.requirements:
+ if ABCommonTests.requirements[uid] is not None:
+ self.assertNotIdentical(
+ None, (yield txn.addressbookHomeWithUID(uid))
+ )
+ # Successfully migrated addressbook homes are deleted
+ self.assertFalse(self.filesPath.child("addressbooks").child(
+ "__uids__").child("ho").child("me").child("home1").exists())
+
+ # Want metadata preserved
+ home = (yield txn.addressbookHomeWithUID("home1"))
+ adbk = (yield home.addressbookWithName("addressbook_1"))
+ for name, md5 in (
+ ("1.vcf", ABCommonTests.md5Values[0]),
+ ("2.vcf", ABCommonTests.md5Values[1]),
+ ("3.vcf", ABCommonTests.md5Values[2]),
+ ):
+ object = (yield adbk.addressbookObjectWithName(name))
+ self.assertEquals(object.md5(), md5)
Deleted: CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/util.py
===================================================================
--- CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/util.py 2011-10-13 18:56:29 UTC (rev 8189)
+++ CalendarServer/branches/users/cdaboo/component-set-fixes/txdav/common/datastore/util.py 2011-10-13 20:50:21 UTC (rev 8190)
@@ -1,382 +0,0 @@
-# -*- test-case-name: txdav.common.datastore.test -*-
-##
-# Copyright (c) 2010 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-Utilities, mostly related to upgrading, common to calendar and addresbook
-data stores.
-"""
-
-import os
-import re
-import errno
-import xattr
-
-from twext.python.log import LoggingMixIn
-from twisted.application.service import Service
-from twisted.internet import reactor
-from twisted.internet.defer import inlineCallbacks
-from twisted.python.modules import getModule
-from twisted.python.runtime import platform
-
-from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
-from txdav.carddav.datastore.util import migrateHome as migrateAddressbookHome
-from txdav.common.datastore.file import CommonDataStore as FileStore, TOPPATHS
-from txdav.base.propertystore.xattr import PropertyStore as XattrPropertyStore
-from txdav.base.propertystore.appledouble_xattr import (
- PropertyStore as AppleDoubleStore)
-
-
-class UpgradeToDatabaseService(Service, LoggingMixIn, object):
- """
- Upgrade resources from a filesystem store to a database store.
- """
-
- @classmethod
- def wrapService(cls, path, service, store, uid=None, gid=None):
- """
- Create an L{UpgradeToDatabaseService} if there are still file-based
- calendar or addressbook homes remaining in the given path.
-
- @param path: a path pointing at the document root, where the file-based
- data-store is located.
- @type path: L{CachingFilePath}
-
- @param service: the service to wrap. This service should be started
- when the upgrade is complete. (This is accomplished by returning
- it directly when no upgrade needs to be done, and by adding it to
- the service hierarchy when the upgrade completes; assuming that the
- service parent of the resulting service will be set to a
- L{MultiService} or similar.)
-
- @param store: the SQL storage service.
-
- @type service: L{IService}
-
- @return: a service
- @rtype: L{IService}
- """
- # TODO: TOPPATHS should be computed based on enabled flags in 'store',
- # not hard coded.
- for homeType in TOPPATHS:
- if path.child(homeType).exists():
- if platform.isMacOSX():
- appropriateStoreClass = XattrPropertyStore
- else:
- attrs = xattr.xattr(path.path)
- try:
- attrs.get('user.should-not-be-set')
- except IOError, ioe:
- if ioe.errno == errno.ENODATA:
- # xattrs are supported and enabled on the filesystem
- # where the calendar data lives. this takes some
- # doing (you have to edit fstab), so this means
- # we're trying to migrate some 2.x data from a
- # previous linux installation.
- appropriateStoreClass = XattrPropertyStore
- elif ioe.errno == errno.EOPNOTSUPP:
- # The operation wasn't supported. This is what will
- # usually happen on a naively configured filesystem,
- # so this means we're most likely trying to migrate
- # some data from an untarred archive created on an
- # OS X installation using xattrs.
- appropriateStoreClass = AppleDoubleStore
- else:
- # No need to check for ENOENT and the like; we just
- # checked above to make sure the parent exists.
- # Other errors are not anticipated here, so fail
- # fast.
- raise
-
- appropriateStoreClass = AppleDoubleStore
-
- self = cls(
- FileStore(path, None, True, True,
- propertyStoreClass=appropriateStoreClass),
- store, service, uid=uid, gid=gid,
- )
- return self
- return service
-
-
- def __init__(self, fileStore, sqlStore, service, uid=None, gid=None):
- """
- Initialize the service.
- """
- self.wrappedService = service
- self.fileStore = fileStore
- self.sqlStore = sqlStore
- self.uid = uid
- self.gid = gid
-
-
- @inlineCallbacks
- def doMigration(self):
- """
- Do the migration. Called by C{startService}, but a different method
- because C{startService} should return C{None}, not a L{Deferred}.
-
- @return: a Deferred which fires when the migration is complete.
- """
- self.log_warn("Beginning filesystem -> database upgrade.")
- for homeType, migrateFunc, eachFunc, destFunc, topPathName in [
- ("calendar", migrateCalendarHome,
- self.fileStore.eachCalendarHome,
- lambda txn: txn.calendarHomeWithUID,
- "calendars"),
- ("addressbook", migrateAddressbookHome,
- self.fileStore.eachAddressbookHome,
- lambda txn: txn.addressbookHomeWithUID,
- "addressbooks")
- ]:
- for fileTxn, fileHome in eachFunc():
- uid = fileHome.uid()
- self.log_warn("Migrating %s UID %r" % (homeType, uid))
- sqlTxn = self.sqlStore.newTransaction(migrating=True)
- homeGetter = destFunc(sqlTxn)
- if (yield homeGetter(uid, create=False)) is not None:
- self.log_warn(
- "%s home %r already existed not migrating" % (
- homeType, uid))
- yield sqlTxn.abort()
- yield fileTxn.commit()
- continue
- sqlHome = yield homeGetter(uid, create=True)
- if sqlHome is None:
- raise RuntimeError("THIS SHOULD NOT BE POSSIBLE.")
- yield migrateFunc(fileHome, sqlHome)
- yield fileTxn.commit()
- yield sqlTxn.commit()
- # FIXME: need a public remove...HomeWithUID() for de-
- # provisioning
-
- # Remove file home after migration
- fileHome._path.remove()
- for homeType in TOPPATHS:
- homesPath = self.fileStore._path.child(homeType)
- if homesPath.isdir():
- homesPath.remove()
-
- # Set attachment directory ownership. FIXME: is this still necessary
- # since attachments started living outside the database directory
- # created by initdb? default permissions might be correct now.
- sqlAttachmentsPath = self.sqlStore.attachmentsPath
- if (sqlAttachmentsPath and sqlAttachmentsPath.exists() and
- (self.uid or self.gid)):
- uid = self.uid or -1
- gid = self.gid or -1
- for fp in sqlAttachmentsPath.walk():
- os.chown(fp.path, uid, gid)
-
- self.log_warn(
- "Filesystem upgrade complete, launching database service."
- )
- # see http://twistedmatrix.com/trac/ticket/4649
- reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
-
-
- def startService(self):
- """
- Start the service.
- """
- self.doMigration()
-
-
-class UpgradeDatabaseSchemaService(Service, LoggingMixIn, object):
- """
- Checks and upgrades the database schema. This assumes there are a bunch of
- upgrade files in sql syntax that we can execute against the database to accomplish
- the upgrade.
- """
-
- @classmethod
- def wrapService(cls, service, store, uid=None, gid=None):
- """
- Create an L{UpgradeDatabaseSchemaService} when starting the database
- so we can check the schema version and do any upgrades.
-
- @param service: the service to wrap. This service should be started
- when the upgrade is complete. (This is accomplished by returning
- it directly when no upgrade needs to be done, and by adding it to
- the service hierarchy when the upgrade completes; assuming that the
- service parent of the resulting service will be set to a
- L{MultiService} or similar.)
-
- @param store: the SQL storage service.
-
- @type service: L{IService}
-
- @return: a service
- @rtype: L{IService}
- """
- return cls(store, service, uid=uid, gid=gid,)
-
-
- def __init__(self, sqlStore, service, uid=None, gid=None):
- """
- Initialize the service.
-
- @param sqlStore: The store to operate on. Can be C{None} when doing unit tests.
- @param service: Wrapped service. Can be C{None} when doing unit tests.
- """
- self.wrappedService = service
- self.sqlStore = sqlStore
- self.uid = uid
- self.gid = gid
- self.schemaLocation = getModule(__name__).filePath.sibling("sql_schema")
-
- @inlineCallbacks
- def doUpgrade(self):
- """
- Do the schema check and upgrade if needed. Called by C{startService}, but a different method
- because C{startService} should return C{None}, not a L{Deferred}.
-
- @return: a Deferred which fires when the migration is complete.
- """
- self.log_warn("Beginning database schema check.")
-
- # Retrieve the version number from the schema file
- current_schema = self.schemaLocation.child("current.sql").getContent()
- found = re.search("insert into CALENDARSERVER values \('VERSION', '(\d)+'\);", current_schema)
- if found is None:
- msg = "Schema is missing required schema VERSION insert statement: %s" % (current_schema,)
- self.log_error(msg)
- raise RuntimeError(msg)
- else:
- required_version = int(found.group(1))
- self.log_warn("Required schema version: %s." % (required_version,))
-
- # Get the schema version in the current database
- sqlTxn = self.sqlStore.newTransaction()
- dialect = sqlTxn.dialect
- try:
- actual_version = yield sqlTxn.schemaVersion()
- yield sqlTxn.commit()
- except RuntimeError:
- self.log_error("Database schema version cannot be determined.")
- yield sqlTxn.abort()
- raise
-
- self.log_warn("Actual schema version: %s." % (actual_version,))
-
- if required_version == actual_version:
- self.log_warn("Schema version check complete: no upgrade needed.")
- elif required_version < actual_version:
- msg = "Actual schema version %s is more recent than the expected version %s. The service cannot be started" % (actual_version, required_version,)
- self.log_error(msg)
- raise RuntimeError(msg)
- else:
- yield self.upgradeVersion(actual_version, required_version, dialect)
-
- self.log_warn(
- "Database schema check complete, launching database service."
- )
- # see http://twistedmatrix.com/trac/ticket/4649
- if self.wrappedService is not None:
- reactor.callLater(0, self.wrappedService.setServiceParent, self.parent)
-
- @inlineCallbacks
-
- def upgradeVersion(self, fromVersion, toVersion, dialect):
- """
- Update the database from one version to another (the current one). Do this by
- looking for upgrade_from_X_to_Y.sql files that cover the full range of upgrades.
- """
-
- self.log_warn("Starting schema upgrade from version %d to %d." % (fromVersion, toVersion,))
-
- # Scan for all possible upgrade files - returned sorted
- files = self.scanForUpgradeFiles(dialect)
-
- # Determine upgrade sequence and run each upgrade
- upgrades = self.determineUpgradeSequence(fromVersion, toVersion, files, dialect)
-
- # Use one transaction for the entire set of upgrades
- sqlTxn = self.sqlStore.newTransaction()
- try:
- for fp in upgrades:
- yield self.applyUpgrade(sqlTxn, fp)
- yield sqlTxn.commit()
- except RuntimeError:
- self.log_error("Database upgrade failed:" % (fp.basename(),))
- yield sqlTxn.abort()
- raise
-
- self.log_warn("Schema upgraded from version %d to %d." % (fromVersion, toVersion,))
-
- def scanForUpgradeFiles(self, dialect):
- """
- Scan the module path for upgrade files with the require name.
- """
-
- fp = self.schemaLocation.child("upgrades").child(dialect)
- upgrades = []
- regex = re.compile("upgrade_from_(\d)+_to_(\d)+.sql")
- for child in fp.globChildren("upgrade_*.sql"):
- matched = regex.match(child.basename())
- if matched is not None:
- fromV = int(matched.group(1))
- toV = int(matched.group(2))
- upgrades.append((fromV, toV, child))
-
- upgrades.sort(key=lambda x:(x[0], x[1]))
- return upgrades
-
- def determineUpgradeSequence(self, fromVersion, toVersion, files, dialect):
- """
- Determine the upgrade_from_X_to_Y.sql files that cover the full range of upgrades.
- Note that X and Y may not be consecutive, e.g., we might have an upgrade from 3 to 4,
- 4 to 5, and 3 to 5 - the later because it is more efficient to jump over the intermediate
- step. As a result we will always try and pick the upgrade file that gives the biggest
- jump from one version to another at each step.
- """
-
- # Now find the path from the old version to the current one
- filesByFromVersion = {}
- for fromV, toV, fp in files:
- if fromV not in filesByFromVersion or filesByFromVersion[fromV][1] < toV:
- filesByFromVersion[fromV] = fromV, toV, fp
-
- upgrades = []
- nextVersion = fromVersion
- while nextVersion != toVersion:
- if nextVersion not in filesByFromVersion:
- msg = "Missing upgrade file from version %d with dialect %s" % (nextVersion, dialect,)
- self.log_error(msg)
- raise RuntimeError(msg)
- else:
- upgrades.append(filesByFromVersion[nextVersion][2])
- nextVersion = filesByFromVersion[nextVersion][1]
-
- return upgrades
-
- @inlineCallbacks
- def applyUpgrade(self, sqlTxn, fp):
- """
- Apply the schema upgrade .sql file to the database.
- """
- self.log_warn("Applying schema upgrade: %s" % (fp.basename(),))
- sql = fp.getContent()
- yield sqlTxn.execSQLBlock(sql)
-
- def startService(self):
- """
- Start the service.
- """
- self.doUpgrade()
-
-
-
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20111013/42f7ad7b/attachment-0001.html>
More information about the calendarserver-changes
mailing list