[CalendarServer-changes] [4551] CalendarServer/branches/users/wsanchez/deployment/twistedcaldav
source_changes at macosforge.org
source_changes at macosforge.org
Wed Sep 23 10:19:38 PDT 2009
Revision: 4551
http://trac.macosforge.org/projects/calendarserver/changeset/4551
Author: cdaboo at apple.com
Date: 2009-09-23 10:19:34 -0700 (Wed, 23 Sep 2009)
Log Message:
-----------
Backport freebusy from index change to deployment branch.
Modified Paths:
--------------
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/ical.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/index.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/method/report_common.py
CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_index.py
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/ical.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/ical.py 2009-09-23 14:17:30 UTC (rev 4550)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/ical.py 2009-09-23 17:19:34 UTC (rev 4551)
@@ -651,6 +651,25 @@
else:
return None, None
+ def getFBType(self):
+
+ # Only VEVENTs block time
+ if self.name() not in ("VEVENT", ):
+ return "FREE"
+
+ # If it is TRANSPARENT we always ignore it
+ if self.propertyValue("TRANSP") == "TRANSPARENT":
+ return "FREE"
+
+ # Handle status
+ status = self.propertyValue("STATUS")
+ if status == "CANCELLED":
+ return "FREE"
+ elif status == "TENTATIVE":
+ return "BUSY-TENTATIVE"
+ else:
+ return "BUSY"
+
def addProperty(self, property):
"""
Adds a property to this component.
@@ -1252,7 +1271,7 @@
def tzexpand(tzdata, start, end):
"""
- Expand a timezone to get onset/utc-offset observance tuples withinthe specified
+ Expand a timezone to get onset/utc-offset observance tuples within the specified
time range.
@param tzdata: the iCalendar data containing a VTIMEZONE.
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/index.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/index.py 2009-09-23 14:17:30 UTC (rev 4550)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/index.py 2009-09-23 17:19:34 UTC (rev 4551)
@@ -52,32 +52,40 @@
log = Logger()
db_basename = db_prefix + "sqlite"
-schema_version = "6"
+schema_version = "7"
collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
+icalfbtype_to_indexfbtype = {
+ "FREE" : 'F',
+ "BUSY" : 'B',
+ "BUSY-UNAVAILABLE": 'U',
+ "BUSY-TENTATIVE" : 'T',
+}
+indexfbtype_to_icalfbtype = dict([(v, k) for k,v in icalfbtype_to_indexfbtype.iteritems()])
+
#
-# Duration into the future through which recurrances are expanded in the index
+# Duration into the future through which recurrences are expanded in the index
# by default. This is a caching parameter which affects the size of the index;
# it does not affect search results beyond this period, but it may affect
# performance of such a search.
#
-default_future_expansion_duration = datetime.timedelta(days=356*1)
+default_future_expansion_duration = datetime.timedelta(days=365*1)
#
-# Maximum duration into the future through which recurrances are expanded in the
+# Maximum duration into the future through which recurrences are expanded in the
# index. This is a caching parameter which affects the size of the index; it
# does not affect search results beyond this period, but it may affect
# performance of such a search.
#
-# When a search is performed on a timespan that goes beyond that which is
+# When a search is performed on a time span that goes beyond that which is
# expanded in the index, we have to open each resource which may have data in
# that time period. In order to avoid doing that multiple times, we want to
# cache those results. However, we don't necessarily want to cache all
-# occurances into some obscenely far-in-the-future date, so we cap the caching
+# occurrences into some obscenely far-in-the-future date, so we cap the caching
# period. Searches beyond this period will always be relatively expensive for
-# resources with occurances beyond this period.
+# resources with occurrences beyond this period.
#
-maximum_future_expansion_duration = datetime.timedelta(days=356*5)
+maximum_future_expansion_duration = datetime.timedelta(days=365*5)
class ReservationError(LookupError):
"""
@@ -257,11 +265,11 @@
return qualifiers is not None
- def search(self, filter):
+ def search(self, filter, fbtype=False):
"""
Finds resources matching the given qualifiers.
@param filter: the L{Filter} for the calendar-query to execute.
- @return: an interable iterable of tuples for each resource matching the
+ @return: an iterable of tuples for each resource matching the
given C{qualifiers}. The tuples are C{(name, uid, type)}, where
C{name} is the resource name, C{uid} is the resource UID, and
C{type} is the resource iCalendar component type.x
@@ -274,11 +282,23 @@
qualifiers = calendarquery.sqlcalendarquery(filter)
else:
qualifiers = None
- if qualifiers is not None:
- rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1])
+
+ # Perform the search
+ if qualifiers is None:
+ rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
else:
- rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
+ if fbtype:
+ # For a free-busy time-range query we return all instances
+ rowiter = self._db_execute(
+ "select RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE, RESOURCE.ORGANIZER, TIMESPAN.FLOAT, TIMESPAN.START, TIMESPAN.END, TIMESPAN.FBTYPE" +
+ qualifiers[0],
+ *qualifiers[1]
+ )
+ else:
+ rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1])
+ # Check result for missing resources
+
for row in rowiter:
name = row[0]
if self.resource.getChild(name.encode("utf-8")):
@@ -337,45 +357,42 @@
# NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
# UID: iCalendar UID (may or may not be unique)
# TYPE: iCalendar component type
- # RECURRANCE_MAX: Highest date of recurrance expansion
+ # RECURRANCE_MAX: Highest date of recurrence expansion
+ # ORGANIZER: cu-address of the Organizer of the event
#
- if uidunique:
- q.execute(
- """
- create table RESOURCE (
- NAME text unique,
- UID text unique,
- TYPE text,
- RECURRANCE_MAX date
- )
- """
+ q.execute(
+ """
+ create table RESOURCE (
+ NAME text unique,
+ UID text%s,
+ TYPE text,
+ RECURRANCE_MAX date,
+ ORGANIZER text
)
- else:
- q.execute(
- """
- create table RESOURCE (
- NAME text unique,
- UID text,
- TYPE text,
- RECURRANCE_MAX date
- )
- """
- )
+ """ % (" unique" if uidunique else "",)
+ )
#
- # TIMESPAN table tracks (expanded) timespans for resources
+ # TIMESPAN table tracks (expanded) time spans for resources
# NAME: Related resource (RESOURCE foreign key)
# FLOAT: 'Y' if start/end are floating, 'N' otherwise
# START: Start date
# END: End date
+ # FBTYPE: FBTYPE value:
+ # '?' - unknown
+ # 'F' - free
+ # 'B' - busy
+ # 'U' - busy-unavailable
+ # 'T' - busy-tentative
#
q.execute(
"""
create table TIMESPAN (
- NAME text,
- FLOAT text(1),
- START date,
- END date
+ NAME text,
+ FLOAT text(1),
+ START date,
+ END date,
+ FBTYPE text(1)
)
"""
)
@@ -395,8 +412,25 @@
"""
)
- def _add_to_db(self, name, calendar, cursor = None):
+ def _db_upgrade(self, old_version):
"""
+ Upgrade the database tables.
+ """
+
+ # When going to version 7 all we need to do is add a column to the time-range
+ if old_version < "7":
+ self._db_connection = sqlite.connect(self.dbpath, isolation_level=None)
+ q = self._db_connection.cursor()
+ q.execute("alter table RESOURCE add column ORGANIZER text default '?'")
+ q.execute("alter table TIMESPAN add column FBTYPE text(1) default '?'")
+ self._db_upgrade_schema(q)
+ self._db_close()
+ return self._db()
+ else:
+ return super(AbstractCalendarIndex, self)._db_upgrade(old_version)
+
+ def _add_to_db(self, name, calendar, cursor=None):
+ """
Records the given calendar resource in the index with the given name.
Resource names and UIDs must both be unique; only one resource name may
be associated with any given UID and vice versa.
@@ -407,6 +441,9 @@
contents.
"""
uid = calendar.resourceUID()
+ organizer = calendar.getOrganizer()
+ if not organizer:
+ organizer = ""
# Decide how far to expand based on the component
master = calendar.masterComponent()
@@ -423,12 +460,12 @@
instance = instances[key]
start = instance.start.replace(tzinfo=utc)
end = instance.end.replace(tzinfo=utc)
- float = ('N', 'Y')[instance.start.tzinfo is None]
+ float = 'Y' if instance.start.tzinfo is None else 'N'
self._db_execute(
"""
- insert into TIMESPAN (NAME, FLOAT, START, END)
- values (:1, :2, :3, :4)
- """, name, float, start, end
+ insert into TIMESPAN (NAME, FLOAT, START, END, FBTYPE)
+ values (:1, :2, :3, :4, :5)
+ """, name, float, start, end, icalfbtype_to_indexfbtype.get(instance.component.getFBType(), 'F')
)
# Special - for unbounded recurrence we insert a value for "infinity"
@@ -439,16 +476,16 @@
float = 'N'
self._db_execute(
"""
- insert into TIMESPAN (NAME, FLOAT, START, END)
- values (:1, :2, :3, :4)
- """, name, float, start, end
+ insert into TIMESPAN (NAME, FLOAT, START, END, FBTYPE)
+ values (:1, :2, :3, :4, :5)
+ """, name, float, start, end, '?'
)
self._db_execute(
"""
- insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX)
- values (:1, :2, :3, :4)
- """, name, uid, calendar.resourceType(), instances.limit
+ insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX, ORGANIZER)
+ values (:1, :2, :3, :4, :5)
+ """, name, uid, calendar.resourceType(), instances.limit, organizer
)
def _delete_from_db(self, name, uid):
@@ -626,7 +663,7 @@
def __init__(self, resource):
"""
@param resource: the L{twistedcaldav.static.CalDAVFile} resource to
- index. C{resource} must be a calendar collection (ie.
+ index. C{resource} must be a calendar collection (i.e.
C{resource.isPseudoCalendarCollection()} returns C{True}.)
"""
assert resource.isCalendarCollection(), "non-calendar collection resource %s has no index." % (resource,)
@@ -728,7 +765,7 @@
def __init__(self, resource):
"""
@param resource: the L{twistedcaldav.static.CalDAVFile} resource to
- index. C{resource} must be a calendar collection (ie.
+ index. C{resource} must be a calendar collection (i.e.
C{resource.isPseudoCalendarCollection()} returns C{True}.)
"""
assert resource.isPseudoCalendarCollection() and not resource.isCalendarCollection(), "non-calendar collection resource %s has no index." % (resource,)
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/method/report_common.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/method/report_common.py 2009-09-23 14:17:30 UTC (rev 4550)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/method/report_common.py 2009-09-23 17:19:34 UTC (rev 4551)
@@ -102,8 +102,8 @@
@param request: the L{IRequest} for the current request.
@param responses: the list of responses to append the result of this method to.
- @param href: the L{HRef} element of the resource being targetted.
- @param resource: the L{CalDAVFile} for the targetted resource.
+ @param href: the L{HRef} element of the resource being targeted.
+ @param resource: the L{CalDAVFile} for the targeted resource.
@param calendar: the L{Component} for the calendar for the resource. This may be None
if the calendar has not already been read in, in which case the resource
will be used to get the calendar if needed.
@@ -136,7 +136,7 @@
Return all (non-hidden) properties for the specified resource.
@param request: the L{IRequest} for the current request.
@param prop: the L{PropertyContainer} element for the properties of interest.
- @param resource: the L{CalDAVFile} for the targetted resource.
+ @param resource: the L{CalDAVFile} for the targeted resource.
@param calendar: the L{Component} for the calendar for the resource. This may be None
if the calendar has not already been read in, in which case the resource
will be used to get the calendar if needed.
@@ -157,7 +157,7 @@
Return property names for all properties on the specified resource.
@param request: the L{IRequest} for the current request.
@param prop: the L{PropertyContainer} element for the properties of interest.
- @param resource: the L{CalDAVFile} for the targetted resource.
+ @param resource: the L{CalDAVFile} for the targeted resource.
@param calendar: the L{Component} for the calendar for the resource. This may be None
if the calendar has not already been read in, in which case the resource
will be used to get the calendar if needed.
@@ -181,7 +181,7 @@
Return the specified properties on the specified resource.
@param request: the L{IRequest} for the current request.
@param prop: the L{PropertyContainer} element for the properties of interest.
- @param resource: the L{CalDAVFile} for the targetted resource.
+ @param resource: the L{CalDAVFile} for the targeted resource.
@param calendar: the L{Component} for the calendar for the resource. This may be None
if the calendar has not already been read in, in which case the resource
will be used to get the calendar if needed.
@@ -220,7 +220,7 @@
Return the specified properties on the specified resource.
@param request: the L{IRequest} for the current request.
@param props: a list of property elements or qname tuples for the properties of interest.
- @param resource: the L{CalDAVFile} for the targetted resource.
+ @param resource: the L{CalDAVFile} for the targeted resource.
@param calendar: the L{Component} for the calendar for the resource. This may be None
if the calendar has not already been read in, in which case the resource
will be used to get the calendar if needed.
@@ -283,6 +283,9 @@
_namedPropertiesForResource = deferredGenerator(_namedPropertiesForResource)
+fbtype_mapper = {"BUSY": 0, "BUSY-TENTATIVE": 1, "BUSY-UNAVAILABLE": 2}
+fbtype_index_mapper = {'B': 0, 'T': 1, 'U': 2}
+
def generateFreeBusyInfo(request, calresource, fbinfo, timerange, matchtotal,
excludeuid=None, organizer=None, same_calendar_user=False):
"""
@@ -295,9 +298,9 @@
@param matchtotal: the running total for the number of matches.
@param excludeuid: a C{str} containing a UID value to exclude any components with that
UID from contributing to free-busy.
- @param organizer: a C{str} containing the value of the ORGANIZER proeprty in the VFREEBUSY request.
+ @param organizer: a C{str} containing the value of the ORGANIZER property in the VFREEBUSY request.
This is used in conjunction with the UID value to process exclusions.
- @param same_calendar_user: a C{bool} indicating whether the calendar user requesting tyhe free-busy information
+ @param same_calendar_user: a C{bool} indicating whether the calendar user requesting the free-busy information
is the same as the calendar user being targeted.
"""
@@ -310,6 +313,10 @@
yield matchtotal
return
+ # May need organizer principal
+ organizer_principal = calresource.principalForCalendarUserAddress(organizer) if organizer else None
+ organizer_uid = organizer_principal.principalUID() if organizer_principal else ""
+
#
# What we do is a fake calendar-query for VEVENT/VFREEBUSYs in the specified time-range.
# We then take those results and merge them into one VFREEBUSY component
@@ -340,14 +347,23 @@
tz = None
tzinfo = filter.settimezone(tz)
- # Do some optimisation of access control calculation by determining any inherited ACLs outside of
+ # Do some optimization of access control calculation by determining any inherited ACLs outside of
# the child resource loop and supply those to the checkPrivileges on each child.
filteredaces = waitForDeferred(calresource.inheritedACEsforChildren(request))
yield filteredaces
filteredaces = filteredaces.getResult()
- for name, uid, type in calresource.index().search(filter): #@UnusedVariable
-
+ resources = calresource.index().search(filter, fbtype=True)
+
+ # We care about separate instances for VEVENTs only
+ aggregated_resources = {}
+ for name, uid, type, test_organizer, float, start, end, fbtype in resources:
+ aggregated_resources.setdefault((name, uid, type, test_organizer,), []).append((float, start, end, fbtype,))
+
+ for key in aggregated_resources.iterkeys():
+
+ name, uid, type, test_organizer = key
+
# Check privileges - must have at least CalDAV:read-free-busy
child = waitForDeferred(request.locateChildResource(calresource, name))
yield child
@@ -360,41 +376,92 @@
except AccessDeniedError:
continue
- calendar = calresource.iCalendar(name)
-
- # The calendar may come back as None if the resource is being changed, or was deleted
- # between our initial index query and getting here. For now we will ignore this errror, but in
- # the longer term we need to simplement some form of locking, perhaps.
- if calendar is None:
- log.err("Calendar %s is missing from calendar collection %r" % (name, calresource))
- continue
-
- # Ignore ones of this UID
- if excludeuid:
- # See if we have a UID match
- if (excludeuid == uid):
- # Check that ORGANIZER's match (security requirement)
- if (organizer is None) or (organizer == calendar.getOrganizer()):
+ # Short-cut - if an fbtype exists we can use that
+ if type == "VEVENT" and aggregated_resources[key][0][3] != '?':
+
+ # Look at each instance
+ for float, start, end, fbtype in aggregated_resources[key]:
+ # Ignore free time or unknown
+ if fbtype in ('F', '?'):
continue
- # Check for no ORGANIZER and check by same calendar user
- elif (calendar.getOrganizer() is None) and same_calendar_user:
- continue
+
+ # Ignore ones of this UID
+ if excludeuid:
+ # See if we have a UID match
+ if (excludeuid == uid):
+ test_principal = calresource.principalForCalendarUserAddress(test_organizer) if test_organizer else None
+ test_uid = test_principal.principalUID() if test_principal else ""
- if filter.match(calendar, None):
- # Check size of results is within limit
- matchtotal += 1
- if matchtotal > max_number_of_matches:
- raise NumberOfMatchesWithinLimits
+ # Check that ORGANIZER's match (security requirement)
+ if (organizer is None) or (organizer_uid == test_uid):
+ continue
+ # Check for no ORGANIZER and check by same calendar user
+ elif (test_uid == "") and same_calendar_user:
+ continue
+
+ # Apply a timezone to any floating times
+ fbstart = datetime.datetime.strptime(start[:-6], "%Y-%m-%d %H:%M:%S")
+ if float == 'Y':
+ fbstart = fbstart.replace(tzinfo=tzinfo)
+ else:
+ fbstart = fbstart.replace(tzinfo=utc)
+ fbend =datetime.datetime.strptime(end[:-6], "%Y-%m-%d %H:%M:%S")
+ if float == 'Y':
+ fbend = fbend.replace(tzinfo=tzinfo)
+ else:
+ fbend = fbend.replace(tzinfo=utc)
+
+ # Clip instance to time range
+ clipped = clipPeriod((fbstart, fbend - fbstart), (timerange.start, timerange.end))
- if calendar.mainType() == "VEVENT":
- processEventFreeBusy(calendar, fbinfo, timerange, tzinfo)
- elif calendar.mainType() == "VFREEBUSY":
- processFreeBusyFreeBusy(calendar, fbinfo, timerange)
- elif calendar.mainType() == "VAVAILABILITY":
- processAvailabilityFreeBusy(calendar, fbinfo, timerange)
- else:
- assert "Free-busy query returned unwanted component: %s in %r", (name, calresource,)
+ # Double check for overlap
+ if clipped:
+ fbinfo[fbtype_index_mapper.get(fbtype, 0)].append(clipped)
+
+ else:
+ calendar = calresource.iCalendar(name)
+
+ # The calendar may come back as None if the resource is being changed, or was deleted
+ # between our initial index query and getting here. For now we will ignore this error, but in
+ # the longer term we need to implement some form of locking, perhaps.
+ if calendar is None:
+ log.err("Calendar %s is missing from calendar collection %r" % (name, calresource))
+ continue
+
+ # Ignore ones of this UID
+ if excludeuid:
+ # See if we have a UID match
+ if (excludeuid == uid):
+ test_organizer = calendar.getOrganizer()
+ test_principal = calresource.principalForCalendarUserAddress(test_organizer) if test_organizer else None
+ test_uid = test_principal.principalUID() if test_principal else ""
+ # Check that ORGANIZER's match (security requirement)
+ if (organizer is None) or (organizer_uid == test_uid):
+ continue
+ # Check for no ORGANIZER and check by same calendar user
+ elif (test_organizer is None) and same_calendar_user:
+ continue
+
+ if filter.match(calendar, None):
+ # Check size of results is within limit
+ matchtotal += 1
+ if matchtotal > max_number_of_matches:
+ raise NumberOfMatchesWithinLimits
+
+ if calendar.mainType() == "VEVENT":
+ processEventFreeBusy(calendar, fbinfo, timerange, tzinfo)
+
+ # Lets also force an index rebuild for this resource so that next time we have the fbtype set
+ calresource.index().addResource(name, calendar)
+
+ elif calendar.mainType() == "VFREEBUSY":
+ processFreeBusyFreeBusy(calendar, fbinfo, timerange)
+ elif calendar.mainType() == "VAVAILABILITY":
+ processAvailabilityFreeBusy(calendar, fbinfo, timerange)
+ else:
+ assert "Free-busy query returned unwanted component: %s in %r", (name, calresource,)
+
yield matchtotal
generateFreeBusyInfo = deferredGenerator(generateFreeBusyInfo)
@@ -488,14 +555,13 @@
if fbtype == "FREE":
continue
- # Look at each period in the propert
+ # Look at each period in the property
assert isinstance(fb.value(), list), "FREEBUSY property does not contain a list of values: %r" % (fb,)
for period in fb.value():
# Clip period for this instance
clipped = clipPeriod(period, (timerange.start, timerange.end))
if clipped:
- mapper = {"BUSY": 0, "BUSY-TENTATIVE": 1, "BUSY-UNAVAILABLE": 2}
- fbinfo[mapper.get(fbtype, 0)].append(clipped)
+ fbinfo[fbtype_mapper.get(fbtype, 0)].append(clipped)
def processAvailabilityFreeBusy(calendar, fbinfo, timerange):
"""
@@ -537,8 +603,7 @@
if fbtype is None:
fbtype = "BUSY-UNAVAILABLE"
- mapper = {"BUSY": 0, "BUSY-TENTATIVE": 1, "BUSY-UNAVAILABLE": 2}
- fbinfo[mapper.get(fbtype, 2)].extend(busyperiods)
+ fbinfo[fbtype_mapper.get(fbtype, 2)].extend(busyperiods)
def processAvailablePeriods(calendar, timerange):
@@ -557,7 +622,7 @@
uid = component.propertyValue("UID")
uidmap.setdefault(uid, []).append(component)
- # Then we expand each uid set seperately
+ # Then we expand each uid set separately
for componentSet in uidmap.itervalues():
instances = InstanceList()
instances.expandTimeRanges(componentSet, timerange.end)
Modified: CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_index.py
===================================================================
--- CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_index.py 2009-09-23 14:17:30 UTC (rev 4550)
+++ CalendarServer/branches/users/wsanchez/deployment/twistedcaldav/test/test_index.py 2009-09-23 17:19:34 UTC (rev 4551)
@@ -14,13 +14,23 @@
# limitations under the License.
##
-from twistedcaldav.index import Index
-import twistedcaldav.test.util
-from twistedcaldav.test.util import InMemoryMemcacheProtocol
-from twistedcaldav.index import ReservationError, MemcachedUIDReserver
from twisted.web2.test.test_http import deferLater
+from twistedcaldav import caldavxml
+from twistedcaldav.caldavxml import TimeRange
+from twistedcaldav.ical import Component
+from twistedcaldav.index import Index, default_future_expansion_duration, AbstractCalendarIndex
+from twistedcaldav.index import ReservationError, MemcachedUIDReserver
+from twistedcaldav.test.util import InMemoryMemcacheProtocol
+import twistedcaldav.test.util
+
+from vobject.icalendar import utc
+
+import datetime
+import os
+import sqlite3
+
class SQLIndexTests (twistedcaldav.test.util.TestCase):
"""
Test abstract SQL DB class
@@ -88,7 +98,575 @@
return d
+ def test_index(self):
+ data = (
+ (
+ "#1.1 Simple component",
+ "1.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ True,
+ ),
+ (
+ "#2.1 Recurring component",
+ "2.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ True,
+ ),
+ (
+ "#2.2 Recurring component with override",
+ "2.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.2
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.2
+RECURRENCE-ID:20080608T120000Z
+DTSTART:20080608T120000Z
+DTEND:20080608T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ True,
+ ),
+ (
+ "#2.3 Recurring component with broken override - new",
+ "2.3",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.3
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.3
+RECURRENCE-ID:20080609T120000Z
+DTSTART:20080608T120000Z
+DTEND:20080608T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ False,
+ ),
+ (
+ "#2.4 Recurring component with broken override - existing",
+ "2.4",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.4
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.4
+RECURRENCE-ID:20080609T120000Z
+DTSTART:20080608T120000Z
+DTEND:20080608T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ True,
+ True,
+ ),
+ )
+ for description, name, calendar_txt, reCreate, ok in data:
+ calendar = Component.fromString(calendar_txt)
+ if ok:
+ f = open(os.path.join(self.site.resource.fp.path, name), "w")
+ f.write(calendar_txt)
+ del f
+
+ self.db.addResource(name, calendar)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+ else:
+ self.db.addResource(name, calendar)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ self.db._db_recreate()
+ for description, name, calendar_txt, reCreate, ok in data:
+ if ok:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+ else:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ #self.db.testAndUpdateIndex(datetime.date(2020, 1, 1))
+ for description, name, calendar_txt, reCreate, ok in data:
+ if ok:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+ else:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ def test_index_timespan(self):
+ data = (
+ (
+ "#1.1 Simple component - busy",
+ "1.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080602T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B'),),
+ ),
+ (
+ "#1.2 Simple component - transparent",
+ "1.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.2
+DTSTART:20080602T120000Z
+DTEND:20080602T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+TRANSP:TRANSPARENT
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080602T000000Z", "20080603T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'F'),),
+ ),
+ (
+ "#1.3 Simple component - canceled",
+ "1.3",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.3
+DTSTART:20080603T120000Z
+DTEND:20080603T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+STATUS:CANCELLED
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080603T000000Z", "20080604T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'F'),),
+ ),
+ (
+ "#1.4 Simple component - tentative",
+ "1.4",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.4
+DTSTART:20080604T120000Z
+DTEND:20080604T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+STATUS:TENTATIVE
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080604T000000Z", "20080605T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-04 12:00:00+00:00", "2008-06-04 13:00:00+00:00", 'T'),),
+ ),
+ (
+ "#2.1 Recurring component - busy",
+ "2.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.1
+DTSTART:20080605T120000Z
+DTEND:20080605T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080605T000000Z", "20080607T000000Z",
+ "mailto:user1 at example.com",
+ (
+ ('N', "2008-06-05 12:00:00+00:00", "2008-06-05 13:00:00+00:00", 'B'),
+ ('N', "2008-06-06 12:00:00+00:00", "2008-06-06 13:00:00+00:00", 'B'),
+ ),
+ ),
+ (
+ "#2.2 Recurring component - busy",
+ "2.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.2
+DTSTART:20080607T120000Z
+DTEND:20080607T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.2
+RECURRENCE-ID:20080608T120000Z
+DTSTART:20080608T140000Z
+DTEND:20080608T150000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+TRANSP:TRANSPARENT
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080607T000000Z", "20080609T000000Z",
+ "mailto:user1 at example.com",
+ (
+ ('N', "2008-06-07 12:00:00+00:00", "2008-06-07 13:00:00+00:00", 'B'),
+ ('N', "2008-06-08 14:00:00+00:00", "2008-06-08 15:00:00+00:00", 'F'),
+ ),
+ ),
+ )
+
+ for description, name, calendar_txt, trstart, trend, organizer, instances in data:
+ calendar = Component.fromString(calendar_txt)
+
+ f = open(os.path.join(self.site.resource.fp.path, name), "w")
+ f.write(calendar_txt)
+ del f
+
+ self.db.addResource(name, calendar)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ # Create fake filter element to match time-range
+ filter = caldavxml.Filter(
+ caldavxml.ComponentFilter(
+ caldavxml.ComponentFilter(
+ TimeRange(
+ start=trstart,
+ end=trend,
+ ),
+ name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
+ ),
+ name="VCALENDAR",
+ )
+ )
+
+ resources = self.db.search(filter, fbtype=True)
+ index_results = set()
+ for _ignore_name, _ignore_uid, type, test_organizer, float, start, end, fbtype in resources:
+ self.assertEqual(test_organizer, organizer, msg=description)
+ index_results.add((float, start, end, fbtype,))
+
+ self.assertEqual(set(instances), index_results, msg=description)
+
+class SQLIndexUpgradeTests (twistedcaldav.test.util.TestCase):
+ """
+ Test abstract SQL DB class
+ """
+
+ class OldIndexv6(Index):
+
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return "6"
+
+ def _db_init_data_tables_base(self, q, uidunique):
+ """
+ Initialise the underlying database tables.
+ @param q: a database cursor to use.
+ """
+ #
+ # RESOURCE table is the primary index table
+ # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
+ # UID: iCalendar UID (may or may not be unique)
+ # TYPE: iCalendar component type
+ # RECURRANCE_MAX: Highest date of recurrence expansion
+ #
+ if uidunique:
+ q.execute(
+ """
+ create table RESOURCE (
+ NAME text unique,
+ UID text unique,
+ TYPE text,
+ RECURRANCE_MAX date
+ )
+ """
+ )
+ else:
+ q.execute(
+ """
+ create table RESOURCE (
+ NAME text unique,
+ UID text,
+ TYPE text,
+ RECURRANCE_MAX date
+ )
+ """
+ )
+
+ #
+ # TIMESPAN table tracks (expanded) time spans for resources
+ # NAME: Related resource (RESOURCE foreign key)
+ # FLOAT: 'Y' if start/end are floating, 'N' otherwise
+ # START: Start date
+ # END: End date
+ #
+ q.execute(
+ """
+ create table TIMESPAN (
+ NAME text,
+ FLOAT text(1),
+ START date,
+ END date
+ )
+ """
+ )
+
+ if uidunique:
+ #
+ # RESERVED table tracks reserved UIDs
+ # UID: The UID being reserved
+ # TIME: When the reservation was made
+ #
+ q.execute(
+ """
+ create table RESERVED (
+ UID text unique,
+ TIME date
+ )
+ """
+ )
+
+ def _db_upgrade(self, old_version):
+ """
+ Upgrade the database tables.
+ """
+
+ return super(AbstractCalendarIndex, self)._db_upgrade(old_version)
+
+ def _add_to_db(self, name, calendar, cursor = None, expand_until=None, reCreate=False):
+ """
+ Records the given calendar resource in the index with the given name.
+ Resource names and UIDs must both be unique; only one resource name may
+ be associated with any given UID and vice versa.
+ NB This method does not commit the changes to the db - the caller
+ MUST take care of that
+ @param name: the name of the resource to add.
+ @param calendar: a L{Calendar} object representing the resource
+ contents.
+ """
+ uid = calendar.resourceUID()
+
+ # Decide how far to expand based on the component
+ master = calendar.masterComponent()
+ if master is None or not calendar.isRecurring() and not calendar.isRecurringUnbounded():
+ # When there is no master we have a set of overridden components - index them all.
+ # When there is one instance - index it.
+ # When bounded - index all.
+ expand = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
+ else:
+ if expand_until:
+ expand = expand_until
+ else:
+ expand = datetime.date.today() + default_future_expansion_duration
+
+ instances = calendar.expandTimeRanges(expand)
+
+ self._delete_from_db(name, uid)
+
+ for key in instances:
+ instance = instances[key]
+ start = instance.start.replace(tzinfo=utc)
+ end = instance.end.replace(tzinfo=utc)
+ float = 'Y' if instance.start.tzinfo is None else 'N'
+ self._db_execute(
+ """
+ insert into TIMESPAN (NAME, FLOAT, START, END)
+ values (:1, :2, :3, :4)
+ """, name, float, start, end
+ )
+
+ # Special - for unbounded recurrence we insert a value for "infinity"
+ # that will allow an open-ended time-range to always match it.
+ if calendar.isRecurringUnbounded():
+ start = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
+ end = datetime.datetime(2100, 1, 1, 1, 0, 0, tzinfo=utc)
+ float = 'N'
+ self._db_execute(
+ """
+ insert into TIMESPAN (NAME, FLOAT, START, END)
+ values (:1, :2, :3, :4)
+ """, name, float, start, end
+ )
+
+ self._db_execute(
+ """
+ insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX)
+ values (:1, :2, :3, :4)
+ """, name, uid, calendar.resourceType(), instances.limit
+ )
+
+ def setUp(self):
+ super(SQLIndexUpgradeTests, self).setUp()
+ self.site.resource.isCalendarCollection = lambda: True
+ self.db = Index(self.site.resource)
+ self.olddb = SQLIndexUpgradeTests.OldIndexv6(self.site.resource)
+
+ def prepareOldDB(self):
+ if os.path.exists(self.olddb.dbpath):
+ os.remove(self.olddb.dbpath)
+
+ def test_old_schema(self):
+
+ self.prepareOldDB()
+
+ schema = self.olddb._db_value_for_sql(
+ """
+ select VALUE from CALDAV
+ where KEY = 'SCHEMA_VERSION'
+ """)
+ self.assertEqual(schema, self.olddb._db_version())
+
+ def test_empty_upgrade(self):
+
+ self.prepareOldDB()
+
+ schema = self.olddb._db_value_for_sql(
+ """
+ select VALUE from CALDAV
+ where KEY = 'SCHEMA_VERSION'
+ """)
+ self.assertEqual(schema, self.olddb._db_version())
+
+ self.assertRaises(sqlite3.OperationalError, self.olddb._db_value_for_sql, "select ORGANIZER from RESOURCE")
+ self.assertRaises(sqlite3.OperationalError, self.olddb._db_value_for_sql, "select FBTYPE from TIMESPAN")
+
+ schema = self.db._db_value_for_sql(
+ """
+ select VALUE from CALDAV
+ where KEY = 'SCHEMA_VERSION'
+ """)
+ self.assertEqual(schema, self.db._db_version())
+
+ value = self.db._db_value_for_sql("select ORGANIZER from RESOURCE")
+ self.assertEqual(value, None)
+
+ def test_basic_upgrade(self):
+
+ self.prepareOldDB()
+
+ calendar_name = "1.ics"
+ calendar_data = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+"""
+
+ self.olddb.addResource(calendar_name, Component.fromString(calendar_data))
+ self.assertTrue(self.olddb.resourceExists(calendar_name))
+
+ self.assertRaises(sqlite3.OperationalError, self.olddb._db_value_for_sql, "select ORGANIZER from RESOURCE")
+ self.assertRaises(sqlite3.OperationalError, self.olddb._db_value_for_sql, "select FBTYPE from TIMESPAN")
+
+ value = self.db._db_value_for_sql("select ORGANIZER from RESOURCE where NAME = :1", calendar_name)
+ self.assertEqual(value, "?")
+
+ value = self.db._db_value_for_sql("select FBTYPE from TIMESPAN where NAME = :1", calendar_name)
+ self.assertEqual(value, "?")
+
+ self.db.addResource(calendar_name, Component.fromString(calendar_data))
+ self.assertTrue(self.olddb.resourceExists(calendar_name))
+
+ value = self.db._db_value_for_sql("select ORGANIZER from RESOURCE where NAME = :1", calendar_name)
+ self.assertEqual(value, "mailto:user1 at example.com")
+
+ value = self.db._db_value_for_sql("select FBTYPE from TIMESPAN where NAME = :1", calendar_name)
+ self.assertEqual(value, "B")
+
class MemcacheTests(SQLIndexTests):
def setUp(self):
super(MemcacheTests, self).setUp()
@@ -97,6 +675,6 @@
def tearDown(self):
- for k, v in self.memcache._timeouts.iteritems():
+ for _ignore_k, v in self.memcache._timeouts.iteritems():
if v.active():
v.cancel()
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20090923/8d8e8b00/attachment-0001.html>
More information about the calendarserver-changes
mailing list