[CalendarServer-changes] [3489] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Dec 9 11:05:11 PST 2008
Revision: 3489
http://trac.macosforge.org/projects/calendarserver/changeset/3489
Author: sagen at apple.com
Date: 2008-12-09 11:05:10 -0800 (Tue, 09 Dec 2008)
Log Message:
-----------
Dynamically expand calendar indices for recurrences, as needed (up to 5 years in the future -- beyond that, calendar queries become brute-force)
Modified Paths:
--------------
CalendarServer/trunk/run
CalendarServer/trunk/twistedcaldav/index.py
CalendarServer/trunk/twistedcaldav/method/report_calquery.py
CalendarServer/trunk/twistedcaldav/method/report_common.py
CalendarServer/trunk/twistedcaldav/static.py
CalendarServer/trunk/twistedcaldav/test/test_icalendar.py
CalendarServer/trunk/twistedcaldav/test/test_xml.py
Modified: CalendarServer/trunk/run
===================================================================
--- CalendarServer/trunk/run 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/run 2008-12-09 19:05:10 UTC (rev 3489)
@@ -696,7 +696,7 @@
caldavtester="${top}/CalDAVTester";
-svn_get "CalDAVTester" "${caldavtester}" "${svn_uri_base}/CalDAVTester/trunk" 3432;
+svn_get "CalDAVTester" "${caldavtester}" "${svn_uri_base}/CalDAVTester/trunk" 3486;
#
# Calendar Server
Modified: CalendarServer/trunk/twistedcaldav/index.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/index.py 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/twistedcaldav/index.py 2008-12-09 19:05:10 UTC (rev 3489)
@@ -27,12 +27,14 @@
"MemcachedUIDReserver",
"Index",
"IndexSchedule",
+ "IndexedSearchException",
]
import datetime
import os
import time
import hashlib
+from dateutil.parser import parse as dateparse
try:
import sqlite3 as sqlite
@@ -59,15 +61,15 @@
collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
#
-# Duration into the future through which recurrances are expanded in the index
+# Duration into the future through which recurrences are expanded in the index
# by default. This is a caching parameter which affects the size of the index;
# it does not affect search results beyond this period, but it may affect
# performance of such a search.
#
-default_future_expansion_duration = datetime.timedelta(days=356*1)
+default_future_expansion_duration = datetime.timedelta(days=365*1)
#
-# Maximum duration into the future through which recurrances are expanded in the
+# Maximum duration into the future through which recurrences are expanded in the
# index. This is a caching parameter which affects the size of the index; it
# does not affect search results beyond this period, but it may affect
# performance of such a search.
@@ -80,7 +82,7 @@
# period. Searches beyond this period will always be relatively expensive for
# resources with occurances beyond this period.
#
-maximum_future_expansion_duration = datetime.timedelta(days=356*5)
+maximum_future_expansion_duration = datetime.timedelta(days=365*5)
class ReservationError(LookupError):
"""
@@ -88,7 +90,10 @@
which is not reserved.
"""
-class AbstractCalendarIndex(AbstractSQLDatabase):
+class IndexedSearchException(ValueError):
+ pass
+
+class AbstractCalendarIndex(AbstractSQLDatabase, LoggingMixIn):
"""
Calendar collection index abstract base class that defines the apis for the index.
This will be subclassed for the two types of index behaviour we need: one for
@@ -252,15 +257,16 @@
results = self._db_values_for_sql(statement, *names)
return results
- def searchValid(self, filter):
- if isinstance(filter, caldavxml.Filter):
- qualifiers = calendarquery.sqlcalendarquery(filter)
- else:
- qualifiers = None
- return qualifiers is not None
+ def testAndUpdateIndex(self, minDate):
+ # Find out if the index is expanded far enough
+ names = self.notExpandedBeyond(minDate)
+ # Actually expand recurrence max
+ for name in names:
+ self.log_info("Search falls outside range of index for %s %s" % (name, minDate))
+ self.reExpandResource(name, minDate)
- def search(self, filter):
+ def indexedSearch(self, filter):
"""
Finds resources matching the given qualifiers.
@param filter: the L{Filter} for the calendar-query to execute.
@@ -269,19 +275,38 @@
C{name} is the resource name, C{uid} is the resource UID, and
C{type} is the resource iCalendar component type.x
"""
- # FIXME: Don't forget to use maximum_future_expansion_duration when we
- # start caching...
- # Make sure we have a proper Filter element and get the partial SQL statement to use.
+ # Make sure we have a proper Filter element and get the partial SQL
+ # statement to use.
if isinstance(filter, caldavxml.Filter):
qualifiers = calendarquery.sqlcalendarquery(filter)
+ if qualifiers is not None:
+ if len(qualifiers[1]) > 1:
+ # Bring index up to a given date if it's not already
+ try:
+ # TODO: is there a more reliable way to get the date?
+ minDate = dateparse(qualifiers[1][1]).date()
+ except ValueError:
+ # this isn't a date after all
+ minDate = None
+ if minDate:
+ self.testAndUpdateIndex(minDate)
+ else:
+ # We cannot handler this filter in an indexed search
+ raise IndexedSearchException()
+
else:
qualifiers = None
+
+ # Perform the search
if qualifiers is not None:
rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1])
+
else:
- rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
+ raise IndexedSearchException()
+ # Check result for missing resources
+
for row in rowiter:
name = row[0]
if self.resource.getChild(name.encode("utf-8")):
@@ -291,13 +316,35 @@
% (name, self.resource))
self.deleteResource(name)
+
+
+ def bruteForceSearch(self):
+ """
+ List the whole index and tests for existence, updating the index
+ @return: all resources in the index
+ """
+ # List all resources
+ rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
+
+ # Check result for missing resources:
+
+ for row in rowiter:
+ name = row[0]
+ if self.resource.getChild(name.encode("utf-8")):
+ yield row
+ else:
+ log.err("Calendar resource %s is missing from %s. Removing from index."
+ % (name, self.resource))
+ self.deleteResource(name)
+
+
def _db_version(self):
"""
@return: the schema version assigned to this index.
"""
return schema_version
- def _add_to_db(self, name, calendar, cursor = None):
+ def _add_to_db(self, name, calendar, cursor = None, expand_until=None):
"""
Records the given calendar resource in the index with the given name.
Resource names and UIDs must both be unique; only one resource name may
@@ -340,7 +387,7 @@
# NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
# UID: iCalendar UID (may or may not be unique)
# TYPE: iCalendar component type
- # RECURRANCE_MAX: Highest date of recurrance expansion
+ # RECURRANCE_MAX: Highest date of recurrence expansion
#
if uidunique:
q.execute(
@@ -398,8 +445,24 @@
"""
)
- def _add_to_db(self, name, calendar, cursor = None):
+ def notExpandedBeyond(self, minDate):
"""
+ Gives all resources which have not been expanded beyond a given date
+ in the index
+ """
+ return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX < :1", minDate)
+
+ def reExpandResource(self, name, expand_until):
+ """
+ Given a resource name, remove it from the database and re-add it
+ with a longer expansion.
+ """
+ calendar = self.resource.getChild(name).iCalendar()
+ self._add_to_db(name, calendar, expand_until=expand_until)
+ self._db_commit()
+
+ def _add_to_db(self, name, calendar, cursor = None, expand_until=None):
+ """
Records the given calendar resource in the index with the given name.
Resource names and UIDs must both be unique; only one resource name may
be associated with any given UID and vice versa.
@@ -411,9 +474,18 @@
"""
uid = calendar.resourceUID()
- expand_max = datetime.date.today() + default_future_expansion_duration
+ if expand_until:
+ expand = expand_until
+ else:
+ expand = datetime.date.today() + default_future_expansion_duration
- instances = calendar.expandTimeRanges(expand_max)
+ if expand > (datetime.date.today() + maximum_future_expansion_duration):
+ raise IndexedSearchException
+
+ instances = calendar.expandTimeRanges(expand)
+
+ self._delete_from_db(name, uid)
+
for key in instances:
instance = instances[key]
start = instance.start.replace(tzinfo=utc)
Modified: CalendarServer/trunk/twistedcaldav/method/report_calquery.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_calquery.py 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/twistedcaldav/method/report_calquery.py 2008-12-09 19:05:10 UTC (rev 3489)
@@ -34,6 +34,7 @@
from twistedcaldav.caldavxml import caldav_namespace
from twistedcaldav.customxml import TwistedCalendarAccessProperty
from twistedcaldav.method import report_common
+from twistedcaldav.index import IndexedSearchException
from twistedcaldav.log import Logger
log = Logger()
@@ -163,11 +164,17 @@
# Check for disabled access
if filteredaces is not None:
- # See whether the filter is valid for an index only query
- index_query_ok = calresource.index().searchValid(filter)
-
- # Get list of children that match the search and have read access
- names = [name for name, ignore_uid, ignore_type in calresource.index().search(filter)]
+ index_query_ok = True
+ try:
+ # Get list of children that match the search and have read
+ # access
+ names = [name for name, ignore_uid, ignore_type
+ in calresource.index().indexedSearch(filter)]
+ except IndexedSearchException:
+ names = [name for name, ignore_uid, ignore_type
+ in calresource.index().bruteForceSearch()]
+ index_query_ok = False
+
if not names:
return
Modified: CalendarServer/trunk/twistedcaldav/method/report_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_common.py 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/twistedcaldav/method/report_common.py 2008-12-09 19:05:10 UTC (rev 3489)
@@ -56,6 +56,7 @@
from twistedcaldav.dateops import clipPeriod, normalizePeriodList, timeRangesOverlap
from twistedcaldav.ical import Component, Property, iCalendarProductID
from twistedcaldav.instance import InstanceList
+from twistedcaldav.index import IndexedSearchException
from twistedcaldav.log import Logger
log = Logger()
@@ -342,8 +343,13 @@
# the child resource loop and supply those to the checkPrivileges on each child.
filteredaces = (yield calresource.inheritedACEsforChildren(request))
- for name, uid, type in calresource.index().search(filter): #@UnusedVariable
-
+ try:
+ resources = calresource.index().indexedSearch(filter)
+ except IndexedSearchException:
+ resources = calresource.index().bruteForceSearch()
+
+ for name, uid, type in resources: #@UnusedVariable
+
# Check privileges - must have at least CalDAV:read-free-busy
child = (yield request.locateChildResource(calresource, name))
Modified: CalendarServer/trunk/twistedcaldav/static.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/static.py 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/twistedcaldav/static.py 2008-12-09 19:05:10 UTC (rev 3489)
@@ -252,7 +252,7 @@
tzids = set()
isowner = (yield self.isOwner(request))
- for name, uid, type in self.index().search(None): #@UnusedVariable
+ for name, uid, type in self.index().bruteForceSearch(): #@UnusedVariable
try:
child = yield request.locateChildResource(self, name)
child = IDAVResource(child)
Modified: CalendarServer/trunk/twistedcaldav/test/test_icalendar.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_icalendar.py 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/twistedcaldav/test/test_icalendar.py 2008-12-09 19:05:10 UTC (rev 3489)
@@ -169,7 +169,7 @@
self.assertEqual(end, datetime.datetime(2004, 11, 27))
break;
- #test_component_timerange.todo = "recurrance expansion should give us no end date here"
+ #test_component_timerange.todo = "recurrence expansion should give us no end date here"
def test_parse_date(self):
"""
Modified: CalendarServer/trunk/twistedcaldav/test/test_xml.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_xml.py 2008-12-09 18:47:26 UTC (rev 3488)
+++ CalendarServer/trunk/twistedcaldav/test/test_xml.py 2008-12-09 19:05:10 UTC (rev 3489)
@@ -115,7 +115,7 @@
("20020102", "20020103", False),
("20011201", "20020101", False), # End is non-inclusive
- # Expanded recurrance
+ # Expanded recurrence
("20030101T000000Z", "20030101T000001Z", True),
("20030101T000000Z", "20030101T000000Z", True), # Timespan of zero duration
("20030101", "20030101", True), # Timespan of zero duration
@@ -136,4 +136,4 @@
)).match(self.calendar):
self.fail("Calendar has %sVEVENT with timerange %s?" % (no, (start, end)))
- test_TimeRange.todo = "recurrance expansion"
+ test_TimeRange.todo = "recurrence expansion"
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20081209/59ed9a45/attachment-0001.html>
More information about the calendarserver-changes
mailing list