[CalendarServer-changes] [6335] CalendarServer/branches/users/glyph/more-deferreds-6
source_changes at macosforge.org
source_changes at macosforge.org
Tue Sep 21 21:00:17 PDT 2010
Revision: 6335
http://trac.macosforge.org/projects/calendarserver/changeset/6335
Author: glyph at apple.com
Date: 2010-09-21 21:00:15 -0700 (Tue, 21 Sep 2010)
Log Message:
-----------
Move file indexing modules into the datastore modules, because they're part of the data store. Eliminate some dead code. Make a few more things async. Eliminate unnecessary/unused 'name' argument to iCalendar and iCalendarText methods (though it remains on iCalendarForUser)
Modified Paths:
--------------
CalendarServer/branches/users/glyph/more-deferreds-6/twext/web2/server.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/icaldav.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_addressbook_common.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_common.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_calendar_query.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_common.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/resource.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/storebridge.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookmultiget.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookquery.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/file.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/sql.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/file.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/datastore/sql_legacy.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/icommondatastore.py
Added Paths:
-----------
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py
CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py
Removed Paths:
-------------
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/index.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_index.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_vcardindex.py
CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/vcardindex.py
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twext/web2/server.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twext/web2/server.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twext/web2/server.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -471,10 +471,10 @@
@raise NoURLForResourceError: if C{resource} has no URL in this request
(because it was not obtained from the request).
"""
- resource = self._urlsByResource.get(resource, None)
- if resource is None:
+ url = self._urlsByResource.get(resource, None)
+ if url is None:
raise NoURLForResourceError(resource)
- return resource
+ return url
def locateResource(self, url):
"""
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/icaldav.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/icaldav.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/icaldav.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -67,29 +67,25 @@
Create a calendar collection for this resource.
"""
- def iCalendar(name=None):
+ def iCalendar():
"""
Instantiate an iCalendar component object representing this resource or
its child with the given name.
The behavior of this method is not specified if it is called on a
resource that is not a calendar collection or a calendar resource within
a calendar collection.
- @param name: the name of the desired child of this resource, or None
- if this resource is desired. Must be None if this resource is
- not a calendar collection.
+
@return: a L{twistedcaldav.ical.Component} of type C{"VCALENDAR"}.
"""
- def iCalendarText(name=None):
+ def iCalendarText():
"""
Obtains the iCalendar text representing this resource or its child with
the given name.
The behavior of this method is not specified if it is called on a
resource that is not a calendar collection or a calendar resource within
a calendar collection.
- @param name: the name of the desired child of this resource, or None
- if this resource is desired. Must be None if this resource is
- not a calendar collection.
+
@return: a string containing iCalendar text with a top-level component
of type C{"VCALENDAR"}.
"""
Deleted: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/index.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/index.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/index.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -1,1156 +0,0 @@
-# -*- test-case-name: twistedcaldav.test.test_index -*-
-##
-# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-CalDAV Index.
-
-This API is considered private to static.py and is therefore subject to
-change.
-"""
-
-__all__ = [
- "db_basename",
- "ReservationError",
- "MemcachedUIDReserver",
- "Index",
- "IndexSchedule",
- "IndexedSearchException",
-]
-
-import datetime
-import time
-import hashlib
-
-try:
- import sqlite3 as sqlite
-except ImportError:
- from pysqlite2 import dbapi2 as sqlite
-
-from vobject.icalendar import utc
-
-from twisted.internet.defer import maybeDeferred, succeed
-
-from twext.python.log import Logger, LoggingMixIn
-
-from twistedcaldav.ical import Component
-from twistedcaldav.query import calendarquery, calendarqueryfilter
-from twistedcaldav.sql import AbstractSQLDatabase
-from twistedcaldav.sql import db_prefix
-from twistedcaldav.instance import InvalidOverriddenInstanceError
-from twistedcaldav.config import config
-from twistedcaldav.memcachepool import CachePoolUserMixIn
-
-log = Logger()
-
-db_basename = db_prefix + "sqlite"
-schema_version = "10"
-collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
-
-icalfbtype_to_indexfbtype = {
- "FREE" : 'F',
- "BUSY" : 'B',
- "BUSY-UNAVAILABLE": 'U',
- "BUSY-TENTATIVE" : 'T',
-}
-indexfbtype_to_icalfbtype = dict([(v, k) for k,v in icalfbtype_to_indexfbtype.iteritems()])
-
-#
-# Duration into the future through which recurrences are expanded in the index
-# by default. This is a caching parameter which affects the size of the index;
-# it does not affect search results beyond this period, but it may affect
-# performance of such a search.
-#
-default_future_expansion_duration = datetime.timedelta(days=365*1)
-
-#
-# Maximum duration into the future through which recurrences are expanded in the
-# index. This is a caching parameter which affects the size of the index; it
-# does not affect search results beyond this period, but it may affect
-# performance of such a search.
-#
-# When a search is performed on a time span that goes beyond that which is
-# expanded in the index, we have to open each resource which may have data in
-# that time period. In order to avoid doing that multiple times, we want to
-# cache those results. However, we don't necessarily want to cache all
-# occurrences into some obscenely far-in-the-future date, so we cap the caching
-# period. Searches beyond this period will always be relatively expensive for
-# resources with occurrences beyond this period.
-#
-maximum_future_expansion_duration = datetime.timedelta(days=365*5)
-
-class ReservationError(LookupError):
- """
- Attempt to reserve a UID which is already reserved or to unreserve a UID
- which is not reserved.
- """
-
-class IndexedSearchException(ValueError):
- pass
-
-class SyncTokenValidException(ValueError):
- pass
-
-class AbstractCalendarIndex(AbstractSQLDatabase, LoggingMixIn):
- """
- Calendar collection index abstract base class that defines the apis for the index.
- This will be subclassed for the two types of index behaviour we need: one for
- regular calendar collections, one for schedule calendar collections.
- """
-
- def __init__(self, resource):
- """
- @param resource: the L{CalDAVResource} resource to
- index. C{resource} must be a calendar collection (ie.
- C{resource.isPseudoCalendarCollection()} returns C{True}.)
- """
- self.resource = resource
- db_filename = self.resource.fp.child(db_basename).path
- super(AbstractCalendarIndex, self).__init__(db_filename, False)
-
- def create(self):
- """
- Create the index and initialize it.
- """
- self._db()
-
- def reserveUID(self, uid):
- """
- Reserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is already reserved
- """
- raise NotImplementedError
-
- def unreserveUID(self, uid):
- """
- Unreserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is not reserved
- """
- raise NotImplementedError
-
- def isReservedUID(self, uid):
- """
- Check to see whether a UID is reserved.
- @param uid: the UID to check
- @return: True if C{uid} is reserved, False otherwise.
- """
- raise NotImplementedError
-
- def isAllowedUID(self, uid, *names):
- """
- Checks to see whether to allow an operation with adds the the specified
- UID is allowed to the index. Specifically, the operation may not
- violate the constraint that UIDs must be unique, and the UID must not
- be reserved.
- @param uid: the UID to check
- @param names: the names of resources being replaced or deleted by the
- operation; UIDs associated with these resources are not checked.
- @return: True if the UID is not in the index and is not reserved,
- False otherwise.
- """
- raise NotImplementedError
-
- def resourceNamesForUID(self, uid):
- """
- Looks up the names of the resources with the given UID.
- @param uid: the UID of the resources to look up.
- @return: a list of resource names
- """
- names = self._db_values_for_sql("select NAME from RESOURCE where UID = :1", uid)
-
- #
- # Check that each name exists as a child of self.resource. If not, the
- # resource record is stale.
- #
- resources = []
- for name in names:
- name_utf8 = name.encode("utf-8")
- if name is not None and self.resource.getChild(name_utf8) is None:
- # Clean up
- log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
- self._delete_from_db(name, uid, False)
- self._db_commit()
- else:
- resources.append(name_utf8)
-
- return resources
-
- def resourceNameForUID(self, uid):
- """
- Looks up the name of the resource with the given UID.
- @param uid: the UID of the resource to look up.
- @return: If the resource is found, its name; C{None} otherwise.
- """
- result = None
-
- for name in self.resourceNamesForUID(uid):
- assert result is None, "More than one resource with UID %s in calendar collection %r" % (uid, self)
- result = name
-
- return result
-
- def resourceUIDForName(self, name):
- """
- Looks up the UID of the resource with the given name.
- @param name: the name of the resource to look up.
- @return: If the resource is found, the UID of the resource; C{None}
- otherwise.
- """
- uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
-
- return uid
-
- def addResource(self, name, calendar, fast=False, reCreate=False):
- """
- Adding or updating an existing resource.
- To check for an update we attempt to get an existing UID
- for the resource name. If present, then the index entries for
- that UID are removed. After that the new index entries are added.
- @param name: the name of the resource to add.
- @param calendar: a L{Calendar} object representing the resource
- contents.
- @param fast: if C{True} do not do commit, otherwise do commit.
- """
- oldUID = self.resourceUIDForName(name)
- if oldUID is not None:
- self._delete_from_db(name, oldUID, False)
- self._add_to_db(name, calendar, reCreate=reCreate)
- if not fast:
- self._db_commit()
-
- def deleteResource(self, name):
- """
- Remove this resource from the index.
- @param name: the name of the resource to add.
- @param uid: the UID of the calendar component in the resource.
- """
- uid = self.resourceUIDForName(name)
- if uid is not None:
- self._delete_from_db(name, uid)
- self._db_commit()
-
- def resourceExists(self, name):
- """
- Determines whether the specified resource name exists in the index.
- @param name: the name of the resource to test
- @return: True if the resource exists, False if not
- """
- uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
- return uid is not None
-
- def resourcesExist(self, names):
- """
- Determines whether the specified resource name exists in the index.
- @param names: a C{list} containing the names of the resources to test
- @return: a C{list} of all names that exist
- """
- statement = "select NAME from RESOURCE where NAME in ("
- for ctr in (item[0] for item in enumerate(names)):
- if ctr != 0:
- statement += ", "
- statement += ":%s" % (ctr,)
- statement += ")"
- results = self._db_values_for_sql(statement, *names)
- return results
-
-
- def testAndUpdateIndex(self, minDate):
- # Find out if the index is expanded far enough
- names = self.notExpandedBeyond(minDate)
- # Actually expand recurrence max
- for name in names:
- self.log_info("Search falls outside range of index for %s %s" % (name, minDate))
- self.reExpandResource(name, minDate)
-
- def whatchanged(self, revision):
-
- results = [(name.encode("utf-8"), deleted) for name, deleted in self._db_execute("select NAME, DELETED from REVISIONS where REVISION > :1", revision)]
- results.sort(key=lambda x:x[1])
-
- changed = []
- deleted = []
- for name, wasdeleted in results:
- if name:
- if wasdeleted == 'Y':
- if revision:
- deleted.append(name)
- else:
- changed.append(name)
- else:
- raise SyncTokenValidException
-
- return changed, deleted,
-
- def lastRevision(self):
- return self._db_value_for_sql(
- "select REVISION from REVISION_SEQUENCE"
- )
-
- def bumpRevision(self, fast=False):
- self._db_execute(
- """
- update REVISION_SEQUENCE set REVISION = REVISION + 1
- """,
- )
- self._db_commit()
- return self._db_value_for_sql(
- """
- select REVISION from REVISION_SEQUENCE
- """,
- )
-
- def indexedSearch(self, filter, useruid="", fbtype=False):
- """
- Finds resources matching the given qualifiers.
- @param filter: the L{Filter} for the calendar-query to execute.
- @return: an iterable of tuples for each resource matching the
- given C{qualifiers}. The tuples are C{(name, uid, type)}, where
- C{name} is the resource name, C{uid} is the resource UID, and
- C{type} is the resource iCalendar component type.
- """
-
- # Make sure we have a proper Filter element and get the partial SQL
- # statement to use.
- if isinstance(filter, calendarqueryfilter.Filter):
- if fbtype:
- # Lookup the useruid - try the empty (default) one if needed
- dbuseruid = self._db_value_for_sql(
- "select PERUSERID from PERUSER where USERUID == :1",
- useruid,
- )
- else:
- dbuseruid = ""
-
- qualifiers = calendarquery.sqlcalendarquery(filter, None, dbuseruid)
- if qualifiers is not None:
- # Determine how far we need to extend the current expansion of
- # events. If we have an open-ended time-range we will expand one
- # year past the start. That should catch bounded recurrences - unbounded
- # will have been indexed with an "infinite" value always included.
- maxDate, isStartDate = filter.getmaxtimerange()
- if maxDate:
- maxDate = maxDate.date()
- if isStartDate:
- maxDate += datetime.timedelta(days=365)
- self.testAndUpdateIndex(maxDate)
- else:
- # We cannot handler this filter in an indexed search
- raise IndexedSearchException()
-
- else:
- qualifiers = None
-
- # Perform the search
- if qualifiers is None:
- rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
- else:
- if fbtype:
- # Lookup the useruid - try the empty (default) one if needed
- dbuseruid = self._db_value_for_sql(
- "select PERUSERID from PERUSER where USERUID == :1",
- useruid,
- )
-
- # For a free-busy time-range query we return all instances
- rowiter = self._db_execute(
- "select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE, RESOURCE.ORGANIZER, TIMESPAN.FLOAT, TIMESPAN.START, TIMESPAN.END, TIMESPAN.FBTYPE, TIMESPAN.TRANSPARENT, TRANSPARENCY.TRANSPARENT" +
- qualifiers[0],
- *qualifiers[1]
- )
- else:
- rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1])
-
- # Check result for missing resources
-
- for row in rowiter:
- name = row[0]
- if self.resource.getChild(name.encode("utf-8")):
- if fbtype:
- row = list(row)
- if row[9]:
- row[8] = row[9]
- del row[9]
- yield row
- else:
- log.err("Calendar resource %s is missing from %s. Removing from index."
- % (name, self.resource))
- self.deleteResource(name)
-
- def bruteForceSearch(self):
- """
- List the whole index and tests for existence, updating the index
- @return: all resources in the index
- """
- # List all resources
- rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
-
- # Check result for missing resources:
-
- for row in rowiter:
- name = row[0]
- if self.resource.getChild(name.encode("utf-8")):
- yield row
- else:
- log.err("Calendar resource %s is missing from %s. Removing from index."
- % (name, self.resource))
- self.deleteResource(name)
-
-
- def _db_version(self):
- """
- @return: the schema version assigned to this index.
- """
- return schema_version
-
- def _add_to_db(self, name, calendar, cursor=None, expand_until=None, reCreate=False):
- """
- Records the given calendar resource in the index with the given name.
- Resource names and UIDs must both be unique; only one resource name may
- be associated with any given UID and vice versa.
- NB This method does not commit the changes to the db - the caller
- MUST take care of that
- @param name: the name of the resource to add.
- @param calendar: a L{Calendar} object representing the resource
- contents.
- """
- raise NotImplementedError
-
- def _delete_from_db(self, name, uid, dorevision=True):
- """
- Deletes the specified entry from all dbs.
- @param name: the name of the resource to delete.
- @param uid: the uid of the resource to delete.
- """
- raise NotImplementedError
-
-class CalendarIndex (AbstractCalendarIndex):
- """
- Calendar index - abstract class for indexer that indexes calendar objects in a collection.
- """
-
- def __init__(self, resource):
- """
- @param resource: the L{CalDAVResource} resource to
- index.
- """
- super(CalendarIndex, self).__init__(resource)
-
- def _db_init_data_tables_base(self, q, uidunique):
- """
- Initialise the underlying database tables.
- @param q: a database cursor to use.
- """
- #
- # RESOURCE table is the primary index table
- # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
- # UID: iCalendar UID (may or may not be unique)
- # TYPE: iCalendar component type
- # RECURRANCE_MAX: Highest date of recurrence expansion
- # ORGANIZER: cu-address of the Organizer of the event
- #
- q.execute(
- """
- create table RESOURCE (
- RESOURCEID integer primary key autoincrement,
- NAME text unique,
- UID text%s,
- TYPE text,
- RECURRANCE_MAX date,
- ORGANIZER text
- )
- """ % (" unique" if uidunique else "",)
- )
-
- #
- # TIMESPAN table tracks (expanded) time spans for resources
- # NAME: Related resource (RESOURCE foreign key)
- # FLOAT: 'Y' if start/end are floating, 'N' otherwise
- # START: Start date
- # END: End date
- # FBTYPE: FBTYPE value:
- # '?' - unknown
- # 'F' - free
- # 'B' - busy
- # 'U' - busy-unavailable
- # 'T' - busy-tentative
- # TRANSPARENT: Y if transparent, N if opaque (default non-per-user value)
- #
- q.execute(
- """
- create table TIMESPAN (
- INSTANCEID integer primary key autoincrement,
- RESOURCEID integer,
- FLOAT text(1),
- START date,
- END date,
- FBTYPE text(1),
- TRANSPARENT text(1)
- )
- """
- )
- q.execute(
- """
- create index STARTENDFLOAT on TIMESPAN (START, END, FLOAT)
- """
- )
-
- #
- # PERUSER table tracks per-user ids
- # PERUSERID: autoincrement primary key
- # UID: User ID used in calendar data
- #
- q.execute(
- """
- create table PERUSER (
- PERUSERID integer primary key autoincrement,
- USERUID text
- )
- """
- )
- q.execute(
- """
- create index PERUSER_UID on PERUSER (USERUID)
- """
- )
-
- #
- # TRANSPARENCY table tracks per-user per-instance transparency
- # PERUSERID: user id key
- # INSTANCEID: instance id key
- # TRANSPARENT: Y if transparent, N if opaque
- #
- q.execute(
- """
- create table TRANSPARENCY (
- PERUSERID integer,
- INSTANCEID integer,
- TRANSPARENT text(1)
- )
- """
- )
-
- #
- # REVISIONS table tracks changes
- # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
- # REVISION: revision number
- # WASDELETED: Y if revision deleted, N if added or changed
- #
- q.execute(
- """
- create table REVISION_SEQUENCE (
- REVISION integer
- )
- """
- )
- q.execute(
- """
- insert into REVISION_SEQUENCE (REVISION) values (0)
- """
- )
- q.execute(
- """
- create table REVISIONS (
- NAME text unique,
- REVISION integer,
- DELETED text(1)
- )
- """
- )
- q.execute(
- """
- create index REVISION on REVISIONS (REVISION)
- """
- )
-
- if uidunique:
- #
- # RESERVED table tracks reserved UIDs
- # UID: The UID being reserved
- # TIME: When the reservation was made
- #
- q.execute(
- """
- create table RESERVED (
- UID text unique,
- TIME date
- )
- """
- )
-
- # Cascading triggers to help on delete
- q.execute(
- """
- create trigger resourceDelete after delete on RESOURCE
- for each row
- begin
- delete from TIMESPAN where TIMESPAN.RESOURCEID = OLD.RESOURCEID;
- end
- """
- )
- q.execute(
- """
- create trigger timespanDelete after delete on TIMESPAN
- for each row
- begin
- delete from TRANSPARENCY where INSTANCEID = OLD.INSTANCEID;
- end
- """
- )
-
- def _db_can_upgrade(self, old_version):
- """
- Can we do an in-place upgrade
- """
-
- # v10 is a big change - no upgrade possible
- return False
-
- def _db_upgrade_data_tables(self, q, old_version):
- """
- Upgrade the data from an older version of the DB.
- """
-
- # v10 is a big change - no upgrade possible
- pass
-
- def notExpandedBeyond(self, minDate):
- """
- Gives all resources which have not been expanded beyond a given date
- in the index
- """
- return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX < :1", minDate)
-
- def reExpandResource(self, name, expand_until):
- """
- Given a resource name, remove it from the database and re-add it
- with a longer expansion.
- """
- calendar = self.resource.getChild(name).iCalendar()
- self._add_to_db(name, calendar, expand_until=expand_until, reCreate=True)
- self._db_commit()
-
- def _add_to_db(self, name, calendar, cursor = None, expand_until=None, reCreate=False):
- """
- Records the given calendar resource in the index with the given name.
- Resource names and UIDs must both be unique; only one resource name may
- be associated with any given UID and vice versa.
- NB This method does not commit the changes to the db - the caller
- MUST take care of that
- @param name: the name of the resource to add.
- @param calendar: a L{Calendar} object representing the resource
- contents.
- """
- uid = calendar.resourceUID()
- organizer = calendar.getOrganizer()
- if not organizer:
- organizer = ""
-
- # Decide how far to expand based on the component
- master = calendar.masterComponent()
- if master is None or not calendar.isRecurring() and not calendar.isRecurringUnbounded():
- # When there is no master we have a set of overridden components - index them all.
- # When there is one instance - index it.
- # When bounded - index all.
- expand = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
- else:
- if expand_until:
- expand = expand_until
- else:
- expand = datetime.date.today() + default_future_expansion_duration
-
- if expand > (datetime.date.today() + maximum_future_expansion_duration):
- raise IndexedSearchException
-
- try:
- instances = calendar.expandTimeRanges(expand, ignoreInvalidInstances=reCreate)
- except InvalidOverriddenInstanceError, e:
- log.err("Invalid instance %s when indexing %s in %s" % (e.rid, name, self.resource,))
- raise
-
- self._delete_from_db(name, uid, False)
-
- # Add RESOURCE item
- self._db_execute(
- """
- insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX, ORGANIZER)
- values (:1, :2, :3, :4, :5)
- """, name, uid, calendar.resourceType(), instances.limit, organizer
- )
- resourceid = self.lastrowid
-
- # Get a set of all referenced per-user UIDs and map those to entries already
- # in the DB and add new ones as needed
- useruids = calendar.allPerUserUIDs()
- useruids.add("")
- useruidmap = {}
- for useruid in useruids:
- peruserid = self._db_value_for_sql(
- "select PERUSERID from PERUSER where USERUID = :1",
- useruid
- )
- if peruserid is None:
- self._db_execute(
- """
- insert into PERUSER (USERUID)
- values (:1)
- """, useruid
- )
- peruserid = self.lastrowid
- useruidmap[useruid] = peruserid
-
- for key in instances:
- instance = instances[key]
- start = instance.start.replace(tzinfo=utc)
- end = instance.end.replace(tzinfo=utc)
- float = 'Y' if instance.start.tzinfo is None else 'N'
- transp = 'T' if instance.component.propertyValue("TRANSP") == "TRANSPARENT" else 'F'
- self._db_execute(
- """
- insert into TIMESPAN (RESOURCEID, FLOAT, START, END, FBTYPE, TRANSPARENT)
- values (:1, :2, :3, :4, :5, :6)
- """,
- resourceid,
- float,
- start,
- end,
- icalfbtype_to_indexfbtype.get(instance.component.getFBType(), 'F'),
- transp
- )
- instanceid = self.lastrowid
- peruserdata = calendar.perUserTransparency(instance.rid)
- for useruid, transp in peruserdata:
- peruserid = useruidmap[useruid]
- self._db_execute(
- """
- insert into TRANSPARENCY (PERUSERID, INSTANCEID, TRANSPARENT)
- values (:1, :2, :3)
- """, peruserid, instanceid, 'T' if transp else 'F'
- )
-
-
- # Special - for unbounded recurrence we insert a value for "infinity"
- # that will allow an open-ended time-range to always match it.
- if calendar.isRecurringUnbounded():
- start = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
- end = datetime.datetime(2100, 1, 1, 1, 0, 0, tzinfo=utc)
- float = 'N'
- self._db_execute(
- """
- insert into TIMESPAN (RESOURCEID, FLOAT, START, END, FBTYPE, TRANSPARENT)
- values (:1, :2, :3, :4, :5, :6)
- """, resourceid, float, start, end, '?', '?'
- )
- instanceid = self.lastrowid
- peruserdata = calendar.perUserTransparency(None)
- for useruid, transp in peruserdata:
- peruserid = useruidmap[useruid]
- self._db_execute(
- """
- insert into TRANSPARENCY (PERUSERID, INSTANCEID, TRANSPARENT)
- values (:1, :2, :3)
- """, peruserid, instanceid, 'T' if transp else 'F'
- )
-
- self._db_execute(
- """
- insert or replace into REVISIONS (NAME, REVISION, DELETED)
- values (:1, :2, :3)
- """, name, self.bumpRevision(fast=True), 'N',
- )
-
- def _delete_from_db(self, name, uid, dorevision=True):
- """
- Deletes the specified entry from all dbs.
- @param name: the name of the resource to delete.
- @param uid: the uid of the resource to delete.
- """
- self._db_execute("delete from RESOURCE where NAME = :1", name)
- if dorevision:
- self._db_execute(
- """
- update REVISIONS SET REVISION = :1, DELETED = :2
- where NAME = :3
- """, self.bumpRevision(fast=True), 'Y', name
- )
-
-
-def wrapInDeferred(f):
- def _(*args, **kwargs):
- return maybeDeferred(f, *args, **kwargs)
-
- return _
-
-
-class MemcachedUIDReserver(CachePoolUserMixIn, LoggingMixIn):
- def __init__(self, index, cachePool=None):
- self.index = index
- self._cachePool = cachePool
-
- def _key(self, uid):
- return 'reservation:%s' % (
- hashlib.md5('%s:%s' % (uid,
- self.index.resource.fp.path)).hexdigest())
-
- def reserveUID(self, uid):
- uid = uid.encode('utf-8')
- self.log_debug("Reserving UID %r @ %r" % (
- uid,
- self.index.resource.fp.path))
-
- def _handleFalse(result):
- if result is False:
- raise ReservationError(
- "UID %s already reserved for calendar collection %s."
- % (uid, self.index.resource)
- )
-
- d = self.getCachePool().add(self._key(uid),
- 'reserved',
- expireTime=config.UIDReservationTimeOut)
- d.addCallback(_handleFalse)
- return d
-
-
- def unreserveUID(self, uid):
- uid = uid.encode('utf-8')
- self.log_debug("Unreserving UID %r @ %r" % (
- uid,
- self.index.resource.fp.path))
-
- def _handleFalse(result):
- if result is False:
- raise ReservationError(
- "UID %s is not reserved for calendar collection %s."
- % (uid, self.index.resource)
- )
-
- d =self.getCachePool().delete(self._key(uid))
- d.addCallback(_handleFalse)
- return d
-
-
- def isReservedUID(self, uid):
- uid = uid.encode('utf-8')
- self.log_debug("Is reserved UID %r @ %r" % (
- uid,
- self.index.resource.fp.path))
-
- def _checkValue((flags, value)):
- if value is None:
- return False
- else:
- return True
-
- d = self.getCachePool().get(self._key(uid))
- d.addCallback(_checkValue)
- return d
-
-
-
-class SQLUIDReserver(object):
- def __init__(self, index):
- self.index = index
-
- @wrapInDeferred
- def reserveUID(self, uid):
- """
- Reserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is already reserved
- """
-
- try:
- self.index._db_execute("insert into RESERVED (UID, TIME) values (:1, :2)", uid, datetime.datetime.now())
- self.index._db_commit()
- except sqlite.IntegrityError:
- self.index._db_rollback()
- raise ReservationError(
- "UID %s already reserved for calendar collection %s."
- % (uid, self.index.resource)
- )
- except sqlite.Error, e:
- log.err("Unable to reserve UID: %s", (e,))
- self.index._db_rollback()
- raise
-
- def unreserveUID(self, uid):
- """
- Unreserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is not reserved
- """
-
- def _cb(result):
- if result == False:
- raise ReservationError(
- "UID %s is not reserved for calendar collection %s."
- % (uid, self.index.resource)
- )
- else:
- try:
- self.index._db_execute(
- "delete from RESERVED where UID = :1", uid)
- self.index._db_commit()
- except sqlite.Error, e:
- log.err("Unable to unreserve UID: %s", (e,))
- self.index._db_rollback()
- raise
-
- d = self.isReservedUID(uid)
- d.addCallback(_cb)
- return d
-
-
- @wrapInDeferred
- def isReservedUID(self, uid):
- """
- Check to see whether a UID is reserved.
- @param uid: the UID to check
- @return: True if C{uid} is reserved, False otherwise.
- """
-
- rowiter = self.index._db_execute("select UID, TIME from RESERVED where UID = :1", uid)
- for uid, attime in rowiter:
- # Double check that the time is within a reasonable period of now
- # otherwise we probably have a stale reservation
- tm = time.strptime(attime[:19], "%Y-%m-%d %H:%M:%S")
- dt = datetime.datetime(year=tm.tm_year, month=tm.tm_mon, day=tm.tm_mday, hour=tm.tm_hour, minute=tm.tm_min, second = tm.tm_sec)
- if datetime.datetime.now() - dt > datetime.timedelta(seconds=config.UIDReservationTimeOut):
- try:
- self.index._db_execute("delete from RESERVED where UID = :1", uid)
- self.index._db_commit()
- except sqlite.Error, e:
- log.err("Unable to unreserve UID: %s", (e,))
- self.index._db_rollback()
- raise
- return False
- else:
- return True
-
- return False
-
-
-
-class Index (CalendarIndex):
- """
- Calendar collection index - regular collection that enforces CalDAV UID uniqueness requirement.
- """
-
- def __init__(self, resource):
- """
- @param resource: the L{CalDAVResource} resource to
- index. C{resource} must be a calendar collection (i.e.
- C{resource.isPseudoCalendarCollection()} returns C{True}.)
- """
- assert resource.isCalendarCollection(), "non-calendar collection resource %s has no index." % (resource,)
- super(Index, self).__init__(resource)
-
- if (
- hasattr(config, "Memcached") and
- config.Memcached.Pools.Default.ClientEnabled
- ):
- self.reserver = MemcachedUIDReserver(self)
- else:
- self.reserver = SQLUIDReserver(self)
-
- #
- # A dict of sets. The dict keys are calendar collection paths,
- # and the sets contains reserved UIDs for each path.
- #
-
- def reserveUID(self, uid):
- return self.reserver.reserveUID(uid)
-
-
- def unreserveUID(self, uid):
- return self.reserver.unreserveUID(uid)
-
-
- def isReservedUID(self, uid):
- return self.reserver.isReservedUID(uid)
-
-
- def isAllowedUID(self, uid, *names):
- """
- Checks to see whether to allow an operation which would add the
- specified UID to the index. Specifically, the operation may not
- violate the constraint that UIDs must be unique.
- @param uid: the UID to check
- @param names: the names of resources being replaced or deleted by the
- operation; UIDs associated with these resources are not checked.
- @return: True if the UID is not in the index and is not reserved,
- False otherwise.
- """
- rname = self.resourceNameForUID(uid)
- return (rname is None or rname in names)
-
- def _db_type(self):
- """
- @return: the collection type assigned to this index.
- """
- return collection_types["Calendar"]
-
- def _db_init_data_tables(self, q):
- """
- Initialise the underlying database tables.
- @param q: a database cursor to use.
- """
-
- # Create database where the RESOURCE table has unique UID column.
- self._db_init_data_tables_base(q, True)
-
- def _db_recreate(self, do_commit=True):
- """
- Re-create the database tables from existing calendar data.
- """
-
- #
- # Populate the DB with data from already existing resources.
- # This allows for index recovery if the DB file gets
- # deleted.
- #
- fp = self.resource.fp
- for name in fp.listdir():
- if name.startswith("."):
- continue
-
- try:
- stream = fp.child(name).open()
- except (IOError, OSError), e:
- log.err("Unable to open resource %s: %s" % (name, e))
- continue
-
- # FIXME: This is blocking I/O
- try:
- calendar = Component.fromStream(stream)
- calendar.validateForCalDAV()
- except ValueError:
- log.err("Non-calendar resource: %s" % (name,))
- else:
- #log.msg("Indexing resource: %s" % (name,))
- self.addResource(name, calendar, True, reCreate=True)
- finally:
- stream.close()
-
- # Do commit outside of the loop for better performance
- if do_commit:
- self._db_commit()
-
-class IndexSchedule (CalendarIndex):
- """
- Schedule collection index - does not require UID uniqueness.
- """
-
- def reserveUID(self, uid): #@UnusedVariable
- """
- Reserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is already reserved
- """
-
- # iTIP does not require unique UIDs
- return succeed(None)
-
- def unreserveUID(self, uid): #@UnusedVariable
- """
- Unreserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is not reserved
- """
-
- # iTIP does not require unique UIDs
- return succeed(None)
-
- def isReservedUID(self, uid): #@UnusedVariable
- """
- Check to see whether a UID is reserved.
- @param uid: the UID to check
- @return: True if C{uid} is reserved, False otherwise.
- """
-
- # iTIP does not require unique UIDs
- return succeed(False)
-
- def isAllowedUID(self, uid, *names): #@UnusedVariable
- """
- Checks to see whether to allow an operation with adds the the specified
- UID is allowed to the index. Specifically, the operation may not
- violate the constraint that UIDs must be unique, and the UID must not
- be reserved.
- @param uid: the UID to check
- @param names: the names of resources being replaced or deleted by the
- operation; UIDs associated with these resources are not checked.
- @return: True if the UID is not in the index and is not reserved,
- False otherwise.
- """
-
- # iTIP does not require unique UIDs
- return True
-
- def _db_type(self):
- """
- @return: the collection type assigned to this index.
- """
- return collection_types["iTIP"]
-
- def _db_init_data_tables(self, q):
- """
- Initialise the underlying database tables.
- @param q: a database cursor to use.
- """
-
- # Create database where the RESOURCE table has a UID column that is not unique.
- self._db_init_data_tables_base(q, False)
-
- def _db_recreate(self, do_commit=True):
- """
- Re-create the database tables from existing calendar data.
- """
-
- #
- # Populate the DB with data from already existing resources.
- # This allows for index recovery if the DB file gets
- # deleted.
- #
- fp = self.resource.fp
- for name in fp.listdir():
- if name.startswith("."):
- continue
-
- try:
- stream = fp.child(name).open()
- except (IOError, OSError), e:
- log.err("Unable to open resource %s: %s" % (name, e))
- continue
-
- # FIXME: This is blocking I/O
- try:
- calendar = Component.fromStream(stream)
- calendar.validCalendarForCalDAV()
- calendar.validateComponentsForCalDAV(True)
- except ValueError:
- log.err("Non-calendar resource: %s" % (name,))
- else:
- #log.msg("Indexing resource: %s" % (name,))
- self.addResource(name, calendar, True, reCreate=True)
- finally:
- stream.close()
-
- # Do commit outside of the loop for better performance
- if do_commit:
- self._db_commit()
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_addressbook_common.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_addressbook_common.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_addressbook_common.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -25,6 +25,8 @@
from twisted.internet import reactor
from twisted.python.failure import Failure
+from txdav.common.icommondatastore import ReservationError
+
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.defer import returnValue
from twext.web2 import responsecode
@@ -42,7 +44,6 @@
from twistedcaldav.config import config
from twistedcaldav.carddavxml import NoUIDConflict, carddav_namespace
from twistedcaldav.vcard import Component
-from twistedcaldav.vcardindex import ReservationError
from twext.python.log import Logger
log = Logger()
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_common.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_common.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/put_common.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -46,6 +46,8 @@
from twext.python.log import Logger
from twext.web2.dav.http import ErrorResponse
+from txdav.common.icommondatastore import ReservationError
+
from twistedcaldav.config import config
from twistedcaldav.caldavxml import ScheduleTag, NoUIDConflict
from twistedcaldav.caldavxml import NumberOfRecurrencesWithinLimits
@@ -57,7 +59,6 @@
from twistedcaldav.datafilters.peruserdata import PerUserDataFilter
from twistedcaldav.ical import Component, Property
-from twistedcaldav.index import ReservationError
from twistedcaldav.instance import TooManyInstancesError,\
InvalidOverriddenInstanceError
from twistedcaldav.memcachelock import MemcacheLock, MemcacheLockTimeoutError
@@ -698,12 +699,14 @@
@inlineCallbacks
def mergePerUserData(self):
-
if self.calendar:
accessUID = (yield self.destination.resourceOwnerPrincipal(self.request))
accessUID = accessUID.principalUID() if accessUID else ""
- oldCal = self.destination.iCalendar() if self.destination.exists() and self.destinationcal else None
-
+ if self.destination.exists() and self.destinationcal:
+ oldCal = self.destination.iCalendar()
+ else:
+ oldCal = None
+
# Duplicate before we do the merge because someone else may "own" the calendar object
# and we should not change it. This is not ideal as we may duplicate it unnecessarily
# but we currently have no api to let the caller tell us whether it cares about the
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_calendar_query.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_calendar_query.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_calendar_query.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -38,7 +38,7 @@
NumberOfRecurrencesWithinLimits
from twistedcaldav.config import config
from twistedcaldav.customxml import TwistedCalendarAccessProperty
-from twistedcaldav.index import IndexedSearchException
+from txdav.common.icommondatastore import IndexedSearchException
from twistedcaldav.instance import TooManyInstancesError
from twistedcaldav.method import report_common
from twistedcaldav.query import calendarqueryfilter
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_common.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_common.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/method/report_common.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -64,9 +64,11 @@
from twistedcaldav.dateops import clipPeriod, normalizePeriodList, timeRangesOverlap
from twistedcaldav.ical import Component, Property, iCalendarProductID
from twistedcaldav.instance import InstanceList
-from twistedcaldav.index import IndexedSearchException
+
from twistedcaldav.query import calendarqueryfilter
+from txdav.common.icommondatastore import IndexedSearchException
+
log = Logger()
COLLECTION_TYPE_REGULAR = "collection"
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/resource.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/resource.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -72,19 +72,17 @@
from twistedcaldav.extensions import DAVResource, DAVPrincipalResource,\
PropertyNotFoundError, DAVResourceWithChildrenMixin
from twistedcaldav.ical import Component
-from twistedcaldav.ical import Component as iComponent
-from twistedcaldav.ical import Property as iProperty
+
from twistedcaldav.ical import allowedComponents
from twistedcaldav.icaldav import ICalDAVResource, ICalendarPrincipalResource
-from twistedcaldav.index import SyncTokenValidException, Index
from twistedcaldav.linkresource import LinkResource
from twistedcaldav.notify import getPubSubConfiguration, getPubSubPath,\
getPubSubXMPPURI, getPubSubHeartbeatURI
from twistedcaldav.sharing import SharedCollectionMixin, SharedHomeMixin
from twistedcaldav.vcard import Component as vComponent
-from twistedcaldav.vcardindex import AddressBookIndex
-from txdav.common.icommondatastore import InternalDataStoreError
+from txdav.common.icommondatastore import InternalDataStoreError, \
+ SyncTokenValidException
##
# Sharing Conts
@@ -839,6 +837,7 @@
else:
returnValue(None)
+
@inlineCallbacks
def resourceOwnerPrincipal(self, request):
"""
@@ -850,13 +849,16 @@
if isVirt:
returnValue(self._shareePrincipal)
else:
- parent = (yield self.locateParent(request, request.urlForResource(self)))
+ parent = (yield self.locateParent(
+ request, request.urlForResource(self)
+ ))
if parent and isinstance(parent, CalDAVResource):
result = (yield parent.resourceOwnerPrincipal(request))
returnValue(result)
else:
returnValue(None)
+
def isOwner(self, request, adminprincipals=False, readprincipals=False):
"""
Determine whether the DAV:owner of this resource matches the currently authorized principal
@@ -1104,35 +1106,19 @@
returnValue(False)
- def iCalendar(self, name=None):
- """
- See L{ICalDAVResource.iCalendar}.
- This implementation returns the an object created from the data returned
- by L{iCalendarText} when given the same arguments.
-
- Note that L{iCalendarText} by default calls this method, which creates
- an infinite loop. A subclass must override one of both of these
- methods.
- """
-
- try:
- calendar_data = self.iCalendarText(name)
- except InternalDataStoreError:
- return None
-
- if calendar_data is None: return None
-
- try:
- return iComponent.fromString(calendar_data)
- except ValueError:
- return None
-
@inlineCallbacks
def iCalendarForUser(self, request, name=None):
-
- caldata = self.iCalendar(name)
-
+ if name is not None:
+ # FIXME: this is really the caller's job; why am I looking up sub-
+ # resources?
+ returnValue(
+ (yield (yield request.locateChildResource(self, name)
+ ).iCalendarForUser(request))
+ )
+
+ caldata = self.iCalendar()
+
accessUID = (yield self.resourceOwnerPrincipal(request))
if accessUID is None:
accessUID = ""
@@ -1141,6 +1127,7 @@
returnValue(PerUserDataFilter(accessUID).filter(caldata))
+
def iCalendarAddressDoNormalization(self, ical):
"""
Normalize calendar user addresses in the supplied iCalendar object into their
@@ -1445,124 +1432,13 @@
return fail(NotImplementedError())
- def createSpecialCollection(self, resourceType=None):
- #
- # Create the collection once we know it is safe to do so
- #
- def onCollection(status):
- if status != responsecode.CREATED:
- raise HTTPError(status)
+ def iCalendarRolledup(self):
+ """
+ Only implemented by calendar collections; see storebridge.
+ """
+
- self.writeDeadProperty(resourceType)
- return status
- def onError(f):
- try:
- rmdir(self.fp)
- except Exception, e:
- log.err("Unable to clean up after failed MKCOL (special resource type: %s): %s" % (e, resourceType,))
- return f
-
- d = mkcollection(self.fp)
- if resourceType is not None:
- d.addCallback(onCollection)
- d.addErrback(onError)
- return d
-
- @inlineCallbacks
- def iCalendarRolledup(self, request):
- if self.isPseudoCalendarCollection():
-
-
-# FIXME: move cache implementation!
- # Determine the cache key
-# isvirt = self.isVirtualShare()
-# if isvirt:
-# principal = (yield self.resourceOwnerPrincipal(request))
-# if principal:
-# cacheKey = principal.principalUID()
-# else:
-# cacheKey = "unknown"
-# else:
-# isowner = (yield self.isOwner(request, adminprincipals=True, readprincipals=True))
-# cacheKey = "owner" if isowner else "notowner"
-
- # Now check for a cached .ics
-# rolled = self.fp.child(".subscriptions")
-# if not rolled.exists():
-# try:
-# rolled.makedirs()
-# except IOError, e:
-# self.log_error("Unable to create internet calendar subscription cache directory: %s because of: %s" % (rolled.path, e,))
-# raise HTTPError(ErrorResponse(responsecode.INTERNAL_SERVER_ERROR))
-# cached = rolled.child(cacheKey)
-# if cached.exists():
-# try:
-# cachedData = cached.open().read()
-# except IOError, e:
-# self.log_error("Unable to open or read internet calendar subscription cache file: %s because of: %s" % (cached.path, e,))
-# else:
-# # Check the cache token
-# token, data = cachedData.split("\r\n", 1)
-# if token == self.getSyncToken():
-# returnValue(data)
-
- # Generate a monolithic calendar
- calendar = iComponent("VCALENDAR")
- calendar.addProperty(iProperty("VERSION", "2.0"))
-
- # Do some optimisation of access control calculation by determining any inherited ACLs outside of
- # the child resource loop and supply those to the checkPrivileges on each child.
- filteredaces = (yield self.inheritedACEsforChildren(request))
-
- tzids = set()
- isowner = (yield self.isOwner(request, adminprincipals=True, readprincipals=True))
- accessPrincipal = (yield self.resourceOwnerPrincipal(request))
-
- for name, uid, type in self.index().bruteForceSearch(): #@UnusedVariable
- try:
- child = yield request.locateChildResource(self, name)
- except TypeError:
- child = None
-
- if child is not None:
- # Check privileges of child - skip if access denied
- try:
- yield child.checkPrivileges(request, (davxml.Read(),), inherited_aces=filteredaces)
- except AccessDeniedError:
- continue
-
- # Get the access filtered view of the data
- caldata = child.iCalendarTextFiltered(isowner, accessPrincipal.principalUID() if accessPrincipal else "")
- try:
- subcalendar = iComponent.fromString(caldata)
- except ValueError:
- continue
- assert subcalendar.name() == "VCALENDAR"
-
- for component in subcalendar.subcomponents():
-
- # Only insert VTIMEZONEs once
- if component.name() == "VTIMEZONE":
- tzid = component.propertyValue("TZID")
- if tzid in tzids:
- continue
- tzids.add(tzid)
-
- calendar.addComponent(component)
-
- # Cache the data
- data = str(calendar)
- data = self.getSyncToken() + "\r\n" + data
-# try:
-# cached.open(mode='w').write(data)
-# except IOError, e:
-# self.log_error("Unable to open or write internet calendar subscription cache file: %s because of: %s" % (cached.path, e,))
-
- returnValue(calendar)
-
- raise HTTPError(ErrorResponse(responsecode.BAD_REQUEST))
-
def iCalendarTextFiltered(self, isowner, accessUID=None):
try:
access = self.readDeadProperty(TwistedCalendarAccessProperty)
@@ -1575,23 +1451,16 @@
caldata = PerUserDataFilter(accessUID).filter(caldata)
return str(caldata)
- def iCalendarText(self, name=None):
- if self.isPseudoCalendarCollection():
- if name is None:
- return str(self.iCalendar())
- calendar_resource = self.getChild(name)
- return calendar_resource.iCalendarText()
+ def iCalendarText(self):
+ # storebridge handles this method
+ raise NotImplementedError()
- elif self.isCollection():
- return None
- else:
- if name is not None:
- raise AssertionError("name must be None for non-collection calendar resource")
+ def iCalendar(self):
+ # storebridge handles this method
+ raise NotImplementedError()
- # FIXME: StoreBridge handles this case
- raise NotImplementedError
def createAddressBook(self, request):
"""
@@ -1691,18 +1560,6 @@
return super(CalDAVResource, self).supportedPrivileges(request)
- def index(self):
- """
- Obtains the index for a calendar collection resource.
- @return: the index object for this resource.
- @raise AssertionError: if this resource is not a calendar collection
- resource.
- """
- if self.isAddressBookCollection():
- return AddressBookIndex(self)
- else:
- return Index(self)
-
##
# Quota
##
@@ -1815,9 +1672,6 @@
def isCalendarCollection(self):
return False
- def isPseudoCalendarCollection(self):
- return False
-
def isAddressBookCollection(self):
return False
@@ -2550,7 +2404,7 @@
"""
def resourceType(self):
- return davxml.ResourceType.sharedaddressbook
+ return davxml.ResourceType.sharedaddressbook #@UndefinedVariable
def defaultAccessControlList(self):
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/storebridge.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/storebridge.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -35,7 +35,7 @@
from twext.web2.dav import davxml
from twext.web2.dav.element.base import dav_namespace
from twext.web2.dav.http import ErrorResponse, ResponseQueue
-from twext.web2.dav.resource import TwistedACLInheritable
+from twext.web2.dav.resource import TwistedACLInheritable, AccessDeniedError
from twext.web2.dav.util import parentForURL, allDataFromStream, joinURL, \
davXMLFromStream
from twext.web2.http import HTTPError, StatusResponse, Response
@@ -709,6 +709,62 @@
return True
+ @inlineCallbacks
+ def iCalendarRolledup(self, request):
+ # FIXME: uncached: implement cache in the storage layer
+
+ # Generate a monolithic calendar
+ calendar = vcomponent.VComponent("VCALENDAR")
+ calendar.addProperty(vcomponent.VProperty("VERSION", "2.0"))
+
+ # Do some optimisation of access control calculation by determining any
+ # inherited ACLs outside of the child resource loop and supply those to
+ # the checkPrivileges on each child.
+ filteredaces = (yield self.inheritedACEsforChildren(request))
+
+ tzids = set()
+ isowner = (yield self.isOwner(request, adminprincipals=True, readprincipals=True))
+ accessPrincipal = (yield self.resourceOwnerPrincipal(request))
+
+ for name, uid, type in self.index().bruteForceSearch(): #@UnusedVariable
+ try:
+ child = yield request.locateChildResource(self, name)
+ except TypeError:
+ child = None
+
+ if child is not None:
+ # Check privileges of child - skip if access denied
+ try:
+ yield child.checkPrivileges(request, (davxml.Read(),), inherited_aces=filteredaces)
+ except AccessDeniedError:
+ continue
+
+ # Get the access filtered view of the data
+ caldata = child.iCalendarTextFiltered(isowner, accessPrincipal.principalUID() if accessPrincipal else "")
+ try:
+ subcalendar = vcomponent.VComponent.fromString(caldata)
+ except ValueError:
+ continue
+ assert subcalendar.name() == "VCALENDAR"
+
+ for component in subcalendar.subcomponents():
+
+ # Only insert VTIMEZONEs once
+ if component.name() == "VTIMEZONE":
+ tzid = component.propertyValue("TZID")
+ if tzid in tzids:
+ continue
+ tzids.add(tzid)
+
+ calendar.addComponent(component)
+
+ # Cache the data
+ data = str(calendar)
+ data = self.getSyncToken() + "\r\n" + data
+
+ returnValue(calendar)
+
+
@requiresPermissions(fromParent=[davxml.Unbind()])
@inlineCallbacks
def http_DELETE(self, request):
@@ -1006,11 +1062,14 @@
return succeed(len(self._newStoreObject.iCalendarText()))
- def iCalendarText(self, ignored=None):
- assert ignored is None, "This is a calendar object, not a calendar"
+ def iCalendarText(self):
return self._newStoreObject.iCalendarText()
+ def iCalendar(self):
+ return self._newStoreObject.component()
+
+
def text(self):
return self.iCalendarText()
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookmultiget.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookmultiget.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookmultiget.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -27,7 +27,11 @@
from twistedcaldav import carddavxml
from twistedcaldav import vcard
-from twistedcaldav.index import db_basename
+
+# FIXME: remove this, we should not be importing this module, we should be
+# testing the public API. See comments below about cheating.
+from txdav.carddav.datastore.index_file import db_basename
+
from twistedcaldav.config import config
from twistedcaldav.test.util import AddressBookHomeTestCase
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookquery.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookquery.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_addressbookquery.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -25,8 +25,11 @@
from twext.web2.dav.util import davXMLFromStream
from twext.web2.test.test_server import SimpleRequest
+# FIXME: remove this, we should not be importing this module, we should be
+# testing the public API. See comments below about cheating.
+from txdav.carddav.datastore.index_file import db_basename
+
from twistedcaldav import carddavxml, vcard
-from twistedcaldav.index import db_basename
from twistedcaldav.config import config
from twistedcaldav.test.util import AddressBookHomeTestCase
Deleted: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_index.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_index.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_index.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -1,935 +0,0 @@
-##
-# Copyright (c) 2010 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from twisted.internet import reactor
-from twisted.internet.task import deferLater
-
-from twistedcaldav import caldavxml
-from twistedcaldav.caldavxml import TimeRange
-from twistedcaldav.ical import Component
-from twistedcaldav.index import Index
-from twistedcaldav.index import ReservationError, MemcachedUIDReserver
-from twistedcaldav.instance import InvalidOverriddenInstanceError
-from twistedcaldav.query import calendarqueryfilter
-from twistedcaldav.test.util import InMemoryMemcacheProtocol
-import twistedcaldav.test.util
-
-import datetime
-import os
-
-
-class MinimalResourceReplacement(object):
- """
- Provide the minimal set of attributes and methods from CalDAVFile required
- by L{Index}.
- """
-
- def __init__(self, filePath):
- self.fp = filePath
-
-
- def isCalendarCollection(self):
- return True
-
-
- def getChild(self, name):
- # FIXME: this should really return something with a child method
- return self.fp.child(name)
-
-
- def initSyncToken(self):
- pass
-
-
-
-class SQLIndexTests (twistedcaldav.test.util.TestCase):
- """
- Test abstract SQL DB class
- """
-
- def setUp(self):
- super(SQLIndexTests, self).setUp()
- self.site.resource.isCalendarCollection = lambda: True
- self.indexDirPath = self.site.resource.fp
- # FIXME: since this resource lies about isCalendarCollection, it doesn't
- # have all the associated backend machinery to actually get children.
- self.db = Index(MinimalResourceReplacement(self.indexDirPath))
-
-
- def test_reserve_uid_ok(self):
- uid = "test-test-test"
- d = self.db.isReservedUID(uid)
- d.addCallback(self.assertFalse)
- d.addCallback(lambda _: self.db.reserveUID(uid))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertTrue)
- d.addCallback(lambda _: self.db.unreserveUID(uid))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertFalse)
-
- return d
-
-
- def test_reserve_uid_twice(self):
- uid = "test-test-test"
- d = self.db.reserveUID(uid)
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertTrue)
- d.addCallback(lambda _:
- self.assertFailure(self.db.reserveUID(uid),
- ReservationError))
- return d
-
-
- def test_unreserve_unreserved(self):
- uid = "test-test-test"
- return self.assertFailure(self.db.unreserveUID(uid),
- ReservationError)
-
-
- def test_reserve_uid_timeout(self):
- # WARNING: This test is fundamentally flawed and will fail
- # intermittently because it uses the real clock.
- uid = "test-test-test"
- from twistedcaldav.config import config
- old_timeout = config.UIDReservationTimeOut
- config.UIDReservationTimeOut = 1
-
- def _finally():
- config.UIDReservationTimeOut = old_timeout
-
- d = self.db.isReservedUID(uid)
- d.addCallback(self.assertFalse)
- d.addCallback(lambda _: self.db.reserveUID(uid))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertTrue)
- d.addCallback(lambda _: deferLater(reactor, 2, lambda: None))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertFalse)
- self.addCleanup(_finally)
-
- return d
-
-
- def test_index(self):
- data = (
- (
- "#1.1 Simple component",
- "1.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-END:VCALENDAR
-""",
- False,
- True,
- ),
- (
- "#2.1 Recurring component",
- "2.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=WEEKLY;COUNT=2
-END:VEVENT
-END:VCALENDAR
-""",
- False,
- True,
- ),
- (
- "#2.2 Recurring component with override",
- "2.2",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.2
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=WEEKLY;COUNT=2
-END:VEVENT
-BEGIN:VEVENT
-UID:12345-67890-2.2
-RECURRENCE-ID:20080608T120000Z
-DTSTART:20080608T120000Z
-DTEND:20080608T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-END:VCALENDAR
-""",
- False,
- True,
- ),
- (
- "#2.3 Recurring component with broken override - new",
- "2.3",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.3
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=WEEKLY;COUNT=2
-END:VEVENT
-BEGIN:VEVENT
-UID:12345-67890-2.3
-RECURRENCE-ID:20080609T120000Z
-DTSTART:20080608T120000Z
-DTEND:20080608T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-END:VCALENDAR
-""",
- False,
- False,
- ),
- (
- "#2.4 Recurring component with broken override - existing",
- "2.4",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.4
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=WEEKLY;COUNT=2
-END:VEVENT
-BEGIN:VEVENT
-UID:12345-67890-2.4
-RECURRENCE-ID:20080609T120000Z
-DTSTART:20080608T120000Z
-DTEND:20080608T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-END:VCALENDAR
-""",
- True,
- True,
- ),
- )
-
- for description, name, calendar_txt, reCreate, ok in data:
- calendar = Component.fromString(calendar_txt)
- if ok:
- f = open(os.path.join(self.indexDirPath.path, name), "w")
- f.write(calendar_txt)
- del f
-
- self.db.addResource(name, calendar, reCreate=reCreate)
- self.assertTrue(self.db.resourceExists(name), msg=description)
- else:
- self.assertRaises(InvalidOverriddenInstanceError, self.db.addResource, name, calendar)
- self.assertFalse(self.db.resourceExists(name), msg=description)
-
- self.db._db_recreate()
- for description, name, calendar_txt, reCreate, ok in data:
- if ok:
- self.assertTrue(self.db.resourceExists(name), msg=description)
- else:
- self.assertFalse(self.db.resourceExists(name), msg=description)
-
- self.db.testAndUpdateIndex(datetime.date(2020, 1, 1))
- for description, name, calendar_txt, reCreate, ok in data:
- if ok:
- self.assertTrue(self.db.resourceExists(name), msg=description)
- else:
- self.assertFalse(self.db.resourceExists(name), msg=description)
-
- def test_index_timespan(self):
- data = (
- (
- "#1.1 Simple component - busy",
- "1.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080602T000000Z",
- "mailto:user1 at example.com",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
- ),
- (
- "#1.2 Simple component - transparent",
- "1.2",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.2
-DTSTART:20080602T120000Z
-DTEND:20080602T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-TRANSP:TRANSPARENT
-END:VEVENT
-END:VCALENDAR
-""",
- "20080602T000000Z", "20080603T000000Z",
- "mailto:user1 at example.com",
- (('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),),
- ),
- (
- "#1.3 Simple component - canceled",
- "1.3",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.3
-DTSTART:20080603T120000Z
-DTEND:20080603T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-STATUS:CANCELLED
-END:VEVENT
-END:VCALENDAR
-""",
- "20080603T000000Z", "20080604T000000Z",
- "mailto:user1 at example.com",
- (('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'F', 'F'),),
- ),
- (
- "#1.4 Simple component - tentative",
- "1.4",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.4
-DTSTART:20080604T120000Z
-DTEND:20080604T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-STATUS:TENTATIVE
-END:VEVENT
-END:VCALENDAR
-""",
- "20080604T000000Z", "20080605T000000Z",
- "mailto:user1 at example.com",
- (('N', "2008-06-04 12:00:00+00:00", "2008-06-04 13:00:00+00:00", 'T', 'F'),),
- ),
- (
- "#2.1 Recurring component - busy",
- "2.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.1
-DTSTART:20080605T120000Z
-DTEND:20080605T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=DAILY;COUNT=2
-END:VEVENT
-END:VCALENDAR
-""",
- "20080605T000000Z", "20080607T000000Z",
- "mailto:user1 at example.com",
- (
- ('N', "2008-06-05 12:00:00+00:00", "2008-06-05 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-06 12:00:00+00:00", "2008-06-06 13:00:00+00:00", 'B', 'F'),
- ),
- ),
- (
- "#2.2 Recurring component - busy",
- "2.2",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.2
-DTSTART:20080607T120000Z
-DTEND:20080607T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=DAILY;COUNT=2
-END:VEVENT
-BEGIN:VEVENT
-UID:12345-67890-2.2
-RECURRENCE-ID:20080608T120000Z
-DTSTART:20080608T140000Z
-DTEND:20080608T150000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-TRANSP:TRANSPARENT
-END:VEVENT
-END:VCALENDAR
-""",
- "20080607T000000Z", "20080609T000000Z",
- "mailto:user1 at example.com",
- (
- ('N', "2008-06-07 12:00:00+00:00", "2008-06-07 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-08 14:00:00+00:00", "2008-06-08 15:00:00+00:00", 'B', 'T'),
- ),
- ),
- )
-
- for description, name, calendar_txt, trstart, trend, organizer, instances in data:
- calendar = Component.fromString(calendar_txt)
-
- f = open(os.path.join(self.indexDirPath.path, name), "w")
- f.write(calendar_txt)
- del f
-
- self.db.addResource(name, calendar)
- self.assertTrue(self.db.resourceExists(name), msg=description)
-
- # Create fake filter element to match time-range
- filter = caldavxml.Filter(
- caldavxml.ComponentFilter(
- caldavxml.ComponentFilter(
- TimeRange(
- start=trstart,
- end=trend,
- ),
- name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
- ),
- name="VCALENDAR",
- )
- )
- filter = calendarqueryfilter.Filter(filter)
-
- resources = self.db.indexedSearch(filter, fbtype=True)
- index_results = set()
- for _ignore_name, _ignore_uid, type, test_organizer, float, start, end, fbtype, transp in resources:
- self.assertEqual(test_organizer, organizer, msg=description)
- index_results.add((float, start, end, fbtype, transp,))
-
- self.assertEqual(set(instances), index_results, msg=description)
-
- def test_index_timespan_per_user(self):
- data = (
- (
- "#1.1 Single per-user non-recurring component",
- "1.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.1
-X-CALENDARSERVER-PERUSER-UID:user01
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080602T000000Z",
- "mailto:user1 at example.com",
- (
- (
- "user01",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),),
- ),
- (
- "user02",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
- ),
- ),
- ),
- (
- "#1.2 Two per-user non-recurring component",
- "1.2",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.2
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.2
-X-CALENDARSERVER-PERUSER-UID:user01
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.2
-X-CALENDARSERVER-PERUSER-UID:user02
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080602T000000Z",
- "mailto:user1 at example.com",
- (
- (
- "user01",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),),
- ),
- (
- "user02",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
- ),
- (
- "user03",
- (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
- ),
- ),
- ),
- (
- "#2.1 Single per-user simple recurring component",
- "2.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=DAILY;COUNT=10
-END:VEVENT
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.1
-X-CALENDARSERVER-PERUSER-UID:user01
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080603T000000Z",
- "mailto:user1 at example.com",
- (
- (
- "user01",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),
- ),
- ),
- (
- "user02",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
- ),
- ),
- ),
- ),
- (
- "#2.2 Two per-user simple recurring component",
- "2.2",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.2
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=DAILY;COUNT=10
-END:VEVENT
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.2
-X-CALENDARSERVER-PERUSER-UID:user01
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.2
-X-CALENDARSERVER-PERUSER-UID:user02
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080603T000000Z",
- "mailto:user1 at example.com",
- (
- (
- "user01",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),
- ),
- ),
- (
- "user02",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
- ),
- ),
- (
- "user03",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
- ),
- ),
- ),
- ),
- (
- "#3.1 Single per-user complex recurring component",
- "3.1",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=DAILY;COUNT=10
-END:VEVENT
-BEGIN:VEVENT
-UID:12345-67890-1.1
-RECURRENCE-ID:20080602T120000Z
-DTSTART:20080602T130000Z
-DTEND:20080602T140000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.1
-X-CALENDARSERVER-PERUSER-UID:user01
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-RECURRENCE-ID:20080602T120000Z
-TRANSP:OPAQUE
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080604T000000Z",
- "mailto:user1 at example.com",
- (
- (
- "user01",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
- ),
- ),
- (
- "user02",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'F'),
- ),
- ),
- ),
- ),
- (
- "#3.2 Two per-user complex recurring component",
- "3.2",
- """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.2
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=DAILY;COUNT=10
-END:VEVENT
-BEGIN:VEVENT
-UID:12345-67890-1.2
-RECURRENCE-ID:20080602T120000Z
-DTSTART:20080602T130000Z
-DTEND:20080602T140000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.2
-X-CALENDARSERVER-PERUSER-UID:user01
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-RECURRENCE-ID:20080602T120000Z
-TRANSP:OPAQUE
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-BEGIN:X-CALENDARSERVER-PERUSER
-UID:12345-67890-1.2
-X-CALENDARSERVER-PERUSER-UID:user02
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-BEGIN:VALARM
-ACTION:DISPLAY
-DESCRIPTION:Test
-TRIGGER;RELATED=START:-PT10M
-END:VALARM
-END:X-CALENDARSERVER-PERINSTANCE
-BEGIN:X-CALENDARSERVER-PERINSTANCE
-RECURRENCE-ID:20080603T120000Z
-TRANSP:TRANSPARENT
-END:X-CALENDARSERVER-PERINSTANCE
-END:X-CALENDARSERVER-PERUSER
-END:VCALENDAR
-""",
- "20080601T000000Z", "20080604T000000Z",
- "mailto:user1 at example.com",
- (
- (
- "user01",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
- ),
- ),
- (
- "user02",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
- ),
- ),
- (
- "user03",
- (
- ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
- ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'F'),
- ),
- ),
- ),
- ),
- )
-
- for description, name, calendar_txt, trstart, trend, organizer, peruserinstances in data:
- calendar = Component.fromString(calendar_txt)
-
- f = open(os.path.join(self.indexDirPath.path, name), "w")
- f.write(calendar_txt)
- del f
-
- self.db.addResource(name, calendar)
- self.assertTrue(self.db.resourceExists(name), msg=description)
-
- # Create fake filter element to match time-range
- filter = caldavxml.Filter(
- caldavxml.ComponentFilter(
- caldavxml.ComponentFilter(
- TimeRange(
- start=trstart,
- end=trend,
- ),
- name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
- ),
- name="VCALENDAR",
- )
- )
- filter = calendarqueryfilter.Filter(filter)
-
- for useruid, instances in peruserinstances:
- resources = self.db.indexedSearch(filter, useruid=useruid, fbtype=True)
- index_results = set()
- for _ignore_name, _ignore_uid, type, test_organizer, float, start, end, fbtype, transp in resources:
- self.assertEqual(test_organizer, organizer, msg=description)
- index_results.add((str(float), str(start), str(end), str(fbtype), str(transp),))
-
- self.assertEqual(set(instances), index_results, msg="%s, user:%s" % (description, useruid,))
-
- self.db.deleteResource(name)
-
- def test_index_revisions(self):
- data1 = """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-1.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-END:VEVENT
-END:VCALENDAR
-"""
- data2 = """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.1
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=WEEKLY;COUNT=2
-END:VEVENT
-END:VCALENDAR
-"""
- data3 = """BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-BEGIN:VEVENT
-UID:12345-67890-2.3
-DTSTART:20080601T120000Z
-DTEND:20080601T130000Z
-ORGANIZER;CN="User 01":mailto:user1 at example.com
-ATTENDEE:mailto:user1 at example.com
-ATTENDEE:mailto:user2 at example.com
-RRULE:FREQ=WEEKLY;COUNT=2
-END:VEVENT
-END:VCALENDAR
-"""
-
- calendar = Component.fromString(data1)
- self.db.addResource("data1.ics", calendar)
- calendar = Component.fromString(data2)
- self.db.addResource("data2.ics", calendar)
- calendar = Component.fromString(data3)
- self.db.addResource("data3.ics", calendar)
- self.db.deleteResource("data3.ics")
-
- tests = (
- (0, (["data1.ics", "data2.ics",], [],)),
- (1, (["data2.ics",], ["data3.ics",],)),
- (2, ([], ["data3.ics",],)),
- (3, ([], ["data3.ics",],)),
- (4, ([], [],)),
- (5, ([], [],)),
- )
-
- for revision, results in tests:
- self.assertEquals(self.db.whatchanged(revision), results, "Mismatched results for whatchanged with revision %d" % (revision,))
-
-class MemcacheTests(SQLIndexTests):
- def setUp(self):
- super(MemcacheTests, self).setUp()
- self.memcache = InMemoryMemcacheProtocol()
- self.db.reserver = MemcachedUIDReserver(self.db, self.memcache)
-
-
- def tearDown(self):
- for _ignore_k, v in self.memcache._timeouts.iteritems():
- if v.active():
- v.cancel()
Deleted: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_vcardindex.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_vcardindex.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_vcardindex.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -1,209 +0,0 @@
-##
-# Copyright (c) 2010 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-from twisted.internet import reactor
-from twisted.internet.task import deferLater
-
-from twistedcaldav.test.util import InMemoryMemcacheProtocol
-from twistedcaldav.vcard import Component
-from twistedcaldav.vcardindex import AddressBookIndex, MemcachedUIDReserver, ReservationError
-import twistedcaldav.test.util
-
-import os
-
-class MinimalResourceReplacement(object):
- """
- Provide the minimal set of attributes and methods from CalDAVFile required
- by L{Index}.
- """
-
- def __init__(self, filePath):
- self.fp = filePath
-
-
- def isAddressBookCollection(self):
- return True
-
-
- def getChild(self, name):
- # FIXME: this should really return something with a child method
- return self.fp.child(name)
-
-
- def initSyncToken(self):
- pass
-
-
-
-class SQLIndexTests (twistedcaldav.test.util.TestCase):
- """
- Test abstract SQL DB class
- """
-
- def setUp(self):
- super(SQLIndexTests, self).setUp()
- self.site.resource.isAddressBookCollection = lambda: True
- self.indexDirPath = self.site.resource.fp
- # FIXME: since this resource lies about isCalendarCollection, it doesn't
- # have all the associated backend machinery to actually get children.
- self.db = AddressBookIndex(MinimalResourceReplacement(self.indexDirPath))
-
-
- def test_reserve_uid_ok(self):
- uid = "test-test-test"
- d = self.db.isReservedUID(uid)
- d.addCallback(self.assertFalse)
- d.addCallback(lambda _: self.db.reserveUID(uid))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertTrue)
- d.addCallback(lambda _: self.db.unreserveUID(uid))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertFalse)
-
- return d
-
-
- def test_reserve_uid_twice(self):
- uid = "test-test-test"
- d = self.db.reserveUID(uid)
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertTrue)
- d.addCallback(lambda _:
- self.assertFailure(self.db.reserveUID(uid),
- ReservationError))
- return d
-
-
- def test_unreserve_unreserved(self):
- uid = "test-test-test"
- return self.assertFailure(self.db.unreserveUID(uid),
- ReservationError)
-
-
- def test_reserve_uid_timeout(self):
- # WARNING: This test is fundamentally flawed and will fail
- # intermittently because it uses the real clock.
- uid = "test-test-test"
- from twistedcaldav.config import config
- old_timeout = config.UIDReservationTimeOut
- config.UIDReservationTimeOut = 1
-
- def _finally():
- config.UIDReservationTimeOut = old_timeout
-
- d = self.db.isReservedUID(uid)
- d.addCallback(self.assertFalse)
- d.addCallback(lambda _: self.db.reserveUID(uid))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertTrue)
- d.addCallback(lambda _: deferLater(reactor, 2, lambda: None))
- d.addCallback(lambda _: self.db.isReservedUID(uid))
- d.addCallback(self.assertFalse)
- self.addCleanup(_finally)
-
- return d
-
-
- def test_index(self):
- data = (
- (
- "#1.1 Simple component",
- "1.1",
- """BEGIN:VCARD
-VERSION:3.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-UID:12345-67890-1.1
-FN:Cyrus Daboo
-N:Daboo;Cyrus
-EMAIL;TYPE=INTERNET,PREF:cyrus at example.com
-END:VCARD
-""",
- ),
- )
-
- for description, name, vcard_txt in data:
- calendar = Component.fromString(vcard_txt)
- f = open(os.path.join(self.site.resource.fp.path, name), "w")
- f.write(vcard_txt)
- del f
-
- self.db.addResource(name, calendar)
- self.assertTrue(self.db.resourceExists(name), msg=description)
-
- self.db._db_recreate()
- for description, name, vcard_txt in data:
- self.assertTrue(self.db.resourceExists(name), msg=description)
-
- def test_index_revisions(self):
- data1 = """BEGIN:VCARD
-VERSION:3.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-UID:12345-67890-1-1.1
-FN:Cyrus Daboo
-N:Daboo;Cyrus
-EMAIL;TYPE=INTERNET,PREF:cyrus at example.com
-END:VCARD
-"""
- data2 = """BEGIN:VCARD
-VERSION:3.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-UID:12345-67890-2-1.1
-FN:Wilfredo Sanchez
-N:Sanchez;Wilfredo
-EMAIL;TYPE=INTERNET,PREF:wsanchez at example.com
-END:VCARD
-"""
- data3 = """BEGIN:VCARD
-VERSION:3.0
-PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
-UID:12345-67890-3-1.1
-FN:Bruce Gaya
-N:Gaya;Bruce
-EMAIL;TYPE=INTERNET,PREF:bruce at example.com
-END:VCARD
-"""
-
- vcard = Component.fromString(data1)
- self.db.addResource("data1.vcf", vcard)
- vcard = Component.fromString(data2)
- self.db.addResource("data2.vcf", vcard)
- vcard = Component.fromString(data3)
- self.db.addResource("data3.vcf", vcard)
- self.db.deleteResource("data3.vcf")
-
- tests = (
- (0, (["data1.vcf", "data2.vcf",], [],)),
- (1, (["data2.vcf",], ["data3.vcf",],)),
- (2, ([], ["data3.vcf",],)),
- (3, ([], ["data3.vcf",],)),
- (4, ([], [],)),
- (5, ([], [],)),
- )
-
- for revision, results in tests:
- self.assertEquals(self.db.whatchanged(revision), results, "Mismatched results for whatchanged with revision %d" % (revision,))
-
-class MemcacheTests(SQLIndexTests):
- def setUp(self):
- super(MemcacheTests, self).setUp()
- self.memcache = InMemoryMemcacheProtocol()
- self.db.reserver = MemcachedUIDReserver(self.db, self.memcache)
-
-
- def tearDown(self):
- for _ignore_k, v in self.memcache._timeouts.iteritems():
- if v.active():
- v.cancel()
Deleted: CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/vcardindex.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/vcardindex.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/vcardindex.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -1,701 +0,0 @@
-##
-# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-
-"""
-CardDAV Index.
-
-This API is considered private to static.py and is therefore subject to
-change.
-"""
-
-__all__ = [
- "AddressBookIndex",
-]
-
-import datetime
-import os
-import time
-import hashlib
-
-try:
- import sqlite3 as sqlite
-except ImportError:
- from pysqlite2 import dbapi2 as sqlite
-
-from twisted.internet.defer import maybeDeferred
-
-from twistedcaldav import carddavxml
-from twistedcaldav.index import SyncTokenValidException
-from twistedcaldav.query import addressbookquery
-from twistedcaldav.sql import AbstractSQLDatabase
-from twistedcaldav.sql import db_prefix
-from twistedcaldav.vcard import Component
-
-from twext.python.log import Logger, LoggingMixIn
-from twistedcaldav.config import config
-from twistedcaldav.memcachepool import CachePoolUserMixIn
-
-log = Logger()
-
-db_basename = db_prefix + "sqlite"
-schema_version = "2"
-
-class ReservationError(LookupError):
- """
- Attempt to reserve a UID which is already reserved or to unreverse a UID
- which is not reserved.
- """
-
-def wrapInDeferred(f):
- def _(*args, **kwargs):
- return maybeDeferred(f, *args, **kwargs)
-
- return _
-
-
-class MemcachedUIDReserver(CachePoolUserMixIn, LoggingMixIn):
- def __init__(self, index, cachePool=None):
- self.index = index
- self._cachePool = cachePool
-
- def _key(self, uid):
- return 'reservation:%s' % (
- hashlib.md5('%s:%s' % (uid,
- self.index.resource.fp.path)).hexdigest())
-
- def reserveUID(self, uid):
- uid = uid.encode('utf-8')
- self.log_debug("Reserving UID %r @ %r" % (
- uid,
- self.index.resource.fp.path))
-
- def _handleFalse(result):
- if result is False:
- raise ReservationError(
- "UID %s already reserved for address book collection %s."
- % (uid, self.index.resource)
- )
-
- d = self.getCachePool().add(self._key(uid),
- 'reserved',
- expireTime=config.UIDReservationTimeOut)
- d.addCallback(_handleFalse)
- return d
-
-
- def unreserveUID(self, uid):
- uid = uid.encode('utf-8')
- self.log_debug("Unreserving UID %r @ %r" % (
- uid,
- self.index.resource.fp.path))
-
- def _handleFalse(result):
- if result is False:
- raise ReservationError(
- "UID %s is not reserved for address book collection %s."
- % (uid, self.index.resource)
- )
-
- d =self.getCachePool().delete(self._key(uid))
- d.addCallback(_handleFalse)
- return d
-
-
- def isReservedUID(self, uid):
- uid = uid.encode('utf-8')
- self.log_debug("Is reserved UID %r @ %r" % (
- uid,
- self.index.resource.fp.path))
-
- def _checkValue((flags, value)):
- if value is None:
- return False
- else:
- return True
-
- d = self.getCachePool().get(self._key(uid))
- d.addCallback(_checkValue)
- return d
-
-
-
-class SQLUIDReserver(object):
- def __init__(self, index):
- self.index = index
-
- @wrapInDeferred
- def reserveUID(self, uid):
- """
- Reserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is already reserved
- """
-
- try:
- self.index._db_execute("insert into RESERVED (UID, TIME) values (:1, :2)", uid, datetime.datetime.now())
- self.index._db_commit()
- except sqlite.IntegrityError:
- self.index._db_rollback()
- raise ReservationError(
- "UID %s already reserved for address book collection %s."
- % (uid, self.index.resource)
- )
- except sqlite.Error, e:
- log.err("Unable to reserve UID: %s", (e,))
- self.index._db_rollback()
- raise
-
- def unreserveUID(self, uid):
- """
- Unreserve a UID for this index's resource.
- @param uid: the UID to reserve
- @raise ReservationError: if C{uid} is not reserved
- """
-
- def _cb(result):
- if result == False:
- raise ReservationError(
- "UID %s is not reserved for address book collection %s."
- % (uid, self.index.resource)
- )
- else:
- try:
- self.index._db_execute(
- "delete from RESERVED where UID = :1", uid)
- self.index._db_commit()
- except sqlite.Error, e:
- log.err("Unable to unreserve UID: %s", (e,))
- self.index._db_rollback()
- raise
-
- d = self.isReservedUID(uid)
- d.addCallback(_cb)
- return d
-
-
- @wrapInDeferred
- def isReservedUID(self, uid):
- """
- Check to see whether a UID is reserved.
- @param uid: the UID to check
- @return: True if C{uid} is reserved, False otherwise.
- """
-
- rowiter = self.index._db_execute("select UID, TIME from RESERVED where UID = :1", uid)
- for uid, attime in rowiter:
- # Double check that the time is within a reasonable period of now
- # otherwise we probably have a stale reservation
- tm = time.strptime(attime[:19], "%Y-%m-%d %H:%M:%S")
- dt = datetime.datetime(year=tm.tm_year, month=tm.tm_mon, day=tm.tm_mday, hour=tm.tm_hour, minute=tm.tm_min, second = tm.tm_sec)
- if datetime.datetime.now() - dt > datetime.timedelta(seconds=config.UIDReservationTimeOut):
- try:
- self.index._db_execute("delete from RESERVED where UID = :1", uid)
- self.index._db_commit()
- except sqlite.Error, e:
- log.err("Unable to unreserve UID: %s", (e,))
- self.index._db_rollback()
- raise
- return False
- else:
- return True
-
- return False
-
-class AddressBookIndex(AbstractSQLDatabase):
- """
- AddressBook collection index abstract base class that defines the apis for the index.
- """
-
- def __init__(self, resource):
- """
- @param resource: the L{CalDAVResource} resource to
- index. C{resource} must be an addressbook collection (ie.
- C{resource.isAddressBookCollection()} returns C{True}.)
- """
- assert resource.isAddressBookCollection(), "non-addressbook collection resource %s has no index." % (resource,)
- self.resource = resource
- db_filename = os.path.join(self.resource.fp.path, db_basename)
- super(AddressBookIndex, self).__init__(db_filename, False)
-
- if (
- hasattr(config, "Memcached") and
- config.Memcached.Pools.Default.ClientEnabled
- ):
- self.reserver = MemcachedUIDReserver(self)
- else:
- self.reserver = SQLUIDReserver(self)
-
- def create(self):
- """
- Create the index and initialize it.
- """
- self._db()
-
- def recreate(self):
- """
- Delete the database and re-create it
- """
- try:
- os.remove(self.dbpath)
- except OSError:
- pass
- self.create()
-
- #
- # A dict of sets. The dict keys are address book collection paths,
- # and the sets contains reserved UIDs for each path.
- #
-
- def reserveUID(self, uid):
- return self.reserver.reserveUID(uid)
-
- def unreserveUID(self, uid):
- return self.reserver.unreserveUID(uid)
-
- def isReservedUID(self, uid):
- return self.reserver.isReservedUID(uid)
-
- def isAllowedUID(self, uid, *names):
- """
- Checks to see whether to allow an operation which would add the
- specified UID to the index. Specifically, the operation may not
- violate the constraint that UIDs must be unique.
- @param uid: the UID to check
- @param names: the names of resources being replaced or deleted by the
- operation; UIDs associated with these resources are not checked.
- @return: True if the UID is not in the index and is not reserved,
- False otherwise.
- """
- rname = self.resourceNameForUID(uid)
- return (rname is None or rname in names)
-
- def resourceNamesForUID(self, uid):
- """
- Looks up the names of the resources with the given UID.
- @param uid: the UID of the resources to look up.
- @return: a list of resource names
- """
- names = self._db_values_for_sql("select NAME from RESOURCE where UID = :1", uid)
-
- #
- # Check that each name exists as a child of self.resource. If not, the
- # resource record is stale.
- #
- resources = []
- for name in names:
- name_utf8 = name.encode("utf-8")
- if name is not None and self.resource.getChild(name_utf8) is None:
- # Clean up
- log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
- self._delete_from_db(name, uid, False)
- self._db_commit()
- else:
- resources.append(name_utf8)
-
- return resources
-
- def resourceNameForUID(self, uid):
- """
- Looks up the name of the resource with the given UID.
- @param uid: the UID of the resource to look up.
- @return: If the resource is found, its name; C{None} otherwise.
- """
- result = None
-
- for name in self.resourceNamesForUID(uid):
- assert result is None, "More than one resource with UID %s in address book collection %r" % (uid, self)
- result = name
-
- return result
-
- def resourceUIDForName(self, name):
- """
- Looks up the UID of the resource with the given name.
- @param name: the name of the resource to look up.
- @return: If the resource is found, the UID of the resource; C{None}
- otherwise.
- """
- uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
-
- return uid
-
- def addResource(self, name, vcard, fast=False):
- """
- Adding or updating an existing resource.
- To check for an update we attempt to get an existing UID
- for the resource name. If present, then the index entries for
- that UID are removed. After that the new index entries are added.
- @param name: the name of the resource to add.
- @param vCard: a L{Component} object representing the resource
- contents.
- @param fast: if C{True} do not do commit, otherwise do commit.
- """
- oldUID = self.resourceUIDForName(name)
- if oldUID is not None:
- self._delete_from_db(name, oldUID, False)
- self._add_to_db(name, vcard)
- if not fast:
- self._db_commit()
-
- def deleteResource(self, name):
- """
- Remove this resource from the index.
- @param name: the name of the resource to add.
- @param uid: the UID of the vcard component in the resource.
- """
- uid = self.resourceUIDForName(name)
- if uid is not None:
- self._delete_from_db(name, uid)
- self._db_commit()
-
- def resourceExists(self, name):
- """
- Determines whether the specified resource name exists in the index.
- @param name: the name of the resource to test
- @return: True if the resource exists, False if not
- """
- uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
- return uid is not None
-
- def resourcesExist(self, names):
- """
- Determines whether the specified resource name exists in the index.
- @param names: a C{list} containing the names of the resources to test
- @return: a C{list} of all names that exist
- """
- statement = "select NAME from RESOURCE where NAME in ("
- for ctr, ignore_name in enumerate(names):
- if ctr != 0:
- statement += ", "
- statement += ":%s" % (ctr,)
- statement += ")"
- results = self._db_values_for_sql(statement, *names)
- return results
-
- def whatchanged(self, revision):
-
- results = [(name.encode("utf-8"), deleted) for name, deleted in self._db_execute("select NAME, DELETED from REVISIONS where REVISION > :1", revision)]
- results.sort(key=lambda x:x[1])
-
- changed = []
- deleted = []
- for name, wasdeleted in results:
- if name:
- if wasdeleted == 'Y':
- if revision:
- deleted.append(name)
- else:
- changed.append(name)
- else:
- raise SyncTokenValidException
-
- return changed, deleted,
-
- def lastRevision(self):
- return self._db_value_for_sql(
- "select REVISION from REVISION_SEQUENCE"
- )
-
- def bumpRevision(self, fast=False):
- self._db_execute(
- """
- update REVISION_SEQUENCE set REVISION = REVISION + 1
- """,
- )
- if not fast:
- self._db_commit()
- return self._db_value_for_sql(
- """
- select REVISION from REVISION_SEQUENCE
- """,
- )
-
- def searchValid(self, filter):
- if isinstance(filter, carddavxml.Filter):
- qualifiers = addressbookquery.sqladdressbookquery(filter)
- else:
- qualifiers = None
-
- return qualifiers is not None
-
- def search(self, filter):
- """
- Finds resources matching the given qualifiers.
- @param filter: the L{Filter} for the addressbook-query to execute.
- @return: an interable iterable of tuples for each resource matching the
- given C{qualifiers}. The tuples are C{(name, uid, type)}, where
- C{name} is the resource name, C{uid} is the resource UID, and
- C{type} is the resource iCalendar component type.x
- """
- # FIXME: Don't forget to use maximum_future_expansion_duration when we
- # start caching...
-
- # Make sure we have a proper Filter element and get the partial SQL statement to use.
- if isinstance(filter, carddavxml.Filter):
- qualifiers = addressbookquery.sqladdressbookquery(filter)
- else:
- qualifiers = None
- if qualifiers is not None:
- rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID" + qualifiers[0], *qualifiers[1])
- else:
- rowiter = self._db_execute("select NAME, UID from RESOURCE")
-
- for row in rowiter:
- name = row[0]
- if self.resource.getChild(name.encode("utf-8")):
- yield row
- else:
- log.err("vCard resource %s is missing from %s. Removing from index."
- % (name, self.resource))
- self.deleteResource(name, None)
-
- def bruteForceSearch(self):
- """
- List the whole index and tests for existence, updating the index
- @return: all resources in the index
- """
- # List all resources
- rowiter = self._db_execute("select NAME, UID from RESOURCE")
-
- # Check result for missing resources:
-
- for row in rowiter:
- name = row[0]
- if self.resource.getChild(name.encode("utf-8")):
- yield row
- else:
- log.err("AddressBook resource %s is missing from %s. Removing from index."
- % (name, self.resource))
- self.deleteResource(name)
-
-
- def _db_version(self):
- """
- @return: the schema version assigned to this index.
- """
- return schema_version
-
- def _db_type(self):
- """
- @return: the collection type assigned to this index.
- """
- return "AddressBook"
-
- def _db_init_data_tables(self, q):
- """
- Initialise the underlying database tables.
- @param q: a database cursor to use.
- """
-
- # Create database where the RESOURCE table has unique UID column.
- self._db_init_data_tables_base(q, True)
-
- def _db_init_data_tables_base(self, q, uidunique):
- """
- Initialise the underlying database tables.
- @param q: a database cursor to use.
- """
- #
- # RESOURCE table is the primary index table
- # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
- # UID: iCalendar UID (may or may not be unique)
- #
- q.execute(
- """
- create table RESOURCE (
- NAME text unique,
- UID text unique
- )
- """
- )
-
- #
- # REVISIONS table tracks changes
- # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
- # REVISION: revision number
- # WASDELETED: Y if revision deleted, N if added or changed
- #
- q.execute(
- """
- create table REVISION_SEQUENCE (
- REVISION integer
- )
- """
- )
- q.execute(
- """
- insert into REVISION_SEQUENCE (REVISION) values (0)
- """
- )
- q.execute(
- """
- create table REVISIONS (
- NAME text unique,
- REVISION integer default 0,
- DELETED text(1) default "N"
- )
- """
- )
- q.execute(
- """
- create index REVISION on REVISIONS (REVISION)
- """
- )
-
- #
- # RESERVED table tracks reserved UIDs
- # UID: The UID being reserved
- # TIME: When the reservation was made
- #
- q.execute(
- """
- create table RESERVED (
- UID text unique,
- TIME date
- )
- """
- )
-
- def _db_recreate(self, do_commit=True):
- """
- Re-create the database tables from existing address book data.
- """
-
- #
- # Populate the DB with data from already existing resources.
- # This allows for index recovery if the DB file gets
- # deleted.
- #
- fp = self.resource.fp
- for name in fp.listdir():
- if name.startswith("."):
- continue
-
- try:
- stream = fp.child(name).open()
- except (IOError, OSError), e:
- log.err("Unable to open resource %s: %s" % (name, e))
- continue
-
- try:
- # FIXME: This is blocking I/O
- try:
- vcard = Component.fromStream(stream)
- vcard.validForCardDAV()
- except ValueError:
- log.err("Non-addressbook resource: %s" % (name,))
- else:
- #log.msg("Indexing resource: %s" % (name,))
- self.addResource(name, vcard, True)
- finally:
- stream.close()
-
- # Do commit outside of the loop for better performance
- if do_commit:
- self._db_commit()
-
- def _db_can_upgrade(self, old_version):
- """
- Can we do an in-place upgrade
- """
-
- # v2 is a minor change
- return True
-
- def _db_upgrade_data_tables(self, q, old_version):
- """
- Upgrade the data from an older version of the DB.
- """
-
- # When going to version 2+ all we need to do is add revision table and index
- if old_version < 2:
- q.execute(
- """
- create table REVISION_SEQUENCE (
- REVISION integer
- )
- """
- )
- q.execute(
- """
- insert into REVISION_SEQUENCE (REVISION) values (0)
- """
- )
- q.execute(
- """
- create table REVISIONS (
- NAME text unique,
- REVISION integer default 0,
- CREATEDREVISION integer default 0,
- WASDELETED text(1) default "N"
- )
- """
- )
- q.execute(
- """
- create index REVISION on REVISIONS (REVISION)
- """
- )
-
- self._db_execute(
- """
- insert into REVISIONS (NAME)
- select NAME from RESOURCE
- """
- )
-
-
- def _add_to_db(self, name, vcard, cursor = None):
- """
- Records the given address book resource in the index with the given name.
- Resource names and UIDs must both be unique; only one resource name may
- be associated with any given UID and vice versa.
- NB This method does not commit the changes to the db - the caller
- MUST take care of that
- @param name: the name of the resource to add.
- @param vcard: a L{AddressBook} object representing the resource
- contents.
- """
- uid = vcard.resourceUID()
-
- self._db_execute(
- """
- insert into RESOURCE (NAME, UID)
- values (:1, :2)
- """, name, uid,
- )
-
- self._db_execute(
- """
- insert or replace into REVISIONS (NAME, REVISION, DELETED)
- values (:1, :2, :3)
- """, name, self.bumpRevision(fast=True), 'N',
- )
-
- def _delete_from_db(self, name, uid, dorevision=True):
- """
- Deletes the specified entry from all dbs.
- @param name: the name of the resource to delete.
- @param uid: the uid of the resource to delete.
- """
- self._db_execute("delete from RESOURCE where NAME = :1", name)
- if dorevision:
- self._db_execute(
- """
- update REVISIONS SET REVISION = :1, DELETED = :2
- where NAME = :3
- """, self.bumpRevision(fast=True), 'Y', name
- )
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/file.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/file.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/file.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -44,13 +44,14 @@
from twistedcaldav import caldavxml, customxml
from twistedcaldav.caldavxml import ScheduleCalendarTransp, Opaque
-from twistedcaldav.index import Index as OldIndex, IndexSchedule as OldInboxIndex
from twistedcaldav.sharing import InvitesDatabase
from txdav.caldav.icalendarstore import IAttachment
from txdav.caldav.icalendarstore import ICalendar, ICalendarObject
from txdav.caldav.icalendarstore import ICalendarHome
+from txdav.caldav.datastore.index_file import Index as OldIndex,\
+ IndexSchedule as OldInboxIndex
from txdav.caldav.datastore.util import (
validateCalendarComponent, dropboxIDFromCalendarObject
)
Copied: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py (from rev 6322, CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/index.py)
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py (rev 0)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/index_file.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -0,0 +1,1148 @@
+# -*- test-case-name: twistedcaldav.test.test_index -*-
+##
+# Copyright (c) 2005-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+CalDAV Index.
+
+This API is considered private to static.py and is therefore subject to
+change.
+"""
+
+__all__ = [
+ "db_basename",
+ "ReservationError",
+ "MemcachedUIDReserver",
+ "Index",
+ "IndexSchedule",
+ "IndexedSearchException",
+]
+
+import datetime
+import time
+import hashlib
+
+try:
+ import sqlite3 as sqlite
+except ImportError:
+ from pysqlite2 import dbapi2 as sqlite
+
+from vobject.icalendar import utc
+
+from twisted.internet.defer import maybeDeferred, succeed
+
+from twext.python.log import Logger, LoggingMixIn
+
+from txdav.common.icommondatastore import SyncTokenValidException,\
+ ReservationError, IndexedSearchException
+
+from twistedcaldav.ical import Component
+from twistedcaldav.query import calendarquery, calendarqueryfilter
+from twistedcaldav.sql import AbstractSQLDatabase
+from twistedcaldav.sql import db_prefix
+from twistedcaldav.instance import InvalidOverriddenInstanceError
+from twistedcaldav.config import config
+from twistedcaldav.memcachepool import CachePoolUserMixIn
+
+log = Logger()
+
+db_basename = db_prefix + "sqlite"
+schema_version = "10"
+collection_types = {"Calendar": "Regular Calendar Collection", "iTIP": "iTIP Calendar Collection"}
+
+icalfbtype_to_indexfbtype = {
+ "FREE" : 'F',
+ "BUSY" : 'B',
+ "BUSY-UNAVAILABLE": 'U',
+ "BUSY-TENTATIVE" : 'T',
+}
+indexfbtype_to_icalfbtype = dict([(v, k) for k,v in icalfbtype_to_indexfbtype.iteritems()])
+
+#
+# Duration into the future through which recurrences are expanded in the index
+# by default. This is a caching parameter which affects the size of the index;
+# it does not affect search results beyond this period, but it may affect
+# performance of such a search.
+#
+default_future_expansion_duration = datetime.timedelta(days=365*1)
+
+#
+# Maximum duration into the future through which recurrences are expanded in the
+# index. This is a caching parameter which affects the size of the index; it
+# does not affect search results beyond this period, but it may affect
+# performance of such a search.
+#
+# When a search is performed on a time span that goes beyond that which is
+# expanded in the index, we have to open each resource which may have data in
+# that time period. In order to avoid doing that multiple times, we want to
+# cache those results. However, we don't necessarily want to cache all
+# occurrences into some obscenely far-in-the-future date, so we cap the caching
+# period. Searches beyond this period will always be relatively expensive for
+# resources with occurrences beyond this period.
+#
+maximum_future_expansion_duration = datetime.timedelta(days=365*5)
+
+
+class AbstractCalendarIndex(AbstractSQLDatabase, LoggingMixIn):
+ """
+ Calendar collection index abstract base class that defines the apis for the index.
+ This will be subclassed for the two types of index behaviour we need: one for
+ regular calendar collections, one for schedule calendar collections.
+ """
+
+ def __init__(self, resource):
+ """
+ @param resource: the L{CalDAVResource} resource to
+ index. C{resource} must be a calendar collection (ie.
+ C{resource.isPseudoCalendarCollection()} returns C{True}.)
+ """
+ self.resource = resource
+ db_filename = self.resource.fp.child(db_basename).path
+ super(AbstractCalendarIndex, self).__init__(db_filename, False)
+
+ def create(self):
+ """
+ Create the index and initialize it.
+ """
+ self._db()
+
+ def reserveUID(self, uid):
+ """
+ Reserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is already reserved
+ """
+ raise NotImplementedError
+
+ def unreserveUID(self, uid):
+ """
+ Unreserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is not reserved
+ """
+ raise NotImplementedError
+
+ def isReservedUID(self, uid):
+ """
+ Check to see whether a UID is reserved.
+ @param uid: the UID to check
+ @return: True if C{uid} is reserved, False otherwise.
+ """
+ raise NotImplementedError
+
+ def isAllowedUID(self, uid, *names):
+ """
+ Checks to see whether to allow an operation with adds the the specified
+ UID is allowed to the index. Specifically, the operation may not
+ violate the constraint that UIDs must be unique, and the UID must not
+ be reserved.
+ @param uid: the UID to check
+ @param names: the names of resources being replaced or deleted by the
+ operation; UIDs associated with these resources are not checked.
+ @return: True if the UID is not in the index and is not reserved,
+ False otherwise.
+ """
+ raise NotImplementedError
+
+ def resourceNamesForUID(self, uid):
+ """
+ Looks up the names of the resources with the given UID.
+ @param uid: the UID of the resources to look up.
+ @return: a list of resource names
+ """
+ names = self._db_values_for_sql("select NAME from RESOURCE where UID = :1", uid)
+
+ #
+ # Check that each name exists as a child of self.resource. If not, the
+ # resource record is stale.
+ #
+ resources = []
+ for name in names:
+ name_utf8 = name.encode("utf-8")
+ if name is not None and self.resource.getChild(name_utf8) is None:
+ # Clean up
+ log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
+ self._delete_from_db(name, uid, False)
+ self._db_commit()
+ else:
+ resources.append(name_utf8)
+
+ return resources
+
+ def resourceNameForUID(self, uid):
+ """
+ Looks up the name of the resource with the given UID.
+ @param uid: the UID of the resource to look up.
+ @return: If the resource is found, its name; C{None} otherwise.
+ """
+ result = None
+
+ for name in self.resourceNamesForUID(uid):
+ assert result is None, "More than one resource with UID %s in calendar collection %r" % (uid, self)
+ result = name
+
+ return result
+
+ def resourceUIDForName(self, name):
+ """
+ Looks up the UID of the resource with the given name.
+ @param name: the name of the resource to look up.
+ @return: If the resource is found, the UID of the resource; C{None}
+ otherwise.
+ """
+ uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
+
+ return uid
+
+ def addResource(self, name, calendar, fast=False, reCreate=False):
+ """
+ Adding or updating an existing resource.
+ To check for an update we attempt to get an existing UID
+ for the resource name. If present, then the index entries for
+ that UID are removed. After that the new index entries are added.
+ @param name: the name of the resource to add.
+ @param calendar: a L{Calendar} object representing the resource
+ contents.
+ @param fast: if C{True} do not do commit, otherwise do commit.
+ """
+ oldUID = self.resourceUIDForName(name)
+ if oldUID is not None:
+ self._delete_from_db(name, oldUID, False)
+ self._add_to_db(name, calendar, reCreate=reCreate)
+ if not fast:
+ self._db_commit()
+
+ def deleteResource(self, name):
+ """
+ Remove this resource from the index.
+ @param name: the name of the resource to add.
+ @param uid: the UID of the calendar component in the resource.
+ """
+ uid = self.resourceUIDForName(name)
+ if uid is not None:
+ self._delete_from_db(name, uid)
+ self._db_commit()
+
+ def resourceExists(self, name):
+ """
+ Determines whether the specified resource name exists in the index.
+ @param name: the name of the resource to test
+ @return: True if the resource exists, False if not
+ """
+ uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
+ return uid is not None
+
+ def resourcesExist(self, names):
+ """
+ Determines whether the specified resource name exists in the index.
+ @param names: a C{list} containing the names of the resources to test
+ @return: a C{list} of all names that exist
+ """
+ statement = "select NAME from RESOURCE where NAME in ("
+ for ctr in (item[0] for item in enumerate(names)):
+ if ctr != 0:
+ statement += ", "
+ statement += ":%s" % (ctr,)
+ statement += ")"
+ results = self._db_values_for_sql(statement, *names)
+ return results
+
+
+ def testAndUpdateIndex(self, minDate):
+ # Find out if the index is expanded far enough
+ names = self.notExpandedBeyond(minDate)
+ # Actually expand recurrence max
+ for name in names:
+ self.log_info("Search falls outside range of index for %s %s" % (name, minDate))
+ self.reExpandResource(name, minDate)
+
+ def whatchanged(self, revision):
+
+ results = [(name.encode("utf-8"), deleted) for name, deleted in self._db_execute("select NAME, DELETED from REVISIONS where REVISION > :1", revision)]
+ results.sort(key=lambda x:x[1])
+
+ changed = []
+ deleted = []
+ for name, wasdeleted in results:
+ if name:
+ if wasdeleted == 'Y':
+ if revision:
+ deleted.append(name)
+ else:
+ changed.append(name)
+ else:
+ raise SyncTokenValidException
+
+ return changed, deleted,
+
+ def lastRevision(self):
+ return self._db_value_for_sql(
+ "select REVISION from REVISION_SEQUENCE"
+ )
+
+ def bumpRevision(self, fast=False):
+ self._db_execute(
+ """
+ update REVISION_SEQUENCE set REVISION = REVISION + 1
+ """,
+ )
+ self._db_commit()
+ return self._db_value_for_sql(
+ """
+ select REVISION from REVISION_SEQUENCE
+ """,
+ )
+
+ def indexedSearch(self, filter, useruid="", fbtype=False):
+ """
+ Finds resources matching the given qualifiers.
+ @param filter: the L{Filter} for the calendar-query to execute.
+ @return: an iterable of tuples for each resource matching the
+ given C{qualifiers}. The tuples are C{(name, uid, type)}, where
+ C{name} is the resource name, C{uid} is the resource UID, and
+ C{type} is the resource iCalendar component type.
+ """
+
+ # Make sure we have a proper Filter element and get the partial SQL
+ # statement to use.
+ if isinstance(filter, calendarqueryfilter.Filter):
+ if fbtype:
+ # Lookup the useruid - try the empty (default) one if needed
+ dbuseruid = self._db_value_for_sql(
+ "select PERUSERID from PERUSER where USERUID == :1",
+ useruid,
+ )
+ else:
+ dbuseruid = ""
+
+ qualifiers = calendarquery.sqlcalendarquery(filter, None, dbuseruid)
+ if qualifiers is not None:
+ # Determine how far we need to extend the current expansion of
+ # events. If we have an open-ended time-range we will expand one
+ # year past the start. That should catch bounded recurrences - unbounded
+ # will have been indexed with an "infinite" value always included.
+ maxDate, isStartDate = filter.getmaxtimerange()
+ if maxDate:
+ maxDate = maxDate.date()
+ if isStartDate:
+ maxDate += datetime.timedelta(days=365)
+ self.testAndUpdateIndex(maxDate)
+ else:
+ # We cannot handler this filter in an indexed search
+ raise IndexedSearchException()
+
+ else:
+ qualifiers = None
+
+ # Perform the search
+ if qualifiers is None:
+ rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
+ else:
+ if fbtype:
+ # Lookup the useruid - try the empty (default) one if needed
+ dbuseruid = self._db_value_for_sql(
+ "select PERUSERID from PERUSER where USERUID == :1",
+ useruid,
+ )
+
+ # For a free-busy time-range query we return all instances
+ rowiter = self._db_execute(
+ "select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE, RESOURCE.ORGANIZER, TIMESPAN.FLOAT, TIMESPAN.START, TIMESPAN.END, TIMESPAN.FBTYPE, TIMESPAN.TRANSPARENT, TRANSPARENCY.TRANSPARENT" +
+ qualifiers[0],
+ *qualifiers[1]
+ )
+ else:
+ rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1])
+
+ # Check result for missing resources
+
+ for row in rowiter:
+ name = row[0]
+ if self.resource.getChild(name.encode("utf-8")):
+ if fbtype:
+ row = list(row)
+ if row[9]:
+ row[8] = row[9]
+ del row[9]
+ yield row
+ else:
+ log.err("Calendar resource %s is missing from %s. Removing from index."
+ % (name, self.resource))
+ self.deleteResource(name)
+
+ def bruteForceSearch(self):
+ """
+ List the whole index and tests for existence, updating the index
+ @return: all resources in the index
+ """
+ # List all resources
+ rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE")
+
+ # Check result for missing resources:
+
+ for row in rowiter:
+ name = row[0]
+ if self.resource.getChild(name.encode("utf-8")):
+ yield row
+ else:
+ log.err("Calendar resource %s is missing from %s. Removing from index."
+ % (name, self.resource))
+ self.deleteResource(name)
+
+
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return schema_version
+
+ def _add_to_db(self, name, calendar, cursor=None, expand_until=None, reCreate=False):
+ """
+ Records the given calendar resource in the index with the given name.
+ Resource names and UIDs must both be unique; only one resource name may
+ be associated with any given UID and vice versa.
+ NB This method does not commit the changes to the db - the caller
+ MUST take care of that
+ @param name: the name of the resource to add.
+ @param calendar: a L{Calendar} object representing the resource
+ contents.
+ """
+ raise NotImplementedError
+
+ def _delete_from_db(self, name, uid, dorevision=True):
+ """
+ Deletes the specified entry from all dbs.
+ @param name: the name of the resource to delete.
+ @param uid: the uid of the resource to delete.
+ """
+ raise NotImplementedError
+
+class CalendarIndex (AbstractCalendarIndex):
+ """
+ Calendar index - abstract class for indexer that indexes calendar objects in a collection.
+ """
+
+ def __init__(self, resource):
+ """
+ @param resource: the L{CalDAVResource} resource to
+ index.
+ """
+ super(CalendarIndex, self).__init__(resource)
+
+ def _db_init_data_tables_base(self, q, uidunique):
+ """
+ Initialise the underlying database tables.
+ @param q: a database cursor to use.
+ """
+ #
+ # RESOURCE table is the primary index table
+ # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
+ # UID: iCalendar UID (may or may not be unique)
+ # TYPE: iCalendar component type
+ # RECURRANCE_MAX: Highest date of recurrence expansion
+ # ORGANIZER: cu-address of the Organizer of the event
+ #
+ q.execute(
+ """
+ create table RESOURCE (
+ RESOURCEID integer primary key autoincrement,
+ NAME text unique,
+ UID text%s,
+ TYPE text,
+ RECURRANCE_MAX date,
+ ORGANIZER text
+ )
+ """ % (" unique" if uidunique else "",)
+ )
+
+ #
+ # TIMESPAN table tracks (expanded) time spans for resources
+ # NAME: Related resource (RESOURCE foreign key)
+ # FLOAT: 'Y' if start/end are floating, 'N' otherwise
+ # START: Start date
+ # END: End date
+ # FBTYPE: FBTYPE value:
+ # '?' - unknown
+ # 'F' - free
+ # 'B' - busy
+ # 'U' - busy-unavailable
+ # 'T' - busy-tentative
+ # TRANSPARENT: Y if transparent, N if opaque (default non-per-user value)
+ #
+ q.execute(
+ """
+ create table TIMESPAN (
+ INSTANCEID integer primary key autoincrement,
+ RESOURCEID integer,
+ FLOAT text(1),
+ START date,
+ END date,
+ FBTYPE text(1),
+ TRANSPARENT text(1)
+ )
+ """
+ )
+ q.execute(
+ """
+ create index STARTENDFLOAT on TIMESPAN (START, END, FLOAT)
+ """
+ )
+
+ #
+ # PERUSER table tracks per-user ids
+ # PERUSERID: autoincrement primary key
+ # UID: User ID used in calendar data
+ #
+ q.execute(
+ """
+ create table PERUSER (
+ PERUSERID integer primary key autoincrement,
+ USERUID text
+ )
+ """
+ )
+ q.execute(
+ """
+ create index PERUSER_UID on PERUSER (USERUID)
+ """
+ )
+
+ #
+ # TRANSPARENCY table tracks per-user per-instance transparency
+ # PERUSERID: user id key
+ # INSTANCEID: instance id key
+ # TRANSPARENT: Y if transparent, N if opaque
+ #
+ q.execute(
+ """
+ create table TRANSPARENCY (
+ PERUSERID integer,
+ INSTANCEID integer,
+ TRANSPARENT text(1)
+ )
+ """
+ )
+
+ #
+ # REVISIONS table tracks changes
+ # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
+ # REVISION: revision number
+ # WASDELETED: Y if revision deleted, N if added or changed
+ #
+ q.execute(
+ """
+ create table REVISION_SEQUENCE (
+ REVISION integer
+ )
+ """
+ )
+ q.execute(
+ """
+ insert into REVISION_SEQUENCE (REVISION) values (0)
+ """
+ )
+ q.execute(
+ """
+ create table REVISIONS (
+ NAME text unique,
+ REVISION integer,
+ DELETED text(1)
+ )
+ """
+ )
+ q.execute(
+ """
+ create index REVISION on REVISIONS (REVISION)
+ """
+ )
+
+ if uidunique:
+ #
+ # RESERVED table tracks reserved UIDs
+ # UID: The UID being reserved
+ # TIME: When the reservation was made
+ #
+ q.execute(
+ """
+ create table RESERVED (
+ UID text unique,
+ TIME date
+ )
+ """
+ )
+
+ # Cascading triggers to help on delete
+ q.execute(
+ """
+ create trigger resourceDelete after delete on RESOURCE
+ for each row
+ begin
+ delete from TIMESPAN where TIMESPAN.RESOURCEID = OLD.RESOURCEID;
+ end
+ """
+ )
+ q.execute(
+ """
+ create trigger timespanDelete after delete on TIMESPAN
+ for each row
+ begin
+ delete from TRANSPARENCY where INSTANCEID = OLD.INSTANCEID;
+ end
+ """
+ )
+
+ def _db_can_upgrade(self, old_version):
+ """
+ Can we do an in-place upgrade
+ """
+
+ # v10 is a big change - no upgrade possible
+ return False
+
+ def _db_upgrade_data_tables(self, q, old_version):
+ """
+ Upgrade the data from an older version of the DB.
+ """
+
+ # v10 is a big change - no upgrade possible
+ pass
+
+ def notExpandedBeyond(self, minDate):
+ """
+ Gives all resources which have not been expanded beyond a given date
+ in the index
+ """
+ return self._db_values_for_sql("select NAME from RESOURCE where RECURRANCE_MAX < :1", minDate)
+
+ def reExpandResource(self, name, expand_until):
+ """
+ Given a resource name, remove it from the database and re-add it
+ with a longer expansion.
+ """
+ calendar = self.resource.getChild(name).iCalendar()
+ self._add_to_db(name, calendar, expand_until=expand_until, reCreate=True)
+ self._db_commit()
+
+ def _add_to_db(self, name, calendar, cursor = None, expand_until=None, reCreate=False):
+ """
+ Records the given calendar resource in the index with the given name.
+ Resource names and UIDs must both be unique; only one resource name may
+ be associated with any given UID and vice versa.
+ NB This method does not commit the changes to the db - the caller
+ MUST take care of that
+ @param name: the name of the resource to add.
+ @param calendar: a L{Calendar} object representing the resource
+ contents.
+ """
+ uid = calendar.resourceUID()
+ organizer = calendar.getOrganizer()
+ if not organizer:
+ organizer = ""
+
+ # Decide how far to expand based on the component
+ master = calendar.masterComponent()
+ if master is None or not calendar.isRecurring() and not calendar.isRecurringUnbounded():
+ # When there is no master we have a set of overridden components - index them all.
+ # When there is one instance - index it.
+ # When bounded - index all.
+ expand = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
+ else:
+ if expand_until:
+ expand = expand_until
+ else:
+ expand = datetime.date.today() + default_future_expansion_duration
+
+ if expand > (datetime.date.today() + maximum_future_expansion_duration):
+ raise IndexedSearchException
+
+ try:
+ instances = calendar.expandTimeRanges(expand, ignoreInvalidInstances=reCreate)
+ except InvalidOverriddenInstanceError, e:
+ log.err("Invalid instance %s when indexing %s in %s" % (e.rid, name, self.resource,))
+ raise
+
+ self._delete_from_db(name, uid, False)
+
+ # Add RESOURCE item
+ self._db_execute(
+ """
+ insert into RESOURCE (NAME, UID, TYPE, RECURRANCE_MAX, ORGANIZER)
+ values (:1, :2, :3, :4, :5)
+ """, name, uid, calendar.resourceType(), instances.limit, organizer
+ )
+ resourceid = self.lastrowid
+
+ # Get a set of all referenced per-user UIDs and map those to entries already
+ # in the DB and add new ones as needed
+ useruids = calendar.allPerUserUIDs()
+ useruids.add("")
+ useruidmap = {}
+ for useruid in useruids:
+ peruserid = self._db_value_for_sql(
+ "select PERUSERID from PERUSER where USERUID = :1",
+ useruid
+ )
+ if peruserid is None:
+ self._db_execute(
+ """
+ insert into PERUSER (USERUID)
+ values (:1)
+ """, useruid
+ )
+ peruserid = self.lastrowid
+ useruidmap[useruid] = peruserid
+
+ for key in instances:
+ instance = instances[key]
+ start = instance.start.replace(tzinfo=utc)
+ end = instance.end.replace(tzinfo=utc)
+ float = 'Y' if instance.start.tzinfo is None else 'N'
+ transp = 'T' if instance.component.propertyValue("TRANSP") == "TRANSPARENT" else 'F'
+ self._db_execute(
+ """
+ insert into TIMESPAN (RESOURCEID, FLOAT, START, END, FBTYPE, TRANSPARENT)
+ values (:1, :2, :3, :4, :5, :6)
+ """,
+ resourceid,
+ float,
+ start,
+ end,
+ icalfbtype_to_indexfbtype.get(instance.component.getFBType(), 'F'),
+ transp
+ )
+ instanceid = self.lastrowid
+ peruserdata = calendar.perUserTransparency(instance.rid)
+ for useruid, transp in peruserdata:
+ peruserid = useruidmap[useruid]
+ self._db_execute(
+ """
+ insert into TRANSPARENCY (PERUSERID, INSTANCEID, TRANSPARENT)
+ values (:1, :2, :3)
+ """, peruserid, instanceid, 'T' if transp else 'F'
+ )
+
+
+ # Special - for unbounded recurrence we insert a value for "infinity"
+ # that will allow an open-ended time-range to always match it.
+ if calendar.isRecurringUnbounded():
+ start = datetime.datetime(2100, 1, 1, 0, 0, 0, tzinfo=utc)
+ end = datetime.datetime(2100, 1, 1, 1, 0, 0, tzinfo=utc)
+ float = 'N'
+ self._db_execute(
+ """
+ insert into TIMESPAN (RESOURCEID, FLOAT, START, END, FBTYPE, TRANSPARENT)
+ values (:1, :2, :3, :4, :5, :6)
+ """, resourceid, float, start, end, '?', '?'
+ )
+ instanceid = self.lastrowid
+ peruserdata = calendar.perUserTransparency(None)
+ for useruid, transp in peruserdata:
+ peruserid = useruidmap[useruid]
+ self._db_execute(
+ """
+ insert into TRANSPARENCY (PERUSERID, INSTANCEID, TRANSPARENT)
+ values (:1, :2, :3)
+ """, peruserid, instanceid, 'T' if transp else 'F'
+ )
+
+ self._db_execute(
+ """
+ insert or replace into REVISIONS (NAME, REVISION, DELETED)
+ values (:1, :2, :3)
+ """, name, self.bumpRevision(fast=True), 'N',
+ )
+
+ def _delete_from_db(self, name, uid, dorevision=True):
+ """
+ Deletes the specified entry from all dbs.
+ @param name: the name of the resource to delete.
+ @param uid: the uid of the resource to delete.
+ """
+ self._db_execute("delete from RESOURCE where NAME = :1", name)
+ if dorevision:
+ self._db_execute(
+ """
+ update REVISIONS SET REVISION = :1, DELETED = :2
+ where NAME = :3
+ """, self.bumpRevision(fast=True), 'Y', name
+ )
+
+
+def wrapInDeferred(f):
+ def _(*args, **kwargs):
+ return maybeDeferred(f, *args, **kwargs)
+
+ return _
+
+
+class MemcachedUIDReserver(CachePoolUserMixIn, LoggingMixIn):
+ def __init__(self, index, cachePool=None):
+ self.index = index
+ self._cachePool = cachePool
+
+ def _key(self, uid):
+ return 'reservation:%s' % (
+ hashlib.md5('%s:%s' % (uid,
+ self.index.resource.fp.path)).hexdigest())
+
+ def reserveUID(self, uid):
+ uid = uid.encode('utf-8')
+ self.log_debug("Reserving UID %r @ %r" % (
+ uid,
+ self.index.resource.fp.path))
+
+ def _handleFalse(result):
+ if result is False:
+ raise ReservationError(
+ "UID %s already reserved for calendar collection %s."
+ % (uid, self.index.resource)
+ )
+
+ d = self.getCachePool().add(self._key(uid),
+ 'reserved',
+ expireTime=config.UIDReservationTimeOut)
+ d.addCallback(_handleFalse)
+ return d
+
+
+ def unreserveUID(self, uid):
+ uid = uid.encode('utf-8')
+ self.log_debug("Unreserving UID %r @ %r" % (
+ uid,
+ self.index.resource.fp.path))
+
+ def _handleFalse(result):
+ if result is False:
+ raise ReservationError(
+ "UID %s is not reserved for calendar collection %s."
+ % (uid, self.index.resource)
+ )
+
+ d =self.getCachePool().delete(self._key(uid))
+ d.addCallback(_handleFalse)
+ return d
+
+
+ def isReservedUID(self, uid):
+ uid = uid.encode('utf-8')
+ self.log_debug("Is reserved UID %r @ %r" % (
+ uid,
+ self.index.resource.fp.path))
+
+ def _checkValue((flags, value)):
+ if value is None:
+ return False
+ else:
+ return True
+
+ d = self.getCachePool().get(self._key(uid))
+ d.addCallback(_checkValue)
+ return d
+
+
+
+class SQLUIDReserver(object):
+ def __init__(self, index):
+ self.index = index
+
+ @wrapInDeferred
+ def reserveUID(self, uid):
+ """
+ Reserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is already reserved
+ """
+
+ try:
+ self.index._db_execute("insert into RESERVED (UID, TIME) values (:1, :2)", uid, datetime.datetime.now())
+ self.index._db_commit()
+ except sqlite.IntegrityError:
+ self.index._db_rollback()
+ raise ReservationError(
+ "UID %s already reserved for calendar collection %s."
+ % (uid, self.index.resource)
+ )
+ except sqlite.Error, e:
+ log.err("Unable to reserve UID: %s", (e,))
+ self.index._db_rollback()
+ raise
+
+ def unreserveUID(self, uid):
+ """
+ Unreserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is not reserved
+ """
+
+ def _cb(result):
+ if result == False:
+ raise ReservationError(
+ "UID %s is not reserved for calendar collection %s."
+ % (uid, self.index.resource)
+ )
+ else:
+ try:
+ self.index._db_execute(
+ "delete from RESERVED where UID = :1", uid)
+ self.index._db_commit()
+ except sqlite.Error, e:
+ log.err("Unable to unreserve UID: %s", (e,))
+ self.index._db_rollback()
+ raise
+
+ d = self.isReservedUID(uid)
+ d.addCallback(_cb)
+ return d
+
+
+ @wrapInDeferred
+ def isReservedUID(self, uid):
+ """
+ Check to see whether a UID is reserved.
+ @param uid: the UID to check
+ @return: True if C{uid} is reserved, False otherwise.
+ """
+
+ rowiter = self.index._db_execute("select UID, TIME from RESERVED where UID = :1", uid)
+ for uid, attime in rowiter:
+ # Double check that the time is within a reasonable period of now
+ # otherwise we probably have a stale reservation
+ tm = time.strptime(attime[:19], "%Y-%m-%d %H:%M:%S")
+ dt = datetime.datetime(year=tm.tm_year, month=tm.tm_mon, day=tm.tm_mday, hour=tm.tm_hour, minute=tm.tm_min, second = tm.tm_sec)
+ if datetime.datetime.now() - dt > datetime.timedelta(seconds=config.UIDReservationTimeOut):
+ try:
+ self.index._db_execute("delete from RESERVED where UID = :1", uid)
+ self.index._db_commit()
+ except sqlite.Error, e:
+ log.err("Unable to unreserve UID: %s", (e,))
+ self.index._db_rollback()
+ raise
+ return False
+ else:
+ return True
+
+ return False
+
+
+
+class Index (CalendarIndex):
+ """
+ Calendar collection index - regular collection that enforces CalDAV UID uniqueness requirement.
+ """
+
+ def __init__(self, resource):
+ """
+ @param resource: the L{CalDAVResource} resource to
+ index. C{resource} must be a calendar collection (i.e.
+ C{resource.isPseudoCalendarCollection()} returns C{True}.)
+ """
+ assert resource.isCalendarCollection(), "non-calendar collection resource %s has no index." % (resource,)
+ super(Index, self).__init__(resource)
+
+ if (
+ hasattr(config, "Memcached") and
+ config.Memcached.Pools.Default.ClientEnabled
+ ):
+ self.reserver = MemcachedUIDReserver(self)
+ else:
+ self.reserver = SQLUIDReserver(self)
+
+ #
+ # A dict of sets. The dict keys are calendar collection paths,
+ # and the sets contains reserved UIDs for each path.
+ #
+
+ def reserveUID(self, uid):
+ return self.reserver.reserveUID(uid)
+
+
+ def unreserveUID(self, uid):
+ return self.reserver.unreserveUID(uid)
+
+
+ def isReservedUID(self, uid):
+ return self.reserver.isReservedUID(uid)
+
+
+ def isAllowedUID(self, uid, *names):
+ """
+ Checks to see whether to allow an operation which would add the
+ specified UID to the index. Specifically, the operation may not
+ violate the constraint that UIDs must be unique.
+ @param uid: the UID to check
+ @param names: the names of resources being replaced or deleted by the
+ operation; UIDs associated with these resources are not checked.
+ @return: True if the UID is not in the index and is not reserved,
+ False otherwise.
+ """
+ rname = self.resourceNameForUID(uid)
+ return (rname is None or rname in names)
+
+ def _db_type(self):
+ """
+ @return: the collection type assigned to this index.
+ """
+ return collection_types["Calendar"]
+
+ def _db_init_data_tables(self, q):
+ """
+ Initialise the underlying database tables.
+ @param q: a database cursor to use.
+ """
+
+ # Create database where the RESOURCE table has unique UID column.
+ self._db_init_data_tables_base(q, True)
+
+ def _db_recreate(self, do_commit=True):
+ """
+ Re-create the database tables from existing calendar data.
+ """
+
+ #
+ # Populate the DB with data from already existing resources.
+ # This allows for index recovery if the DB file gets
+ # deleted.
+ #
+ fp = self.resource.fp
+ for name in fp.listdir():
+ if name.startswith("."):
+ continue
+
+ try:
+ stream = fp.child(name).open()
+ except (IOError, OSError), e:
+ log.err("Unable to open resource %s: %s" % (name, e))
+ continue
+
+ # FIXME: This is blocking I/O
+ try:
+ calendar = Component.fromStream(stream)
+ calendar.validateForCalDAV()
+ except ValueError:
+ log.err("Non-calendar resource: %s" % (name,))
+ else:
+ #log.msg("Indexing resource: %s" % (name,))
+ self.addResource(name, calendar, True, reCreate=True)
+ finally:
+ stream.close()
+
+ # Do commit outside of the loop for better performance
+ if do_commit:
+ self._db_commit()
+
+class IndexSchedule (CalendarIndex):
+ """
+ Schedule collection index - does not require UID uniqueness.
+ """
+
+ def reserveUID(self, uid): #@UnusedVariable
+ """
+ Reserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is already reserved
+ """
+
+ # iTIP does not require unique UIDs
+ return succeed(None)
+
+ def unreserveUID(self, uid): #@UnusedVariable
+ """
+ Unreserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is not reserved
+ """
+
+ # iTIP does not require unique UIDs
+ return succeed(None)
+
+ def isReservedUID(self, uid): #@UnusedVariable
+ """
+ Check to see whether a UID is reserved.
+ @param uid: the UID to check
+ @return: True if C{uid} is reserved, False otherwise.
+ """
+
+ # iTIP does not require unique UIDs
+ return succeed(False)
+
+ def isAllowedUID(self, uid, *names): #@UnusedVariable
+ """
+ Checks to see whether to allow an operation with adds the the specified
+ UID is allowed to the index. Specifically, the operation may not
+ violate the constraint that UIDs must be unique, and the UID must not
+ be reserved.
+ @param uid: the UID to check
+ @param names: the names of resources being replaced or deleted by the
+ operation; UIDs associated with these resources are not checked.
+ @return: True if the UID is not in the index and is not reserved,
+ False otherwise.
+ """
+
+ # iTIP does not require unique UIDs
+ return True
+
+ def _db_type(self):
+ """
+ @return: the collection type assigned to this index.
+ """
+ return collection_types["iTIP"]
+
+ def _db_init_data_tables(self, q):
+ """
+ Initialise the underlying database tables.
+ @param q: a database cursor to use.
+ """
+
+ # Create database where the RESOURCE table has a UID column that is not unique.
+ self._db_init_data_tables_base(q, False)
+
+ def _db_recreate(self, do_commit=True):
+ """
+ Re-create the database tables from existing calendar data.
+ """
+
+ #
+ # Populate the DB with data from already existing resources.
+ # This allows for index recovery if the DB file gets
+ # deleted.
+ #
+ fp = self.resource.fp
+ for name in fp.listdir():
+ if name.startswith("."):
+ continue
+
+ try:
+ stream = fp.child(name).open()
+ except (IOError, OSError), e:
+ log.err("Unable to open resource %s: %s" % (name, e))
+ continue
+
+ # FIXME: This is blocking I/O
+ try:
+ calendar = Component.fromStream(stream)
+ calendar.validCalendarForCalDAV()
+ calendar.validateComponentsForCalDAV(True)
+ except ValueError:
+ log.err("Non-calendar resource: %s" % (name,))
+ else:
+ #log.msg("Indexing resource: %s" % (name,))
+ self.addResource(name, calendar, True, reCreate=True)
+ finally:
+ stream.close()
+
+ # Do commit outside of the loop for better performance
+ if do_commit:
+ self._db_commit()
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/sql.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/sql.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -33,7 +33,7 @@
from twistedcaldav import caldavxml, customxml
from twistedcaldav.caldavxml import ScheduleCalendarTransp, Opaque
from twistedcaldav.dateops import normalizeForIndex, datetimeMktime
-from twistedcaldav.index import IndexedSearchException
+from txdav.common.icommondatastore import IndexedSearchException
from twistedcaldav.instance import InvalidOverriddenInstanceError
from txdav.caldav.datastore.util import validateCalendarComponent,\
Copied: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py (from rev 6322, CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_index.py)
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py (rev 0)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/caldav/datastore/test/test_index_file.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -0,0 +1,936 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet import reactor
+from twisted.internet.task import deferLater
+
+from txdav.caldav.datastore.index_file import Index, MemcachedUIDReserver
+from txdav.common.icommondatastore import ReservationError
+
+from twistedcaldav import caldavxml
+from twistedcaldav.caldavxml import TimeRange
+from twistedcaldav.ical import Component
+from twistedcaldav.instance import InvalidOverriddenInstanceError
+from twistedcaldav.query import calendarqueryfilter
+from twistedcaldav.test.util import InMemoryMemcacheProtocol
+import twistedcaldav.test.util
+
+import datetime
+import os
+
+
+class MinimalResourceReplacement(object):
+ """
+ Provide the minimal set of attributes and methods from CalDAVFile required
+ by L{Index}.
+ """
+
+ def __init__(self, filePath):
+ self.fp = filePath
+
+
+ def isCalendarCollection(self):
+ return True
+
+
+ def getChild(self, name):
+ # FIXME: this should really return something with a child method
+ return self.fp.child(name)
+
+
+ def initSyncToken(self):
+ pass
+
+
+
+class SQLIndexTests (twistedcaldav.test.util.TestCase):
+ """
+ Test abstract SQL DB class
+ """
+
+ def setUp(self):
+ super(SQLIndexTests, self).setUp()
+ self.site.resource.isCalendarCollection = lambda: True
+ self.indexDirPath = self.site.resource.fp
+ # FIXME: since this resource lies about isCalendarCollection, it doesn't
+ # have all the associated backend machinery to actually get children.
+ self.db = Index(MinimalResourceReplacement(self.indexDirPath))
+
+
+ def test_reserve_uid_ok(self):
+ uid = "test-test-test"
+ d = self.db.isReservedUID(uid)
+ d.addCallback(self.assertFalse)
+ d.addCallback(lambda _: self.db.reserveUID(uid))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertTrue)
+ d.addCallback(lambda _: self.db.unreserveUID(uid))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertFalse)
+
+ return d
+
+
+ def test_reserve_uid_twice(self):
+ uid = "test-test-test"
+ d = self.db.reserveUID(uid)
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertTrue)
+ d.addCallback(lambda _:
+ self.assertFailure(self.db.reserveUID(uid),
+ ReservationError))
+ return d
+
+
+ def test_unreserve_unreserved(self):
+ uid = "test-test-test"
+ return self.assertFailure(self.db.unreserveUID(uid),
+ ReservationError)
+
+
+ def test_reserve_uid_timeout(self):
+ # WARNING: This test is fundamentally flawed and will fail
+ # intermittently because it uses the real clock.
+ uid = "test-test-test"
+ from twistedcaldav.config import config
+ old_timeout = config.UIDReservationTimeOut
+ config.UIDReservationTimeOut = 1
+
+ def _finally():
+ config.UIDReservationTimeOut = old_timeout
+
+ d = self.db.isReservedUID(uid)
+ d.addCallback(self.assertFalse)
+ d.addCallback(lambda _: self.db.reserveUID(uid))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertTrue)
+ d.addCallback(lambda _: deferLater(reactor, 2, lambda: None))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertFalse)
+ self.addCleanup(_finally)
+
+ return d
+
+
+ def test_index(self):
+ data = (
+ (
+ "#1.1 Simple component",
+ "1.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ True,
+ ),
+ (
+ "#2.1 Recurring component",
+ "2.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ True,
+ ),
+ (
+ "#2.2 Recurring component with override",
+ "2.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.2
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.2
+RECURRENCE-ID:20080608T120000Z
+DTSTART:20080608T120000Z
+DTEND:20080608T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ True,
+ ),
+ (
+ "#2.3 Recurring component with broken override - new",
+ "2.3",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.3
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.3
+RECURRENCE-ID:20080609T120000Z
+DTSTART:20080608T120000Z
+DTEND:20080608T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ False,
+ False,
+ ),
+ (
+ "#2.4 Recurring component with broken override - existing",
+ "2.4",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.4
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.4
+RECURRENCE-ID:20080609T120000Z
+DTSTART:20080608T120000Z
+DTEND:20080608T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ True,
+ True,
+ ),
+ )
+
+ for description, name, calendar_txt, reCreate, ok in data:
+ calendar = Component.fromString(calendar_txt)
+ if ok:
+ f = open(os.path.join(self.indexDirPath.path, name), "w")
+ f.write(calendar_txt)
+ del f
+
+ self.db.addResource(name, calendar, reCreate=reCreate)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+ else:
+ self.assertRaises(InvalidOverriddenInstanceError, self.db.addResource, name, calendar)
+ self.assertFalse(self.db.resourceExists(name), msg=description)
+
+ self.db._db_recreate()
+ for description, name, calendar_txt, reCreate, ok in data:
+ if ok:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+ else:
+ self.assertFalse(self.db.resourceExists(name), msg=description)
+
+ self.db.testAndUpdateIndex(datetime.date(2020, 1, 1))
+ for description, name, calendar_txt, reCreate, ok in data:
+ if ok:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+ else:
+ self.assertFalse(self.db.resourceExists(name), msg=description)
+
+ def test_index_timespan(self):
+ data = (
+ (
+ "#1.1 Simple component - busy",
+ "1.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080602T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ ),
+ (
+ "#1.2 Simple component - transparent",
+ "1.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.2
+DTSTART:20080602T120000Z
+DTEND:20080602T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+TRANSP:TRANSPARENT
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080602T000000Z", "20080603T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),),
+ ),
+ (
+ "#1.3 Simple component - canceled",
+ "1.3",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.3
+DTSTART:20080603T120000Z
+DTEND:20080603T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+STATUS:CANCELLED
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080603T000000Z", "20080604T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'F', 'F'),),
+ ),
+ (
+ "#1.4 Simple component - tentative",
+ "1.4",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.4
+DTSTART:20080604T120000Z
+DTEND:20080604T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+STATUS:TENTATIVE
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080604T000000Z", "20080605T000000Z",
+ "mailto:user1 at example.com",
+ (('N', "2008-06-04 12:00:00+00:00", "2008-06-04 13:00:00+00:00", 'T', 'F'),),
+ ),
+ (
+ "#2.1 Recurring component - busy",
+ "2.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.1
+DTSTART:20080605T120000Z
+DTEND:20080605T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080605T000000Z", "20080607T000000Z",
+ "mailto:user1 at example.com",
+ (
+ ('N', "2008-06-05 12:00:00+00:00", "2008-06-05 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-06 12:00:00+00:00", "2008-06-06 13:00:00+00:00", 'B', 'F'),
+ ),
+ ),
+ (
+ "#2.2 Recurring component - busy",
+ "2.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.2
+DTSTART:20080607T120000Z
+DTEND:20080607T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=2
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-2.2
+RECURRENCE-ID:20080608T120000Z
+DTSTART:20080608T140000Z
+DTEND:20080608T150000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+TRANSP:TRANSPARENT
+END:VEVENT
+END:VCALENDAR
+""",
+ "20080607T000000Z", "20080609T000000Z",
+ "mailto:user1 at example.com",
+ (
+ ('N', "2008-06-07 12:00:00+00:00", "2008-06-07 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-08 14:00:00+00:00", "2008-06-08 15:00:00+00:00", 'B', 'T'),
+ ),
+ ),
+ )
+
+ for description, name, calendar_txt, trstart, trend, organizer, instances in data:
+ calendar = Component.fromString(calendar_txt)
+
+ f = open(os.path.join(self.indexDirPath.path, name), "w")
+ f.write(calendar_txt)
+ del f
+
+ self.db.addResource(name, calendar)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ # Create fake filter element to match time-range
+ filter = caldavxml.Filter(
+ caldavxml.ComponentFilter(
+ caldavxml.ComponentFilter(
+ TimeRange(
+ start=trstart,
+ end=trend,
+ ),
+ name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
+ ),
+ name="VCALENDAR",
+ )
+ )
+ filter = calendarqueryfilter.Filter(filter)
+
+ resources = self.db.indexedSearch(filter, fbtype=True)
+ index_results = set()
+ for _ignore_name, _ignore_uid, type, test_organizer, float, start, end, fbtype, transp in resources:
+ self.assertEqual(test_organizer, organizer, msg=description)
+ index_results.add((float, start, end, fbtype, transp,))
+
+ self.assertEqual(set(instances), index_results, msg=description)
+
+ def test_index_timespan_per_user(self):
+ data = (
+ (
+ "#1.1 Single per-user non-recurring component",
+ "1.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.1
+X-CALENDARSERVER-PERUSER-UID:user01
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080602T000000Z",
+ "mailto:user1 at example.com",
+ (
+ (
+ "user01",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),),
+ ),
+ (
+ "user02",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ ),
+ ),
+ ),
+ (
+ "#1.2 Two per-user non-recurring component",
+ "1.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.2
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.2
+X-CALENDARSERVER-PERUSER-UID:user01
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.2
+X-CALENDARSERVER-PERUSER-UID:user02
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080602T000000Z",
+ "mailto:user1 at example.com",
+ (
+ (
+ "user01",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),),
+ ),
+ (
+ "user02",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ ),
+ (
+ "user03",
+ (('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),),
+ ),
+ ),
+ ),
+ (
+ "#2.1 Single per-user simple recurring component",
+ "2.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=10
+END:VEVENT
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.1
+X-CALENDARSERVER-PERUSER-UID:user01
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080603T000000Z",
+ "mailto:user1 at example.com",
+ (
+ (
+ "user01",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),
+ ),
+ ),
+ (
+ "user02",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
+ ),
+ ),
+ ),
+ ),
+ (
+ "#2.2 Two per-user simple recurring component",
+ "2.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.2
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=10
+END:VEVENT
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.2
+X-CALENDARSERVER-PERUSER-UID:user01
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.2
+X-CALENDARSERVER-PERUSER-UID:user02
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080603T000000Z",
+ "mailto:user1 at example.com",
+ (
+ (
+ "user01",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'T'),
+ ),
+ ),
+ (
+ "user02",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
+ ),
+ ),
+ (
+ "user03",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-02 12:00:00+00:00", "2008-06-02 13:00:00+00:00", 'B', 'F'),
+ ),
+ ),
+ ),
+ ),
+ (
+ "#3.1 Single per-user complex recurring component",
+ "3.1",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=10
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1.1
+RECURRENCE-ID:20080602T120000Z
+DTSTART:20080602T130000Z
+DTEND:20080602T140000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.1
+X-CALENDARSERVER-PERUSER-UID:user01
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+RECURRENCE-ID:20080602T120000Z
+TRANSP:OPAQUE
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080604T000000Z",
+ "mailto:user1 at example.com",
+ (
+ (
+ "user01",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
+ ),
+ ),
+ (
+ "user02",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'F'),
+ ),
+ ),
+ ),
+ ),
+ (
+ "#3.2 Two per-user complex recurring component",
+ "3.2",
+ """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.2
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=DAILY;COUNT=10
+END:VEVENT
+BEGIN:VEVENT
+UID:12345-67890-1.2
+RECURRENCE-ID:20080602T120000Z
+DTSTART:20080602T130000Z
+DTEND:20080602T140000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.2
+X-CALENDARSERVER-PERUSER-UID:user01
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+RECURRENCE-ID:20080602T120000Z
+TRANSP:OPAQUE
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+BEGIN:X-CALENDARSERVER-PERUSER
+UID:12345-67890-1.2
+X-CALENDARSERVER-PERUSER-UID:user02
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+BEGIN:VALARM
+ACTION:DISPLAY
+DESCRIPTION:Test
+TRIGGER;RELATED=START:-PT10M
+END:VALARM
+END:X-CALENDARSERVER-PERINSTANCE
+BEGIN:X-CALENDARSERVER-PERINSTANCE
+RECURRENCE-ID:20080603T120000Z
+TRANSP:TRANSPARENT
+END:X-CALENDARSERVER-PERINSTANCE
+END:X-CALENDARSERVER-PERUSER
+END:VCALENDAR
+""",
+ "20080601T000000Z", "20080604T000000Z",
+ "mailto:user1 at example.com",
+ (
+ (
+ "user01",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'T'),
+ ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
+ ),
+ ),
+ (
+ "user02",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'T'),
+ ),
+ ),
+ (
+ "user03",
+ (
+ ('N', "2008-06-01 12:00:00+00:00", "2008-06-01 13:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-02 13:00:00+00:00", "2008-06-02 14:00:00+00:00", 'B', 'F'),
+ ('N', "2008-06-03 12:00:00+00:00", "2008-06-03 13:00:00+00:00", 'B', 'F'),
+ ),
+ ),
+ ),
+ ),
+ )
+
+ for description, name, calendar_txt, trstart, trend, organizer, peruserinstances in data:
+ calendar = Component.fromString(calendar_txt)
+
+ f = open(os.path.join(self.indexDirPath.path, name), "w")
+ f.write(calendar_txt)
+ del f
+
+ self.db.addResource(name, calendar)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ # Create fake filter element to match time-range
+ filter = caldavxml.Filter(
+ caldavxml.ComponentFilter(
+ caldavxml.ComponentFilter(
+ TimeRange(
+ start=trstart,
+ end=trend,
+ ),
+ name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
+ ),
+ name="VCALENDAR",
+ )
+ )
+ filter = calendarqueryfilter.Filter(filter)
+
+ for useruid, instances in peruserinstances:
+ resources = self.db.indexedSearch(filter, useruid=useruid, fbtype=True)
+ index_results = set()
+ for _ignore_name, _ignore_uid, type, test_organizer, float, start, end, fbtype, transp in resources:
+ self.assertEqual(test_organizer, organizer, msg=description)
+ index_results.add((str(float), str(start), str(end), str(fbtype), str(transp),))
+
+ self.assertEqual(set(instances), index_results, msg="%s, user:%s" % (description, useruid,))
+
+ self.db.deleteResource(name)
+
+ def test_index_revisions(self):
+ data1 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-1.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+END:VEVENT
+END:VCALENDAR
+"""
+ data2 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.1
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+"""
+ data3 = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+BEGIN:VEVENT
+UID:12345-67890-2.3
+DTSTART:20080601T120000Z
+DTEND:20080601T130000Z
+ORGANIZER;CN="User 01":mailto:user1 at example.com
+ATTENDEE:mailto:user1 at example.com
+ATTENDEE:mailto:user2 at example.com
+RRULE:FREQ=WEEKLY;COUNT=2
+END:VEVENT
+END:VCALENDAR
+"""
+
+ calendar = Component.fromString(data1)
+ self.db.addResource("data1.ics", calendar)
+ calendar = Component.fromString(data2)
+ self.db.addResource("data2.ics", calendar)
+ calendar = Component.fromString(data3)
+ self.db.addResource("data3.ics", calendar)
+ self.db.deleteResource("data3.ics")
+
+ tests = (
+ (0, (["data1.ics", "data2.ics",], [],)),
+ (1, (["data2.ics",], ["data3.ics",],)),
+ (2, ([], ["data3.ics",],)),
+ (3, ([], ["data3.ics",],)),
+ (4, ([], [],)),
+ (5, ([], [],)),
+ )
+
+ for revision, results in tests:
+ self.assertEquals(self.db.whatchanged(revision), results, "Mismatched results for whatchanged with revision %d" % (revision,))
+
+class MemcacheTests(SQLIndexTests):
+ def setUp(self):
+ super(MemcacheTests, self).setUp()
+ self.memcache = InMemoryMemcacheProtocol()
+ self.db.reserver = MemcachedUIDReserver(self.db, self.memcache)
+
+
+ def tearDown(self):
+ for _ignore_k, v in self.memcache._timeouts.iteritems():
+ if v.active():
+ v.cancel()
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/file.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/file.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/file.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -33,7 +33,7 @@
from twistedcaldav.sharing import InvitesDatabase
from twistedcaldav.vcard import Component as VComponent, InvalidVCardDataError
-from twistedcaldav.vcardindex import AddressBookIndex as OldIndex
+from txdav.carddav.datastore.index_file import AddressBookIndex as OldIndex
from txdav.carddav.datastore.util import validateAddressBookComponent
from txdav.carddav.iaddressbookstore import IAddressBook, IAddressBookObject
Copied: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py (from rev 6322, CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/vcardindex.py)
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py (rev 0)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/index_file.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -0,0 +1,696 @@
+##
+# Copyright (c) 2005-2009 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+CardDAV Index.
+
+This API is considered private to static.py and is therefore subject to
+change.
+"""
+
+__all__ = [
+ "AddressBookIndex",
+]
+
+import datetime
+import os
+import time
+import hashlib
+
+try:
+ import sqlite3 as sqlite
+except ImportError:
+ from pysqlite2 import dbapi2 as sqlite
+
+from twisted.internet.defer import maybeDeferred
+
+from twistedcaldav import carddavxml
+from txdav.common.icommondatastore import SyncTokenValidException,\
+ ReservationError
+from twistedcaldav.query import addressbookquery
+from twistedcaldav.sql import AbstractSQLDatabase
+from twistedcaldav.sql import db_prefix
+from twistedcaldav.vcard import Component
+
+from twext.python.log import Logger, LoggingMixIn
+from twistedcaldav.config import config
+from twistedcaldav.memcachepool import CachePoolUserMixIn
+
+log = Logger()
+
+db_basename = db_prefix + "sqlite"
+schema_version = "2"
+
+def wrapInDeferred(f):
+ def _(*args, **kwargs):
+ return maybeDeferred(f, *args, **kwargs)
+
+ return _
+
+
+class MemcachedUIDReserver(CachePoolUserMixIn, LoggingMixIn):
+ def __init__(self, index, cachePool=None):
+ self.index = index
+ self._cachePool = cachePool
+
+ def _key(self, uid):
+ return 'reservation:%s' % (
+ hashlib.md5('%s:%s' % (uid,
+ self.index.resource.fp.path)).hexdigest())
+
+ def reserveUID(self, uid):
+ uid = uid.encode('utf-8')
+ self.log_debug("Reserving UID %r @ %r" % (
+ uid,
+ self.index.resource.fp.path))
+
+ def _handleFalse(result):
+ if result is False:
+ raise ReservationError(
+ "UID %s already reserved for address book collection %s."
+ % (uid, self.index.resource)
+ )
+
+ d = self.getCachePool().add(self._key(uid),
+ 'reserved',
+ expireTime=config.UIDReservationTimeOut)
+ d.addCallback(_handleFalse)
+ return d
+
+
+ def unreserveUID(self, uid):
+ uid = uid.encode('utf-8')
+ self.log_debug("Unreserving UID %r @ %r" % (
+ uid,
+ self.index.resource.fp.path))
+
+ def _handleFalse(result):
+ if result is False:
+ raise ReservationError(
+ "UID %s is not reserved for address book collection %s."
+ % (uid, self.index.resource)
+ )
+
+ d =self.getCachePool().delete(self._key(uid))
+ d.addCallback(_handleFalse)
+ return d
+
+
+ def isReservedUID(self, uid):
+ uid = uid.encode('utf-8')
+ self.log_debug("Is reserved UID %r @ %r" % (
+ uid,
+ self.index.resource.fp.path))
+
+ def _checkValue((flags, value)):
+ if value is None:
+ return False
+ else:
+ return True
+
+ d = self.getCachePool().get(self._key(uid))
+ d.addCallback(_checkValue)
+ return d
+
+
+
+class SQLUIDReserver(object):
+ def __init__(self, index):
+ self.index = index
+
+ @wrapInDeferred
+ def reserveUID(self, uid):
+ """
+ Reserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is already reserved
+ """
+
+ try:
+ self.index._db_execute("insert into RESERVED (UID, TIME) values (:1, :2)", uid, datetime.datetime.now())
+ self.index._db_commit()
+ except sqlite.IntegrityError:
+ self.index._db_rollback()
+ raise ReservationError(
+ "UID %s already reserved for address book collection %s."
+ % (uid, self.index.resource)
+ )
+ except sqlite.Error, e:
+ log.err("Unable to reserve UID: %s", (e,))
+ self.index._db_rollback()
+ raise
+
+ def unreserveUID(self, uid):
+ """
+ Unreserve a UID for this index's resource.
+ @param uid: the UID to reserve
+ @raise ReservationError: if C{uid} is not reserved
+ """
+
+ def _cb(result):
+ if result == False:
+ raise ReservationError(
+ "UID %s is not reserved for address book collection %s."
+ % (uid, self.index.resource)
+ )
+ else:
+ try:
+ self.index._db_execute(
+ "delete from RESERVED where UID = :1", uid)
+ self.index._db_commit()
+ except sqlite.Error, e:
+ log.err("Unable to unreserve UID: %s", (e,))
+ self.index._db_rollback()
+ raise
+
+ d = self.isReservedUID(uid)
+ d.addCallback(_cb)
+ return d
+
+
+ @wrapInDeferred
+ def isReservedUID(self, uid):
+ """
+ Check to see whether a UID is reserved.
+ @param uid: the UID to check
+ @return: True if C{uid} is reserved, False otherwise.
+ """
+
+ rowiter = self.index._db_execute("select UID, TIME from RESERVED where UID = :1", uid)
+ for uid, attime in rowiter:
+ # Double check that the time is within a reasonable period of now
+ # otherwise we probably have a stale reservation
+ tm = time.strptime(attime[:19], "%Y-%m-%d %H:%M:%S")
+ dt = datetime.datetime(year=tm.tm_year, month=tm.tm_mon, day=tm.tm_mday, hour=tm.tm_hour, minute=tm.tm_min, second = tm.tm_sec)
+ if datetime.datetime.now() - dt > datetime.timedelta(seconds=config.UIDReservationTimeOut):
+ try:
+ self.index._db_execute("delete from RESERVED where UID = :1", uid)
+ self.index._db_commit()
+ except sqlite.Error, e:
+ log.err("Unable to unreserve UID: %s", (e,))
+ self.index._db_rollback()
+ raise
+ return False
+ else:
+ return True
+
+ return False
+
+class AddressBookIndex(AbstractSQLDatabase):
+ """
+ AddressBook collection index abstract base class that defines the apis for the index.
+ """
+
+ def __init__(self, resource):
+ """
+ @param resource: the L{CalDAVResource} resource to
+ index. C{resource} must be an addressbook collection (ie.
+ C{resource.isAddressBookCollection()} returns C{True}.)
+ """
+ assert resource.isAddressBookCollection(), "non-addressbook collection resource %s has no index." % (resource,)
+ self.resource = resource
+ db_filename = os.path.join(self.resource.fp.path, db_basename)
+ super(AddressBookIndex, self).__init__(db_filename, False)
+
+ if (
+ hasattr(config, "Memcached") and
+ config.Memcached.Pools.Default.ClientEnabled
+ ):
+ self.reserver = MemcachedUIDReserver(self)
+ else:
+ self.reserver = SQLUIDReserver(self)
+
+ def create(self):
+ """
+ Create the index and initialize it.
+ """
+ self._db()
+
+ def recreate(self):
+ """
+ Delete the database and re-create it
+ """
+ try:
+ os.remove(self.dbpath)
+ except OSError:
+ pass
+ self.create()
+
+ #
+ # A dict of sets. The dict keys are address book collection paths,
+ # and the sets contains reserved UIDs for each path.
+ #
+
+ def reserveUID(self, uid):
+ return self.reserver.reserveUID(uid)
+
+ def unreserveUID(self, uid):
+ return self.reserver.unreserveUID(uid)
+
+ def isReservedUID(self, uid):
+ return self.reserver.isReservedUID(uid)
+
+ def isAllowedUID(self, uid, *names):
+ """
+ Checks to see whether to allow an operation which would add the
+ specified UID to the index. Specifically, the operation may not
+ violate the constraint that UIDs must be unique.
+ @param uid: the UID to check
+ @param names: the names of resources being replaced or deleted by the
+ operation; UIDs associated with these resources are not checked.
+ @return: True if the UID is not in the index and is not reserved,
+ False otherwise.
+ """
+ rname = self.resourceNameForUID(uid)
+ return (rname is None or rname in names)
+
+ def resourceNamesForUID(self, uid):
+ """
+ Looks up the names of the resources with the given UID.
+ @param uid: the UID of the resources to look up.
+ @return: a list of resource names
+ """
+ names = self._db_values_for_sql("select NAME from RESOURCE where UID = :1", uid)
+
+ #
+ # Check that each name exists as a child of self.resource. If not, the
+ # resource record is stale.
+ #
+ resources = []
+ for name in names:
+ name_utf8 = name.encode("utf-8")
+ if name is not None and self.resource.getChild(name_utf8) is None:
+ # Clean up
+ log.err("Stale resource record found for child %s with UID %s in %s" % (name, uid, self.resource))
+ self._delete_from_db(name, uid, False)
+ self._db_commit()
+ else:
+ resources.append(name_utf8)
+
+ return resources
+
+ def resourceNameForUID(self, uid):
+ """
+ Looks up the name of the resource with the given UID.
+ @param uid: the UID of the resource to look up.
+ @return: If the resource is found, its name; C{None} otherwise.
+ """
+ result = None
+
+ for name in self.resourceNamesForUID(uid):
+ assert result is None, "More than one resource with UID %s in address book collection %r" % (uid, self)
+ result = name
+
+ return result
+
+ def resourceUIDForName(self, name):
+ """
+ Looks up the UID of the resource with the given name.
+ @param name: the name of the resource to look up.
+ @return: If the resource is found, the UID of the resource; C{None}
+ otherwise.
+ """
+ uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
+
+ return uid
+
+ def addResource(self, name, vcard, fast=False):
+ """
+ Adding or updating an existing resource.
+ To check for an update we attempt to get an existing UID
+ for the resource name. If present, then the index entries for
+ that UID are removed. After that the new index entries are added.
+ @param name: the name of the resource to add.
+ @param vCard: a L{Component} object representing the resource
+ contents.
+ @param fast: if C{True} do not do commit, otherwise do commit.
+ """
+ oldUID = self.resourceUIDForName(name)
+ if oldUID is not None:
+ self._delete_from_db(name, oldUID, False)
+ self._add_to_db(name, vcard)
+ if not fast:
+ self._db_commit()
+
+ def deleteResource(self, name):
+ """
+ Remove this resource from the index.
+ @param name: the name of the resource to add.
+ @param uid: the UID of the vcard component in the resource.
+ """
+ uid = self.resourceUIDForName(name)
+ if uid is not None:
+ self._delete_from_db(name, uid)
+ self._db_commit()
+
+ def resourceExists(self, name):
+ """
+ Determines whether the specified resource name exists in the index.
+ @param name: the name of the resource to test
+ @return: True if the resource exists, False if not
+ """
+ uid = self._db_value_for_sql("select UID from RESOURCE where NAME = :1", name)
+ return uid is not None
+
+ def resourcesExist(self, names):
+ """
+ Determines whether the specified resource name exists in the index.
+ @param names: a C{list} containing the names of the resources to test
+ @return: a C{list} of all names that exist
+ """
+ statement = "select NAME from RESOURCE where NAME in ("
+ for ctr, ignore_name in enumerate(names):
+ if ctr != 0:
+ statement += ", "
+ statement += ":%s" % (ctr,)
+ statement += ")"
+ results = self._db_values_for_sql(statement, *names)
+ return results
+
+ def whatchanged(self, revision):
+
+ results = [(name.encode("utf-8"), deleted) for name, deleted in self._db_execute("select NAME, DELETED from REVISIONS where REVISION > :1", revision)]
+ results.sort(key=lambda x:x[1])
+
+ changed = []
+ deleted = []
+ for name, wasdeleted in results:
+ if name:
+ if wasdeleted == 'Y':
+ if revision:
+ deleted.append(name)
+ else:
+ changed.append(name)
+ else:
+ raise SyncTokenValidException
+
+ return changed, deleted,
+
+ def lastRevision(self):
+ return self._db_value_for_sql(
+ "select REVISION from REVISION_SEQUENCE"
+ )
+
+ def bumpRevision(self, fast=False):
+ self._db_execute(
+ """
+ update REVISION_SEQUENCE set REVISION = REVISION + 1
+ """,
+ )
+ if not fast:
+ self._db_commit()
+ return self._db_value_for_sql(
+ """
+ select REVISION from REVISION_SEQUENCE
+ """,
+ )
+
+ def searchValid(self, filter):
+ if isinstance(filter, carddavxml.Filter):
+ qualifiers = addressbookquery.sqladdressbookquery(filter)
+ else:
+ qualifiers = None
+
+ return qualifiers is not None
+
+ def search(self, filter):
+ """
+ Finds resources matching the given qualifiers.
+ @param filter: the L{Filter} for the addressbook-query to execute.
+ @return: an interable iterable of tuples for each resource matching the
+ given C{qualifiers}. The tuples are C{(name, uid, type)}, where
+ C{name} is the resource name, C{uid} is the resource UID, and
+ C{type} is the resource iCalendar component type.x
+ """
+ # FIXME: Don't forget to use maximum_future_expansion_duration when we
+ # start caching...
+
+ # Make sure we have a proper Filter element and get the partial SQL statement to use.
+ if isinstance(filter, carddavxml.Filter):
+ qualifiers = addressbookquery.sqladdressbookquery(filter)
+ else:
+ qualifiers = None
+ if qualifiers is not None:
+ rowiter = self._db_execute("select DISTINCT RESOURCE.NAME, RESOURCE.UID" + qualifiers[0], *qualifiers[1])
+ else:
+ rowiter = self._db_execute("select NAME, UID from RESOURCE")
+
+ for row in rowiter:
+ name = row[0]
+ if self.resource.getChild(name.encode("utf-8")):
+ yield row
+ else:
+ log.err("vCard resource %s is missing from %s. Removing from index."
+ % (name, self.resource))
+ self.deleteResource(name, None)
+
+ def bruteForceSearch(self):
+ """
+ List the whole index and tests for existence, updating the index
+ @return: all resources in the index
+ """
+ # List all resources
+ rowiter = self._db_execute("select NAME, UID from RESOURCE")
+
+ # Check result for missing resources:
+
+ for row in rowiter:
+ name = row[0]
+ if self.resource.getChild(name.encode("utf-8")):
+ yield row
+ else:
+ log.err("AddressBook resource %s is missing from %s. Removing from index."
+ % (name, self.resource))
+ self.deleteResource(name)
+
+
+ def _db_version(self):
+ """
+ @return: the schema version assigned to this index.
+ """
+ return schema_version
+
+ def _db_type(self):
+ """
+ @return: the collection type assigned to this index.
+ """
+ return "AddressBook"
+
+ def _db_init_data_tables(self, q):
+ """
+ Initialise the underlying database tables.
+ @param q: a database cursor to use.
+ """
+
+ # Create database where the RESOURCE table has unique UID column.
+ self._db_init_data_tables_base(q, True)
+
+ def _db_init_data_tables_base(self, q, uidunique):
+ """
+ Initialise the underlying database tables.
+ @param q: a database cursor to use.
+ """
+ #
+ # RESOURCE table is the primary index table
+ # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
+ # UID: iCalendar UID (may or may not be unique)
+ #
+ q.execute(
+ """
+ create table RESOURCE (
+ NAME text unique,
+ UID text unique
+ )
+ """
+ )
+
+ #
+ # REVISIONS table tracks changes
+ # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key)
+ # REVISION: revision number
+ # WASDELETED: Y if revision deleted, N if added or changed
+ #
+ q.execute(
+ """
+ create table REVISION_SEQUENCE (
+ REVISION integer
+ )
+ """
+ )
+ q.execute(
+ """
+ insert into REVISION_SEQUENCE (REVISION) values (0)
+ """
+ )
+ q.execute(
+ """
+ create table REVISIONS (
+ NAME text unique,
+ REVISION integer default 0,
+ DELETED text(1) default "N"
+ )
+ """
+ )
+ q.execute(
+ """
+ create index REVISION on REVISIONS (REVISION)
+ """
+ )
+
+ #
+ # RESERVED table tracks reserved UIDs
+ # UID: The UID being reserved
+ # TIME: When the reservation was made
+ #
+ q.execute(
+ """
+ create table RESERVED (
+ UID text unique,
+ TIME date
+ )
+ """
+ )
+
+ def _db_recreate(self, do_commit=True):
+ """
+ Re-create the database tables from existing address book data.
+ """
+
+ #
+ # Populate the DB with data from already existing resources.
+ # This allows for index recovery if the DB file gets
+ # deleted.
+ #
+ fp = self.resource.fp
+ for name in fp.listdir():
+ if name.startswith("."):
+ continue
+
+ try:
+ stream = fp.child(name).open()
+ except (IOError, OSError), e:
+ log.err("Unable to open resource %s: %s" % (name, e))
+ continue
+
+ try:
+ # FIXME: This is blocking I/O
+ try:
+ vcard = Component.fromStream(stream)
+ vcard.validForCardDAV()
+ except ValueError:
+ log.err("Non-addressbook resource: %s" % (name,))
+ else:
+ #log.msg("Indexing resource: %s" % (name,))
+ self.addResource(name, vcard, True)
+ finally:
+ stream.close()
+
+ # Do commit outside of the loop for better performance
+ if do_commit:
+ self._db_commit()
+
+ def _db_can_upgrade(self, old_version):
+ """
+ Can we do an in-place upgrade
+ """
+
+ # v2 is a minor change
+ return True
+
+ def _db_upgrade_data_tables(self, q, old_version):
+ """
+ Upgrade the data from an older version of the DB.
+ """
+
+ # When going to version 2+ all we need to do is add revision table and index
+ if old_version < 2:
+ q.execute(
+ """
+ create table REVISION_SEQUENCE (
+ REVISION integer
+ )
+ """
+ )
+ q.execute(
+ """
+ insert into REVISION_SEQUENCE (REVISION) values (0)
+ """
+ )
+ q.execute(
+ """
+ create table REVISIONS (
+ NAME text unique,
+ REVISION integer default 0,
+ CREATEDREVISION integer default 0,
+ WASDELETED text(1) default "N"
+ )
+ """
+ )
+ q.execute(
+ """
+ create index REVISION on REVISIONS (REVISION)
+ """
+ )
+
+ self._db_execute(
+ """
+ insert into REVISIONS (NAME)
+ select NAME from RESOURCE
+ """
+ )
+
+
+ def _add_to_db(self, name, vcard, cursor = None):
+ """
+ Records the given address book resource in the index with the given name.
+ Resource names and UIDs must both be unique; only one resource name may
+ be associated with any given UID and vice versa.
+ NB This method does not commit the changes to the db - the caller
+ MUST take care of that
+ @param name: the name of the resource to add.
+ @param vcard: a L{AddressBook} object representing the resource
+ contents.
+ """
+ uid = vcard.resourceUID()
+
+ self._db_execute(
+ """
+ insert into RESOURCE (NAME, UID)
+ values (:1, :2)
+ """, name, uid,
+ )
+
+ self._db_execute(
+ """
+ insert or replace into REVISIONS (NAME, REVISION, DELETED)
+ values (:1, :2, :3)
+ """, name, self.bumpRevision(fast=True), 'N',
+ )
+
+ def _delete_from_db(self, name, uid, dorevision=True):
+ """
+ Deletes the specified entry from all dbs.
+ @param name: the name of the resource to delete.
+ @param uid: the uid of the resource to delete.
+ """
+ self._db_execute("delete from RESOURCE where NAME = :1", name)
+ if dorevision:
+ self._db_execute(
+ """
+ update REVISIONS SET REVISION = :1, DELETED = :2
+ where NAME = :3
+ """, self.bumpRevision(fast=True), 'Y', name
+ )
Copied: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py (from rev 6322, CalendarServer/branches/users/glyph/more-deferreds-6/twistedcaldav/test/test_vcardindex.py)
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py (rev 0)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/carddav/datastore/test/test_index_file.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -0,0 +1,212 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from twisted.internet import reactor
+from twisted.internet.task import deferLater
+
+from txdav.common.icommondatastore import ReservationError
+from txdav.carddav.datastore.index_file import AddressBookIndex,\
+ MemcachedUIDReserver
+
+from twistedcaldav.test.util import InMemoryMemcacheProtocol
+from twistedcaldav.vcard import Component
+import twistedcaldav.test.util
+
+import os
+
+class MinimalResourceReplacement(object):
+ """
+ Provide the minimal set of attributes and methods from CalDAVFile required
+ by L{Index}.
+ """
+
+ def __init__(self, filePath):
+ self.fp = filePath
+
+
+ def isAddressBookCollection(self):
+ return True
+
+
+ def getChild(self, name):
+ # FIXME: this should really return something with a child method
+ return self.fp.child(name)
+
+
+ def initSyncToken(self):
+ pass
+
+
+
+class SQLIndexTests (twistedcaldav.test.util.TestCase):
+ """
+ Test abstract SQL DB class
+ """
+
+ def setUp(self):
+ super(SQLIndexTests, self).setUp()
+ self.site.resource.isAddressBookCollection = lambda: True
+ self.indexDirPath = self.site.resource.fp
+ # FIXME: since this resource lies about isCalendarCollection, it doesn't
+ # have all the associated backend machinery to actually get children.
+ self.db = AddressBookIndex(MinimalResourceReplacement(self.indexDirPath))
+
+
+ def test_reserve_uid_ok(self):
+ uid = "test-test-test"
+ d = self.db.isReservedUID(uid)
+ d.addCallback(self.assertFalse)
+ d.addCallback(lambda _: self.db.reserveUID(uid))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertTrue)
+ d.addCallback(lambda _: self.db.unreserveUID(uid))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertFalse)
+
+ return d
+
+
+ def test_reserve_uid_twice(self):
+ uid = "test-test-test"
+ d = self.db.reserveUID(uid)
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertTrue)
+ d.addCallback(lambda _:
+ self.assertFailure(self.db.reserveUID(uid),
+ ReservationError))
+ return d
+
+
+ def test_unreserve_unreserved(self):
+ uid = "test-test-test"
+ return self.assertFailure(self.db.unreserveUID(uid),
+ ReservationError)
+
+
+ def test_reserve_uid_timeout(self):
+ # WARNING: This test is fundamentally flawed and will fail
+ # intermittently because it uses the real clock.
+ uid = "test-test-test"
+ from twistedcaldav.config import config
+ old_timeout = config.UIDReservationTimeOut
+ config.UIDReservationTimeOut = 1
+
+ def _finally():
+ config.UIDReservationTimeOut = old_timeout
+
+ d = self.db.isReservedUID(uid)
+ d.addCallback(self.assertFalse)
+ d.addCallback(lambda _: self.db.reserveUID(uid))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertTrue)
+ d.addCallback(lambda _: deferLater(reactor, 2, lambda: None))
+ d.addCallback(lambda _: self.db.isReservedUID(uid))
+ d.addCallback(self.assertFalse)
+ self.addCleanup(_finally)
+
+ return d
+
+
+ def test_index(self):
+ data = (
+ (
+ "#1.1 Simple component",
+ "1.1",
+ """BEGIN:VCARD
+VERSION:3.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+UID:12345-67890-1.1
+FN:Cyrus Daboo
+N:Daboo;Cyrus
+EMAIL;TYPE=INTERNET,PREF:cyrus at example.com
+END:VCARD
+""",
+ ),
+ )
+
+ for description, name, vcard_txt in data:
+ calendar = Component.fromString(vcard_txt)
+ f = open(os.path.join(self.site.resource.fp.path, name), "w")
+ f.write(vcard_txt)
+ del f
+
+ self.db.addResource(name, calendar)
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ self.db._db_recreate()
+ for description, name, vcard_txt in data:
+ self.assertTrue(self.db.resourceExists(name), msg=description)
+
+ def test_index_revisions(self):
+ data1 = """BEGIN:VCARD
+VERSION:3.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+UID:12345-67890-1-1.1
+FN:Cyrus Daboo
+N:Daboo;Cyrus
+EMAIL;TYPE=INTERNET,PREF:cyrus at example.com
+END:VCARD
+"""
+ data2 = """BEGIN:VCARD
+VERSION:3.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+UID:12345-67890-2-1.1
+FN:Wilfredo Sanchez
+N:Sanchez;Wilfredo
+EMAIL;TYPE=INTERNET,PREF:wsanchez at example.com
+END:VCARD
+"""
+ data3 = """BEGIN:VCARD
+VERSION:3.0
+PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
+UID:12345-67890-3-1.1
+FN:Bruce Gaya
+N:Gaya;Bruce
+EMAIL;TYPE=INTERNET,PREF:bruce at example.com
+END:VCARD
+"""
+
+ vcard = Component.fromString(data1)
+ self.db.addResource("data1.vcf", vcard)
+ vcard = Component.fromString(data2)
+ self.db.addResource("data2.vcf", vcard)
+ vcard = Component.fromString(data3)
+ self.db.addResource("data3.vcf", vcard)
+ self.db.deleteResource("data3.vcf")
+
+ tests = (
+ (0, (["data1.vcf", "data2.vcf",], [],)),
+ (1, (["data2.vcf",], ["data3.vcf",],)),
+ (2, ([], ["data3.vcf",],)),
+ (3, ([], ["data3.vcf",],)),
+ (4, ([], [],)),
+ (5, ([], [],)),
+ )
+
+ for revision, results in tests:
+ self.assertEquals(self.db.whatchanged(revision), results, "Mismatched results for whatchanged with revision %d" % (revision,))
+
+class MemcacheTests(SQLIndexTests):
+ def setUp(self):
+ super(MemcacheTests, self).setUp()
+ self.memcache = InMemoryMemcacheProtocol()
+ self.db.reserver = MemcachedUIDReserver(self.db, self.memcache)
+
+
+ def tearDown(self):
+ for _ignore_k, v in self.memcache._timeouts.iteritems():
+ if v.active():
+ v.cancel()
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/datastore/sql_legacy.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/datastore/sql_legacy.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/datastore/sql_legacy.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -33,7 +33,6 @@
from twistedcaldav import carddavxml
from twistedcaldav.config import config
from twistedcaldav.dateops import normalizeForIndex
-from twistedcaldav.index import IndexedSearchException, ReservationError
from twistedcaldav.memcachepool import CachePoolUserMixIn
from twistedcaldav.notifications import NotificationRecord
from twistedcaldav.query import calendarqueryfilter, calendarquery, \
@@ -41,6 +40,8 @@
from twistedcaldav.query.sqlgenerator import sqlgenerator
from twistedcaldav.sharing import Invite
+from txdav.common.icommondatastore import IndexedSearchException, \
+ ReservationError
from txdav.common.datastore.sql_tables import \
_BIND_MODE_OWN, _BIND_MODE_READ, _BIND_MODE_WRITE, _BIND_MODE_DIRECT, \
_BIND_STATUS_INVITED, _BIND_STATUS_ACCEPTED, _BIND_STATUS_DECLINED, _BIND_STATUS_INVALID,\
Modified: CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/icommondatastore.py
===================================================================
--- CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/icommondatastore.py 2010-09-22 01:16:04 UTC (rev 6334)
+++ CalendarServer/branches/users/glyph/more-deferreds-6/txdav/common/icommondatastore.py 2010-09-22 04:00:15 UTC (rev 6335)
@@ -108,6 +108,20 @@
Uh, oh.
"""
+# Indexing / sync tokens
+
+class ReservationError(LookupError):
+ """
+ Attempt to reserve a UID which is already reserved or to unreserve a UID
+ which is not reserved.
+ """
+
+class IndexedSearchException(ValueError):
+ pass
+
+class SyncTokenValidException(ValueError):
+ pass
+
#
# Interfaces
#
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100921/9177520c/attachment-0001.html>
More information about the calendarserver-changes
mailing list