[CalendarServer-changes] [7030] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Wed Feb 16 13:59:19 PST 2011


Revision: 7030
          http://trac.macosforge.org/projects/calendarserver/changeset/7030
Author:   cdaboo at apple.com
Date:     2011-02-16 13:59:18 -0800 (Wed, 16 Feb 2011)
Log Message:
-----------
Add a free-busy cache (per-user/per-calendar) for a fixed range (7 days in the past, 12 weeks in the future).

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/dateops.py
    CalendarServer/trunk/twistedcaldav/method/report_common.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/txdav/base/propertystore/sql.py
    CalendarServer/trunk/txdav/caldav/datastore/sql.py
    CalendarServer/trunk/txdav/carddav/datastore/sql.py

Modified: CalendarServer/trunk/twistedcaldav/dateops.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/dateops.py	2011-02-16 20:13:34 UTC (rev 7029)
+++ CalendarServer/trunk/twistedcaldav/dateops.py	2011-02-16 21:59:18 UTC (rev 7030)
@@ -54,6 +54,21 @@
     else:
         return datetime.datetime.fromordinal(dt.toordinal())
 
+def normalizeToUTC(dt):
+    """
+    Normalize a L{datetime.date} or L{datetime.datetime} object to UTC.
+    """
+    if not isinstance(dt, datetime.date):
+        raise TypeError("%r is not a datetime.date instance" % (dt,))
+    
+    if isinstance(dt, datetime.datetime):
+        if dt.tzinfo is not None:
+            return dt.astimezone(utc)
+        else:
+            return dt.replace(tzinfo=utc)
+    else:
+        return datetime.datetime.fromordinal(dt.toordinal()).replace(tzinfo=utc)
+
 def floatoffset(dt, tzinfo):
     """
     Apply the timezone offset to the supplied time, then force tz to utc. This gives the local

Modified: CalendarServer/trunk/twistedcaldav/method/report_common.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/method/report_common.py	2011-02-16 20:13:34 UTC (rev 7029)
+++ CalendarServer/trunk/twistedcaldav/method/report_common.py	2011-02-16 21:59:18 UTC (rev 7030)
@@ -37,7 +37,7 @@
 except ImportError:
     from md5 import new as md5
 
-from vobject.icalendar import utc
+from vobject.icalendar import utc, dateTimeToString
 
 from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred
 from twisted.python.failure import Failure
@@ -55,14 +55,17 @@
 
 from twistedcaldav import caldavxml
 from twistedcaldav import carddavxml
-from twistedcaldav.caldavxml import caldav_namespace, CalendarData
+from twistedcaldav.caldavxml import caldav_namespace, CalendarData, TimeRange
 from twistedcaldav.carddavxml import AddressData
+from twistedcaldav.config import config
 from twistedcaldav.datafilters.calendardata import CalendarDataFilter
 from twistedcaldav.datafilters.privateevents import PrivateEventFilter
 from twistedcaldav.datafilters.addressdata import AddressDataFilter
-from twistedcaldav.dateops import clipPeriod, normalizePeriodList, timeRangesOverlap
+from twistedcaldav.dateops import clipPeriod, normalizePeriodList, timeRangesOverlap,\
+    compareDateTime, normalizeToUTC
 from twistedcaldav.ical import Component, Property, iCalendarProductID
 from twistedcaldav.instance import InstanceList
+from twistedcaldav.memcacher import Memcacher
 
 from twistedcaldav.query import calendarqueryfilter
 
@@ -377,8 +380,52 @@
 fbtype_mapper = {"BUSY": 0, "BUSY-TENTATIVE": 1, "BUSY-UNAVAILABLE": 2}
 fbtype_index_mapper = {'B': 0, 'T': 1, 'U': 2}
 
+fbcacher = Memcacher("FBCache", pickle=True)
 
+class FBCacheEntry(object):
+    
+    CACHE_DAYS_FLOATING_ADJUST = 1
+    CACHE_DAYS_BACK = 7             # Must be greater than CACHE_DAYS_FLOATING_ADJUST
+    CACHE_DAYS_FORWARD = 12 * 7     # 12 weeks must be greater than CACHE_DAYS_FLOATING_ADJUST
+    
+    def __init__(self, key, token, timerange, fbresults):
+        self.key = key
+        self.token = token
+        self.timerange = timerange
+        self.fbresults = fbresults
+    
+    @classmethod
+    @inlineCallbacks
+    def getCacheEntry(cls, calresource, useruid, timerange):
+        
+        key = calresource.resourceID() + "/" + useruid
+        token = (yield calresource.getSyncToken())
+        entry = (yield fbcacher.get(key))
+        
+        if entry:
+    
+            # Offset one day at either end to account for floating
+            cached_start = entry.timerange.start + datetime.timedelta(days=FBCacheEntry.CACHE_DAYS_FLOATING_ADJUST)
+            cached_end = entry.timerange.end - datetime.timedelta(days=FBCacheEntry.CACHE_DAYS_FLOATING_ADJUST)
 
+            # Verify that the requested timerange lies within the cache timerange
+            if compareDateTime(timerange.end, cached_end) <= 0 and compareDateTime(timerange.start, cached_start) >= 0:
+                
+                # Verify that cached entry is still valid
+                if token == entry.token:
+                    returnValue(entry.fbresults)
+        
+        returnValue(None) 
+
+    @classmethod
+    @inlineCallbacks
+    def makeCacheEntry(cls, calresource, useruid, timerange, fbresults):
+
+        key = calresource.resourceID() + "/" + useruid
+        token = (yield calresource.getSyncToken())
+        entry = cls(key, token, timerange, fbresults)
+        yield fbcacher.set(key, entry)
+
 @inlineCallbacks
 def generateFreeBusyInfo(request, calresource, fbinfo, timerange, matchtotal,
                          excludeuid=None, organizer=None,
@@ -416,50 +463,76 @@
     organizer_principal = calresource.principalForCalendarUserAddress(organizer) if organizer else None
     organizer_uid = organizer_principal.principalUID() if organizer_principal else ""
 
-    #
-    # What we do is a fake calendar-query for VEVENT/VFREEBUSYs in the specified time-range.
-    # We then take those results and merge them into one VFREEBUSY component
-    # with appropriate FREEBUSY properties, and return that single item as iCal data.
-    #
-
-    # Create fake filter element to match time-range
-    filter =  caldavxml.Filter(
-                  caldavxml.ComponentFilter(
-                      caldavxml.ComponentFilter(
-                          timerange,
-                          name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
-                      ),
-                      name="VCALENDAR",
-                   )
-              )
-    filter = calendarqueryfilter.Filter(filter)
-
-    # Get the timezone property from the collection, and store in the query filter
-    # for use during the query itself.
-    has_prop = (yield calresource.hasProperty((caldav_namespace, "calendar-timezone"), request))
-    if has_prop:
-        tz = (yield calresource.readProperty((caldav_namespace, "calendar-timezone"), request))
-    else:
-        tz = None
-    tzinfo = filter.settimezone(tz)
-
-    # Do some optimization of access control calculation by determining any
-    # inherited ACLs outside of the child resource loop and supply those to the
-    # checkPrivileges on each child.
-    filteredaces = (yield calresource.inheritedACEsforChildren(request))
-
+    # Free busy is per-user 
     userPrincipal = (yield calresource.resourceOwnerPrincipal(request))
     if userPrincipal:
         useruid = userPrincipal.principalUID()
     else:
         useruid = ""
-    try:
-        resources = yield maybeDeferred(calresource.index().indexedSearch,
-            filter, useruid=useruid, fbtype=True
-        )
-    except IndexedSearchException:
-        resources = yield maybeDeferred(calresource.index().bruteForceSearch)
+            
+    # Try cache
+    resources = (yield FBCacheEntry.getCacheEntry(calresource, useruid, timerange)) if config.EnableFreeBusyCache else None
 
+    if resources is None:
+        
+        caching = False
+        if config.EnableFreeBusyCache:
+            # Log extended item
+            if not hasattr(request, "extendedLogItems"):
+                request.extendedLogItems = {}
+            request.extendedLogItems["fb-uncached"] = request.extendedLogItems.get("fb-uncached", 0) + 1
+
+            # We want to cache a large range of time based on the current date
+            cache_start = normalizeToUTC(datetime.date.today() - datetime.timedelta(days=FBCacheEntry.CACHE_DAYS_BACK))
+            cache_end = normalizeToUTC(datetime.date.today() + datetime.timedelta(days=FBCacheEntry.CACHE_DAYS_FORWARD))
+            
+            # If the requested timerange would fit in our allowed cache range, trigger the cache creation
+            if compareDateTime(timerange.start, cache_start) >= 0 and compareDateTime(timerange.end, cache_end) <= 0:
+                cache_timerange = TimeRange(start=dateTimeToString(cache_start), end=dateTimeToString(cache_end))
+                caching = True
+ 
+        #
+        # What we do is a fake calendar-query for VEVENT/VFREEBUSYs in the specified time-range.
+        # We then take those results and merge them into one VFREEBUSY component
+        # with appropriate FREEBUSY properties, and return that single item as iCal data.
+        #
+    
+        # Create fake filter element to match time-range
+        filter =  caldavxml.Filter(
+                      caldavxml.ComponentFilter(
+                          caldavxml.ComponentFilter(
+                              cache_timerange if caching else timerange,
+                              name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
+                          ),
+                          name="VCALENDAR",
+                       )
+                  )
+        filter = calendarqueryfilter.Filter(filter)
+    
+        # Get the timezone property from the collection, and store in the query filter
+        # for use during the query itself.
+        has_prop = (yield calresource.hasProperty((caldav_namespace, "calendar-timezone"), request))
+        if has_prop:
+            tz = (yield calresource.readProperty((caldav_namespace, "calendar-timezone"), request))
+        else:
+            tz = None
+        tzinfo = filter.settimezone(tz)
+    
+        try:
+            resources = yield maybeDeferred(calresource.index().indexedSearch,
+                filter, useruid=useruid, fbtype=True
+            )
+            if caching:
+                yield FBCacheEntry.makeCacheEntry(calresource, useruid, cache_timerange, resources)
+        except IndexedSearchException:
+            resources = yield maybeDeferred(calresource.index().bruteForceSearch)
+            
+    else:
+        # Log extended item
+        if not hasattr(request, "extendedLogItems"):
+            request.extendedLogItems = {}
+        request.extendedLogItems["fb-cached"] = request.extendedLogItems.get("fb-cached", 0) + 1
+
     # We care about separate instances for VEVENTs only
     aggregated_resources = {}
     for name, uid, type, test_organizer, float, start, end, fbtype, transp in resources:
@@ -471,16 +544,6 @@
 
         name, uid, type, test_organizer = key
 
-        # Check privileges - must have at least CalDAV:read-free-busy
-        child = (yield request.locateChildResource(calresource, name))
-
-        # TODO: for server-to-server we bypass this right now as we have no way to authorize external users.
-        if not servertoserver:
-            try:
-                yield child.checkPrivileges(request, (caldavxml.ReadFreeBusy(),), inherited_aces=filteredaces, principal=organizerPrincipal)
-            except AccessDeniedError:
-                continue
-
         # Short-cut - if an fbtype exists we can use that
         if type == "VEVENT" and aggregated_resources[key][0][3] != '?':
 
@@ -533,6 +596,7 @@
                     raise NumberOfMatchesWithinLimits(max_number_of_matches)
                 
         else:
+            child = (yield request.locateChildResource(calresource, name))
             calendar = (yield child.iCalendarForUser(request))
             
             # The calendar may come back as None if the resource is being changed, or was deleted

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2011-02-16 20:13:34 UTC (rev 7029)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2011-02-16 21:59:18 UTC (rev 7030)
@@ -660,13 +660,19 @@
                     "Default",
                 ]
             },
-#            "ProxyDB": {
+#            "Shared": {
 #                "ClientEnabled": True,
 #                "ServerEnabled": True,
 #                "BindAddress": "127.0.0.1",
 #                "Port": 11211,
 #                "HandleCacheTypes": [
-#                    "ProxyDB", "PrincipalToken",
+#                    "ProxyDB",
+#                    "PrincipalToken",
+#                    "FBCache",
+#                    "ScheduleAddressMapper",
+#                    "SQL.props",
+#                    "SQL.calhome",
+#                    "SQL.adbkhome",
 #                ]
 #            },
         },
@@ -691,6 +697,8 @@
     "EnableResponseCache":  True,
     "ResponseCacheTimeout": 30, # Minutes
 
+    "EnableFreeBusyCache":  True,
+
     # Specify which opendirectory module to use:
     # "opendirectory" is PyOpenDirectory (the old one which uses
     # DirectoryService.framework)

Modified: CalendarServer/trunk/txdav/base/propertystore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/base/propertystore/sql.py	2011-02-16 20:13:34 UTC (rev 7029)
+++ CalendarServer/trunk/txdav/base/propertystore/sql.py	2011-02-16 21:59:18 UTC (rev 7030)
@@ -42,7 +42,7 @@
 
 class PropertyStore(AbstractPropertyStore):
 
-    _cacher = Memcacher("propertystore.sql", pickle=True, key_normalization=False)
+    _cacher = Memcacher("SQL.props", pickle=True, key_normalization=False)
 
     def __init__(self, *a, **kw):
         raise NotImplementedError(

Modified: CalendarServer/trunk/txdav/caldav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/sql.py	2011-02-16 20:13:34 UTC (rev 7029)
+++ CalendarServer/trunk/txdav/caldav/datastore/sql.py	2011-02-16 21:59:18 UTC (rev 7030)
@@ -94,7 +94,7 @@
     _notifierPrefix = "CalDAV"
     _revisionsTable = CALENDAR_OBJECT_REVISIONS_TABLE
 
-    _cacher = Memcacher("datastore.calhome", pickle=True, key_normalization=False)
+    _cacher = Memcacher("SQL.calhome", pickle=True, key_normalization=False)
 
     def __init__(self, transaction, ownerUID, notifiers):
 

Modified: CalendarServer/trunk/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/trunk/txdav/carddav/datastore/sql.py	2011-02-16 20:13:34 UTC (rev 7029)
+++ CalendarServer/trunk/txdav/carddav/datastore/sql.py	2011-02-16 21:59:18 UTC (rev 7030)
@@ -80,7 +80,7 @@
     _notifierPrefix = "CardDAV"
     _revisionsTable = ADDRESSBOOK_OBJECT_REVISIONS_TABLE
 
-    _cacher = Memcacher("datastore.adbkhome", pickle=True, key_normalization=False)
+    _cacher = Memcacher("SQL.adbkhome", pickle=True, key_normalization=False)
 
     def __init__(self, transaction, ownerUID, notifiers):
 
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20110216/d37a91d1/attachment-0001.html>


More information about the calendarserver-changes mailing list