[CalendarServer-changes] [13831] CalendarServer/trunk/txdav/who

[source_changes at macosforge.org]

Revision: 13831
          http://trac.calendarserver.org//changeset/13831
Author:   sagen at apple.com
Date:     2014-08-04 12:12:09 -0700 (Mon, 04 Aug 2014)
Log Message:
-----------
Purge expired records after every 100 lookups

Modified Paths:
--------------
    CalendarServer/trunk/txdav/who/cache.py
    CalendarServer/trunk/txdav/who/test/test_cache.py

Modified: CalendarServer/trunk/txdav/who/cache.py
===================================================================
--- CalendarServer/trunk/txdav/who/cache.py	2014-08-04 18:07:35 UTC (rev 13830)
+++ CalendarServer/trunk/txdav/who/cache.py	2014-08-04 19:12:09 UTC (rev 13831)
@@ -44,6 +44,10 @@
 )
 from twisted.python.constants import Values, ValueConstant
 
+# After this many lookup calls, we scan the cache for expired records
+# to purge
+SCAN_AFTER_LOOKUP_COUNT = 100
+
 log = Logger()
 
 
@@ -96,9 +100,16 @@
         }
         self._hitCount = 0
         self._requestCount = 0
+        self._lookupsUntilScan = SCAN_AFTER_LOOKUP_COUNT
 
 
     def setTestTime(self, timestamp):
+        """
+        Only used for unit tests to override the notion of "now"
+
+        @param timestamp: seconds
+        @type timestamp: C{float}
+        """
         self._test_time = timestamp
 
 
@@ -138,9 +149,26 @@
                 pass
 
 
+    def purgeExpiredRecords(self):
+        """
+        Scans the cache for expired records and deletes them
+        """
+        if hasattr(self, "_test_time"):
+            now = self._test_time
+        else:
+            now = time.time()
+
+        for indexType in self._cache:
+            for key, (cachedTime, record) in self._cache[indexType].items():
+                if now - self._expireSeconds > cachedTime:
+                    del self._cache[indexType][key]
+
+
     def lookupRecord(self, indexType, key):
         """
-        Looks for a record in the specified index, under the specified key
+        Looks for a record in the specified index, under the specified key.
+        After every SCAN_AFTER_LOOKUP_COUNT lookups are done,
+        purgeExpiredRecords() is called.
 
         @param index: an index type
         @type indexType: L{IndexType}
@@ -152,6 +180,12 @@
         @rtype: L{DirectoryRecord}
         """
 
+        if self._lookupsUntilScan == 0:
+            self._lookupsUntilScan = SCAN_AFTER_LOOKUP_COUNT
+            self.purgeExpiredRecords()
+        else:
+            self._lookupsUntilScan -= 1
+
         self._requestCount += 1
         if key in self._cache[indexType]:
 

Modified: CalendarServer/trunk/txdav/who/test/test_cache.py
===================================================================
--- CalendarServer/trunk/txdav/who/test/test_cache.py	2014-08-04 18:07:35 UTC (rev 13830)
+++ CalendarServer/trunk/txdav/who/test/test_cache.py	2014-08-04 19:12:09 UTC (rev 13831)
@@ -20,7 +20,9 @@
 
 from twisted.internet.defer import inlineCallbacks
 from twistedcaldav.test.util import StoreTestCase
-from txdav.who.cache import CachingDirectoryService
+from txdav.who.cache import (
+    CachingDirectoryService, IndexType, SCAN_AFTER_LOOKUP_COUNT
+)
 from twext.who.idirectory import (
     RecordType
 )
@@ -228,3 +230,35 @@
         self.assertEquals(record.uid, u"cache-uid-1")
         self.assertEquals(dir._hitCount, 1)
         self.assertEquals(dir._requestCount, 4)
+
+
+    @inlineCallbacks
+    def test_cachePurging(self):
+        """
+        Verify records are purged from cache after a certain amount of requests
+        """
+
+        dir = self.cachingDirectory
+
+        dir.setTestTime(1.0)
+
+        record = yield dir.recordWithUID(u"cache-uid-1")
+        self.assertEquals(record.uid, u"cache-uid-1")
+        self.assertEquals(dir._hitCount, 0)
+        self.assertEquals(dir._requestCount, 1)
+
+        # 60 seconds later, the record has expired, but is still present
+        dir.setTestTime(60.0)
+
+        self.assertTrue(u"cache-uid-1" in dir._cache[IndexType.uid])
+
+        # After SCAN_AFTER_LOOKUP_COUNT requests, however, that expired entry
+        # will be removed
+
+        for i in xrange(SCAN_AFTER_LOOKUP_COUNT):
+            yield dir.recordWithUID(u"cache-uid-2")
+
+        # cache-uid-1 no longer in cache
+        self.assertFalse(u"cache-uid-1" in dir._cache[IndexType.uid])
+        # cache-uid-2 still in cache
+        self.assertTrue(u"cache-uid-2" in dir._cache[IndexType.uid])
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140804/d74c38f8/attachment.html>