[CalendarServer-changes] [2177] CalendarServer/trunk/twistedcaldav/directory/digest.py

[source_changes at macosforge.org]

Revision: 2177
          http://trac.macosforge.org/projects/calendarserver/changeset/2177
Author:   cdaboo at apple.com
Date:     2008-02-25 13:22:10 -0800 (Mon, 25 Feb 2008)

Log Message:
-----------
Make the clean-up process occur much more quickly by turning off the auto-commit option. That
also requires adding explicit commit()/rollback() calls. A new api to do a batch delete of items
in one transaction was added.

Modified Paths:
--------------
    CalendarServer/trunk/twistedcaldav/directory/digest.py

Modified: CalendarServer/trunk/twistedcaldav/directory/digest.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/digest.py	2008-02-23 04:20:49 UTC (rev 2176)
+++ CalendarServer/trunk/twistedcaldav/directory/digest.py	2008-02-25 21:22:10 UTC (rev 2177)
@@ -87,6 +87,15 @@
         """
         pass
 
+    def deleteMany(self, keys):
+        """
+        Remove the records associated with the supplied keys.
+
+        @param key:        the key to remove.
+        @type key:         C{str}
+        """
+        pass
+
     def keys(self):
         """
         Return all the keys currently available.
@@ -95,6 +104,14 @@
         """
         pass
     
+    def items(self):
+        """
+        Return all the key/value pairs currently available.
+        
+        @return:    a C{list} of C{tuple} for each key/value currently in the database.
+        """
+        pass
+    
 class DigestCredentialsMap(object):
 
     implements(IDigestCredentialsDatabase)
@@ -130,12 +147,26 @@
         if self.db.has_key(key):
             del self.db[key]
 
+    def deleteMany(self, keys):
+        """
+        See IDigestCredentialsDatabase.
+        """
+        for key in keys:
+            if self.db.has_key(key):
+                del self.db[key]
+
     def keys(self):
         """
         See IDigestCredentialsDatabase.
         """
         return self.db.keys()
 
+    def items(self):
+        """
+        See IDigestCredentialsDatabase.
+        """
+        return self.db.items()
+
 class DigestCredentialsDB(AbstractSQLDatabase):
 
     implements(IDigestCredentialsDatabase)
@@ -159,7 +190,7 @@
 
     def __init__(self, path):
         db_path = os.path.join(path, DigestCredentialsDB.dbFilename)
-        super(DigestCredentialsDB, self).__init__(db_path, autocommit=True)
+        super(DigestCredentialsDB, self).__init__(db_path, autocommit=False)
         self.exceptions = 0
     
     def has_key(self, key):
@@ -189,8 +220,10 @@
         try:
             pvalue = pickle.dumps(value)
             self._set_in_db(key, pvalue)
+            self._db_commit()
             self.exceptions = 0
         except OperationalError, e:
+            self._db_rollback()
             self.exceptions += 1
             if self.exceptions >= self.exceptionLimit:
                 self._db_close()
@@ -224,14 +257,33 @@
         """
         try:
             self._delete_from_db(key)
+            self._db_commit()
             self.exceptions = 0
         except OperationalError, e:
+            self._db_rollback()
             self.exceptions += 1
             if self.exceptions >= self.exceptionLimit:
                 self._db_close()
                 log.err("Reset digest credentials database connection: %s" % (e,))
             raise
 
+    def deleteMany(self, keys):
+        """
+        See IDigestCredentialsDatabase.
+        """
+        try:
+            for key in keys:
+                self._delete_from_db(key)
+            self._db_commit()
+            self.exceptions = 0
+        except OperationalError, e:
+            self._db_rollback()
+            self.exceptions += 1
+            if self.exceptions >= self.exceptionLimit:
+                self._db_close()
+                log.err("Reset digest credentials database connection: %s" % (e,))
+            raise
+
     def keys(self):
         """
         See IDigestCredentialsDatabase.
@@ -250,6 +302,24 @@
                 log.err("Reset digest credentials database connection: %s" % (e,))
             raise
 
+    def items(self):
+        """
+        See IDigestCredentialsDatabase.
+        """
+        try:
+            result = []
+            for key in self._db_execute("select KEY, VALUE from DIGESTCREDENTIALS"):
+                result.append((str(key[0]), pickle.loads(str(key[1])),))
+            
+            self.exceptions = 0
+            return result
+        except OperationalError, e:
+            self.exceptions += 1
+            if self.exceptions >= self.exceptionLimit:
+                self._db_close()
+                log.err("Reset digest credentials database connection: %s" % (e,))
+            raise
+
     def _set_in_db(self, key, value):
         """
         Insert the specified entry into the database, replacing any that might already exist.
@@ -500,16 +570,16 @@
         """
         This should be called at regular intervals to remove expired credentials from the cache.
         """
-        keys = self.db.keys()
+        items = self.db.items()
         oldest_allowed = time.time() - DigestCredentialFactory.CHALLENGE_LIFETIME_SECS
-        for key in keys:
-            try:
-                value = self.db.get(key)
-                if value is not None:
-                    ignore_clientip, ignore_cnonce, db_timestamp = value
-                    if db_timestamp <= oldest_allowed:
-                        self.invalidate(key)
-            except Exception, e:
-                # Clean-up errors can be logged but we should ignore them
-                log.err("Error cleaning digest credentials: %s" % (e,))
-                pass
+        delete_keys = []
+        for key, value in items:
+            ignore_clientip, ignore_cnonce, db_timestamp = value
+            if db_timestamp <= oldest_allowed:
+                delete_keys.append(key)
+
+        try:
+            self.db.deleteMany(delete_keys)
+        except Exception, e:
+            # Clean-up errors can be logged but we should ignore them
+            log.err("Failed to clean digest credentials: %s" % (e,))

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20080225/120b260f/attachment.html