[CalendarServer-changes] [15506] CalendarServer/trunk/txdav

source_changes at macosforge.org source_changes at macosforge.org
Mon Apr 11 17:07:53 PDT 2016


Revision: 15506
          http://trac.calendarserver.org//changeset/15506
Author:   sagen at apple.com
Date:     2016-04-11 17:07:53 -0700 (Mon, 11 Apr 2016)
Log Message:
-----------
Batch when cleaning out unused or missing groups

Modified Paths:
--------------
    CalendarServer/trunk/txdav/common/datastore/sql_directory.py
    CalendarServer/trunk/txdav/who/groups.py

Modified: CalendarServer/trunk/txdav/common/datastore/sql_directory.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_directory.py	2016-04-11 21:09:14 UTC (rev 15505)
+++ CalendarServer/trunk/txdav/common/datastore/sql_directory.py	2016-04-12 00:07:53 UTC (rev 15506)
@@ -52,7 +52,21 @@
         )
 
 
+    @classmethod
+    def allGroups(cls, txn):
 
+        return GroupsRecord.query(txn, None)
+
+
+    @classmethod
+    def groupsMissingSince(cls, txn, since):
+
+        return GroupsRecord.query(
+            txn,
+            (cls.extant == 0).And(cls.modified < since)
+        )
+
+
 class GroupMembershipRecord(SerializableRecord, fromTable(schema.GROUP_MEMBERSHIP)):
     """
     @DynamicAttrs
@@ -452,7 +466,40 @@
         returnValue(set([group.groupUID.decode("utf-8") for group in groups]))
 
 
+    @inlineCallbacks
+    def allGroups(self):
+        """
+        Return the UIDs of all groups.
 
+        @returns: the UIDs of all groups
+        @rtype: a Deferred resulting in a set
+        """
+
+        results = yield GroupsRecord.allGroups(self)
+        groups = set([record.groupUID.decode("utf-8") for record in results])
+
+        returnValue(groups)
+
+
+    @inlineCallbacks
+    def groupsMissingSince(self, since):
+        """
+        @param since: datetime to compare against
+        @type since: C{datetime.datetime}
+
+        Return the UIDs of all groups marked as non-extant who have not been
+            modified since the given datetime.
+
+        @returns: UIDs
+        @rtype: a Deferred resulting in a set
+        """
+
+        results = yield GroupsRecord.groupsMissingSince(self, since)
+        groups = set([record.groupUID.decode("utf-8") for record in results])
+
+        returnValue(groups)
+
+
 class DelegatesAPIMixin(object):
     """
     A mixin for L{CommonStoreTransaction} that covers the delegates API.

Modified: CalendarServer/trunk/txdav/who/groups.py
===================================================================
--- CalendarServer/trunk/txdav/who/groups.py	2016-04-11 21:09:14 UTC (rev 15505)
+++ CalendarServer/trunk/txdav/who/groups.py	2016-04-12 00:07:53 UTC (rev 15506)
@@ -272,31 +272,40 @@
         #     "Groups to refresh: {g}", g=groupUIDs
         # )
 
-        if config.AutomaticPurging.Enabled and groupUIDs:
-            # remove unused groups and groups that have not been seen in a while
+        # Get the set of all known groups in the DB
+        knownGroupUIDs = yield txn.allGroups()
+
+        # We'll want to remove groups no longer in use
+        groupsToRemove = knownGroupUIDs - groupUIDs
+
+        # Also look for groups which have been marked as missing for a while
+        if config.AutomaticPurging.Enabled:
             dateLimit = (
                 datetime.datetime.utcnow() -
                 datetime.timedelta(seconds=float(config.AutomaticPurging.GroupPurgeIntervalSeconds))
             )
+            missingGroups = yield txn.groupsMissingSince(dateLimit)
+            groupsToRemove |= missingGroups
+
+        # Delete the groups in batches
+        groupsToRemove = list(groupsToRemove)
+        batchSize = 100
+        deletedGroupUIDs = []
+        while groupsToRemove:
+            batch = groupsToRemove[:batchSize]
+            del groupsToRemove[:batchSize]
+
             rows = yield GroupsRecord.deletesome(
                 txn,
-                (
-                    (GroupsRecord.extant == 0).And(GroupsRecord.modified < dateLimit)
-                ).Or(
-                    GroupsRecord.groupUID.NotIn(groupUIDs)
-                ),
+                GroupsRecord.groupUID.In(batch),
                 returnCols=GroupsRecord.groupUID,
             )
-        else:
-            # remove unused groups
-            rows = yield GroupsRecord.deletesome(
-                txn,
-                GroupsRecord.groupUID.NotIn(groupUIDs) if groupUIDs else None,
-                returnCols=GroupsRecord.groupUID,
+            deletedGroupUIDs.extend([row[0] for row in rows])
+
+        if deletedGroupUIDs:
+            self.log.debug(
+                "Deleted old or unused groups {d}", d=deletedGroupUIDs
             )
-        deletedGroupUIDs = [row[0] for row in rows]
-        if deletedGroupUIDs:
-            self.log.debug("Deleted old or unused groups {d}", d=deletedGroupUIDs)
 
         # For each of those groups, create a per-group refresh work item
         for groupUID in set(groupUIDs) - set(deletedGroupUIDs):
@@ -390,10 +399,10 @@
     @inlineCallbacks
     def refreshGroup(self, txn, groupUID):
         """
-            Does the work of a per-group refresh work item
-            Faults in the flattened membership of a group, as UIDs
-            and updates the GROUP_MEMBERSHIP table
-            WorkProposal is returned for tests
+        Does the work of a per-group refresh work item
+        Faults in the flattened membership of a group, as UIDs
+        and updates the GROUP_MEMBERSHIP table
+        WorkProposal is returned for tests
         """
         self.log.debug("Refreshing group: {g}", g=groupUID)
 
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160411/7feebf7b/attachment-0001.html>


More information about the calendarserver-changes mailing list