[CalendarServer-changes] [15506] CalendarServer/trunk/txdav
source_changes at macosforge.org
source_changes at macosforge.org
Mon Apr 11 17:07:53 PDT 2016
Revision: 15506
http://trac.calendarserver.org//changeset/15506
Author: sagen at apple.com
Date: 2016-04-11 17:07:53 -0700 (Mon, 11 Apr 2016)
Log Message:
-----------
Batch when cleaning out unused or missing groups
Modified Paths:
--------------
CalendarServer/trunk/txdav/common/datastore/sql_directory.py
CalendarServer/trunk/txdav/who/groups.py
Modified: CalendarServer/trunk/txdav/common/datastore/sql_directory.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_directory.py 2016-04-11 21:09:14 UTC (rev 15505)
+++ CalendarServer/trunk/txdav/common/datastore/sql_directory.py 2016-04-12 00:07:53 UTC (rev 15506)
@@ -52,7 +52,21 @@
)
+ @classmethod
+ def allGroups(cls, txn):
+ return GroupsRecord.query(txn, None)
+
+
+ @classmethod
+ def groupsMissingSince(cls, txn, since):
+
+ return GroupsRecord.query(
+ txn,
+ (cls.extant == 0).And(cls.modified < since)
+ )
+
+
class GroupMembershipRecord(SerializableRecord, fromTable(schema.GROUP_MEMBERSHIP)):
"""
@DynamicAttrs
@@ -452,7 +466,40 @@
returnValue(set([group.groupUID.decode("utf-8") for group in groups]))
+ @inlineCallbacks
+ def allGroups(self):
+ """
+ Return the UIDs of all groups.
+ @returns: the UIDs of all groups
+ @rtype: a Deferred resulting in a set
+ """
+
+ results = yield GroupsRecord.allGroups(self)
+ groups = set([record.groupUID.decode("utf-8") for record in results])
+
+ returnValue(groups)
+
+
+ @inlineCallbacks
+ def groupsMissingSince(self, since):
+ """
+ @param since: datetime to compare against
+ @type since: C{datetime.datetime}
+
+ Return the UIDs of all groups marked as non-extant who have not been
+ modified since the given datetime.
+
+ @returns: UIDs
+ @rtype: a Deferred resulting in a set
+ """
+
+ results = yield GroupsRecord.groupsMissingSince(self, since)
+ groups = set([record.groupUID.decode("utf-8") for record in results])
+
+ returnValue(groups)
+
+
class DelegatesAPIMixin(object):
"""
A mixin for L{CommonStoreTransaction} that covers the delegates API.
Modified: CalendarServer/trunk/txdav/who/groups.py
===================================================================
--- CalendarServer/trunk/txdav/who/groups.py 2016-04-11 21:09:14 UTC (rev 15505)
+++ CalendarServer/trunk/txdav/who/groups.py 2016-04-12 00:07:53 UTC (rev 15506)
@@ -272,31 +272,40 @@
# "Groups to refresh: {g}", g=groupUIDs
# )
- if config.AutomaticPurging.Enabled and groupUIDs:
- # remove unused groups and groups that have not been seen in a while
+ # Get the set of all known groups in the DB
+ knownGroupUIDs = yield txn.allGroups()
+
+ # We'll want to remove groups no longer in use
+ groupsToRemove = knownGroupUIDs - groupUIDs
+
+ # Also look for groups which have been marked as missing for a while
+ if config.AutomaticPurging.Enabled:
dateLimit = (
datetime.datetime.utcnow() -
datetime.timedelta(seconds=float(config.AutomaticPurging.GroupPurgeIntervalSeconds))
)
+ missingGroups = yield txn.groupsMissingSince(dateLimit)
+ groupsToRemove |= missingGroups
+
+ # Delete the groups in batches
+ groupsToRemove = list(groupsToRemove)
+ batchSize = 100
+ deletedGroupUIDs = []
+ while groupsToRemove:
+ batch = groupsToRemove[:batchSize]
+ del groupsToRemove[:batchSize]
+
rows = yield GroupsRecord.deletesome(
txn,
- (
- (GroupsRecord.extant == 0).And(GroupsRecord.modified < dateLimit)
- ).Or(
- GroupsRecord.groupUID.NotIn(groupUIDs)
- ),
+ GroupsRecord.groupUID.In(batch),
returnCols=GroupsRecord.groupUID,
)
- else:
- # remove unused groups
- rows = yield GroupsRecord.deletesome(
- txn,
- GroupsRecord.groupUID.NotIn(groupUIDs) if groupUIDs else None,
- returnCols=GroupsRecord.groupUID,
+ deletedGroupUIDs.extend([row[0] for row in rows])
+
+ if deletedGroupUIDs:
+ self.log.debug(
+ "Deleted old or unused groups {d}", d=deletedGroupUIDs
)
- deletedGroupUIDs = [row[0] for row in rows]
- if deletedGroupUIDs:
- self.log.debug("Deleted old or unused groups {d}", d=deletedGroupUIDs)
# For each of those groups, create a per-group refresh work item
for groupUID in set(groupUIDs) - set(deletedGroupUIDs):
@@ -390,10 +399,10 @@
@inlineCallbacks
def refreshGroup(self, txn, groupUID):
"""
- Does the work of a per-group refresh work item
- Faults in the flattened membership of a group, as UIDs
- and updates the GROUP_MEMBERSHIP table
- WorkProposal is returned for tests
+ Does the work of a per-group refresh work item
+ Faults in the flattened membership of a group, as UIDs
+ and updates the GROUP_MEMBERSHIP table
+ WorkProposal is returned for tests
"""
self.log.debug("Refreshing group: {g}", g=groupUID)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20160411/7feebf7b/attachment-0001.html>
More information about the calendarserver-changes
mailing list