<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[15629] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/15629">15629</a></dd>
<dt>Author</dt> <dd>cdaboo@apple.com</dd>
<dt>Date</dt> <dd>2016-05-21 06:30:05 -0700 (Sat, 21 May 2016)</dd>
</dl>
<h3>Log Message</h3>
<pre>OPtimize the inbox cleanup process by staggering the work and reducing the type of queries done to detect what needs to be cleaned.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkconfcaldavdstdconfigplist">CalendarServer/trunk/conf/caldavd-stdconfig.plist</a></li>
<li><a href="#CalendarServertrunktwistedcaldavstdconfigpy">CalendarServer/trunk/twistedcaldav/stdconfig.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresqlpy">CalendarServer/trunk/txdav/common/datastore/sql.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemacurrentoracledialectsql">CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemacurrentsql">CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_60_to_61sql">CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_60_to_61.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_60_to_61sql">CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_60_to_61.sql</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoreworkinbox_cleanuppy">CalendarServer/trunk/txdav/common/datastore/work/inbox_cleanup.py</a></li>
<li><a href="#CalendarServertrunktxdavcommondatastoreworktesttest_inbox_cleanuppy">CalendarServer/trunk/txdav/common/datastore/work/test/test_inbox_cleanup.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkconfcaldavdstdconfigplist"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/conf/caldavd-stdconfig.plist (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/conf/caldavd-stdconfig.plist 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/conf/caldavd-stdconfig.plist 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -1069,14 +1069,27 @@
</span><span class="cx"> <key>ItemLifetimeDays</key>
</span><span class="cx"> <real>14.0</real>
</span><span class="cx">
</span><del>- <!-- Number of days to keep an inbox item past the time when its referenced
- event ends -->
- <key>ItemLifeBeyondEventEndDays</key>
- <real>14.0</real>
-
</del><span class="cx"> <!-- Number of days between inbox cleanups -->
</span><span class="cx"> <key>CleanupPeriodDays</key>
</span><span class="cx"> <real>2.0</real>
</span><ins>+
+ <!-- Number of seconds before CleanupOneInboxWork starts after
+ InboxCleanupWork -->
+ <key>StartDelaySeconds</key>
+ <integer>300</integer>
+
+ <!-- Number of seconds between each CleanupOneInboxWork (fractional) -->
+ <key>StaggerSeconds</key>
+ <real>0.5</real>
+
+ <!-- Number of items above which inbox removals will be deferred to a work
+ item -->
+ <key>InboxRemoveWorkThreshold</key>
+ <integer>5</integer>
+
+ <!-- Number of seconds between each InboxRemoveWork -->
+ <key>RemovalStaggerSeconds</key>
+ <real>0.5</real>
</ins><span class="cx"> </dict>
</span><span class="cx">
</span><span class="cx"> <!-- CardDAV Features -->
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavstdconfigpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -645,9 +645,12 @@
</span><span class="cx">
</span><span class="cx"> "InboxCleanup": {
</span><span class="cx"> "Enabled": True,
</span><del>- "ItemLifetimeDays" : 14.0, # Number of days before deleting a new inbox item
- "ItemLifeBeyondEventEndDays" : 14.0, # Number of days to keep an inbox item past the time when its referenced event ends
- "CleanupPeriodDays" : 2.0, # Number of days between inbox cleanups
</del><ins>+ "ItemLifetimeDays" : 14.0, # Number of days before deleting a new inbox item
+ "CleanupPeriodDays" : 2.0, # Number of days between inbox cleanups
+ "StartDelaySeconds": 5 * 60, # Number of seconds before CleanupOneInboxWork starts after InboxCleanupWork
+ "StaggerSeconds": 0.5, # Number of seconds between each CleanupOneInboxWork (fractional)
+ "InboxRemoveWorkThreshold": 5, # Number of items above which inbox removals will be deferred to a work item
+ "RemovalStaggerSeconds": 0.5, # Number of seconds between each InboxRemoveWork
</ins><span class="cx"> },
</span><span class="cx">
</span><span class="cx"> # CardDAV Features
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresqlpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql.py (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql.py 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/sql.py 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -1472,97 +1472,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @classproperty
</span><del>- def _orphanedInboxItemsInHomeIDQuery(cls):
- """
- DAL query to select inbox items that refer to nonexistent events in a
- given home identified by the home resource ID.
- """
- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
- return Select(
- [co.RESOURCE_NAME],
- From=co.join(cb),
- Where=(
- cb.CALENDAR_HOME_RESOURCE_ID == Parameter("homeID")).And(
- cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
- cb.BIND_MODE == _BIND_MODE_OWN).And(
- cb.CALENDAR_RESOURCE_NAME == 'inbox').And(
- co.ICALENDAR_UID.NotIn(
- Select(
- [co.ICALENDAR_UID],
- From=co.join(cb),
- Where=(
- cb.CALENDAR_HOME_RESOURCE_ID == Parameter("homeID")).And(
- cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
- cb.BIND_MODE == _BIND_MODE_OWN).And(
- cb.CALENDAR_RESOURCE_NAME != 'inbox')
- )
- )
- ),
- )
-
-
- @inlineCallbacks
- def orphanedInboxItemsInHomeID(self, homeID):
- """
- Find inbox item names that refer to nonexistent events in a given home.
-
- Returns a deferred to a list of orphaned inbox item names
- """
- rows = yield self._orphanedInboxItemsInHomeIDQuery.on(self, homeID=homeID)
- names = [row[0] for row in rows]
- returnValue(names)
-
-
- @classproperty
- def _inboxItemsInHomeIDForEventsBeforeCutoffQuery(cls):
- """
- DAL query to select inbox items that refer to events in a before a
- given date.
- """
- co = schema.CALENDAR_OBJECT
- cb = schema.CALENDAR_BIND
- tr = schema.TIME_RANGE
- return Select(
- [co.RESOURCE_NAME],
- From=co.join(cb),
- Where=(
- cb.CALENDAR_HOME_RESOURCE_ID == Parameter("homeID")).And(
- cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
- cb.BIND_MODE == _BIND_MODE_OWN).And(
- cb.CALENDAR_RESOURCE_NAME == 'inbox').And(
- co.ICALENDAR_UID.In(
- Select(
- [co.ICALENDAR_UID],
- From=tr.join(co.join(cb)),
- Where=(
- cb.CALENDAR_HOME_RESOURCE_ID == Parameter("homeID")).And(
- cb.CALENDAR_RESOURCE_ID == co.CALENDAR_RESOURCE_ID).And(
- cb.BIND_MODE == _BIND_MODE_OWN).And(
- cb.CALENDAR_RESOURCE_NAME != 'inbox').And(
- tr.CALENDAR_OBJECT_RESOURCE_ID == co.RESOURCE_ID).And(
- tr.END_DATE < Parameter("cutoff"))
- )
- )
- ),
- )
-
-
- @inlineCallbacks
- def listInboxItemsInHomeForEventsBefore(self, homeID, cutoff):
- """
- return a list of inbox item names that refer to events before a given
- date in a given home.
-
- Returns a deferred to a list of orphaned inbox item names
- """
- rows = yield self._inboxItemsInHomeIDForEventsBeforeCutoffQuery.on(
- self, homeID=homeID, cutoff=cutoff)
- names = [row[0] for row in rows]
- returnValue(names)
-
-
- @classproperty
</del><span class="cx"> def _inboxItemsInHomeIDCreatedBeforeCutoffQuery(cls):
</span><span class="cx"> """
</span><span class="cx"> DAL query to select inbox items created before a given date.
</span><span class="lines">@@ -1584,7 +1493,7 @@
</span><span class="cx"> @inlineCallbacks
</span><span class="cx"> def listInboxItemsInHomeCreatedBefore(self, homeID, cutoff):
</span><span class="cx"> """
</span><del>- return a list of inbox item names that creaed before a given date in a
</del><ins>+ return a list of inbox item names that created before a given date in a
</ins><span class="cx"> given home.
</span><span class="cx">
</span><span class="cx"> Returns a deferred to a list of orphaned inbox item names
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemacurrentoracledialectsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -557,6 +557,14 @@
</span><span class="cx"> "HOME_ID" integer not null unique references CALENDAR_HOME on delete cascade
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create table INBOX_REMOVE_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "HOME_ID" integer not null references CALENDAR_HOME on delete cascade,
+ "RESOURCE_NAME" nvarchar2(255),
+ unique ("HOME_ID", "RESOURCE_NAME")
+);
+
</ins><span class="cx"> create table SCHEDULE_WORK (
</span><span class="cx"> "WORK_ID" integer primary key,
</span><span class="cx"> "JOB_ID" integer not null references JOB,
</span><span class="lines">@@ -971,6 +979,10 @@
</span><span class="cx"> "JOB_ID"
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+create index INBOX_REMOVE_WORK_JOB_4b627f1e on INBOX_REMOVE_WORK (
+ "JOB_ID"
+);
+
</ins><span class="cx"> create index SCHEDULE_WORK_JOB_ID_65e810ee on SCHEDULE_WORK (
</span><span class="cx"> "JOB_ID"
</span><span class="cx"> );
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemacurrentsql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -1053,6 +1053,18 @@
</span><span class="cx"> create index CLEANUP_ONE_INBOX_WORK_JOB_ID on
</span><span class="cx"> CLEANUP_ONE_INBOX_WORK(JOB_ID);
</span><span class="cx">
</span><ins>+create table INBOX_REMOVE_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ HOME_ID integer not null references CALENDAR_HOME on delete cascade,
+ RESOURCE_NAME varchar(255) not null,
+
+ unique (HOME_ID, RESOURCE_NAME) -- implicit index
+);
+
+create index INBOX_REMOVE_WORK_JOB_ID on
+ INBOX_REMOVE_WORK(JOB_ID);
+
</ins><span class="cx"> -------------------
</span><span class="cx"> -- Schedule Work --
</span><span class="cx"> -------------------
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemaupgradesoracledialectupgrade_from_60_to_61sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_60_to_61.sql (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_60_to_61.sql 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_60_to_61.sql 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -31,6 +31,18 @@
</span><span class="cx"> "REVISION"
</span><span class="cx"> );
</span><span class="cx">
</span><ins>+-- New table
+create table INBOX_REMOVE_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "HOME_ID" integer not null references CALENDAR_HOME on delete cascade,
+ "RESOURCE_NAME" nvarchar2(255),
+ unique ("HOME_ID", "RESOURCE_NAME")
+);
</ins><span class="cx">
</span><ins>+create index INBOX_REMOVE_WORK_JOB_4b627f1e on INBOX_REMOVE_WORK (
+ "JOB_ID"
+);
+
</ins><span class="cx"> -- update the version
</span><span class="cx"> update CALENDARSERVER set VALUE = '61' where NAME = 'VERSION';
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoresql_schemaupgradespostgresdialectupgrade_from_60_to_61sql"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_60_to_61.sql (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_60_to_61.sql 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_60_to_61.sql 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -28,5 +28,18 @@
</span><span class="cx"> create index NOTIFICATION_OBJECT_REVISIONS_REVISION
</span><span class="cx"> on NOTIFICATION_OBJECT_REVISIONS(REVISION);
</span><span class="cx">
</span><ins>+-- New table
+create table INBOX_REMOVE_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ HOME_ID integer not null references CALENDAR_HOME on delete cascade,
+ RESOURCE_NAME varchar(255) not null,
+
+ unique (HOME_ID, RESOURCE_NAME) -- implicit index
+);
+
+create index INBOX_REMOVE_WORK_JOB_ID on
+ INBOX_REMOVE_WORK(JOB_ID);
+
</ins><span class="cx"> -- update the version
</span><span class="cx"> update CALENDARSERVER set VALUE = '61' where NAME = 'VERSION';
</span></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoreworkinbox_cleanuppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/work/inbox_cleanup.py (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/work/inbox_cleanup.py 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/work/inbox_cleanup.py 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -77,8 +77,11 @@
</span><span class="cx"> Where=ch.STATUS == _HOME_STATUS_NORMAL,
</span><span class="cx"> ).on(self.transaction)
</span><span class="cx">
</span><ins>+ # Add an initial delay to the start of the first work item, then add an offset between each item
+ seconds = config.InboxCleanup.StartDelaySeconds
</ins><span class="cx"> for homeRow in homeRows:
</span><del>- yield CleanupOneInboxWork.reschedule(self.transaction, seconds=0, homeID=homeRow[0])
</del><ins>+ yield CleanupOneInboxWork.reschedule(self.transaction, seconds=seconds, homeID=homeRow[0])
+ seconds += config.InboxCleanup.StaggerSeconds
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="lines">@@ -91,51 +94,46 @@
</span><span class="cx">
</span><span class="cx"> # No need to delete other work items. They are unique
</span><span class="cx">
</span><del>- # get orphan names
- orphanNames = set((
- yield self.transaction.orphanedInboxItemsInHomeID(self.homeID)
- ))
- if orphanNames:
- home = yield self.transaction.calendarHomeWithResourceID(self.homeID)
- log.info(
- "Inbox cleanup work in home: {homeUID}, deleting orphaned items: {orphanNames}",
- homeUID=home.uid(), orphanNames=orphanNames,
- )
-
</del><span class="cx"> # get old item names
</span><span class="cx"> if float(config.InboxCleanup.ItemLifetimeDays) >= 0: # use -1 to disable; 0 is test case
</span><span class="cx"> cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=float(config.InboxCleanup.ItemLifetimeDays))
</span><span class="cx"> oldItemNames = set((
</span><span class="cx"> yield self.transaction.listInboxItemsInHomeCreatedBefore(self.homeID, cutoff)
</span><span class="cx"> ))
</span><del>- newDeleters = oldItemNames - orphanNames
- if newDeleters:
</del><ins>+ if oldItemNames:
</ins><span class="cx"> home = yield self.transaction.calendarHomeWithResourceID(self.homeID)
</span><span class="cx"> log.info(
</span><del>- "Inbox cleanup work in home: {homeUID}, deleting old items: {newDeleters}",
- homeUID=home.uid(), newDeleters=newDeleters,
</del><ins>+ "Inbox cleanup work in home: {homeUID}, deleting old items: {oldItemNames}",
+ homeUID=home.uid(), newDeleters=oldItemNames,
</ins><span class="cx"> )
</span><del>- else:
- oldItemNames = set()
</del><span class="cx">
</span><del>- # get item name for old events
- if float(config.InboxCleanup.ItemLifeBeyondEventEndDays) >= 0: # use -1 to disable; 0 is test case
- cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=float(config.InboxCleanup.ItemLifeBeyondEventEndDays))
- itemNamesForOldEvents = set((
- yield self.transaction.listInboxItemsInHomeForEventsBefore(self.homeID, cutoff)
- ))
- newDeleters = itemNamesForOldEvents - oldItemNames - orphanNames
- if newDeleters:
- home = yield self.transaction.calendarHomeWithResourceID(self.homeID)
- log.info(
- "Inbox cleanup work in home: {homeUID}, deleting items for old events: {newDeleters}",
- homeUID=home.uid(), newDeleters=newDeleters,
- )
- else:
- itemNamesForOldEvents = set()
</del><ins>+ # If the number to delete is below our threshold then delete right away,
+ # otherwise queue up more work items to delete these
+ if len(oldItemNames) < config.InboxCleanup.InboxRemoveWorkThreshold:
+ inbox = yield home.childWithName("inbox")
+ for item in (yield inbox.objectResourcesWithNames(oldItemNames)):
+ yield item.remove()
+ else:
+ seconds = config.InboxCleanup.RemovalStaggerSeconds
+ for item in oldItemNames:
+ yield InboxRemoveWork.reschedule(self.transaction, seconds=seconds, homeID=self.homeID, resourceName=item)
+ seconds += config.InboxCleanup.RemovalStaggerSeconds
</ins><span class="cx">
</span><del>- itemNamesToDelete = orphanNames | itemNamesForOldEvents | oldItemNames
- if itemNamesToDelete:
</del><ins>+
+
+class InboxRemoveWork(WorkItem, fromTable(schema.INBOX_REMOVE_WORK)):
+
+ group = property(lambda self: (self.table.HOME_ID == self.homeID).And(self.table.RESOURCE_NAME == self.resourceName))
+
+ @inlineCallbacks
+ def doWork(self):
+
+ # Some of the resources may no longer exist by the time this work item runs
+ # so simply ignore that and let the work complete without doing anything
+ home = yield self.transaction.calendarHomeWithResourceID(self.homeID)
+ if home is not None:
</ins><span class="cx"> inbox = yield home.childWithName("inbox")
</span><del>- for item in (yield inbox.objectResourcesWithNames(itemNamesToDelete)):
- yield item.remove()
</del><ins>+ if inbox is not None:
+ item = yield inbox.objectResourceWithName(self.resourceName)
+ if item is not None:
+ yield item.remove()
</ins></span></pre></div>
<a id="CalendarServertrunktxdavcommondatastoreworktesttest_inbox_cleanuppy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/common/datastore/work/test/test_inbox_cleanup.py (15628 => 15629)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/common/datastore/work/test/test_inbox_cleanup.py 2016-05-21 13:29:05 UTC (rev 15628)
+++ CalendarServer/trunk/txdav/common/datastore/work/test/test_inbox_cleanup.py 2016-05-21 13:30:05 UTC (rev 15629)
</span><span class="lines">@@ -155,36 +155,10 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_orphans(self):
- """
- Verify that orphaned Inbox items are removed
- """
- self.patch(config.InboxCleanup, "ItemLifetimeDays", -1)
- self.patch(config.InboxCleanup, "ItemLifeBeyondEventEndDays", -1)
-
- # create orphans by deleting events
- cal = yield self.calendarUnderTest(home="user01", name="calendar")
- for item in (yield cal.objectResourcesWithNames(["cal1.ics", "cal3.ics"])):
- yield item.purge()
-
- # do cleanup
- yield self.transactionUnderTest().enqueue(CleanupOneInboxWork, homeID=cal.ownerHome()._resourceID, notBefore=datetime.datetime.utcnow())
- yield self.commit()
- yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60)
-
- # check that orphans are deleted
- inbox = yield self.calendarUnderTest(home="user01", name="inbox")
- items = yield inbox.objectResources()
- names = [item.name() for item in items]
- self.assertEqual(set(names), set(["cal2.ics"]))
-
-
- @inlineCallbacks
</del><span class="cx"> def test_old(self):
</span><span class="cx"> """
</span><span class="cx"> Verify that old inbox items are removed
</span><span class="cx"> """
</span><del>- self.patch(config.InboxCleanup, "ItemLifeBeyondEventEndDays", -1)
</del><span class="cx">
</span><span class="cx"> # Predate some inbox items
</span><span class="cx"> inbox = yield self.calendarUnderTest(home="user01", name="inbox")
</span><span class="lines">@@ -211,21 +185,28 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>- def test_referenceOldEvent(self):
</del><ins>+ def test_old_queued(self):
</ins><span class="cx"> """
</span><del>- Verify that inbox items references old events are removed
</del><ins>+ Verify that old inbox items are removed
</ins><span class="cx"> """
</span><del>- # events are already too old, so make one event end now
- calendar = yield self.calendarUnderTest(home="user01", name="calendar")
- cal3Event = yield calendar.objectResourceWithName("cal3.ics")
</del><span class="cx">
</span><del>- tr = schema.TIME_RANGE
</del><ins>+ # Patch to force remove work items
+ self.patch(config.InboxCleanup, "InboxRemoveWorkThreshold", 0)
+
+ # Predate some inbox items
+ inbox = yield self.calendarUnderTest(home="user01", name="inbox")
+ oldDate = datetime.datetime.utcnow() - datetime.timedelta(days=float(config.InboxCleanup.ItemLifetimeDays), seconds=10)
+
+ itemsToPredate = ["cal2.ics", "cal3.ics"]
+ co = schema.CALENDAR_OBJECT
</ins><span class="cx"> yield Update(
</span><del>- {tr.END_DATE: datetime.datetime.utcnow()},
- Where=tr.CALENDAR_OBJECT_RESOURCE_ID == cal3Event._resourceID
- ).on(self.transactionUnderTest())
</del><ins>+ {co.CREATED: oldDate},
+ Where=co.RESOURCE_NAME.In(Parameter("itemsToPredate", len(itemsToPredate))).And(
+ co.CALENDAR_RESOURCE_ID == inbox._resourceID)
+ ).on(self.transactionUnderTest(), itemsToPredate=itemsToPredate)
+
</ins><span class="cx"> # do cleanup
</span><del>- yield self.transactionUnderTest().enqueue(CleanupOneInboxWork, homeID=calendar.ownerHome()._resourceID, notBefore=datetime.datetime.utcnow())
</del><ins>+ yield self.transactionUnderTest().enqueue(CleanupOneInboxWork, homeID=inbox.ownerHome()._resourceID, notBefore=datetime.datetime.utcnow())
</ins><span class="cx"> yield self.commit()
</span><span class="cx"> yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60)
</span><span class="cx">
</span><span class="lines">@@ -233,4 +214,4 @@
</span><span class="cx"> inbox = yield self.calendarUnderTest(home="user01", name="inbox")
</span><span class="cx"> items = yield inbox.objectResources()
</span><span class="cx"> names = [item.name() for item in items]
</span><del>- self.assertEqual(set(names), set(["cal3.ics"]))
</del><ins>+ self.assertEqual(set(names), set(["cal1.ics"]))
</ins></span></pre>
</div>
</div>
</body>
</html>