[CalendarServer-changes] [12193] CalendarServer/branches/users/gaya/cleanrevisions
source_changes at macosforge.org
source_changes at macosforge.org
Wed Mar 12 11:23:44 PDT 2014
Revision: 12193
http://trac.calendarserver.org//changeset/12193
Author: gaya at apple.com
Date: 2013-12-23 13:49:21 -0800 (Mon, 23 Dec 2013)
Log Message:
-----------
Works in simple tests; clean up names
Modified Paths:
--------------
CalendarServer/branches/users/gaya/cleanrevisions/calendarserver/tap/caldav.py
CalendarServer/branches/users/gaya/cleanrevisions/twistedcaldav/stdconfig.py
CalendarServer/branches/users/gaya/cleanrevisions/txdav/carddav/datastore/sql.py
CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql.py
CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current.sql
CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_30_to_31.sql
CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_30_to_31.sql
CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/work/revision_cleanup.py
Modified: CalendarServer/branches/users/gaya/cleanrevisions/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/calendarserver/tap/caldav.py 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/calendarserver/tap/caldav.py 2013-12-23 21:49:21 UTC (rev 12193)
@@ -551,7 +551,7 @@
yield scheduleNextMailPoll(self.store, int(config.LogID) if config.LogID else 5)
if self.doGroupCaching:
yield scheduleNextGroupCachingUpdate(self.store, int(config.LogID) if config.LogID else 5)
- yield scheduleFirstFindMinRevision(self.store)
+ yield scheduleFirstFindMinRevision(self.store, int(config.LogID) if config.LogID else 5)
Modified: CalendarServer/branches/users/gaya/cleanrevisions/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/twistedcaldav/stdconfig.py 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/twistedcaldav/stdconfig.py 2013-12-23 21:49:21 UTC (rev 12193)
@@ -625,9 +625,8 @@
"RemoveDuplicatePrivateComments": False, # Remove duplicate private comments on PUT
- "SyncTokenLifetimeDays" : 16, # Number of days that a client sync report token is valid
+ "SyncTokenLifetimeDays" : 14, # Number of days that a client sync report token is valid
"RevisionCleanupPeriodDays" : 2, # Number of days between revision cleanups
- "RevisionCleanupPhaseDays" : 0.25, # Number of days between startup and the first revision cleanup
# CardDAV Features
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/carddav/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/carddav/datastore/sql.py 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/carddav/datastore/sql.py 2013-12-23 21:49:21 UTC (rev 12193)
@@ -569,13 +569,13 @@
if revision:
cs = schema.CALENDARSERVER
- minRevision = int((yield Select(
+ minValidRevision = int((yield Select(
[cs.VALUE],
From=cs,
- Where=(cs.NAME == "MIN-REVISION")
+ Where=(cs.NAME == "MIN-VALID-REVISION")
).on(self._txn))[0][0])
- if revision < minRevision:
+ if revision < minValidRevision:
raise SyncTokenValidException
# call sharedChildResourceNamesSinceRevision() and filter results
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql.py
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql.py 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql.py 2013-12-23 21:49:21 UTC (rev 12193)
@@ -2035,13 +2035,13 @@
deleted = set()
if revision:
cs = schema.CALENDARSERVER
- minRevision = int((yield Select(
+ minValidRevision = int((yield Select(
[cs.VALUE],
From=cs,
- Where=(cs.NAME == "MIN-REVISION")
+ Where=(cs.NAME == "MIN-VALID-REVISION")
).on(self._txn))[0][0])
- if revision < minRevision:
+ if revision < minValidRevision:
raise SyncTokenValidException
results = [
@@ -2478,13 +2478,13 @@
deleted = []
if revision:
cs = schema.CALENDARSERVER
- minRevision = int((yield Select(
+ minValidRevision = int((yield Select(
[cs.VALUE],
From=cs,
- Where=(cs.NAME == "MIN-REVISION")
+ Where=(cs.NAME == "MIN-VALID-REVISION")
).on(self._txn))[0][0])
- if revision < minRevision:
+ if revision < minValidRevision:
raise SyncTokenValidException
results = [
@@ -6226,15 +6226,19 @@
# get groups where this object was once a member and version info
aboMembers = schema.ABO_MEMBERS
- groupRows = yield Select([aboMembers.GROUP_ID, aboMembers.MEMBER_ID, aboMembers.DELETED, aboMembers.REVISION],
+ groupRows = yield Select(
+ [aboMembers.GROUP_ID,
+ aboMembers.MEMBER_ID,
+ aboMembers.REMOVED,
+ aboMembers.REVISION],
From=aboMembers,
).on(txn)
# group results by group, member, and revisionInfo
groupIDToMemberIDMap = {}
for groupRow in groupRows:
- groupID, memberID, deleted, revision = groupRow
- revisionInfo = [deleted, revision]
+ groupID, memberID, removed, revision = groupRow
+ revisionInfo = [removed, revision]
if groupID not in groupIDToMemberIDMap:
groupIDToMemberIDMap[groupID] = {}
memberIDToRevisionsMap = groupIDToMemberIDMap[groupID]
@@ -6243,7 +6247,7 @@
revisionInfoList = memberIDToRevisionsMap[memberID]
revisionInfoList.append(revisionInfo)
- # go though list an delete old revisions, leaving at least one undeleted member
+ # go though list an delete old revisions, leaving at least one unremoved member
for groupID, memberIDToRevisionsMap in groupIDToMemberIDMap.iteritems():
for memberID, revisionInfoList in memberIDToRevisionsMap.iteritems():
@@ -6251,7 +6255,7 @@
maxRevisionInfoToRemove = None
revisionsToSave = []
for revisionInfo in revisionInfoList:
- deleted, revision = revisionInfo
+ removed, revision = revisionInfo
if revision < minRevision:
revisionsToRemove.append(revision)
if not maxRevisionInfoToRemove or revision > maxRevisionInfoToRemove[1]:
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2013-12-23 21:49:21 UTC (rev 12193)
@@ -372,6 +372,11 @@
"RESOURCE_ID" integer not null references CALENDAR_OBJECT on delete cascade
);
+create table FIND_MIN_VALID_REVISION_WORK (
+ "WORK_ID" integer primary key not null,
+ "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
+
create table REVISION_CLEANUP_WORK (
"WORK_ID" integer primary key not null,
"NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
@@ -386,7 +391,7 @@
insert into CALENDARSERVER (NAME, VALUE) values ('CALENDAR-DATAVERSION', '5');
insert into CALENDARSERVER (NAME, VALUE) values ('ADDRESSBOOK-DATAVERSION', '2');
insert into CALENDARSERVER (NAME, VALUE) values ('NOTIFICATION-DATAVERSION', '1');
-insert into CALENDARSERVER (NAME, VALUE) values ('MIN-REVISION', '1');
+insert into CALENDARSERVER (NAME, VALUE) values ('MIN-VALID-REVISION', '1');
create index CALENDAR_HOME_METADAT_3cb9049e on CALENDAR_HOME_METADATA (
DEFAULT_EVENTS
);
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current.sql 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/current.sql 2013-12-23 21:49:21 UTC (rev 12193)
@@ -710,6 +710,11 @@
-- Revision Cleaner Work --
---------------------------
+create table FIND_MIN_VALID_REVISION_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ') not null, -- implicit index
+ NOT_BEFORE timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
+
create table REVISION_CLEANUP_WORK (
WORK_ID integer primary key default nextval('WORKITEM_SEQ') not null, -- implicit index
NOT_BEFORE timestamp default timezone('UTC', CURRENT_TIMESTAMP)
@@ -728,4 +733,4 @@
insert into CALENDARSERVER values ('CALENDAR-DATAVERSION', '5');
insert into CALENDARSERVER values ('ADDRESSBOOK-DATAVERSION', '2');
insert into CALENDARSERVER values ('NOTIFICATION-DATAVERSION', '1');
-insert into CALENDARSERVER values ('MIN-REVISION', '1');
+insert into CALENDARSERVER values ('MIN-VALID-REVISION', '1');
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_30_to_31.sql
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_30_to_31.sql 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_30_to_31.sql 2013-12-23 21:49:21 UTC (rev 12193)
@@ -29,8 +29,12 @@
alter table NOTIFICATION_OBJECT_REVISIONS
add ("MODIFIED" timestamp default CURRENT_TIMESTAMP at time zone 'UTC');
- -- Add cleanup work table --
+ -- Add cleanup work tables --
+create table FIND_MIN_VALID_REVISION_WORK (
+ "WORK_ID" integer primary key not null,
+ "NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
+);
create table REVISION_CLEANUP_WORK (
"WORK_ID" integer primary key not null,
"NOT_BEFORE" timestamp default CURRENT_TIMESTAMP at time zone 'UTC'
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_30_to_31.sql
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_30_to_31.sql 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_30_to_31.sql 2013-12-23 21:49:21 UTC (rev 12193)
@@ -29,8 +29,12 @@
alter table NOTIFICATION_OBJECT_REVISIONS
add column MODIFIED timestamp default timezone('UTC', CURRENT_TIMESTAMP);
- -- Add cleanup work table --
+ -- Add cleanup work tables --
+create table FIND_MIN_VALID_REVISION_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ') not null,
+ NOT_BEFORE timestamp default timezone('UTC', CURRENT_TIMESTAMP)
+);
create table REVISION_CLEANUP_WORK (
WORK_ID integer primary key default nextval('WORKITEM_SEQ') not null,
NOT_BEFORE timestamp default timezone('UTC', CURRENT_TIMESTAMP)
Modified: CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/work/revision_cleanup.py
===================================================================
--- CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/work/revision_cleanup.py 2013-12-23 21:32:37 UTC (rev 12192)
+++ CalendarServer/branches/users/gaya/cleanrevisions/txdav/common/datastore/work/revision_cleanup.py 2013-12-23 21:49:21 UTC (rev 12193)
@@ -32,8 +32,8 @@
log = Logger()
-class FindMinRevisionWork(WorkItem,
- fromTable(schema.REVISION_CLEANUP_WORK)):
+class FindMinValidRevisionWork(WorkItem,
+ fromTable(schema.FIND_MIN_VALID_REVISION_WORK)):
group = "find_min_revision"
@@ -45,16 +45,16 @@
# Get the minimum valid revision
cs = schema.CALENDARSERVER
- minRevision = int((yield Select(
+ minValidRevision = int((yield Select(
[cs.VALUE],
From=cs,
- Where=(cs.NAME == "MIN-REVISION")
+ Where=(cs.NAME == "MIN-VALID-REVISION")
).on(self.transaction))[0][0])
# get max revision on table rows before dateLimit
dateLimit = (datetime.datetime.utcnow() -
datetime.timedelta(days=float(config.SyncTokenLifetimeDays)))
- maxRevision = 0
+ maxRevOlderThanDate = 0
# TODO: Use one Select statement
for table in (
@@ -65,23 +65,21 @@
):
revisionRows = yield Select(
[Max(table.REVISION)],
- From=Select(
- [table.REVISION],
- From=table,
- Where=(table.MODIFIED < dateLimit),
- ).on(self.transaction)
- )
+ From=table,
+ Where=(table.MODIFIED < dateLimit),
+ ).on(self.transaction)
+
if revisionRows:
tableMaxRevision = revisionRows[0][0]
- if tableMaxRevision > maxRevision:
- maxRevison = tableMaxRevision
+ if tableMaxRevision > maxRevOlderThanDate:
+ maxRevOlderThanDate = tableMaxRevision
- if maxRevision > minRevision:
+ if maxRevOlderThanDate > minValidRevision:
# save it
cs = schema.CALENDARSERVER
yield Update(
- {cs.VALUE: minRevision},
- Where=cs.NAME == "MIN-REVISION",
+ {cs.VALUE: maxRevOlderThanDate},
+ Where=cs.NAME == "MIN-VALID-REVISION",
).on(self.transaction)
# Schedule revision cleanup
@@ -95,8 +93,8 @@
# Schedule next update
notBefore = (datetime.datetime.utcnow() +
datetime.timedelta(days=float(config.RevisionCleanupPeriodDays)))
- log.debug("Rescheduling find minimum revision work: %s" % (notBefore,))
- yield self.transaction.enqueue(FindMinRevisionWork,
+ log.debug("Rescheduling find minimum valid revision work: %s" % (notBefore,))
+ yield self.transaction.enqueue(FindMinValidRevisionWork,
notBefore=notBefore)
@@ -114,30 +112,29 @@
# Get the minimum valid revision
cs = schema.CALENDARSERVER
- minRevision = int((yield Select(
+ minValidRevision = int((yield Select(
[cs.VALUE],
From=cs,
- Where=(cs.NAME == "MIN-REVISION")
+ Where=(cs.NAME == "MIN-VALID-REVISION")
).on(self.transaction))[0][0])
# delete revisions
- yield deleteRevisionsBefore(self.transaction, minRevision)
+ yield deleteRevisionsBefore(self.transaction, minValidRevision)
# Schedule next update
notBefore = (datetime.datetime.utcnow() +
datetime.timedelta(days=float(config.RevisionCleanupPeriodDays)))
- log.debug("Rescheduling find minimum revision work: %s" % (notBefore,))
- yield self.transaction.enqueue(FindMinRevisionWork,
+ log.debug("Rescheduling find minimum valid revision work: %s" % (notBefore,))
+ yield self.transaction.enqueue(FindMinValidRevisionWork,
notBefore=notBefore)
@inlineCallbacks
-def scheduleFirstFindMinRevision(store):
+def scheduleFirstFindMinRevision(store, seconds):
txn = store.newTransaction()
- notBefore = (datetime.datetime.utcnow() +
- datetime.timedelta(days=float(config.RevisionCleanupPhaseDays)))
- log.debug("Scheduling find minimum revision work: %s" % (notBefore,))
- wp = (yield txn.enqueue(FindMinRevisionWork, notBefore=notBefore))
+ notBefore = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)
+ log.debug("Scheduling first find minimum valid revision work: %s" % (notBefore,))
+ wp = (yield txn.enqueue(FindMinValidRevisionWork, notBefore=notBefore))
yield txn.commit()
returnValue(wp)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140312/d625f653/attachment.html>
More information about the calendarserver-changes
mailing list