[CalendarServer-changes] [14779] CalendarServer/trunk/txdav/common/datastore
source_changes at macosforge.org
source_changes at macosforge.org
Tue May 12 08:15:18 PDT 2015
Revision: 14779
http://trac.calendarserver.org//changeset/14779
Author: cdaboo at apple.com
Date: 2015-05-12 08:15:17 -0700 (Tue, 12 May 2015)
Log Message:
-----------
Clean-up migration meta-data.
Modified Paths:
--------------
CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py
CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py
CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py
CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql
CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql
CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql
Modified: CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/home_sync.py 2015-05-12 15:15:17 UTC (rev 14779)
@@ -22,7 +22,8 @@
from twistedcaldav.accounting import emitAccounting
from txdav.caldav.icalendarstore import ComponentUpdateState
from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord, \
- CalendarObjectMigrationRecord, AttachmentMigrationRecord
+ CalendarObjectMigrationRecord, AttachmentMigrationRecord, \
+ MigrationCleanupWork
from txdav.caldav.datastore.sql import ManagedAttachment, CalendarBindRecord
from txdav.common.datastore.sql_external import NotificationCollectionExternal
from txdav.common.datastore.sql_notification import NotificationCollection
@@ -279,8 +280,12 @@
# Unpause work items
yield newhome.unpauseWork()
- # TODO: remove migration state
- pass
+ # Remove migration state
+ yield MigrationCleanupWork.reschedule(
+ txn,
+ MigrationCleanupWork.notBeforeDelay,
+ homeResourceID=newhome.id(),
+ )
# TODO: purge the old ones
pass
Modified: CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/sync_metadata.py 2015-05-12 15:15:17 UTC (rev 14779)
@@ -16,8 +16,9 @@
from twext.enterprise.dal.record import Record, fromTable
from twext.enterprise.dal.syntax import Parameter, Delete
+from twext.enterprise.jobqueue import SingletonWorkItem
+from twisted.internet.defer import inlineCallbacks
from txdav.common.datastore.sql_tables import schema
-from twisted.internet.defer import inlineCallbacks
"""
Module that manages store-level metadata objects used during the migration process.
@@ -56,3 +57,30 @@
L{Record} for L{schema.ATTACHMENT_MIGRATION}.
"""
pass
+
+
+
+class MigrationCleanupWork(SingletonWorkItem, fromTable(schema.MIGRATION_CLEANUP_WORK)):
+
+ group = "homeResourceID"
+
+ notBeforeDelay = 300 # 5 minutes
+
+ @inlineCallbacks
+ def doWork(self):
+ """
+ Delete all the corresponding migration records.
+ """
+
+ yield CalendarMigrationRecord.deletesome(
+ self.transaction,
+ CalendarMigrationRecord.calendarHomeResourceID == self.homeResourceID,
+ )
+ yield CalendarObjectMigrationRecord.deletesome(
+ self.transaction,
+ CalendarObjectMigrationRecord.calendarHomeResourceID == self.homeResourceID,
+ )
+ yield AttachmentMigrationRecord.deletesome(
+ self.transaction,
+ AttachmentMigrationRecord.calendarHomeResourceID == self.homeResourceID,
+ )
Modified: CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/podding/migration/test/test_migration.py 2015-05-12 15:15:17 UTC (rev 14779)
@@ -15,23 +15,33 @@
##
from pycalendar.datetime import DateTime
+
+from twext.enterprise.jobqueue import JobItem
+
+from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.python.filepath import FilePath
+
from twistedcaldav.config import config
from twistedcaldav.ical import Component
+
+from txdav.caldav.datastore.scheduling.ischedule.delivery import IScheduleRequest
+from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
+from txdav.caldav.datastore.test.common import CaptureProtocol
from txdav.common.datastore.podding.migration.home_sync import CrossPodHomeSync
+from txdav.common.datastore.podding.migration.sync_metadata import CalendarMigrationRecord, \
+ AttachmentMigrationRecord, CalendarObjectMigrationRecord, \
+ MigrationCleanupWork
from txdav.common.datastore.podding.test.util import MultiStoreConduitTest
from txdav.common.datastore.sql_tables import _BIND_MODE_READ, \
_HOME_STATUS_DISABLED, _HOME_STATUS_NORMAL, _HOME_STATUS_EXTERNAL, \
_HOME_STATUS_MIGRATING
from txdav.common.datastore.test.util import populateCalendarsFrom
from txdav.who.delegates import Delegates
+
+from txweb2.dav.test.util import SimpleRequest
from txweb2.http_headers import MimeType
from txweb2.stream import MemoryStream
-from txdav.caldav.datastore.scheduling.ischedule.delivery import IScheduleRequest
-from txdav.caldav.datastore.scheduling.ischedule.resource import IScheduleInboxResource
-from txweb2.dav.test.util import SimpleRequest
-from txdav.caldav.datastore.test.common import CaptureProtocol
class TestCompleteMigrationCycle(MultiStoreConduitTest):
@@ -66,7 +76,10 @@
yield super(TestCompleteMigrationCycle, self).setUp()
yield self.populate()
+ # Speed up work
+ self.patch(MigrationCleanupWork, "notBeforeDelay", 1)
+
def configure(self):
super(TestCompleteMigrationCycle, self).configure()
config.GroupAttendees.Enabled = True
@@ -648,7 +661,14 @@
data = yield self.attachmentToString(attachment)
self.assertEqual(data, "Here is some text #1.")
+ # No migration data left
+ txn = self.theTransactionUnderTest(1)
+ yield JobItem.waitEmpty(self.theStoreUnderTest(1).newTransaction, reactor, 60)
+ for migrationType in (CalendarMigrationRecord, CalendarObjectMigrationRecord, AttachmentMigrationRecord,):
+ records = yield migrationType.all(txn)
+ self.assertEqual(len(records), 0, msg=migrationType.__name__)
+
@inlineCallbacks
def test_migration(self):
"""
Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current-oracle-dialect.sql 2015-05-12 15:15:17 UTC (rev 14779)
@@ -646,6 +646,12 @@
"HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade
);
+create table MIGRATION_CLEANUP_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade
+);
+
create table CALENDARSERVER (
"NAME" nvarchar2(255) primary key,
"VALUE" nvarchar2(255)
@@ -997,4 +1003,8 @@
"HOME_RESOURCE_ID"
);
+create index MIGRATION_CLEANUP_WOR_8c23cc35 on MIGRATION_CLEANUP_WORK (
+ "JOB_ID"
+);
+
-- Extra schema to add to current-oracle-dialect.sql
Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/current.sql 2015-05-12 15:15:17 UTC (rev 14779)
@@ -1219,6 +1219,19 @@
PRINCIPAL_PURGE_HOME_WORK(HOME_RESOURCE_ID);
+----------------------------
+-- Migration Cleanup Work --
+----------------------------
+
+create table MIGRATION_CLEANUP_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ HOME_RESOURCE_ID integer not null references CALENDAR_HOME on delete cascade
+);
+
+create index MIGRATION_CLEANUP_WORK_JOB_ID on
+ MIGRATION_CLEANUP_WORK(JOB_ID);
+
--------------------
-- Schema Version --
--------------------
Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/oracle-dialect/upgrade_from_54_to_55.sql 2015-05-12 15:15:17 UTC (rev 14779)
@@ -22,5 +22,16 @@
alter table JOB
add ("PAUSE" integer default 0);
+-- New table
+create table MIGRATION_CLEANUP_WORK (
+ "WORK_ID" integer primary key,
+ "JOB_ID" integer not null references JOB,
+ "HOME_RESOURCE_ID" integer not null references CALENDAR_HOME on delete cascade
+);
+
+create index MIGRATION_CLEANUP_WOR_8c23cc35 on MIGRATION_CLEANUP_WORK (
+ "JOB_ID"
+);
+
-- update the version
update CALENDARSERVER set VALUE = '55' where NAME = 'VERSION';
Modified: CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql
===================================================================
--- CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql 2015-05-12 00:44:44 UTC (rev 14778)
+++ CalendarServer/trunk/txdav/common/datastore/sql_schema/upgrades/postgres-dialect/upgrade_from_54_to_55.sql 2015-05-12 15:15:17 UTC (rev 14779)
@@ -22,5 +22,15 @@
alter table JOB
add column PAUSE integer default 0;
+-- New Table
+create table MIGRATION_CLEANUP_WORK (
+ WORK_ID integer primary key default nextval('WORKITEM_SEQ'), -- implicit index
+ JOB_ID integer references JOB not null,
+ HOME_RESOURCE_ID integer not null references CALENDAR_HOME on delete cascade
+);
+
+create index MIGRATION_CLEANUP_WORK_JOB_ID on
+ MIGRATION_CLEANUP_WORK(JOB_ID);
+
-- update the version
update CALENDARSERVER set VALUE = '55' where NAME = 'VERSION';
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20150512/4487ec08/attachment-0001.html>
More information about the calendarserver-changes
mailing list