[CalendarServer-changes] [13814] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jul 31 19:18:10 PDT 2014
Revision: 13814
http://trac.calendarserver.org//changeset/13814
Author: cdaboo at apple.com
Date: 2014-07-31 19:18:10 -0700 (Thu, 31 Jul 2014)
Log Message:
-----------
Fix proxy and mailgateway migration steps.
Modified Paths:
--------------
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
CalendarServer/trunk/twistedcaldav/directory/calendaruserproxyloader.py
CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
CalendarServer/trunk/twistedcaldav/upgrade.py
CalendarServer/trunk/txdav/caldav/datastore/scheduling/imip/mailgateway.py
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2014-08-01 01:26:51 UTC (rev 13813)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxy.py 2014-08-01 02:18:10 UTC (rev 13814)
@@ -880,8 +880,9 @@
def __init__(self, dbpath):
+ self.dbpath = fullServerPath(config.DataRoot, dbpath)
ADBAPISqliteMixin.__init__(self)
- ProxyDB.__init__(self, "Proxies", "sqlite3", (fullServerPath(config.DataRoot, dbpath),))
+ ProxyDB.__init__(self, "Proxies", "sqlite3", (self.dbpath,))
Modified: CalendarServer/trunk/twistedcaldav/directory/calendaruserproxyloader.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/directory/calendaruserproxyloader.py 2014-08-01 01:26:51 UTC (rev 13813)
+++ CalendarServer/trunk/twistedcaldav/directory/calendaruserproxyloader.py 2014-08-01 02:18:10 UTC (rev 13814)
@@ -31,6 +31,8 @@
from twistedcaldav.config import config, fullServerPath
from twistedcaldav.xmlutil import readXML
+from txdav.who.delegates import addDelegate
+
log = Logger()
ELEMENT_PROXIES = "proxies"
@@ -52,11 +54,10 @@
return "<%s %r>" % (self.__class__.__name__, self.xmlFile)
- def __init__(self, xmlFile, service):
+ def __init__(self, xmlFile):
self.items = []
self.xmlFile = fullServerPath(config.DataRoot, xmlFile)
- self.service = service
# Read in XML
try:
@@ -130,10 +131,42 @@
@inlineCallbacks
- def updateProxyDB(self):
+ def updateProxyDB(self, db):
+ """
+ Move the XML file proxies over to the proxyDB.
+ """
- db = self.service
for item in self.items:
guid, write_proxies, read_proxies = item
yield db.setGroupMembers("%s#%s" % (guid, "calendar-proxy-write"), write_proxies)
yield db.setGroupMembers("%s#%s" % (guid, "calendar-proxy-read"), read_proxies)
+
+
+ @inlineCallbacks
+ def updateProxyStore(self, store):
+ """
+ Move the XML file proxies over to the store database.
+ """
+
+ directory = store.directoryService()
+ txn = store.newTransaction(label="xmlDelegatesToStore")
+ for item in self.items:
+ guid, write_proxies, read_proxies = item
+ delegatorRecord = yield directory.recordWithUID(guid)
+ if delegatorRecord is None:
+ continue
+ for proxy in write_proxies:
+ delegateRecord = yield directory.recordWithUID(proxy)
+ if delegateRecord is None:
+ continue
+
+ yield addDelegate(txn, delegatorRecord, delegateRecord, True)
+
+ for proxy in read_proxies:
+ delegateRecord = yield directory.recordWithUID(proxy)
+ if delegateRecord is None:
+ continue
+
+ yield addDelegate(txn, delegatorRecord, delegateRecord, False)
+
+ yield txn.commit()
Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2014-08-01 01:26:51 UTC (rev 13813)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2014-08-01 02:18:10 UTC (rev 13814)
@@ -27,7 +27,7 @@
from twistedcaldav.upgrade import (
xattrname, upgradeData, updateFreeBusySet,
removeIllegalCharacters, normalizeCUAddrs,
- loadDelegatesFromXML, migrateDelegatesToStore,
+ loadDelegatesFromXMLintoProxyDB, migrateDelegatesToStore,
upgradeResourcesXML
)
from txdav.caldav.datastore.index_file import db_basename
@@ -319,7 +319,7 @@
".calendarserver_version" : {
"@contents" : "2",
},
- MailGatewayTokensDatabase.dbFilename : { "@contents" : None },
+ MailGatewayTokensDatabase.dbFilename : {"@contents" : None},
"%s-journal" % (MailGatewayTokensDatabase.dbFilename,) : {
"@contents" : None,
"@optional" : True,
@@ -1477,10 +1477,10 @@
# Load delegates from xml into sqlite
sqliteProxyService = ProxySqliteDB("proxies.sqlite")
proxyFile = os.path.join(config.DataRoot, "proxies.xml")
- yield loadDelegatesFromXML(proxyFile, sqliteProxyService)
+ yield loadDelegatesFromXMLintoProxyDB(proxyFile, sqliteProxyService)
# Load delegates from sqlite into store
- yield migrateDelegatesToStore(sqliteProxyService, store)
+ yield migrateDelegatesToStore(store)
# Check delegates in store
txn = store.newTransaction()
Modified: CalendarServer/trunk/twistedcaldav/upgrade.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/upgrade.py 2014-08-01 01:26:51 UTC (rev 13813)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2014-08-01 02:18:10 UTC (rev 13814)
@@ -600,8 +600,8 @@
# scanCollection(childCollection)
if os.path.isdir(calPath):
- #log.debug("Regular collection scan: %s" % (calPath,))
- #scanCollection(calPath)
+ # log.debug("Regular collection scan: %s" % (calPath,))
+ # scanCollection(calPath)
log.warn("Regular collection hidden: %s" % (calPath,))
os.rename(calPath, os.path.join(calHome, ".collection." + os.path.basename(calPath)))
@@ -652,6 +652,7 @@
raise UpgradeError("Data upgrade failed, see error.log for details")
+
def upgradeResourcesXML(resourcesFilePath):
"""
Convert the old XML format to the twext.who.xml format
@@ -986,14 +987,31 @@
-def loadDelegatesFromXML(xmlFile, service):
- loader = XMLCalendarUserProxyLoader(xmlFile, service)
- return loader.updateProxyDB()
+def loadDelegatesFromXMLintoProxyDB(xmlFile, service):
+ loader = XMLCalendarUserProxyLoader(xmlFile)
+ return loader.updateProxyDB(service)
+def loadDelegatesFromXMLintoStore(xmlFile, store):
+ loader = XMLCalendarUserProxyLoader(xmlFile)
+ return loader.updateProxyStore(store)
+
+
+
@inlineCallbacks
-def migrateDelegatesToStore(service, store):
+def migrateDelegatesToStore(store):
+ """
+ Migrate the old sqlite proxyDB data into the store. Remove the sqlite file
+ afterwards.
+
+ @param store: the store to migrate into
+ @type store: L{CommonDataStore}
+ """
+
+ log.warn("Migrating delegates to the store")
+
+ service = ProxySqliteDB("proxies.sqlite")
directory = store.directoryService()
txn = store.newTransaction(label="migrateDelegatesToStore")
for groupName, memberUID in (
@@ -1018,8 +1036,14 @@
yield txn.commit()
+ # Remove the old file
+ os.remove(service.dbpath)
+ journalPath = service.dbpath + "-journal"
+ if os.path.exists(journalPath):
+ os.remove(journalPath)
+
class UpgradeFileSystemFormatStep(object):
"""
Upgrade filesystem from previous versions.
@@ -1089,24 +1113,43 @@
def stepWithResult(self, result):
if self.doPostImport:
- # Load proxy assignments from XML if specified
- if (
- self.config.ProxyLoadFromFile and
- os.path.exists(self.config.ProxyLoadFromFile) and
- not os.path.exists(
- os.path.join(
- self.config.DataRoot, "proxies.sqlite"
- )
- )
+ # Migrate any proxyDB file that exists - remove it after migration
+ loadDoneFilePath = os.path.join(self.config.DataRoot, "proxies-loaded")
+ if os.path.exists(
+ os.path.join(self.config.DataRoot, "proxies.sqlite")
):
- sqliteProxyService = ProxySqliteDB("proxies.sqlite")
- yield loadDelegatesFromXML(
- self.config.ProxyLoadFromFile,
- sqliteProxyService
+ # Migrate delegate assignments from sqlite to store
+ yield migrateDelegatesToStore(
+ self.store
)
+
+ # If migration happened and the XML load option is on, write a stub file to prevent
+ # it being loaded on the next restart
+ if (
+ self.config.ProxyLoadFromFile and
+ os.path.exists(self.config.ProxyLoadFromFile) and
+ not os.path.exists(loadDoneFilePath)
+ ):
+ # Write stub file as indicator loading is done
+ FilePath(loadDoneFilePath).touch()
+
+ # If no migration, see if there is an XML file to load into the store (once only)
else:
- sqliteProxyService = None
+ if (
+ self.config.ProxyLoadFromFile and
+ os.path.exists(self.config.ProxyLoadFromFile) and
+ not os.path.exists(loadDoneFilePath)
+ ):
+ log.warn("Loading delegate assignments from XML")
+ yield loadDelegatesFromXMLintoStore(
+ self.config.ProxyLoadFromFile,
+ self.store
+ )
+ # Write stub file as indicator loading is done
+ FilePath(loadDoneFilePath).touch()
+
+
# # Populate the group membership cache
# if (self.config.GroupCaching.Enabled and
# self.config.GroupCaching.EnableUpdater):
@@ -1125,11 +1168,6 @@
# useExternalProxies=self.config.GroupCaching.UseExternalProxies)
# yield updater.updateCache(fast=True)
- uid, gid = getCalendarServerIDs(self.config)
- dbPath = os.path.join(self.config.DataRoot, "proxies.sqlite")
- if os.path.exists(dbPath):
- os.chown(dbPath, uid, gid)
-
# Process old inbox items
self.store.setMigrating(True)
yield self.processInboxItems()
@@ -1138,12 +1176,7 @@
# Migrate mail tokens from sqlite to store
yield migrateTokensToStore(self.config.DataRoot, self.store)
- # Migrate delegate assignments from sqlite to store
- if sqliteProxyService is None:
- sqliteProxyService = ProxySqliteDB("proxies.sqlite")
- yield migrateDelegatesToStore(sqliteProxyService, self.store)
-
@inlineCallbacks
def processInboxItems(self):
"""
Modified: CalendarServer/trunk/txdav/caldav/datastore/scheduling/imip/mailgateway.py
===================================================================
--- CalendarServer/trunk/txdav/caldav/datastore/scheduling/imip/mailgateway.py 2014-08-01 01:26:51 UTC (rev 13813)
+++ CalendarServer/trunk/txdav/caldav/datastore/scheduling/imip/mailgateway.py 2014-08-01 02:18:10 UTC (rev 13814)
@@ -229,11 +229,13 @@
@type store: L{CommonDataStore}
"""
oldDB = MailGatewayTokensDatabase(path)
- txn = store.newTransaction(label="migrateTokensToStore")
- for token, organizer, attendee, icaluid in oldDB.getAllTokens():
- yield txn.imipCreateToken(organizer, attendee, icaluid, token=token)
- yield txn.commit()
- os.remove(oldDB.dbpath)
- journalPath = oldDB.dbpath + "-journal"
- if os.path.exists(journalPath):
- os.remove(journalPath)
+ if os.path.exists(oldDB.dbpath):
+ txn = store.newTransaction(label="migrateTokensToStore")
+ for token, organizer, attendee, icaluid in oldDB.getAllTokens():
+ yield txn.imipCreateToken(organizer, attendee, icaluid, token=token)
+ yield txn.commit()
+
+ os.remove(oldDB.dbpath)
+ journalPath = oldDB.dbpath + "-journal"
+ if os.path.exists(journalPath):
+ os.remove(journalPath)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20140731/14f404fa/attachment-0001.html>
More information about the calendarserver-changes
mailing list