<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[13405] CalendarServer/trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.calendarserver.org//changeset/13405">13405</a></dd>
<dt>Author</dt> <dd>sagen@apple.com</dd>
<dt>Date</dt> <dd>2014-05-02 09:51:33 -0700 (Fri, 02 May 2014)</dd>
</dl>
<h3>Log Message</h3>
<pre>Migrating locations and resources works with twext.who; adds InMemoryDirectoryService</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#CalendarServertrunkcalendarservertoolsresourcespy">CalendarServer/trunk/calendarserver/tools/resources.py</a></li>
<li><a href="#CalendarServertrunkcalendarservertoolstesttest_resourcespy">CalendarServer/trunk/calendarserver/tools/test/test_resources.py</a></li>
<li><a href="#CalendarServertrunktwistedcaldavtesttest_upgradepy">CalendarServer/trunk/twistedcaldav/test/test_upgrade.py</a></li>
<li><a href="#CalendarServertrunktwistedcaldavupgradepy">CalendarServer/trunk/twistedcaldav/upgrade.py</a></li>
<li><a href="#CalendarServertrunktxdavwhotesttest_utilpy">CalendarServer/trunk/txdav/who/test/test_util.py</a></li>
<li><a href="#CalendarServertrunktxdavwhoutilpy">CalendarServer/trunk/txdav/who/util.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="CalendarServertrunkcalendarservertoolsresourcespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tools/resources.py (13404 => 13405)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tools/resources.py 2014-05-02 15:44:31 UTC (rev 13404)
+++ CalendarServer/trunk/calendarserver/tools/resources.py 2014-05-02 16:51:33 UTC (rev 13405)
</span><span class="lines">@@ -36,10 +36,10 @@
</span><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><span class="cx"> from twisted.python.util import switchUID
</span><span class="cx"> from twistedcaldav.config import config, ConfigurationError
</span><del>-from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
-from twistedcaldav.directory.directory import DirectoryService, DirectoryError
-from twistedcaldav.directory.xmlfile import XMLDirectoryService
</del><span class="cx"> from txdav.who.util import directoryFromConfig
</span><ins>+from twext.who.opendirectory import (
+ DirectoryService as OpenDirectoryService
+)
</ins><span class="cx">
</span><span class="cx"> log = Logger()
</span><span class="cx">
</span><span class="lines">@@ -142,131 +142,69 @@
</span><span class="cx"> # hierarchy
</span><span class="cx"> setupMemcached(config)
</span><span class="cx">
</span><del>- try:
- config.directory = directoryFromConfig(config)
- except DirectoryError, e:
- abort(e)
</del><ins>+ config.directory = directoryFromConfig(config)
</ins><span class="cx">
</span><span class="cx"> except ConfigurationError, e:
</span><span class="cx"> abort(e)
</span><span class="cx">
</span><del>- # FIXME: this all has to change:
- # Find the opendirectory service
- userService = config.directory.serviceForRecordType("users")
- resourceService = config.directory.serviceForRecordType("resources")
- if (
- not isinstance(userService, OpenDirectoryService) or
- not isinstance(resourceService, XMLDirectoryService)
- ):
- abort(
- "This script only migrates resources and locations from "
- "OpenDirectory to XML; this calendar server does not have such a "
- "configuration."
- )
</del><ins>+ sourceService = OpenDirectoryService()
+ destService = config.directory
</ins><span class="cx">
</span><span class="cx"> #
</span><span class="cx"> # Start the reactor
</span><span class="cx"> #
</span><span class="cx"> reactor.callLater(
</span><del>- 0, migrate, userService, resourceService, verbose=verbose
</del><ins>+ 0, migrate, sourceService, destService, verbose=verbose
</ins><span class="cx"> )
</span><span class="cx"> reactor.run()
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><span class="cx"> @inlineCallbacks
</span><del>-def migrate(sourceService, resourceService, verbose=False):
</del><ins>+def migrate(sourceService, destService, verbose=False):
</ins><span class="cx"> """
</span><span class="cx"> Simply a wrapper around migrateResources in order to stop the reactor
</span><span class="cx"> """
</span><span class="cx">
</span><span class="cx"> try:
</span><del>- yield migrateResources(sourceService, resourceService, verbose=verbose)
</del><ins>+ yield migrateResources(sourceService, destService, verbose=verbose)
</ins><span class="cx"> finally:
</span><span class="cx"> reactor.stop()
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-def queryForType(sourceService, recordType, verbose=False):
- """
- Queries OD for all records of the specified record type
- """
</del><span class="cx">
</span><del>- attrs = [
- "dsAttrTypeStandard:GeneratedUID",
- "dsAttrTypeStandard:RealName",
- ]
</del><span class="cx">
</span><del>- if verbose:
- print("Querying for all %s records" % (recordType,))
</del><span class="cx">
</span><del>- results = list(sourceService.odModule.listAllRecordsWithAttributes_list(
- sourceService.directory,
- recordType,
- attrs,
- ))
-
- if verbose:
- print("Found %d records" % (len(results),))
-
- return results
-
-
-
</del><span class="cx"> @inlineCallbacks
</span><del>-def migrateResources(
- sourceService, destService, autoSchedules=None,
- queryMethod=queryForType, verbose=False
-):
</del><ins>+def migrateResources(sourceService, destService, verbose=False):
</ins><span class="cx">
</span><del>- directoryRecords = []
- augmentRecords = []
</del><ins>+ destRecords = []
</ins><span class="cx">
</span><del>- for recordTypeOD, recordType in (
- ("dsRecTypeStandard:Resources", DirectoryService.recordType_resources),
- ("dsRecTypeStandard:Places", DirectoryService.recordType_locations),
</del><ins>+ for recordType in (
+ sourceService.recordType.resource,
+ sourceService.recordType.location,
</ins><span class="cx"> ):
</span><del>- data = queryMethod(sourceService, recordTypeOD, verbose=verbose)
- for recordName, val in data:
- guid = val.get("dsAttrTypeStandard:GeneratedUID", None)
- fullName = val.get("dsAttrTypeStandard:RealName", None)
- if guid and fullName:
- if not recordName:
- recordName = guid
- record = yield destService.recordWithGUID(guid)
- if record is None:
- if verbose:
- print("Migrating %s (%s)" % (fullName, recordType))
-
- if autoSchedules is not None:
- autoSchedule = autoSchedules.get(guid, 1)
- else:
- autoSchedule = True
- augmentRecord = (
- yield destService.augmentService.getAugmentRecord(
- guid, recordType
</del><ins>+ records = yield sourceService.recordsWithRecordType(recordType)
+ for sourceRecord in records:
+ destRecord = yield destService.recordWithUID(sourceRecord.uid)
+ if destRecord is None:
+ if verbose:
+ print(
+ "Migrating {name} {recordType} {uid}".format(
+ name=sourceRecord.displayName,
+ recordType=recordType.name,
+ uid=sourceRecord.uid
</ins><span class="cx"> )
</span><span class="cx"> )
</span><del>- if autoSchedule:
- augmentRecord.autoScheduleMode = "automatic"
- else:
- augmentRecord.autoScheduleMode = "none"
- augmentRecords.append(augmentRecord)
</del><ins>+ destRecord = type(sourceRecord)(destService, sourceRecord.fields.copy())
+ destRecords.append(destRecord)
</ins><span class="cx">
</span><del>- directoryRecords.append((
- recordType,
- {
- "guid": guid,
- "shortNames": [recordName],
- "fullName": fullName,
- }
- ))
</del><ins>+ if destRecords:
+ yield destService.updateRecords(destRecords, create=True)
</ins><span class="cx">
</span><del>- destService.createRecords(directoryRecords)
</del><span class="cx">
</span><del>- yield destService.augmentService.addAugmentRecords(augmentRecords)
</del><span class="cx">
</span><span class="cx">
</span><del>-
</del><span class="cx"> if __name__ == "__main__":
</span><span class="cx"> main()
</span></span></pre></div>
<a id="CalendarServertrunkcalendarservertoolstesttest_resourcespy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/calendarserver/tools/test/test_resources.py (13404 => 13405)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/calendarserver/tools/test/test_resources.py 2014-05-02 15:44:31 UTC (rev 13404)
+++ CalendarServer/trunk/calendarserver/tools/test/test_resources.py 2014-05-02 16:51:33 UTC (rev 13405)
</span><span class="lines">@@ -15,176 +15,110 @@
</span><span class="cx"> ##
</span><span class="cx">
</span><span class="cx">
</span><del>-try:
- from calendarserver.tools.resources import migrateResources
- from twisted.internet.defer import inlineCallbacks, succeed
- from twistedcaldav.directory.directory import DirectoryService
- from twistedcaldav.test.util import TestCase
- strGUID = "dsAttrTypeStandard:GeneratedUID"
- strName = "dsAttrTypeStandard:RealName"
</del><span class="cx">
</span><del>-except ImportError:
- pass
</del><ins>+from twisted.internet.defer import inlineCallbacks
+from calendarserver.tools.resources import migrateResources
+from twistedcaldav.test.util import StoreTestCase
+from txdav.who.util import InMemoryDirectoryService
+from twext.who.directory import DirectoryRecord
+from txdav.who.idirectory import RecordType as CalRecordType
+from txdav.who.directory import CalendarDirectoryRecordMixin
</ins><span class="cx">
</span><del>-else:
- class StubDirectoryRecord(object):
</del><span class="cx">
</span><del>- def __init__(
- self, recordType, guid=None, shortNames=None, fullName=None
- ):
- self.recordType = recordType
- self.guid = guid
- self.shortNames = shortNames
- self.fullName = fullName
</del><ins>+class TestRecord(DirectoryRecord, CalendarDirectoryRecordMixin):
+ pass
</ins><span class="cx">
</span><span class="cx">
</span><del>- class StubDirectoryService(object):
</del><ins>+class MigrateResourcesTest(StoreTestCase):
</ins><span class="cx">
</span><del>- def __init__(self, augmentService):
- self.records = {}
- self.augmentService = augmentService
</del><ins>+ @inlineCallbacks
+ def setUp(self):
+ yield super(MigrateResourcesTest, self).setUp()
+ self.store = self.storeUnderTest()
</ins><span class="cx">
</span><del>- def recordWithGUID(self, guid):
- return None
</del><ins>+ self.sourceService = InMemoryDirectoryService(None)
+ fieldName = self.sourceService.fieldName
+ records = (
+ TestRecord(
+ self.sourceService,
+ {
+ fieldName.uid: u"location1",
+ fieldName.shortNames: (u"loc1",),
+ fieldName.recordType: CalRecordType.location,
+ }
+ ),
+ TestRecord(
+ self.sourceService,
+ {
+ fieldName.uid: u"location2",
+ fieldName.shortNames: (u"loc2",),
+ fieldName.recordType: CalRecordType.location,
+ }
+ ),
+ TestRecord(
+ self.sourceService,
+ {
+ fieldName.uid: u"resource1",
+ fieldName.shortNames: (u"res1",),
+ fieldName.recordType: CalRecordType.resource,
+ }
+ ),
+ )
+ yield self.sourceService.updateRecords(records, create=True)
</ins><span class="cx">
</span><del>- def createRecords(self, data):
- for recordType, recordData in data:
- guid = recordData["guid"]
- record = StubDirectoryRecord(
- recordType, guid=guid,
- shortNames=recordData["shortNames"],
- fullName=recordData["fullName"]
- )
- self.records[guid] = record
</del><span class="cx">
</span><del>- def updateRecord(
- self, recordType, guid=None, shortNames=None, fullName=None
- ):
- pass
</del><span class="cx">
</span><ins>+ @inlineCallbacks
+ def test_migrateResources(self):
</ins><span class="cx">
</span><del>- class StubAugmentRecord(object):
</del><ins>+ # Record location1 has not been migrated
+ record = yield self.directory.recordWithUID(u"location1")
+ self.assertEquals(record, None)
</ins><span class="cx">
</span><del>- def __init__(self, guid=None):
- self.guid = guid
- self.autoSchedule = True
</del><ins>+ # Migrate location1, location2, and resource1
+ yield migrateResources(self.sourceService, self.directory)
+ record = yield self.directory.recordWithUID(u"location1")
+ self.assertEquals(record.uid, u"location1")
+ self.assertEquals(record.shortNames[0], u"loc1")
+ record = yield self.directory.recordWithUID(u"location2")
+ self.assertEquals(record.uid, u"location2")
+ self.assertEquals(record.shortNames[0], u"loc2")
+ record = yield self.directory.recordWithUID(u"resource1")
+ self.assertEquals(record.uid, u"resource1")
+ self.assertEquals(record.shortNames[0], u"res1")
</ins><span class="cx">
</span><ins>+ # Add a new location to the sourceService, and modify an existing
+ # location
+ fieldName = self.sourceService.fieldName
+ newRecords = (
+ TestRecord(
+ self.sourceService,
+ {
+ fieldName.uid: u"location1",
+ fieldName.shortNames: (u"newloc1",),
+ fieldName.recordType: CalRecordType.location,
+ }
+ ),
+ TestRecord(
+ self.sourceService,
+ {
+ fieldName.uid: u"location3",
+ fieldName.shortNames: (u"loc3",),
+ fieldName.recordType: CalRecordType.location,
+ }
+ ),
+ )
+ yield self.sourceService.updateRecords(newRecords, create=True)
</ins><span class="cx">
</span><del>- class StubAugmentService(object):
</del><ins>+ yield migrateResources(self.sourceService, self.directory)
</ins><span class="cx">
</span><del>- records = {}
</del><ins>+ # Ensure an existing record does not get migrated again; verified by
+ # seeing if shortNames changed, which they should not:
+ record = yield self.directory.recordWithUID(u"location1")
+ self.assertEquals(record.uid, u"location1")
+ self.assertEquals(record.shortNames[0], u"loc1")
</ins><span class="cx">
</span><del>- @classmethod
- def getAugmentRecord(cls, guid, recordType):
- if guid not in cls.records:
- record = StubAugmentRecord(guid=guid)
- cls.records[guid] = record
- return succeed(cls.records[guid])
-
- @classmethod
- def addAugmentRecords(cls, records):
- for record in records:
- cls.records[record.guid] = record
- return succeed(True)
-
-
- class MigrateResourcesTestCase(TestCase):
-
- @inlineCallbacks
- def test_migrateResources(self):
-
- data = {
- "dsRecTypeStandard:Resources":
- [
- ["projector1", {
- strGUID: "6C99E240-E915-4012-82FA-99E0F638D7EF",
- strName: "Projector 1"
- }],
- ["projector2", {
- strGUID: "7C99E240-E915-4012-82FA-99E0F638D7EF",
- strName: "Projector 2"
- }],
- ],
- "dsRecTypeStandard:Places":
- [
- ["office1", {
- strGUID: "8C99E240-E915-4012-82FA-99E0F638D7EF",
- strName: "Office 1"
- }],
- ],
- }
-
- def queryMethod(sourceService, recordType, verbose=False):
- return data[recordType]
-
- directoryService = StubDirectoryService(StubAugmentService())
- yield migrateResources(
- None, directoryService, queryMethod=queryMethod
- )
- for guid, recordType in (
- (
- "6C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_resources
- ),
- (
- "7C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_resources
- ),
- (
- "8C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_locations
- ),
- ):
- self.assertTrue(guid in directoryService.records)
- record = directoryService.records[guid]
- self.assertEquals(record.recordType, recordType)
-
- self.assertTrue(guid in StubAugmentService.records)
-
- #
- # Add more to OD and re-migrate
- #
- data["dsRecTypeStandard:Resources"].append(
- ["projector3", {
- strGUID: "9C99E240-E915-4012-82FA-99E0F638D7EF",
- strName: "Projector 3"
- }]
- )
- data["dsRecTypeStandard:Places"].append(
- ["office2", {
- strGUID: "AC99E240-E915-4012-82FA-99E0F638D7EF",
- strName: "Office 2"
- }]
- )
-
- yield migrateResources(
- None, directoryService, queryMethod=queryMethod
- )
-
- for guid, recordType in (
- (
- "6C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_resources
- ),
- (
- "7C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_resources
- ),
- (
- "9C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_resources
- ),
- (
- "8C99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_locations
- ),
- (
- "AC99E240-E915-4012-82FA-99E0F638D7EF",
- DirectoryService.recordType_locations
- ),
- ):
- self.assertTrue(guid in directoryService.records)
- record = directoryService.records[guid]
- self.assertEquals(record.recordType, recordType)
-
- self.assertTrue(guid in StubAugmentService.records)
</del><ins>+ # Ensure new record does get migrated
+ record = yield self.directory.recordWithUID(u"location3")
+ self.assertEquals(record.uid, u"location3")
+ self.assertEquals(record.shortNames[0], u"loc3")
</ins></span></pre></div>
<a id="CalendarServertrunktwistedcaldavtesttest_upgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/test/test_upgrade.py (13404 => 13405)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2014-05-02 15:44:31 UTC (rev 13404)
+++ CalendarServer/trunk/twistedcaldav/test/test_upgrade.py 2014-05-02 16:51:33 UTC (rev 13405)
</span><span class="lines">@@ -19,14 +19,12 @@
</span><span class="cx"> import zlib
</span><span class="cx"> import cPickle
</span><span class="cx">
</span><del>-from twisted.python.reflect import namedClass
</del><span class="cx"> from twisted.internet.defer import inlineCallbacks, succeed
</span><span class="cx">
</span><span class="cx"> from txdav.xml.parser import WebDAVDocument
</span><span class="cx"> from txdav.caldav.datastore.index_file import db_basename
</span><span class="cx">
</span><span class="cx"> from twistedcaldav.config import config
</span><del>-from twistedcaldav.directory.resourceinfo import ResourceInfoDatabase
</del><span class="cx"> from txdav.caldav.datastore.scheduling.imip.mailgateway import MailGatewayTokensDatabase
</span><span class="cx"> from twistedcaldav.upgrade import (
</span><span class="cx"> xattrname, upgradeData, updateFreeBusySet,
</span><span class="lines">@@ -1399,95 +1397,6 @@
</span><span class="cx"> self.assertTrue(self.verifyHierarchy(root, after))
</span><span class="cx">
</span><span class="cx">
</span><del>- @inlineCallbacks
- def test_migrateResourceInfo(self):
- # Fake getResourceInfo( )
-
- assignments = {
- 'guid1' : (False, None, None),
- 'guid2' : (True, 'guid1', None),
- 'guid3' : (False, 'guid1', 'guid2'),
- 'guid4' : (True, None, 'guid3'),
- }
-
- def _getResourceInfo(ignored):
- results = []
- for guid, info in assignments.iteritems():
- results.append((guid, info[0], info[1], info[2]))
- return results
-
- self.setUpInitialStates()
- # Override the normal getResourceInfo method with our own:
- # XMLDirectoryService.getResourceInfo = _getResourceInfo
- # self.patch(XMLDirectoryService, "getResourceInfo", _getResourceInfo)
-
- before = {
- "trigger_resource_migration" : {
- "@contents" : "x",
- }
- }
- after = {
- ".calendarserver_version" :
- {
- "@contents" : "2",
- },
- NEWPROXYFILE :
- {
- "@contents" : None,
- },
- MailGatewayTokensDatabase.dbFilename :
- {
- "@contents" : None,
- },
- "%s-journal" % (MailGatewayTokensDatabase.dbFilename,) :
- {
- "@contents" : None,
- "@optional" : True,
- },
- ResourceInfoDatabase.dbFilename :
- {
- "@contents" : None,
- },
- "%s-journal" % (ResourceInfoDatabase.dbFilename,) :
- {
- "@contents" : None,
- "@optional" : True,
- }
- }
- root = self.createHierarchy(before)
- config.DocumentRoot = root
- config.DataRoot = root
- config.ServerRoot = root
-
- (yield self.doUpgrade(config))
- self.assertTrue(self.verifyHierarchy(root, after))
-
- proxydbClass = namedClass(config.ProxyDBService.type)
- calendarUserProxyDatabase = proxydbClass(**config.ProxyDBService.params)
- resourceInfoDatabase = ResourceInfoDatabase(root)
-
- for guid, info in assignments.iteritems():
- proxyGroup = "%s#calendar-proxy-write" % (guid,)
- result = (yield calendarUserProxyDatabase.getMembers(proxyGroup))
- if info[1]:
- self.assertTrue(info[1] in result)
- else:
- self.assertTrue(not result)
-
- readOnlyProxyGroup = "%s#calendar-proxy-read" % (guid,)
- result = (yield calendarUserProxyDatabase.getMembers(readOnlyProxyGroup))
- if info[2]:
- self.assertTrue(info[2] in result)
- else:
- self.assertTrue(not result)
-
- autoSchedule = resourceInfoDatabase._db_value_for_sql("select AUTOSCHEDULE from RESOURCEINFO where GUID = :1", guid)
- autoSchedule = autoSchedule == 1
- self.assertEquals(info[0], autoSchedule)
-
- test_migrateResourceInfo.todo = "Need to port to twext.who"
-
-
</del><span class="cx"> def test_removeIllegalCharacters(self):
</span><span class="cx"> """
</span><span class="cx"> Control characters aside from NL and CR are removed.
</span></span></pre></div>
<a id="CalendarServertrunktwistedcaldavupgradepy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/twistedcaldav/upgrade.py (13404 => 13405)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/twistedcaldav/upgrade.py 2014-05-02 15:44:31 UTC (rev 13404)
+++ CalendarServer/trunk/twistedcaldav/upgrade.py 2014-05-02 16:51:33 UTC (rev 13405)
</span><span class="lines">@@ -53,8 +53,6 @@
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.index_file import db_basename
</span><span class="cx">
</span><del>-# from twisted.protocols.amp import AMP, Command, String, Boolean
-
</del><span class="cx"> from calendarserver.tap.util import getRootResource, FakeRequest
</span><span class="cx">
</span><span class="cx"> from txdav.caldav.datastore.scheduling.imip.mailgateway import migrateTokensToStore
</span><span class="lines">@@ -63,6 +61,7 @@
</span><span class="cx"> from txdav.who.idirectory import RecordType as CalRecordType
</span><span class="cx"> from txdav.who.delegates import addDelegate
</span><span class="cx"> from twistedcaldav.directory.calendaruserproxy import ProxySqliteDB
</span><ins>+from calendarserver.tools.resources import migrateResources
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> deadPropertyXattrPrefix = namedAny(
</span><span class="lines">@@ -267,43 +266,6 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-# class UpgradeOneHome(Command):
-# arguments = [('path', String())]
-# response = [('succeeded', Boolean())]
-
-
-
-# class To1Driver(AMP):
-# """
-# Upgrade driver which runs in the parent process.
-# """
-
-# def upgradeHomeInHelper(self, path):
-# return self.callRemote(UpgradeOneHome, path=path).addCallback(
-# operator.itemgetter("succeeded")
-# )
-
-
-
-# class To1Home(AMP):
-# """
-# Upgrade worker which runs in dedicated subprocesses.
-# """
-
-# def __init__(self, config):
-# super(To1Home, self).__init__()
-# self.directory = getDirectory(config)
-# self.cuaCache = {}
-
-
-# @UpgradeOneHome.responder
-# @inlineCallbacks
-# def upgradeOne(self, path):
-# result = yield upgradeCalendarHome(path, self.directory, self.cuaCache)
-# returnValue(dict(succeeded=result))
-
-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def upgrade_to_1(config, directory):
</span><span class="cx"> """
</span><span class="lines">@@ -371,51 +333,8 @@
</span><span class="cx"> os.rename(oldHome, newHome)
</span><span class="cx">
</span><span class="cx">
</span><del>- @inlineCallbacks
- def migrateResourceInfo(config, directory, uid, gid):
- """
- Retrieve delegate assignments and auto-schedule flag from the directory
- service, because in "v1" that's where this info lived.
- """
</del><span class="cx">
</span><del>- print("FIXME, need to port migrateResourceInfo to twext.who")
- returnValue(None)
</del><span class="cx">
</span><del>- log.warn("Fetching delegate assignments and auto-schedule settings from directory")
- resourceInfo = directory.getResourceInfo()
- if len(resourceInfo) == 0:
- # Nothing to migrate, or else not appleopendirectory
- log.warn("No resource info found in directory")
- returnValue(None)
-
- log.warn("Found info for %d resources and locations in directory; applying settings" % (len(resourceInfo),))
-
- resourceInfoDatabase = ResourceInfoDatabase(config.DataRoot)
- proxydbClass = namedClass(config.ProxyDBService.type)
- calendarUserProxyDatabase = proxydbClass(**config.ProxyDBService.params)
-
- for guid, autoSchedule, proxy, readOnlyProxy in resourceInfo:
- resourceInfoDatabase.setAutoScheduleInDatabase(guid, autoSchedule)
- if proxy:
- yield calendarUserProxyDatabase.setGroupMembersInDatabase(
- "%s#calendar-proxy-write" % (guid,),
- [proxy]
- )
- if readOnlyProxy:
- yield calendarUserProxyDatabase.setGroupMembersInDatabase(
- "%s#calendar-proxy-read" % (guid,),
- [readOnlyProxy]
- )
-
- dbPath = os.path.join(config.DataRoot, ResourceInfoDatabase.dbFilename)
- if os.path.exists(dbPath):
- os.chown(dbPath, uid, gid)
-
- dbPath = os.path.join(config.DataRoot, "proxies.sqlite")
- if os.path.exists(dbPath):
- os.chown(dbPath, uid, gid)
-
-
</del><span class="cx"> def createMailTokensDatabase(config, uid, gid):
</span><span class="cx"> # Cause the tokens db to be created on disk so we can set the
</span><span class="cx"> # permissions on it now
</span><span class="lines">@@ -574,9 +493,6 @@
</span><span class="cx"> )
</span><span class="cx"> log.warn("Done processing calendar homes")
</span><span class="cx">
</span><del>- triggerPath = os.path.join(config.ServerRoot, TRIGGER_FILE)
- if os.path.exists(triggerPath):
- yield migrateResourceInfo(config, directory, uid, gid)
</del><span class="cx"> createMailTokensDatabase(config, uid, gid)
</span><span class="cx">
</span><span class="cx"> if errorOccurred:
</span><span class="lines">@@ -752,7 +668,7 @@
</span><span class="cx"> try:
</span><span class="cx"> # Migrate locations/resources now because upgrade_to_1 depends
</span><span class="cx"> # on them being in resources.xml
</span><del>- yield migrateFromOD(config, directory)
</del><ins>+ yield migrateFromOD(directory)
</ins><span class="cx"> except Exception, e:
</span><span class="cx"> raise UpgradeError("Unable to migrate locations and resources from OD: %s" % (e,))
</span><span class="cx">
</span><span class="lines">@@ -937,37 +853,22 @@
</span><span class="cx">
</span><span class="cx">
</span><span class="cx">
</span><del>-# # Deferred
-def migrateFromOD(config, directory):
- # FIXME:
- print("STILL NEED TO IMPLEMENT migrateFromOD")
- return succeed(None)
-# #
-# # Migrates locations and resources from OD
-# #
-# try:
-# from twistedcaldav.directory.appleopendirectory import OpenDirectoryService
-# from calendarserver.tools.resources import migrateResources
-# except ImportError:
-# return succeed(None)
</del><ins>+# Deferred
+def migrateFromOD(directory):
+ #
+ # Migrates locations and resources from OD
+ #
+ log.warn("Migrating locations and resources")
</ins><span class="cx">
</span><del>-# log.warn("Migrating locations and resources")
</del><ins>+ # Create internal copies of resources and locations based on what is
+ # found in OD
+ from twext.who.opendirectory import (
+ DirectoryService as OpenDirectoryService
+ )
+ return migrateResources(OpenDirectoryService(), directory)
</ins><span class="cx">
</span><del>-# userService = directory.serviceForRecordType("users")
-# resourceService = directory.serviceForRecordType("resources")
-# if (
-# not isinstance(userService, OpenDirectoryService) or
-# not isinstance(resourceService, XMLDirectoryService)
-# ):
-# # Configuration requires no migration
-# return succeed(None)
</del><span class="cx">
</span><del>-# # Create internal copies of resources and locations based on what is
-# # found in OD
-# return migrateResources(userService, resourceService)
</del><span class="cx">
</span><del>-
-
</del><span class="cx"> @inlineCallbacks
</span><span class="cx"> def migrateAutoSchedule(config, directory):
</span><span class="cx"> # Fetch the autoSchedule assignments from resourceinfo.sqlite and store
</span></span></pre></div>
<a id="CalendarServertrunktxdavwhotesttest_utilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/who/test/test_util.py (13404 => 13405)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/who/test/test_util.py 2014-05-02 15:44:31 UTC (rev 13404)
+++ CalendarServer/trunk/txdav/who/test/test_util.py 2014-05-02 16:51:33 UTC (rev 13405)
</span><span class="lines">@@ -20,7 +20,7 @@
</span><span class="cx">
</span><span class="cx"> import os
</span><span class="cx">
</span><del>-from txdav.who.util import directoryFromConfig
</del><ins>+from txdav.who.util import directoryFromConfig, InMemoryDirectoryService
</ins><span class="cx"> from twisted.internet.defer import inlineCallbacks
</span><span class="cx"> from twisted.trial.unittest import TestCase
</span><span class="cx"> from twistedcaldav.config import ConfigDict
</span><span class="lines">@@ -32,7 +32,8 @@
</span><span class="cx"> DirectoryService as DelegateDirectoryService,
</span><span class="cx"> RecordType as DelegateRecordType
</span><span class="cx"> )
</span><del>-from twext.who.idirectory import RecordType
</del><ins>+from twext.who.directory import DirectoryRecord
+from twext.who.idirectory import RecordType, NoSuchRecordError
</ins><span class="cx"> from txdav.who.idirectory import RecordType as CalRecordType
</span><span class="cx"> from txdav.who.wiki import (
</span><span class="cx"> DirectoryService as WikiDirectoryService,
</span><span class="lines">@@ -161,3 +162,95 @@
</span><span class="cx"> # And make sure it's functional:
</span><span class="cx"> record = yield service.recordWithUID("group07")
</span><span class="cx"> self.assertEquals(record.fullNames, [u'Group 07'])
</span><ins>+
+
+
+class IndexDirectoryServiceTest(TestCase):
+
+ @inlineCallbacks
+ def test_updateRecords(self):
+ service = InMemoryDirectoryService(u"xyzzy")
+
+ # Record does not exist
+ record = yield service.recordWithUID(u"foo")
+ self.assertEquals(None, record)
+
+ records = (
+ DirectoryRecord(
+ service,
+ {
+ service.fieldName.uid: u"foo",
+ service.fieldName.shortNames: (u"foo1", u"foo2"),
+ service.fieldName.recordType: RecordType.user,
+ }
+ ),
+ )
+ try:
+ # Trying to update a record when it does not exist should fail
+ yield service.updateRecords(records, create=False)
+ except NoSuchRecordError:
+ pass
+ except:
+ self.fail("Did not raise NoSuchRecordError when create=False and record does not exist")
+
+ record = yield service.recordWithUID(u"foo")
+ self.assertEquals(None, record)
+
+ # Create the record
+ yield service.updateRecords(records, create=True)
+
+ record = yield service.recordWithUID(u"foo")
+ self.assertEquals(record.uid, u"foo")
+
+ records = yield service.recordsWithRecordType(RecordType.user)
+ self.assertEquals(len(records), 1)
+ self.assertEquals(list(records)[0].uid, u"foo")
+
+ record = yield service.recordWithShortName(RecordType.user, u"foo1")
+ self.assertEquals(record.uid, u"foo")
+ record = yield service.recordWithShortName(RecordType.user, u"foo2")
+ self.assertEquals(record.uid, u"foo")
+
+ records = (
+ DirectoryRecord(
+ service,
+ {
+ service.fieldName.uid: u"foo",
+ service.fieldName.shortNames: (u"foo3", u"foo4"),
+ service.fieldName.recordType: RecordType.group,
+ }
+ ),
+ DirectoryRecord(
+ service,
+ {
+ service.fieldName.uid: u"bar",
+ service.fieldName.shortNames: (u"bar1", u"bar2"),
+ service.fieldName.recordType: RecordType.user,
+ }
+ ),
+ )
+
+ # Update the existing record and create a new one
+ yield service.updateRecords(records, create=True)
+
+ record = yield service.recordWithUID(u"foo")
+ self.assertEquals(record.uid, u"foo")
+ self.assertEquals(set(record.shortNames), set((u'foo3', u'foo4')))
+
+ records = yield service.recordsWithRecordType(RecordType.group)
+ self.assertEquals(len(records), 1)
+ self.assertEquals(list(records)[0].uid, u"foo")
+
+ records = yield service.recordsWithRecordType(RecordType.user)
+ self.assertEquals(len(records), 1)
+ self.assertEquals(list(records)[0].uid, u"bar")
+
+ record = yield service.recordWithShortName(RecordType.group, u"foo3")
+ self.assertEquals(record.uid, u"foo")
+ record = yield service.recordWithShortName(RecordType.group, u"foo4")
+ self.assertEquals(record.uid, u"foo")
+
+ # Remove a record
+ yield service.removeRecords((u"foo",))
+ record = yield service.recordWithUID(u"foo")
+ self.assertEquals(None, record)
</ins></span></pre></div>
<a id="CalendarServertrunktxdavwhoutilpy"></a>
<div class="modfile"><h4>Modified: CalendarServer/trunk/txdav/who/util.py (13404 => 13405)</h4>
<pre class="diff"><span>
<span class="info">--- CalendarServer/trunk/txdav/who/util.py 2014-05-02 15:44:31 UTC (rev 13404)
+++ CalendarServer/trunk/txdav/who/util.py 2014-05-02 16:51:33 UTC (rev 13405)
</span><span class="lines">@@ -22,8 +22,10 @@
</span><span class="cx"> from twext.python.types import MappingProxyType
</span><span class="cx"> from twext.who.aggregate import DirectoryService as AggregateDirectoryService
</span><span class="cx"> from twext.who.idirectory import (
</span><del>- FieldName as BaseFieldName, RecordType, DirectoryConfigurationError
</del><ins>+ FieldName as BaseFieldName, RecordType, DirectoryConfigurationError,
+ NoSuchRecordError
</ins><span class="cx"> )
</span><ins>+from twext.who.index import DirectoryService as IndexDirectoryService
</ins><span class="cx"> from twext.who.ldap import (
</span><span class="cx"> DirectoryService as LDAPDirectoryService, LDAPAttribute,
</span><span class="cx"> FieldName as LDAPFieldName,
</span><span class="lines">@@ -44,6 +46,7 @@
</span><span class="cx"> from txdav.who.wiki import DirectoryService as WikiDirectoryService
</span><span class="cx"> from txdav.who.xml import DirectoryService as XMLDirectoryService
</span><span class="cx"> from uuid import UUID
</span><ins>+from twisted.internet.defer import succeed, inlineCallbacks
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> log = Logger()
</span><span class="lines">@@ -273,3 +276,49 @@
</span><span class="cx"> return parts[3]
</span><span class="cx">
</span><span class="cx"> return None
</span><ins>+
+
+
+
+class InMemoryDirectoryService(IndexDirectoryService):
+ """
+ An in-memory IDirectoryService. You must call updateRecords( ) if you want
+ to populate this service.
+ """
+
+ recordType = ConstantsContainer(
+ (
+ RecordType.user,
+ RecordType.group,
+ CalRecordType.location,
+ CalRecordType.resource,
+ CalRecordType.address
+ )
+ )
+
+
+ def loadRecords(self):
+ pass
+
+
+ @inlineCallbacks
+ def updateRecords(self, records, create=False):
+ recordsByUID = dict(((record.uid, record) for record in records))
+ if not create:
+ # Make sure all the records already exist
+ for uid, record in recordsByUID.items():
+ if uid not in self._index[self.fieldName.uid]:
+ raise NoSuchRecordError(uid)
+
+ yield self.removeRecords(recordsByUID.keys())
+ self.indexRecords(records)
+
+
+ def removeRecords(self, uids):
+ index = self._index
+ for fieldName in self.indexedFields:
+ for recordSet in index[fieldName].itervalues():
+ for record in list(recordSet):
+ if record.uid in uids:
+ recordSet.remove(record)
+ return succeed(None)
</ins></span></pre>
</div>
</div>
</body>
</html>