[CalendarServer-changes] [5935] CalendarServer/branches/new-store-no-caldavfile
source_changes at macosforge.org
source_changes at macosforge.org
Mon Jul 26 11:23:06 PDT 2010
Revision: 5935
http://trac.macosforge.org/projects/calendarserver/changeset/5935
Author: glyph at apple.com
Date: 2010-07-26 11:23:05 -0700 (Mon, 26 Jul 2010)
Log Message:
-----------
Catch up to new-store branch as it was merged to trunk.
Modified Paths:
--------------
CalendarServer/branches/new-store-no-caldavfile/setup.py
CalendarServer/branches/new-store-no-caldavfile/twext/web2/dav/method/put.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/addressbook.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/calendar.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/util.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/resource.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/storebridge.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_DAV.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_wrapping.py
CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/util.py
CalendarServer/branches/new-store-no-caldavfile/txcaldav/calendarstore/test/common.py
CalendarServer/branches/new-store-no-caldavfile/txcarddav/addressbookstore/test/common.py
CalendarServer/branches/new-store-no-caldavfile/txdav/datastore/file.py
CalendarServer/branches/new-store-no-caldavfile/txdav/idav.py
Added Paths:
-----------
CalendarServer/branches/new-store-no-caldavfile/contrib/tools/dtraceanalyze.py
CalendarServer/branches/new-store-no-caldavfile/contrib/tools/trace.d
Property Changed:
----------------
CalendarServer/branches/new-store-no-caldavfile/
Property changes on: CalendarServer/branches/new-store-no-caldavfile
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:5594-5890
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/new-store:5911-5934
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:5594-5917
Copied: CalendarServer/branches/new-store-no-caldavfile/contrib/tools/dtraceanalyze.py (from rev 5934, CalendarServer/branches/new-store/contrib/tools/dtraceanalyze.py)
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/contrib/tools/dtraceanalyze.py (rev 0)
+++ CalendarServer/branches/new-store-no-caldavfile/contrib/tools/dtraceanalyze.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -0,0 +1,449 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from __future__ import with_statement
+import collections
+import getopt
+import os
+import re
+import sys
+import tables
+
+class Dtrace(object):
+
+ class DtraceLine(object):
+
+ prefix_maps = {
+ "/usr/share/caldavd/lib/python/": "{caldavd}/",
+ "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6": "{Python}",
+ "/System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5": "{Python}",
+ "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python": "{Extras}",
+ "/System/Library/Frameworks/Python.framework/Versions/2.5/Extras/lib/python": "{Extras}",
+ }
+ contains_maps = {
+ "/CalendarServer": "{caldavd}",
+ "/Twisted": "{Twisted}",
+ "/vobject": "{vobject}",
+ }
+
+ def __init__(self, line, lineno):
+
+ self.entering = True
+ self.function_name = ""
+ self.file_location = ""
+ self.parent = None
+ self.children = []
+ self.lineno = lineno
+
+ re_matched = re.match("(..) ([^ ]+) \(([^\)]+)\)", line)
+ if re_matched is None:
+ print line
+ results = re_matched.groups()
+ if results[0] == "<-":
+ self.entering = False
+ elif results[0] == "->":
+ self.entering = True
+ else:
+ raise ValueError("Invalid start of line at %d" % (lineno,))
+
+ self.function_name = results[1]
+ self.file_location = results[2]
+ for key, value in Dtrace.DtraceLine.prefix_maps.iteritems():
+ if self.file_location.startswith(key):
+ self.file_location = value + self.file_location[len(key):]
+ break
+ else:
+ for key, value in Dtrace.DtraceLine.contains_maps.iteritems():
+ found1 = self.file_location.find(key)
+ if found1 != -1:
+ found2 = self.file_location[found1+1:].find('/')
+ if found2 != -1:
+ self.file_location = value + self.file_location[found1+found2+1:]
+ else:
+ self.file_location = value
+ break
+
+ def __repr__(self):
+ return "%s (%s)" % self.getKey()
+
+ def getKey(self):
+ return (self.file_location, self.function_name,)
+
+ def getPartialKey(self):
+ return (self.filePath(), self.function_name,)
+
+ def addChild(self, child):
+ child.parent = self
+ self.children.append(child)
+
+ def checkForCollapse(self, other):
+ if self.entering and not other.entering:
+ if self.function_name == other.function_name and self.function_name != "mainLoop":
+ if self.filePath() == other.filePath():
+ return True
+ return False
+
+ def filePath(self):
+ return self.file_location[0:self.file_location.rfind(':')]
+
+ def prettyPrint(self, indent, indents, sout):
+
+ indenter = ""
+ for level in indents:
+ if level > 0:
+ indenter += "⎢ "
+ elif level < 0:
+ indenter += "⎿ "
+ else:
+ indenter += " "
+ sout.write("%s%s (%s)\n" % (indenter, self.function_name, self.file_location,))
+
+ def stackName(self):
+ return self.function_name, self.filePath()
+
+ class DtraceStack(object):
+
+ def __init__(self, lines, no_collapse):
+ self.start_indent = 0
+ self.stack = []
+ self.called_by = {}
+ self.call_into = {}
+
+ self.processLines(lines, no_collapse)
+
+ def processLines(self, lines, no_collapse):
+
+ new_lines = []
+ last_line = None
+ for line in lines:
+ if last_line:
+ if not no_collapse and line.checkForCollapse(last_line):
+ new_lines.pop()
+ last_line = None
+ continue
+ new_lines.append(line)
+ last_line = line
+
+ indent = 0
+ min_indent = 0
+ current_line = None
+ blocks = [[]]
+ backstack = []
+ for line in new_lines:
+ stackName = line.stackName()
+ if line.entering:
+ if line.function_name == "mainLoop":
+ if min_indent < 0:
+ newstack = []
+ for oldindent, oldline in blocks[-1]:
+ newstack.append((oldindent - min_indent, oldline,))
+ blocks[-1] = newstack
+ min_indent = 0
+ indent = 0
+ blocks.append([])
+ backstack = []
+ else:
+ indent += 1
+ backstack.append(stackName)
+ blocks[-1].append((indent, line,))
+ if current_line:
+ current_line.addChild(line)
+ current_line = line
+ else:
+ if len(blocks) == 1 or line.function_name != "mainLoop" and indent:
+ indent -= 1
+ while backstack and indent and stackName != backstack[-1]:
+ indent -= 1
+ backstack.pop()
+ if backstack: backstack.pop()
+ if indent < 0:
+ print "help"
+ current_line = current_line.parent if current_line else None
+ min_indent = min(min_indent, indent)
+
+ for block in blocks:
+ self.stack.extend(block)
+ if min_indent < 0:
+ self.start_indent = -min_indent
+ else:
+ self.start_indent = 0
+
+ self.generateCallInfo()
+
+ def generateCallInfo(self):
+
+ for _ignore, line in self.stack:
+ key = line.getKey()
+
+ if line.parent:
+ parent_key = line.parent.getKey()
+ parent_calls = self.called_by.setdefault(key, {}).get(parent_key, 0)
+ self.called_by[key][parent_key] = parent_calls + 1
+
+ for child in line.children:
+ child_key = child.getKey()
+ child_calls = self.call_into.setdefault(key, {}).get(child_key, 0)
+ self.call_into[key][child_key] = child_calls + 1
+
+ def prettyPrint(self, sout):
+ indents = [1] * self.start_indent
+ ctr = 0
+ maxctr = len(self.stack) - 1
+ for indent, line in self.stack:
+ current_indent = self.start_indent + indent
+ next_indent = (self.start_indent + self.stack[ctr+1][0]) if ctr < maxctr else 10000
+ if len(indents) == current_indent:
+ pass
+ elif len(indents) < current_indent:
+ indents.append(current_indent)
+ else:
+ indents = indents[0:current_indent]
+ if next_indent < current_indent:
+ indents = indents[0:next_indent] + [-1] * (current_indent - next_indent)
+ line.prettyPrint(self.start_indent + indent, indents, sout)
+ ctr += 1
+
+ def __init__(self, filepath):
+
+ self.filepath = filepath
+ self.calltimes = collections.defaultdict(lambda: [0, 0, 0])
+ self.exclusiveTotal = 0
+
+ def analyze(self, do_stack, no_collapse):
+
+ print "Parsing dtrace output."
+
+ # Parse the trace lines first and look for the start of the call times
+ lines = []
+ traces = True
+ index = -1
+ with file(filepath) as f:
+ for lineno, line in enumerate(f):
+ if traces:
+ if line.strip() and line[0:3] in ("-> ", "<- "):
+ lines.append(Dtrace.DtraceLine(line, lineno + 1))
+ elif line.startswith("Count,"):
+ traces = False
+ else:
+ if line[0] != ' ':
+ continue
+ line = line.strip()
+ if line.startswith("FILE"):
+ index += 1
+ if index >= 0:
+ self.parseCallTimeLine(line, index)
+
+ self.printTraceDetails(lines, do_stack, no_collapse)
+
+ for ctr, title in enumerate(("Sorted by Count", "Sorted by Exclusive", "Sorted by Inclusive",)):
+ print title
+ self.printCallTimeTotals(ctr)
+
+ def printTraceDetails(self, lines, do_stack, no_collapse):
+
+ print "Found %d lines" % (len(lines),)
+ print "============================"
+ print ""
+
+ self.stack = Dtrace.DtraceStack(lines, no_collapse)
+ if do_stack:
+ with file("stacked.txt", "w") as f:
+ self.stack.prettyPrint(f)
+ print "Wrote stack calls to 'stacked.txt'"
+ print "============================"
+ print ""
+
+ # Get stats for each call
+ stats = {}
+ last_exit = None
+ for line in lines:
+ key = line.getKey()
+ if line.entering:
+ counts = stats.get(key, (0, 0))
+ counts = (counts[0] + (1 if no_collapse else 0), counts[1] + (0 if no_collapse else 1))
+ if line.getPartialKey() != last_exit:
+ counts = (counts[0] + (0 if no_collapse else 1), counts[1] + (1 if no_collapse else 0))
+ stats[key] = counts
+ else:
+ last_exit = line.getPartialKey()
+
+ print "Function Call Counts"
+ print ""
+ table = tables.Table()
+ table.addHeader(("Count", "Function", "File",))
+ for key, value in sorted(stats.iteritems(), key=lambda x: x[1][0], reverse=True):
+ table.addRow(("%d (%d)" % value, key[1], key[0],))
+ table.printTable()
+
+ print ""
+ print "Called By Counts"
+ print ""
+ table = tables.Table()
+ table.addHeader(("Function", "Caller", "Count",))
+ for main_key in sorted(self.stack.called_by.keys(), key=lambda x: x[1] + x[0]):
+ first = True
+ for key, value in sorted(self.stack.called_by[main_key].iteritems(), key=lambda x: x[1], reverse=True):
+ table.addRow((
+ ("%s (%s)" % (main_key[1], main_key[0],)) if first else "",
+ "%s (%s)" % (key[1], key[0],),
+ str(value),
+ ))
+ first = False
+ table.printTable()
+
+ print ""
+ print "Call Into Counts"
+ print ""
+ table = tables.Table()
+ table.addHeader(("Function", "Calls", "Count",))
+ for main_key in sorted(self.stack.call_into.keys(), key=lambda x: x[1] + x[0]):
+ first = True
+ for key, value in sorted(self.stack.call_into[main_key].iteritems(), key=lambda x: x[1], reverse=True):
+ table.addRow((
+ ("%s (%s)" % (main_key[1], main_key[0],)) if first else "",
+ "%s (%s)" % (key[1], key[0],),
+ str(value),
+ ))
+ first = False
+ table.printTable()
+ print ""
+
+ def parseCallTimeLine(self, line, index):
+
+ file, type, name, value = line.split()
+ if file in ("-", "FILE"):
+ return
+ else:
+ self.calltimes[(file, name)][index] = int(value)
+ if index == 1:
+ self.exclusiveTotal += int(value)
+
+ def printCallTimeTotals(self, sortIndex):
+
+ table = tables.Table()
+
+ table.setDefaultColumnFormats((
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%s", tables.Table.ColumnFormat.LEFT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ tables.Table.ColumnFormat("%d", tables.Table.ColumnFormat.RIGHT_JUSTIFY),
+ ))
+
+ table.addHeader(("File", "Name", "Count", "Inclusive", "Exclusive", "Children",))
+ for key, value in sorted(self.calltimes.items(), key=lambda x:x[1][sortIndex], reverse=True):
+ table.addRow((
+ key[0],
+ key[1],
+ value[0],
+ value[2],
+ "%s (%6.3f%%)" % (value[1], (100.0 * value[1]) / self.exclusiveTotal),
+ value[2] - value[1],
+ ))
+ table.addRow()
+ table.addRow((
+ "Total:",
+ "",
+ "",
+ "",
+ self.exclusiveTotal,
+ "",
+ ))
+
+ table.printTable()
+ print ""
+
+def usage(error_msg=None):
+ if error_msg:
+ print error_msg
+
+ print """Usage: dtraceanalyze [options] FILE
+Options:
+ -h Print this help and exit
+ --stack Save indented stack to file
+ --raw-count Display call counts based on full trace,
+ else display counts on collapsed values.
+
+Arguments:
+ FILE File name containing dtrace output to analyze
+
+Description:
+ This utility will analyze the output of the trace.d dtrace script to produce
+ useful statistics, and other performance related data.
+
+ To use this do the following (where PID is the pid of the
+ Python process to monitor:
+
+ > sudo ./trace.d PID > results.txt
+ ...
+ > ./dtraceanalyze.py results.txt
+"""
+
+ if error_msg:
+ raise ValueError(error_msg)
+ else:
+ sys.exit(0)
+
+if __name__ == "__main__":
+
+ sys.setrecursionlimit(10000)
+ do_stack = False
+ no_collapse = False
+ try:
+ options, args = getopt.getopt(sys.argv[1:], "h", ["stack", "no-collapse"])
+
+ for option, value in options:
+ if option == "-h":
+ usage()
+ elif option == "--stack":
+ do_stack = True
+ elif option == "--no-collapse":
+ no_collapse = True
+ else:
+ usage("Unrecognized option: %s" % (option,))
+
+ if len(args) == 0:
+ fname = "results.txt"
+ elif len(args) != 1:
+ usage("Must have one argument")
+ else:
+ fname = args[0]
+
+ filepath = os.path.expanduser(fname)
+ if not os.path.exists(filepath):
+ usage("File '%s' does not exist" % (filepath,))
+
+ print "CalendarServer dtrace analysis tool tool"
+ print "====================================="
+ print ""
+ if do_stack:
+ print "Generating nested stack call file."
+ if no_collapse:
+ print "Consecutive function calls will not be removed."
+ else:
+ print "Consecutive function calls will be removed."
+ print "============================"
+ print ""
+
+ Dtrace(filepath).analyze(do_stack, no_collapse)
+
+ except Exception, e:
+ raise
+ sys.exit(str(e))
Copied: CalendarServer/branches/new-store-no-caldavfile/contrib/tools/trace.d (from rev 5934, CalendarServer/branches/new-store/contrib/tools/trace.d)
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/contrib/tools/trace.d (rev 0)
+++ CalendarServer/branches/new-store-no-caldavfile/contrib/tools/trace.d 2010-07-26 18:23:05 UTC (rev 5935)
@@ -0,0 +1,101 @@
+#!/usr/sbin/dtrace -Zs
+/*
+ * trace.d - measure Python on-CPU times and flow for functions.
+ *
+ * This traces Python activity from a specified process.
+ *
+ * USAGE: trace.d <PID>
+ *
+ * FIELDS:
+ * FILE Filename of the Python program
+ * TYPE Type of call (func/total)
+ * NAME Name of call (function name)
+ * TOTAL Total on-CPU time for calls (us)
+ *
+ * Filename and function names are printed if available.
+ *
+ * COPYRIGHT: Copyright (c) 2007 Brendan Gregg.
+ *
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at Docs/cddl1.txt
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * CDDL HEADER END
+ *
+ * 09-Sep-2007 Brendan Gregg Created original py_ scripts.
+ *
+ * This is a combination of py_flow.d and py_cputime.d that takes a pid
+ * as an argument and runs for 10 seconds.
+ */
+
+#pragma D option quiet
+#pragma D option bufsize=100m
+
+dtrace:::BEGIN
+{
+ printf("Tracing... Hit Ctrl-C to end.\n");
+}
+
+python*:::function-entry
+/pid == $1/
+{
+ self->depth++;
+ self->exclude[self->depth] = 0;
+ self->function[self->depth] = vtimestamp;
+
+ printf("%s %s (%s:%d)\n", "->",
+ copyinstr(arg1), copyinstr(arg0), arg2);
+}
+
+python*:::function-return
+/pid == $1 && self->function[self->depth]/
+{
+ this->oncpu_incl = vtimestamp - self->function[self->depth];
+ this->oncpu_excl = this->oncpu_incl - self->exclude[self->depth];
+ self->function[self->depth] = 0;
+ self->exclude[self->depth] = 0;
+ this->file = basename(copyinstr(arg0));
+ this->name = copyinstr(arg1);
+
+ @num[this->file, "func", this->name] = count();
+ @num["-", "total", "-"] = count();
+ @types_incl[this->file, "func", this->name] = sum(this->oncpu_incl);
+ @types_excl[this->file, "func", this->name] = sum(this->oncpu_excl);
+ @types_excl["-", "total", "-"] = sum(this->oncpu_excl);
+
+ self->depth--;
+ self->exclude[self->depth] += this->oncpu_incl;
+
+ printf("%s %s (%s:%d)\n", "<-",
+ copyinstr(arg1), copyinstr(arg0), arg2);
+}
+
+profile:::tick-10s
+{
+ exit(0);
+}
+
+dtrace:::END
+{
+ printf("\nCount,\n");
+ printf(" %-20s %-10s %-32s %8s\n", "FILE", "TYPE", "NAME", "COUNT");
+ printa(" %-20s %-10s %-32s %@8d\n", @num);
+
+ normalize(@types_excl, 1000);
+ printf("\nExclusive function on-CPU times (us),\n");
+ printf(" %-20s %-10s %-32s %8s\n", "FILE", "TYPE", "NAME", "TOTAL");
+ printa(" %-20s %-10s %-32s %@8d\n", @types_excl);
+
+ normalize(@types_incl, 1000);
+ printf("\nInclusive function on-CPU times (us),\n");
+ printf(" %-20s %-10s %-32s %8s\n", "FILE", "TYPE", "NAME", "TOTAL");
+ printa(" %-20s %-10s %-32s %@8d\n", @types_incl);
+}
Modified: CalendarServer/branches/new-store-no-caldavfile/setup.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/setup.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/setup.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -118,6 +118,7 @@
"bin/calendarserver_manage_principals",
"bin/calendarserver_command_gateway",
"bin/calendarserver_purge_events",
+ "bin/calendarserver_migrate_resources",
"bin/carddavd",
],
data_files = [ ("caldavd", ["conf/caldavd.plist"]),
Modified: CalendarServer/branches/new-store-no-caldavfile/twext/web2/dav/method/put.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twext/web2/dav/method/put.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twext/web2/dav/method/put.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -54,6 +54,11 @@
yield parent
parent = parent.getResult()
+ if not parent.exists():
+ raise HTTPError(
+ StatusResponse(
+ responsecode.CONFLICT,
+ "cannot PUT to non-existent parent"))
x = waitForDeferred(parent.authorize(request, (davxml.Bind(),)))
yield x
x.getResult()
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/addressbook.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/addressbook.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/addressbook.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -39,6 +39,7 @@
from twistedcaldav.config import config
from twistedcaldav.directory.idirectory import IDirectoryService
+from twistedcaldav.directory.util import transactionFromRequest
from twistedcaldav.directory.resource import AutoProvisioningResourceMixIn,\
DirectoryReverseProxyResource
from twistedcaldav.extensions import ReadOnlyResourceMixIn, DAVResource,\
@@ -263,11 +264,7 @@
def homeResourceForRecord(self, record, request):
self.provision()
- TRANSACTION_KEY = '_newStoreTransaction'
- transaction = getattr(request, TRANSACTION_KEY, None)
- if transaction is None:
- transaction = self.parent._newStore.newTransaction(repr(request))
- setattr(request, TRANSACTION_KEY, transaction)
+ transaction = transactionFromRequest(request, self.parent._newStore)
name = record.uid
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/calendar.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/calendar.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/calendar.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
+from twistedcaldav.directory.util import transactionFromRequest
"""
Implements a directory-backed calendar hierarchy.
@@ -264,12 +265,7 @@
def homeResourceForRecord(self, record, request):
self.provision()
- TRANSACTION_KEY = '_newStoreTransaction'
- transaction = getattr(request, TRANSACTION_KEY, None)
- if transaction is None:
- transaction = self.parent._newStore.newTransaction(repr(request))
- setattr(request, TRANSACTION_KEY, transaction)
-
+ transaction = transactionFromRequest(request, self.parent._newStore)
name = record.uid
if record is None:
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/util.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/util.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/directory/util.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -41,6 +41,7 @@
import errno
import time
from twext.python.filepath import CachingFilePath as FilePath
+from txdav.idav import AlreadyFinishedError
class NotFilePath(FilePath):
"""
@@ -181,3 +182,43 @@
raise NotImplementedError()
moveTo = _notAllowed
+
+
+
+def transactionFromRequest(request, newStore):
+ """
+ Return the associated transaction from the given HTTP request, creating a
+ new one from the given data store if none has yet been associated.
+
+ Also, if the request was not previously associated with a transaction, add
+ a failsafe transaction-abort response filter to abort any transaction which
+ has not been committed or aborted by the resource which responds to the
+ request.
+
+ @param request: The request to inspect.
+ @type request: L{IRequest}
+
+ @param newStore: The store to create a transaction from.
+ @type newStore: L{IDataStore}
+
+ @return: a transaction that should be used to read and write data
+ associated with the request.
+ @rtype: L{ITransaction} (and possibly L{ICalendarTransaction} and
+ L{IAddressBookTransaction} as well.
+ """
+ TRANSACTION_KEY = '_newStoreTransaction'
+ transaction = getattr(request, TRANSACTION_KEY, None)
+ if transaction is None:
+ transaction = newStore.newTransaction(repr(request))
+ def abortIfUncommitted(request, response):
+ try:
+ transaction.abort()
+ except AlreadyFinishedError:
+ pass
+ return response
+ abortIfUncommitted.handleErrors = True
+ request.addResponseFilter(abortIfUncommitted)
+ setattr(request, TRANSACTION_KEY, transaction)
+ return transaction
+
+
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/resource.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/resource.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/resource.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -274,9 +274,11 @@
raise RuntimeError("No associated transaction to propagate")
otherResource.associateWithTransaction(self._associatedTransaction)
+
def transactionError(self):
self._transactionError = True
+
def renderHTTP(self, request):
"""
Override C{renderHTTP} to commit the transaction when the resource is
@@ -293,13 +295,9 @@
else:
self._associatedTransaction.commit()
return result
- def failed(failure):
- print 'renderHTTP failed! FIXME PLEASE: handle errors here!'
- failure.printTraceback()
- return failure
- # FIXME: needs a failure handler (that rolls back the transaction)
- return d.addCallback(succeeded).addErrback(failed)
+ return d.addCallback(succeeded)
+
# Begin transitional new-store resource interface:
def copyDeadPropertiesTo(self, other):
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/storebridge.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/storebridge.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/storebridge.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -1149,6 +1149,10 @@
returnValue(CREATED)
+ def createSimilarFile(self, name):
+ return None
+
+
def isCollection(self):
return False
@@ -1164,6 +1168,8 @@
# FIXME: tests, workingness
return succeed(0)
+
+
class _AddressBookChildHelper(object):
"""
Methods for things which are like addressbooks.
@@ -1662,9 +1668,14 @@
returnValue(CREATED)
+ def createSimilarFile(self, name):
+ return None
+
+
def isCollection(self):
return False
+
def exists(self):
# FIXME: tests
return False
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_DAV.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_DAV.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_DAV.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -27,10 +27,20 @@
import twext.web2.dav.test.test_put
import twext.web2.dav.test.test_report
import twext.web2.dav.test.test_report_expand
+from twisted.trial.unittest import SkipTest
-class ACL (twext.web2.dav.test.test_acl.ACL ): resource_class = MyResource
+def ignored(self):
+ raise SkipTest("method requires backing store objects, tested elsewhere")
+
+class ACL (twext.web2.dav.test.test_acl.ACL ):
+ resource_class = MyResource
+ test_DELETE = ignored
+
class COPY (twext.web2.dav.test.test_copy.COPY ): resource_class = MyResource
-class DELETE (twext.web2.dav.test.test_delete.DELETE ): resource_class = MyResource
+class DELETE (twext.web2.dav.test.test_delete.DELETE ):
+ resource_class = MyResource
+ test_DELETE = ignored
+
class LOCK_UNLOCK (twext.web2.dav.test.test_lock.LOCK_UNLOCK ): resource_class = MyResource
class MKCOL (twext.web2.dav.test.test_mkcol.MKCOL ): resource_class = MyResource
class MOVE (twext.web2.dav.test.test_move.MOVE ): resource_class = MyResource
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_wrapping.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_wrapping.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/test_wrapping.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -18,8 +18,12 @@
Tests for the interaction between model-level and protocol-level logic.
"""
+from twext.web2.server import Request
+from twext.web2.responsecode import UNAUTHORIZED
+from twext.web2.http_headers import Headers
+from txdav.idav import AlreadyFinishedError
+
from twext.web2.dav import davxml
-from twext.web2.dav.element.base import dav_namespace
from twistedcaldav.config import config
from twisted.internet.defer import inlineCallbacks, returnValue
@@ -39,6 +43,28 @@
from txcarddav.addressbookstore.test.test_file import vcard4_text
+class FakeChanRequest(object):
+ def writeHeaders(self, code, headers):
+ self.code = code
+ self.headers = headers
+ def registerProducer(self, producer, streaming):
+ pass
+ def write(self, data):
+ pass
+ def unregisterProducer(self):
+ pass
+ def abortConnection(self):
+ pass
+ def getHostInfo(self):
+ return '127.0.0.1', False
+ def getRemoteHost(self):
+ return '127.0.0.1'
+ def finish(self):
+ pass
+
+
+
+
class WrappingTests(TestCase):
"""
Tests for L{twistedcaldav.static.CalDAVFile} creating the appropriate type
@@ -65,7 +91,7 @@
@param objectName: The name of a calendar object.
@type objectName: str
@param objectText: Some iCalendar text to populate it with.
- @type objectText: str
+ @type objectText: str
"""
record = self.directoryService.recordWithShortName("users", "wsanchez")
uid = record.uid
@@ -89,7 +115,7 @@
@param objectName: The name of a addressbook object.
@type objectName: str
@param objectText: Some iVcard text to populate it with.
- @type objectText: str
+ @type objectText: str
"""
record = self.directoryService.recordWithShortName("users", "wsanchez")
uid = record.uid
@@ -109,6 +135,8 @@
txn.commit()
+ requestUnderTest = None
+
@inlineCallbacks
def getResource(self, path):
"""
@@ -119,11 +147,15 @@
@type path: C{str}
"""
- segments = path.split("/")
- resource = self.site.resource
- while segments:
- resource, segments = yield resource.locateChild(self, segments)
- returnValue(resource)
+ if self.requestUnderTest is None:
+ req = self.requestForPath(path)
+ self.requestUnderTest = req
+ else:
+ req = self.requestUnderTest
+ aResource = yield req.locateResource(
+ "http://localhost:8008/" + path
+ )
+ returnValue(aResource)
def commit(self):
@@ -132,17 +164,65 @@
an associated transaction. Commit that transaction to bring the
filesystem into a consistent state.
"""
- self._newStoreTransaction.commit()
+ self.requestUnderTest._newStoreTransaction.commit()
+ def requestForPath(self, path):
+ """
+ Get a L{Request} with a L{FakeChanRequest} for a given path.
+ """
+ headers = Headers()
+ headers.addRawHeader("Host", "localhost:8008")
+ chanReq = FakeChanRequest()
+ req = Request(
+ site=self.site,
+ chanRequest=chanReq,
+ command='GET',
+ path=path,
+ version=('1', '1'),
+ contentLength=0,
+ headers=headers
+ )
+ req.credentialFactories = {}
+ return req
+
+
+ @inlineCallbacks
+ def test_autoRevertUnCommitted(self):
+ """
+ Resources that need to read from the back-end in a transaction will be
+ reverted by a response filter in the case where the request does not
+ commit them. This can happen, for example, with resources that are
+ children of non-existent (proto-)resources.
+ """
+ for pathType in ['calendar', 'addressbook']:
+ req = self.requestForPath('/%ss/users/wsanchez/%s/forget/it'
+ % (pathType, pathType))
+ yield req.process()
+ self.assertEquals(req.chanRequest.code, 404)
+ self.assertRaises(AlreadyFinishedError,
+ req._newStoreTransaction.commit)
+
+
+ @inlineCallbacks
+ def test_simpleRequest(self):
+ """
+ Sanity check and integration test: an unauthorized request of calendar
+ and addressbook resources results in an L{UNAUTHORIZED} response code.
+ """
+ for pathType in ['calendar', 'addressbook']:
+ req = self.requestForPath('/%ss/users/wsanchez/%s/'
+ % (pathType, pathType))
+ yield req.process()
+ self.assertEquals(req.chanRequest.code, UNAUTHORIZED)
+
+
def test_createStore(self):
"""
Creating a DirectoryCalendarHomeProvisioningResource will create a paired
CalendarStore.
"""
self.assertIsInstance(self.calendarCollection._newStore, CalendarStore)
- self.assertEquals(self.calendarCollection._newStore._path,
- self.site.resource.fp)
@inlineCallbacks
@@ -154,7 +234,6 @@
"""
calDavFile = yield self.getResource("calendars/users/wsanchez/")
self.commit()
- self.assertEquals(calDavFile.fp, calDavFile._newStoreCalendarHome._path)
self.assertIsInstance(calDavFile._newStoreCalendarHome, CalendarHome)
@@ -185,7 +264,6 @@
"""
calDavFile = yield self.getResource("calendars/users/wsanchez/calendar")
self.commit()
- self.assertEquals(calDavFile.fp, calDavFile._newStoreCalendar._path)
self.assertEquals(calDavFile.resourceType(),
davxml.ResourceType.calendar)
@@ -203,7 +281,6 @@
self.assertIsInstance(calDavFile, ProtoCalendarCollectionResource)
calDavFile.createCalendarCollection()
self.commit()
- self.assertEquals(calDavFile.fp, calDavFile._newStoreCalendar._path)
@inlineCallbacks
@@ -221,6 +298,7 @@
self.assertIdentical(
getattr(calDavFile, "_newStoreCalendar", None), None
)
+ self.commit()
@inlineCallbacks
@@ -235,8 +313,6 @@
"calendars/users/wsanchez/calendar/1.ics"
)
self.commit()
- self.assertEquals(calDavFileCalendar._newStoreObject._path,
- calDavFileCalendar.fp)
self.assertEquals(calDavFileCalendar._principalCollections,
frozenset([self.principalsResource]))
@@ -262,8 +338,6 @@
AddressBookStore.
"""
self.assertIsInstance(self.addressbookCollection._newStore, AddressBookStore)
- self.assertEquals(self.addressbookCollection._newStore._path,
- self.site.resource.fp)
@inlineCallbacks
@@ -275,7 +349,6 @@
"""
calDavFile = yield self.getResource("addressbooks/users/wsanchez/")
self.commit()
- self.assertEquals(calDavFile.fp, calDavFile._newStoreAddressBookHome._path)
self.assertIsInstance(calDavFile._newStoreAddressBookHome, AddressBookHome)
@@ -288,7 +361,8 @@
"""
calDavFile = yield self.getResource("addressbooks/users/wsanchez/addressbook")
self.commit()
- self.assertEquals(calDavFile.fp, calDavFile._newStoreAddressBook._path)
+ self.assertEquals(calDavFile._principalCollections,
+ frozenset([self.principalsResource]))
@inlineCallbacks
@@ -304,7 +378,8 @@
self.assertIsInstance(calDavFile, ProtoAddressBookCollectionResource)
calDavFile.createAddressBookCollection()
self.commit()
- self.assertEquals(calDavFile.fp, calDavFile._newStoreAddressBook._path)
+ self.assertEquals(calDavFile._principalCollections,
+ frozenset([self.principalsResource]))
@inlineCallbacks
@@ -319,8 +394,6 @@
"addressbooks/users/wsanchez/addressbook/1.vcf"
)
self.commit()
- self.assertEquals(calDavFileAddressBook._newStoreObject._path,
- calDavFileAddressBook.fp)
self.assertEquals(calDavFileAddressBook._principalCollections,
frozenset([self.principalsResource]))
Modified: CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/util.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/twistedcaldav/test/util.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -289,6 +289,9 @@
return verifyChildren(root, structure)
+class norequest(object):
+ def addResponseFilter(self, filter):
+ "stub; ignore me"
class HomeTestCase(TestCase):
@@ -330,7 +333,6 @@
it a new transaction.
"""
users = self.homeProvisioner.getChild("users")
- class norequest(object): pass
user, ignored = (yield users.locateChild(norequest(), ["wsanchez"]))
# Force the request to succeed regardless of the implementation of
@@ -396,7 +398,6 @@
it a new transaction.
"""
users = self.homeProvisioner.getChild("users")
- class norequest(object): pass
user, ignored = (yield users.locateChild(norequest(), ["wsanchez"]))
# Force the request to succeed regardless of the implementation of
Modified: CalendarServer/branches/new-store-no-caldavfile/txcaldav/calendarstore/test/common.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/txcaldav/calendarstore/test/common.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/txcaldav/calendarstore/test/common.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##
-from twext.web2.http_headers import MimeType
"""
Tests for common calendar store API functions.
@@ -27,7 +26,7 @@
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.protocol import Protocol
-from txdav.idav import IPropertyStore, IDataStore
+from txdav.idav import IPropertyStore, IDataStore, AlreadyFinishedError
from txdav.propertystore.base import PropertyName
from txdav.common.icommondatastore import HomeChildNameAlreadyExistsError, \
@@ -43,6 +42,7 @@
from twext.python.filepath import CachingFilePath as FilePath
from twext.web2.dav import davxml
+from twext.web2.http_headers import MimeType
from twext.web2.dav.element.base import WebDAVUnknownElement
from twext.python.vcomponent import VComponent
@@ -1012,3 +1012,16 @@
set(home1_calendarNames))
+ def test_finishedOnCommit(self):
+ """
+ Calling L{ITransaction.abort} or L{ITransaction.commit} after
+ L{ITransaction.commit} has already been called raises an
+ L{AlreadyFinishedError}.
+ """
+ self.calendarObjectUnderTest()
+ txn = self.lastTransaction
+ self.commit()
+ self.assertRaises(AlreadyFinishedError, txn.commit)
+ self.assertRaises(AlreadyFinishedError, txn.abort)
+
+
Modified: CalendarServer/branches/new-store-no-caldavfile/txcarddav/addressbookstore/test/common.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/txcarddav/addressbookstore/test/common.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/txcarddav/addressbookstore/test/common.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -124,12 +124,12 @@
def storeUnderTest(self):
"""
- Subclasses must override this to return an L{IAddressBookStore} provider
- which adheres to the structure detailed by L{CommonTests.requirements}.
- This attribute is a dict of dict of dicts; the outermost layer
- representing UIDs mapping to addressbook homes, then addressbook names mapping
- to addressbook collections, and finally addressbook object names mapping to
- addressbook object text.
+ Subclasses must override this to return an L{IAddressBookStore}
+ provider which adheres to the structure detailed by
+ L{CommonTests.requirements}. This attribute is a dict of dict of dicts;
+ the outermost layer representing UIDs mapping to addressbook homes,
+ then addressbook names mapping to addressbook collections, and finally
+ addressbook object names mapping to addressbook object text.
"""
raise NotImplementedError()
@@ -351,7 +351,7 @@
"""
L{IAddressBookHome.createAddressBookWithName} raises
L{AddressBookAlreadyExistsError} when the name conflicts with an already-
- existing
+ existing address book.
"""
for name in home1_addressbookNames:
self.assertRaises(
@@ -751,7 +751,7 @@
propertyContent = WebDAVUnknownElement("sample content")
propertyContent.name = propertyName.name
propertyContent.namespace = propertyName.namespace
-
+
self.addressbookObjectUnderTest().properties()[
propertyName] = propertyContent
self.commit()
Modified: CalendarServer/branches/new-store-no-caldavfile/txdav/datastore/file.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/txdav/datastore/file.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/txdav/datastore/file.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -26,6 +26,7 @@
from txdav.idav import IDataStoreResource
from txdav.propertystore.base import PropertyName
+from txdav.idav import AlreadyFinishedError
from zope.interface.declarations import implements
@@ -175,10 +176,11 @@
@type mode: C{str}
- @raise RuntimeError: This transaction has already been terminated.
+ @raise AlreadyFinishedError: This transaction has already been
+ terminated.
"""
if self._termination is not None:
- raise RuntimeError("already %s" % (self._termination,))
+ raise AlreadyFinishedError("already %s" % (self._termination,))
self._termination = mode
self._tracker.done = True
Modified: CalendarServer/branches/new-store-no-caldavfile/txdav/idav.py
===================================================================
--- CalendarServer/branches/new-store-no-caldavfile/txdav/idav.py 2010-07-23 20:33:33 UTC (rev 5934)
+++ CalendarServer/branches/new-store-no-caldavfile/txdav/idav.py 2010-07-26 18:23:05 UTC (rev 5935)
@@ -21,7 +21,7 @@
__all__ = [
"PropertyStoreError",
"PropertyChangeNotAllowedError",
- "AbortedTransactionError",
+ "AlreadyFinishedError",
"IPropertyName",
"IPropertyStore",
"IDataStore",
@@ -29,7 +29,6 @@
]
from zope.interface import Attribute, Interface
-
from zope.interface.common.mapping import IMapping
#
@@ -42,6 +41,7 @@
"""
+
class PropertyChangeNotAllowedError(PropertyStoreError):
"""
Property cannot be edited.
@@ -51,9 +51,11 @@
self.keys = keys
-class AbortedTransactionError(RuntimeError):
+
+class AlreadyFinishedError(Exception):
"""
- This transaction has aborted.
+ The transaction was already completed via an C{abort} or C{commit} and
+ cannot be aborted or committed again.
"""
@@ -183,10 +185,16 @@
def abort():
"""
Abort this transaction.
+
+ @raise AlreadyFinishedError: The transaction was already finished with
+ an 'abort' or 'commit' and cannot be aborted again.
"""
def commit():
"""
Perform this transaction.
+
+ @raise AlreadyFinishedError: The transaction was already finished with
+ an 'abort' or 'commit' and cannot be committed again.
"""
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100726/30c542a1/attachment-0001.html>
More information about the calendarserver-changes
mailing list