[CalendarServer-changes] [5719] CalendarServer/branches/new-store
source_changes at macosforge.org
source_changes at macosforge.org
Thu Jun 10 13:18:47 PDT 2010
Revision: 5719
http://trac.macosforge.org/projects/calendarserver/changeset/5719
Author: glyph at apple.com
Date: 2010-06-10 13:18:42 -0700 (Thu, 10 Jun 2010)
Log Message:
-----------
conflict-free trunk integration
Modified Paths:
--------------
CalendarServer/branches/new-store/calendarserver/tap/caldav.py
CalendarServer/branches/new-store/calendarserver/tap/util.py
CalendarServer/branches/new-store/calendarserver/tools/gateway.py
CalendarServer/branches/new-store/calendarserver/tools/principals.py
CalendarServer/branches/new-store/calendarserver/tools/test/test_purge.py
CalendarServer/branches/new-store/calendarserver/tools/util.py
CalendarServer/branches/new-store/conf/caldavd-apple.plist
CalendarServer/branches/new-store/contrib/tools/request_monitor.py
CalendarServer/branches/new-store/twext/web2/channel/http.py
CalendarServer/branches/new-store/twext/web2/dav/test/test_xattrprops.py
CalendarServer/branches/new-store/twext/web2/dav/xattrprops.py
CalendarServer/branches/new-store/twistedcaldav/accounting.py
CalendarServer/branches/new-store/twistedcaldav/directory/appleopendirectory.py
CalendarServer/branches/new-store/twistedcaldav/directory/augment.py
CalendarServer/branches/new-store/twistedcaldav/method/__init__.py
CalendarServer/branches/new-store/twistedcaldav/schedule.py
CalendarServer/branches/new-store/twistedcaldav/scheduling/scheduler.py
CalendarServer/branches/new-store/twistedcaldav/sql.py
CalendarServer/branches/new-store/twistedcaldav/static.py
CalendarServer/branches/new-store/twistedcaldav/stdconfig.py
CalendarServer/branches/new-store/twistedcaldav/test/util.py
CalendarServer/branches/new-store/twistedcaldav/util.py
Added Paths:
-----------
CalendarServer/branches/new-store/calendarserver/tap/test/test_util.py
CalendarServer/branches/new-store/twistedcaldav/method/acl.py
Property Changed:
----------------
CalendarServer/branches/new-store/
Property changes on: CalendarServer/branches/new-store
___________________________________________________________________
Modified: svn:mergeinfo
- /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:5594-5677
+ /CalendarServer/branches/config-separation:4379-4443
/CalendarServer/branches/egg-info-351:4589-4625
/CalendarServer/branches/users/cdaboo/cached-subscription-calendars-5692:5693-5702
/CalendarServer/branches/users/cdaboo/directory-cache-on-demand-3627:3628-3644
/CalendarServer/branches/users/cdaboo/more-sharing-5591:5592-5601
/CalendarServer/branches/users/cdaboo/partition-4464:4465-4957
/CalendarServer/branches/users/cdaboo/relative-config-paths-5070:5071-5105
/CalendarServer/branches/users/cdaboo/shared-calendars-5187:5188-5440
/CalendarServer/branches/users/glyph/contacts-server-merge:4971-5080
/CalendarServer/branches/users/glyph/sendfdport:5388-5424
/CalendarServer/branches/users/glyph/use-system-twisted:5084-5149
/CalendarServer/branches/users/sagen/locations-resources:5032-5051
/CalendarServer/branches/users/sagen/locations-resources-2:5052-5061
/CalendarServer/branches/users/sagen/resource-delegates-4038:4040-4067
/CalendarServer/branches/users/sagen/resource-delegates-4066:4068-4075
/CalendarServer/branches/users/sagen/resources-2:5084-5093
/CalendarServer/branches/users/wsanchez/transations:5515-5593
/CalendarServer/trunk:5594-5718
Modified: CalendarServer/branches/new-store/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tap/caldav.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/calendarserver/tap/caldav.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -77,7 +77,6 @@
from twistedcaldav.static import TimezoneServiceFile
from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twistedcaldav.upgrade import upgradeData
-from twistedcaldav.util import getNCPU
from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
@@ -93,7 +92,8 @@
from calendarserver.provision.root import RootResource
from calendarserver.webadmin.resource import WebAdminResource
from calendarserver.webcal.resource import WebCalendarResource
-from calendarserver.tap.util import getRootResource
+from calendarserver.tap.util import getRootResource, computeProcessCount
+from calendarserver.tools.util import checkDirectory
log = Logger()
@@ -250,6 +250,9 @@
config.updateDefaults(self.overrides)
+ def checkDirectory(self, dirpath, description, access=None, create=None):
+ checkDirectory(dirpath, description, access=access, create=create)
+
def checkConfiguration(self):
uid, gid = None, None
@@ -337,53 +340,8 @@
self.log_info("WARNING: changing umask from: 0%03o to 0%03o"
% (oldmask, config.umask))
- def checkDirectory(self, dirpath, description, access=None, create=None):
- if not os.path.exists(dirpath):
- try:
- mode, username, groupname = create
- except TypeError:
- raise ConfigurationError("%s does not exist: %s"
- % (description, dirpath))
- try:
- os.mkdir(dirpath)
- except (OSError, IOError), e:
- self.log_error("Could not create %s: %s" % (dirpath, e))
- raise ConfigurationError(
- "%s does not exist and cannot be created: %s"
- % (description, dirpath)
- )
- if username:
- uid = getpwnam(username).pw_uid
- else:
- uid = -1
- if groupname:
- gid = getgrnam(groupname).gr_gid
- else:
- gid = -1
-
- try:
- os.chmod(dirpath, mode)
- os.chown(dirpath, uid, gid)
- except (OSError, IOError), e:
- self.log_error("Unable to change mode/owner of %s: %s"
- % (dirpath, e))
-
- self.log_info("Created directory: %s" % (dirpath,))
-
- if not os.path.isdir(dirpath):
- raise ConfigurationError("%s is not a directory: %s"
- % (description, dirpath))
-
- if access and not os.access(dirpath, access):
- raise ConfigurationError(
- "Insufficient permissions for server on %s directory: %s"
- % (description, dirpath)
- )
-
-
-
class GroupOwnedUNIXServer(UNIXServer, object):
"""
A L{GroupOwnedUNIXServer} is a L{UNIXServer} which changes the group
@@ -775,30 +733,16 @@
parentEnv["KRB5_KTNAME"] = os.environ["KRB5_KTNAME"]
#
- # Attempt to calculate the number of processes to use 1 per processor
+ # Calculate the number of processes to spawn
#
if config.MultiProcess.ProcessCount == 0:
- try:
- cpuCount = getNCPU()
- except NotImplementedError, e:
- self.log_error("Unable to detect number of CPUs: %s"
- % (str(e),))
- cpuCount = 0
- else:
- if cpuCount < 1:
- self.log_error(
- "%d processors detected, which is hard to believe."
- % (cpuCount,)
- )
-
- processCount = config.MultiProcess.MinProcessCount
- if 2 * cpuCount > processCount:
- processCount = 2 * cpuCount
-
- self.log_info("%d processors found. Configuring %d processes."
- % (cpuCount, processCount))
-
+ processCount = computeProcessCount(
+ config.MultiProcess.MinProcessCount,
+ config.MultiProcess.PerCPU,
+ config.MultiProcess.PerGB,
+ )
config.MultiProcess.ProcessCount = processCount
+ self.log_info("Configuring %d processes." % (processCount,))
# Open the socket(s) to be inherited by the slaves
Copied: CalendarServer/branches/new-store/calendarserver/tap/test/test_util.py (from rev 5718, CalendarServer/trunk/calendarserver/tap/test/test_util.py)
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tap/test/test_util.py (rev 0)
+++ CalendarServer/branches/new-store/calendarserver/tap/test/test_util.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -0,0 +1,49 @@
+##
+# Copyright (c) 2007-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from calendarserver.tap.util import computeProcessCount
+from twistedcaldav.test.util import TestCase
+
+class ProcessCountTestCase(TestCase):
+
+ def test_count(self):
+
+ data = (
+ # minimum, perCPU, perGB, cpu, memory (in GB), expected:
+ (0, 1, 1, 0, 0, 0),
+ (1, 2, 2, 0, 0, 1),
+ (1, 2, 1, 1, 1, 1),
+ (1, 2, 2, 1, 1, 2),
+ (1, 2, 2, 2, 1, 2),
+ (1, 2, 2, 1, 2, 2),
+ (1, 2, 2, 2, 2, 4),
+ (4, 1, 2, 8, 2, 4),
+ (6, 2, 2, 2, 2, 6),
+ (1, 2, 1, 4, 99, 8),
+
+ (2, 1, 2, 2, 2, 2), # 2 cores, 2GB = 2
+ (2, 1, 2, 2, 4, 2), # 2 cores, 4GB = 2
+ (2, 1, 2, 8, 6, 8), # 8 cores, 6GB = 8
+ (2, 1, 2, 8, 16, 8), # 8 cores, 16GB = 8
+ )
+
+ for min, perCPU, perGB, cpu, mem, expected in data:
+ mem *= (1024 * 1024 * 1024)
+
+ self.assertEquals(
+ expected,
+ computeProcessCount(min, perCPU, perGB, cpuCount=cpu, memSize=mem)
+ )
Modified: CalendarServer/branches/new-store/calendarserver/tap/util.py
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tap/util.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/calendarserver/tap/util.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -53,6 +53,7 @@
from twistedcaldav.static import TimezoneServiceFile
from twistedcaldav.static import AddressBookHomeProvisioningFile, DirectoryBackedAddressBookFile
from twistedcaldav.timezones import TimezoneCache
+from twistedcaldav.util import getMemorySize, getNCPU
from twisted.internet.defer import inlineCallbacks, returnValue
try:
@@ -463,7 +464,39 @@
+def computeProcessCount(minimum, perCPU, perGB, cpuCount=None, memSize=None):
+ """
+ Determine how many process to spawn based on installed RAM and CPUs,
+ returning at least "mininum"
+ """
+ if cpuCount is None:
+ try:
+ cpuCount = getNCPU()
+ except NotImplementedError, e:
+ log.error("Unable to detect number of CPUs: %s" % (str(e),))
+ return minimum
+
+ if memSize is None:
+ try:
+ memSize = getMemorySize()
+ except NotImplementedError, e:
+ log.error("Unable to detect amount of installed RAM: %s" % (str(e),))
+ return minimum
+
+ countByCore = perCPU * cpuCount
+ countByMemory = perGB * (memSize / (1024 * 1024 * 1024))
+
+ # Pick the smaller of the two:
+ count = min(countByCore, countByMemory)
+
+ # ...but at least "minimum"
+ return max(count, minimum)
+
+
+
+
+
class FakeRequest(object):
def __init__(self, rootResource, method, path):
Modified: CalendarServer/branches/new-store/calendarserver/tools/gateway.py
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tools/gateway.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/calendarserver/tools/gateway.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -32,7 +32,7 @@
from twistedcaldav.directory.directory import DirectoryError
from twext.web2.dav import davxml
-from calendarserver.tools.util import loadConfig, getDirectory, setupMemcached, setupNotifications
+from calendarserver.tools.util import loadConfig, getDirectory, setupMemcached, setupNotifications, checkDirectory
from calendarserver.tools.principals import (
principalForPrincipalID, proxySubprincipal, addProxy, removeProxy,
ProxyError, ProxyWarning, updateRecord
@@ -95,6 +95,15 @@
try:
loadConfig(configFileName)
+ # Create the DataRoot directory before shedding privileges
+ if config.DataRoot.startswith(config.ServerRoot + os.sep):
+ checkDirectory(
+ config.DataRoot,
+ "Data root",
+ access=os.W_OK,
+ create=(0750, config.UserName, config.GroupName),
+ )
+
# Shed privileges
if config.UserName and config.GroupName and os.getuid() == 0:
uid = getpwnam(config.UserName).pw_uid
Modified: CalendarServer/branches/new-store/calendarserver/tools/principals.py
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tools/principals.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/calendarserver/tools/principals.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -37,7 +37,7 @@
from twistedcaldav.directory.directory import UnknownRecordTypeError, DirectoryError
from twistedcaldav.directory import augment
-from calendarserver.tools.util import loadConfig, getDirectory, setupMemcached, setupNotifications, booleanArgument
+from calendarserver.tools.util import loadConfig, getDirectory, setupMemcached, setupNotifications, booleanArgument, checkDirectory
__all__ = [
"principalForPrincipalID", "proxySubprincipal", "addProxy", "removeProxy",
@@ -217,6 +217,7 @@
# some logging activity at whatever log level the plist says
clearLogLevels()
+
config.DefaultLogLevel = "debug" if verbose else "error"
#
@@ -225,6 +226,14 @@
observer = StandardIOObserver()
observer.start()
+ # Create the DataRoot directory before shedding privileges
+ if config.DataRoot.startswith(config.ServerRoot + os.sep):
+ checkDirectory(
+ config.DataRoot,
+ "Data root",
+ access=os.W_OK,
+ create=(0750, config.UserName, config.GroupName),
+ )
# Shed privileges
if config.UserName and config.GroupName and os.getuid() == 0:
Modified: CalendarServer/branches/new-store/calendarserver/tools/test/test_purge.py
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tools/test/test_purge.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/calendarserver/tools/test/test_purge.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -99,6 +99,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"endless.ics": {
"@contents" : ENDLESS_ICS,
},
@@ -528,6 +532,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"noninvite.ics": {
"@contents" : NON_INVITE_ICS,
},
@@ -542,6 +550,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"*.ics/UID:7ED97931-9A19-4596-9D4D-52B36D6AB803": {
"@contents" : (
"METHOD:CANCEL",
@@ -558,6 +570,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"organizer.ics": {
"@contents" : (
"STATUS:CANCELLED",
@@ -657,6 +673,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"noninvite_past.ics": {
"@contents" : NON_INVITE_PAST_ICS,
},
@@ -671,6 +691,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"*.ics/UID:7ED97931-9A19-4596-9D4D-52B36D6AB803": {
"@contents" : (
"METHOD:CANCEL",
@@ -692,6 +716,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"organizer.ics": {
"@contents" : (
"STATUS:CANCELLED",
@@ -811,6 +839,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"noninvite.ics": { # event in the past
"@contents" : NON_INVITE_ICS_3,
},
@@ -846,6 +878,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"*.ics/UID:7ED97931-9A19-4596-9D4D-52B36D6AB803": {
"@contents" : (
"METHOD:CANCEL",
@@ -862,6 +898,10 @@
".db.sqlite": {
"@contents" : None, # ignore contents
},
+ ".db.sqlite-journal": {
+ "@optional" : None, # not in old sqlite
+ "@contents" : None, # ignore contents
+ },
"organizer.ics": {
# Purging non-existent organizer; has non-existent
# and existent attendees
Modified: CalendarServer/branches/new-store/calendarserver/tools/util.py
===================================================================
--- CalendarServer/branches/new-store/calendarserver/tools/util.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/calendarserver/tools/util.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -22,12 +22,16 @@
"booleanArgument",
]
-import os
+import os, sys
from time import sleep
import socket
+from pwd import getpwnam
+from grp import getgrnam
from twisted.python.reflect import namedClass
+from twext.python.log import Logger
+
from calendarserver.provision.root import RootResource
from twistedcaldav import memcachepool
from twistedcaldav.config import config, ConfigurationError
@@ -38,6 +42,7 @@
from twistedcaldav.static import CalendarHomeProvisioningFile
from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
+log = Logger()
def loadConfig(configFileName):
if configFileName is None:
@@ -96,11 +101,7 @@
# Load augment/proxy db classes now
augmentClass = namedClass(config.AugmentService.type)
- try:
- augment.AugmentService = augmentClass(**config.AugmentService.params)
- except IOError, e:
- # FIXME: Augments DB tries to write to disk, which seems annoying
- raise DirectoryError(e)
+ augment.AugmentService = augmentClass(**config.AugmentService.params)
proxydbClass = namedClass(config.ProxyDBService.type)
calendaruserproxy.ProxyDBService = proxydbClass(**config.ProxyDBService.params)
@@ -205,3 +206,48 @@
config.Notifications.InternalNotificationPort,
)
+def checkDirectory(dirpath, description, access=None, create=None):
+ if not os.path.exists(dirpath):
+ try:
+ mode, username, groupname = create
+ except TypeError:
+ raise ConfigurationError("%s does not exist: %s"
+ % (description, dirpath))
+ try:
+ os.mkdir(dirpath)
+ except (OSError, IOError), e:
+ log.error("Could not create %s: %s" % (dirpath, e))
+ raise ConfigurationError(
+ "%s does not exist and cannot be created: %s"
+ % (description, dirpath)
+ )
+
+ if username:
+ uid = getpwnam(username).pw_uid
+ else:
+ uid = -1
+
+ if groupname:
+ gid = getgrnam(groupname).gr_gid
+ else:
+ gid = -1
+
+ try:
+ os.chmod(dirpath, mode)
+ os.chown(dirpath, uid, gid)
+ except (OSError, IOError), e:
+ log.error("Unable to change mode/owner of %s: %s"
+ % (dirpath, e))
+
+ log.info("Created directory: %s" % (dirpath,))
+
+ if not os.path.isdir(dirpath):
+ raise ConfigurationError("%s is not a directory: %s"
+ % (description, dirpath))
+
+ if access and not os.access(dirpath, access):
+ raise ConfigurationError(
+ "Insufficient permissions for server on %s directory: %s"
+ % (description, dirpath)
+ )
+
Modified: CalendarServer/branches/new-store/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/branches/new-store/conf/caldavd-apple.plist 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/conf/caldavd-apple.plist 2010-06-10 20:18:42 UTC (rev 5719)
@@ -34,6 +34,12 @@
<key>ServerHostName</key>
<string></string> <!-- The hostname clients use when connecting -->
+ <key>EnableCalDAV</key>
+ <true/>
+
+ <key>EnableCardDAV</key>
+ <false/>
+
<!-- HTTP port [0 = disable HTTP] -->
<key>HTTPPort</key>
<integer>8008</integer>
Modified: CalendarServer/branches/new-store/contrib/tools/request_monitor.py
===================================================================
--- CalendarServer/branches/new-store/contrib/tools/request_monitor.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/contrib/tools/request_monitor.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -42,16 +42,16 @@
ssl = int(line.split("/")[0])
elif line.find("8008") != -1:
nonssl = int(line.split("/")[0])
- return "%s+%s" % (ssl, nonssl)
+ return "%s+%s" % (ssl, nonssl), ssl, nonssl
_listenQueueHistory = []
def listenQueueHistory():
global _listenQueueHistory
- latest = listenq()
+ latest, ssl, nonssl = listenq()
_listenQueueHistory.insert(0, latest)
del _listenQueueHistory[12:]
- return _listenQueueHistory
+ return _listenQueueHistory, ssl, nonssl
_idleHistory = []
@@ -149,16 +149,20 @@
def usage():
print "-h print help and exit"
- print "--lines N specifies how many lines to tail from access.log"
+ print "--lines N specifies how many lines to tail from access.log (default: 10000)"
+ print "--procs N specifies how many python processes are expected in the log file (default: 80)"
numLines = 10000
-options, args = getopt.getopt(sys.argv[1:], "h", ["lines=",])
+numProcs = 80
+options, args = getopt.getopt(sys.argv[1:], "h", ["lines=", "procs=",])
for option, value in options:
if option == "-h":
usage()
sys.exit(0)
elif option == "--lines":
numLines = int(value)
+ elif option == "--procs":
+ numProcs = int(value)
while True:
@@ -276,17 +280,18 @@
if len(samples) < 3:
+ avgRequests = 0
avg = ""
else:
samples = samples[1:-1]
total = 0
for sample in samples:
total += sample
- avg = float(total) / len(samples)
- avg = "%.1f average requests per second" % (avg,)
+ avgRequests = float(total) / len(samples)
+ avg = "%.1f average requests per second" % (avgRequests,)
print "- " * 40
- q = listenQueueHistory()
+ q, lqssl, lqnon = listenQueueHistory()
print datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"),
print "Listenq (ssl+non):", q[0], " (Recent", ", ".join(q[1:]), "Oldest)"
q = idleHistory()
@@ -295,7 +300,14 @@
if avg:
print avg, "|",
print "%d requests between %s and %s" % (numLines, startTime.strftime("%H:%M:%S"), endTime.strftime("%H:%M:%S"))
- print "Response time: average %.1f ms, max %.1f ms" % (totalRespTime / numRequests, maxRespTime)
+
+ lqlatency = (lqssl / avgRequests, lqnon / avgRequests,) if avgRequests else (0.0, 0.0,)
+ print "Response time: average %.1f ms, max %.1f ms, listenq latency (ssl+non): %.1f s %.1f s" % (
+ totalRespTime / numRequests,
+ maxRespTime,
+ lqlatency[0],
+ lqlatency[1],
+ )
print "<10ms: %d >10ms: %d >100ms: %d >1s: %d >10s: %d >30s: %d >60s: %d" % (under10ms, over10ms, over100ms, over1s, over10s, over30s, over60s)
print
if errorCount:
@@ -303,7 +315,7 @@
print
print "Proc: Peak outstanding: Seconds of processing (number of requests):"
- for l in xrange(8):
+ for l in xrange(numProcs/10 + 1):
base = l * 10
print "%2d-%2d: " % (base, base+9),
Modified: CalendarServer/branches/new-store/twext/web2/channel/http.py
===================================================================
--- CalendarServer/branches/new-store/twext/web2/channel/http.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twext/web2/channel/http.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -1124,7 +1124,7 @@
if self.logData is not None:
doneTime = time.time()
self.logData.response.append("\r\n\r\n<<<< Response complete at: %.3f (elapsed: %.1f ms)\r\n" % (doneTime, 1000 * (doneTime - self.startTime),))
- accounting.emitAccounting("HTTP", "", "".join(self.logData.request) + "".join(self.logData.response))
+ accounting.emitAccounting("HTTP", "", "".join(self.logData.request) + "".join(self.logData.response), self.command)
HTTPChannel.chanRequestFactory = HTTPLoggingChannelRequest
Modified: CalendarServer/branches/new-store/twext/web2/dav/test/test_xattrprops.py
===================================================================
--- CalendarServer/branches/new-store/twext/web2/dav/test/test_xattrprops.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twext/web2/dav/test/test_xattrprops.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -71,7 +71,26 @@
# of the EPERM failure.
self.assertEquals(error.response.code, FORBIDDEN)
+ def _missingTest(self, method):
+ # Remove access to the directory containing the file so that getting
+ # extended attributes from it fails with EPERM.
+ self.resourcePath.parent().chmod(0)
+ # Make sure to restore access to it later so that it can be deleted
+ # after the test run is finished.
+ self.addCleanup(self.resourcePath.parent().chmod, 0700)
+ # Try to get a property from it - and fail.
+ document = self._makeValue()
+ error = self.assertRaises(
+ HTTPError,
+ getattr(self.propertyStore, method),
+ document.root_element.qname())
+
+ # Make sure that the status is FORBIDDEN, a roughly reasonable mapping
+ # of the EPERM failure.
+ self.assertEquals(error.response.code, FORBIDDEN)
+
+
def test_getErrors(self):
"""
If there is a problem getting the specified property (aside from the
@@ -80,7 +99,25 @@
"""
self._forbiddenTest('get')
+ def test_getMissing(self):
+ """
+ Test missing file.
+ """
+ resourcePath = FilePath(self.mktemp())
+ resource = DAVFile(resourcePath.path)
+ propertyStore = xattrPropertyStore(resource)
+
+ # Try to get a property from it - and fail.
+ document = self._makeValue()
+ error = self.assertRaises(
+ HTTPError,
+ propertyStore.get,
+ document.root_element.qname())
+
+ # Make sure that the status is NOT FOUND.
+ self.assertEquals(error.response.code, NOT_FOUND)
+
def _makeValue(self, uid=None):
"""
Create and return any old WebDAVDocument for use by the get tests.
@@ -274,6 +311,19 @@
self._forbiddenTest('contains')
+ def test_containsMissing(self):
+ """
+ Test missing file.
+ """
+
+ resourcePath = FilePath(self.mktemp())
+ resource = DAVFile(resourcePath.path)
+ propertyStore = xattrPropertyStore(resource)
+
+ # Try to get a property from it - and fail.
+ document = self._makeValue()
+ self.assertFalse(propertyStore.contains(document.root_element.qname()))
+
def test_list(self):
"""
L{xattrPropertyStore.list} returns a C{list} of property names
@@ -309,6 +359,18 @@
# of the EPERM failure.
self.assertEquals(error.response.code, FORBIDDEN)
+ def test_listMissing(self):
+ """
+ Test missing file.
+ """
+
+ resourcePath = FilePath(self.mktemp())
+ resource = DAVFile(resourcePath.path)
+ propertyStore = xattrPropertyStore(resource)
+
+ # Try to get a property from it - and fail.
+ self.assertEqual(propertyStore.list(), [])
+
def test_get_uids(self):
"""
L{xattrPropertyStore.get} accepts a L{WebDAVElement} and stores a
Modified: CalendarServer/branches/new-store/twext/web2/dav/xattrprops.py
===================================================================
--- CalendarServer/branches/new-store/twext/web2/dav/xattrprops.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twext/web2/dav/xattrprops.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -34,6 +34,7 @@
__all__ = ["xattrPropertyStore"]
import urllib
+import os
import sys
import zlib
import errno
@@ -132,6 +133,7 @@
@return: A L{WebDAVDocument} representing the value associated with the
given property.
"""
+
try:
data = self.attrs.get(self._encode(qname, uid))
except KeyError:
@@ -139,7 +141,7 @@
responsecode.NOT_FOUND,
"No such property: {%s}%s" % qname))
except IOError, e:
- if e.errno in _ATTR_MISSING:
+ if e.errno in _ATTR_MISSING or e.errno == errno.ENOENT:
raise HTTPError(StatusResponse(
responsecode.NOT_FOUND,
"No such property: {%s}%s" % qname))
@@ -241,6 +243,7 @@
@return: C{True} if the property exists, C{False} otherwise.
"""
+
key = self._encode(qname, uid)
try:
self.attrs.get(key)
@@ -267,10 +270,13 @@
@return: A C{list} of property names as two-tuples of namespace URI and
local name.
"""
+
prefix = self.deadPropertyXattrPrefix
try:
attrs = iter(self.attrs)
- except IOError:
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ return []
raise HTTPError(
StatusResponse(
statusForFailure(Failure()),
Modified: CalendarServer/branches/new-store/twistedcaldav/accounting.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/accounting.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/accounting.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -69,7 +69,7 @@
return False
-def emitAccounting(category, principal, data):
+def emitAccounting(category, principal, data, tag=None):
"""
Write the supplied data to the appropriate location for the given
category and principal.
@@ -111,10 +111,16 @@
if not os.path.isdir(os.path.join(logRoot, logDirectory)):
os.makedirs(os.path.join(logRoot, logDirectory))
logFilename = "%s-01" % (logFilename,)
+ if tag:
+ logFilename += " (%s)" % (tag,)
+ logFilename += ".txt"
else:
index = 1
while True:
path = "%s-%02d" % (logFilename, index)
+ if tag:
+ path += " (%s)" % (tag,)
+ path += ".txt"
if not os.path.isfile(os.path.join(logRoot, path)):
logFilename = path
break
Modified: CalendarServer/branches/new-store/twistedcaldav/directory/appleopendirectory.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/directory/appleopendirectory.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/directory/appleopendirectory.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -24,6 +24,8 @@
]
import sys
+import time
+from uuid import UUID
import opendirectory
import dsattributes
@@ -55,6 +57,11 @@
"""
@param params: a dictionary containing the following keys:
node: an OpenDirectory node name to bind to.
+ restrictEnabledRecords: C{True} if a group in the
+ directory is to be used to determine which calendar
+ users are enabled.
+ restrictToGroup: C{str} guid or name of group used to
+ restrict enabled users.
cacheTimeout: C{int} number of minutes before cache is invalidated.
@param dosetup: if C{True} then the directory records are initialized,
if C{False} they are not.
@@ -63,17 +70,15 @@
defaults = {
'node' : '/Search',
+ 'restrictEnabledRecords' : False,
+ 'restrictToGroup' : '',
'cacheTimeout' : 30,
'recordTypes' : (
self.recordType_users,
self.recordType_groups,
),
}
- ignored = (
- 'requireComputerRecord',
- 'restrictEnabledRecords',
- 'restrictToGroup'
- )
+ ignored = ('requireComputerRecord',)
params = self.getParams(params, defaults, ignored)
self._recordTypes = params['recordTypes']
@@ -89,9 +94,63 @@
self.realmName = params['node']
self.directory = directory
self.node = params['node']
+ self.restrictEnabledRecords = params['restrictEnabledRecords']
+ self.restrictToGroup = params['restrictToGroup']
+ try:
+ UUID(self.restrictToGroup)
+ except:
+ self.restrictToGUID = False
+ else:
+ self.restrictToGUID = True
+ self.restrictedTimestamp = 0
self._records = {}
self._delayedCalls = set()
+ @property
+ def restrictedGUIDs(self):
+ """
+ Look up (and cache) the set of guids that are members of the
+ restrictToGroup. If restrictToGroup is not set, return None to
+ indicate there are no group restrictions.
+ """
+ if self.restrictEnabledRecords:
+ if time.time() - self.restrictedTimestamp > self.cacheTimeout:
+ attributeToMatch = dsattributes.kDS1AttrGeneratedUID if self.restrictToGUID else dsattributes.kDSNAttrRecordName
+ valueToMatch = self.restrictToGroup
+ self.log_debug("Doing restricted group membership check")
+ self.log_debug("opendirectory.queryRecordsWithAttribute_list(%r,%r,%r,%r,%r,%r,%r)" % (
+ self.directory,
+ attributeToMatch,
+ valueToMatch,
+ dsattributes.eDSExact,
+ False,
+ dsattributes.kDSStdRecordTypeGroups,
+ [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups,],
+ ))
+ results = opendirectory.queryRecordsWithAttribute_list(
+ self.directory,
+ attributeToMatch,
+ valueToMatch,
+ dsattributes.eDSExact,
+ False,
+ dsattributes.kDSStdRecordTypeGroups,
+ [dsattributes.kDSNAttrGroupMembers, dsattributes.kDSNAttrNestedGroups,],
+ )
+
+ if len(results) == 1:
+ members = results[0][1].get(dsattributes.kDSNAttrGroupMembers, [])
+ nestedGroups = results[0][1].get(dsattributes.kDSNAttrNestedGroups, [])
+ else:
+ members = []
+ nestedGroups = []
+ self._cachedRestrictedGUIDs = set(self._expandGroupMembership(members, nestedGroups, returnGroups=True))
+ self.log_debug("Got %d restricted group members" % (len(self._cachedRestrictedGUIDs),))
+ self.restrictedTimestamp = time.time()
+ return self._cachedRestrictedGUIDs
+ else:
+ # No restrictions
+ return None
+
def __cmp__(self, other):
if not isinstance(other, DirectoryRecord):
return super(DirectoryRecord, self).__eq__(other)
@@ -353,6 +412,13 @@
continue
recordGUID = value.get(dsattributes.kDS1AttrGeneratedUID)
+
+ # Skip if group restriction is in place and guid is not
+ # a member
+ if self.restrictedGUIDs is not None:
+ if str(recordGUID) not in self.restrictedGUIDs:
+ continue
+
recordType = value.get(dsattributes.kDSNAttrRecordType)
if isinstance(recordType, list):
recordType = recordType[0]
@@ -470,6 +536,7 @@
return deferred
+
def queryDirectory(self, recordTypes, indexType, indexKey,
lookupMethod=opendirectory.queryRecordsWithAttribute_list):
@@ -591,6 +658,17 @@
% (recordType, recordShortName, recordNodeName))
continue
+ # If restrictToGroup is in effect, all guids which are not a member
+ # of that group are disabled (overriding the augments db).
+ if (
+ self.restrictedGUIDs is not None and
+ config.Scheduling.iMIP.Username != recordShortName
+ ):
+ unrestricted = recordGUID in self.restrictedGUIDs
+ else:
+ unrestricted = True
+
+
# Special case for groups, which have members.
if recordType == self.recordType_groups:
memberGUIDs = value.get(dsattributes.kDSNAttrGroupMembers)
@@ -626,6 +704,11 @@
d = augment.AugmentService.getAugmentRecord(record.guid)
d.addCallback(lambda x:record.addAugmentInformation(x))
+ if not unrestricted:
+ self.log_debug("%s is not enabled because it's not a member of group: %s" % (recordGUID, self.restrictToGroup))
+ record.enabledForCalendaring = False
+ record.enabledForAddressBooks = False
+
if record.enabledForCalendaring:
enabledRecords.append(record)
else:
Modified: CalendarServer/branches/new-store/twistedcaldav/directory/augment.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/directory/augment.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/directory/augment.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -170,61 +170,23 @@
"""
XMLFile based augment database implementation.
"""
-
+
def __init__(self, xmlFiles, cacheTimeout=30):
-
+
self.xmlFiles = [fullServerPath(config.DataRoot, path) for path in xmlFiles]
self.cacheTimeout = cacheTimeout * 60 # Value is mins we want secs
self.lastCached = 0
self.db = {}
-
- # Preflight existence of files
- missing = list()
- for xmlFile in self.xmlFiles:
- if not os.path.exists(xmlFile):
- missing.append(xmlFile)
-
- # For each missing one create an empty xml file
- if missing:
- # If all files are missing, then create one augment file that defaults
- # to all records being enabled
- doDefault = (len(missing) == len(self.xmlFiles))
- for missedFile in missing:
-
- _ignore_etree, root = newElementTreeWithRoot(xmlaugmentsparser.ELEMENT_AUGMENTS)
- if doDefault:
- record = addSubElement(root, xmlaugmentsparser.ELEMENT_RECORD)
- addSubElement(record, xmlaugmentsparser.ELEMENT_UID, "Default")
- addSubElement(record, xmlaugmentsparser.ELEMENT_ENABLE, "true")
- addSubElement(record, xmlaugmentsparser.ELEMENT_ENABLECALENDAR, "true")
- addSubElement(record, xmlaugmentsparser.ELEMENT_ENABLEADDRESSBOOK, "true")
- doDefault = False
- writeXML(missedFile, root)
- # Set permissions
- uid = -1
- if config.UserName:
- try:
- uid = pwd.getpwnam(config.UserName).pw_uid
- except KeyError:
- log.error("User not found: %s" % (config.UserName,))
- gid = -1
- if config.GroupName:
- try:
- gid = grp.getgrnam(config.GroupName).gr_gid
- except KeyError:
- log.error("Group not found: %s" % (config.GroupName,))
- if uid != -1 and gid != -1:
- os.chown(missedFile, uid, gid)
-
try:
self.db = self._parseXML()
except RuntimeError:
log.error("Failed to parse XML augments file - fatal error on startup")
raise
-
+
self.lastCached = time.time()
+
@inlineCallbacks
def getAllUIDs(self):
"""
@@ -284,9 +246,44 @@
return succeed(None)
def _doAddToFile(self, xmlfile, records):
-
+
+ if not os.path.exists(xmlfile):
+
+ # File doesn't yet exist. Create it with items in self.db, and
+ # set file permissions.
+
+ _ignore_etree, augments_node = newElementTreeWithRoot(xmlaugmentsparser.ELEMENT_AUGMENTS)
+ for record in self.db.itervalues():
+ record_node = addSubElement(augments_node, xmlaugmentsparser.ELEMENT_RECORD)
+ addSubElement(record_node, xmlaugmentsparser.ELEMENT_UID, record.uid)
+ addSubElement(record_node, xmlaugmentsparser.ELEMENT_ENABLE, "true" if record.enabled else "false")
+ addSubElement(record_node, xmlaugmentsparser.ELEMENT_HOSTEDAT, record.hostedAt)
+ addSubElement(record_node, xmlaugmentsparser.ELEMENT_ENABLECALENDAR, "true" if record.enabledForCalendaring else "false")
+ addSubElement(record_node, xmlaugmentsparser.ELEMENT_ENABLEADDRESSBOOK, "true" if record.enabledForAddressBooks else "false")
+ addSubElement(record_node, xmlaugmentsparser.ELEMENT_AUTOSCHEDULE, "true" if record.autoSchedule else "false")
+
+
+ writeXML(xmlfile, augments_node)
+
+ # Set permissions
+ uid = -1
+ if config.UserName:
+ try:
+ uid = pwd.getpwnam(config.UserName).pw_uid
+ except KeyError:
+ log.error("User not found: %s" % (config.UserName,))
+ gid = -1
+ if config.GroupName:
+ try:
+ gid = grp.getgrnam(config.GroupName).gr_gid
+ except KeyError:
+ log.error("Group not found: %s" % (config.GroupName,))
+ if uid != -1 and gid != -1:
+ os.chown(xmlfile, uid, gid)
+
+
_ignore_etree, augments_node = readXML(xmlfile)
-
+
# Create new record
for record in records:
record_node = addSubElement(augments_node, xmlaugmentsparser.ELEMENT_RECORD)
@@ -302,6 +299,9 @@
def _doModifyInFile(self, xmlfile, records):
+ if not os.path.exists(xmlfile):
+ return
+
_ignore_etree, augments_node = readXML(xmlfile)
# Map uid->record for fast lookup
@@ -393,15 +393,31 @@
self.removeAugmentRecords(self.db.keys())
return succeed(None)
-
+
def _parseXML(self):
-
+ """
+ Parse self.xmlFiles into AugmentRecords.
+
+ If none of the xmlFiles exist, create a default record.
+ """
+
# Do each file
results = {}
+
+ allMissing = True
for xmlFile in self.xmlFiles:
-
- # Creating a parser does the parse
- XMLAugmentsParser(xmlFile, results)
+ if os.path.exists(xmlFile):
+ # Creating a parser does the parse
+ XMLAugmentsParser(xmlFile, results)
+ allMissing = False
+
+ if allMissing:
+ results["Default"] = AugmentRecord(
+ "Default",
+ enabled=True,
+ enabledForCalendaring=True,
+ enabledForAddressBooks=True,
+ )
return results
Modified: CalendarServer/branches/new-store/twistedcaldav/method/__init__.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/method/__init__.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/method/__init__.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -22,6 +22,7 @@
"""
__all__ = [
+ "acl",
"copymove",
"delete",
"get",
Copied: CalendarServer/branches/new-store/twistedcaldav/method/acl.py (from rev 5718, CalendarServer/trunk/twistedcaldav/method/acl.py)
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/method/acl.py (rev 0)
+++ CalendarServer/branches/new-store/twistedcaldav/method/acl.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -0,0 +1,55 @@
+##
+# Copyright (c) 2006-2009 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+"""
+CalDAV ACL method.
+"""
+
+__all__ = ["http_ACL"]
+
+
+from twext.python.log import Logger
+from twext.web2 import responsecode
+from twext.web2.dav.util import parentForURL
+from twext.web2.http import HTTPError
+
+from twisted.internet.defer import inlineCallbacks, returnValue
+
+from twistedcaldav.resource import isAddressBookCollectionResource,\
+ isPseudoCalendarCollectionResource
+from twistedcaldav.static import AddressBookHomeFile, CalendarHomeFile
+
+log = Logger()
+
+ at inlineCallbacks
+def http_ACL(self, request):
+ #
+ # Override base ACL request handling to ensure that the calendar/address book
+ # homes cannot have ACL's set, and calendar/address object resources too.
+ #
+
+ if self.fp.exists():
+ if isinstance(self, CalendarHomeFile) or isinstance(self, AddressBookHomeFile):
+ raise HTTPError(responsecode.NOT_ALLOWED)
+
+ parentURL = parentForURL(request.uri)
+ parent = (yield request.locateResource(parentURL))
+ if isPseudoCalendarCollectionResource(parent) or isAddressBookCollectionResource(parent):
+ raise HTTPError(responsecode.NOT_ALLOWED)
+
+ # Do normal ACL behavior
+ response = (yield super(CalDAVFile, self).http_ACL(request))
+ returnValue(response)
Modified: CalendarServer/branches/new-store/twistedcaldav/schedule.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/schedule.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/schedule.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -29,6 +29,7 @@
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
from twext.web2 import responsecode
from twext.web2.dav import davxml
+from twext.web2.dav.element.extensions import SyncCollection
from twext.web2.dav.util import joinURL, normalizeURL
from twext.web2.http import HTTPError
from twext.web2.http import Response
@@ -73,6 +74,9 @@
result.append(davxml.Report(caldavxml.CalendarQuery(),))
result.append(davxml.Report(caldavxml.CalendarMultiGet(),))
# free-busy report not allowed
+ if config.EnableSyncReport:
+ # Only allowed on calendar/inbox/addressbook collections
+ result.append(davxml.Report(SyncCollection(),))
return result
class ScheduleInboxResource (CalendarSchedulingCollectionResource):
Modified: CalendarServer/branches/new-store/twistedcaldav/scheduling/scheduler.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/scheduling/scheduler.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/scheduling/scheduler.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -188,9 +188,12 @@
def loadRecipientsFromCalendarData(self):
# Get the ATTENDEEs
- attendees = set()
+ attendees = list()
+ unique_set = set()
for attendee, _ignore in self.calendar.getAttendeesByInstance():
- attendees.add(attendee)
+ if attendee not in unique_set:
+ attendees.append(attendee)
+ unique_set.add(attendee)
if not attendees:
log.err("%s request must have at least one Recipient" % (self.method,))
@@ -391,8 +394,14 @@
partitioned_recipients = []
remote_recipients = []
imip_recipients = []
- for recipient in self.recipients:
+ for ctr, recipient in enumerate(self.recipients):
+ # Check for freebusy limit
+ if freebusy and ctr >= config.Scheduling.Options.LimitFreeBusyAttendees:
+ err = HTTPError(ErrorResponse(responsecode.NOT_FOUND, (caldav_namespace, "recipient-limit")))
+ responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.NO_USER_SUPPORT)
+ continue
+
if self.fakeTheResult:
responses.add(recipient.cuaddr, responsecode.OK, reqstatus=iTIPRequestStatus.SUCCESS if freebusy else iTIPRequestStatus.MESSAGE_DELIVERED)
@@ -411,7 +420,7 @@
else:
err = HTTPError(ErrorResponse(responsecode.NOT_FOUND, (caldav_namespace, "recipient-exists")))
responses.add(recipient.cuaddr, Failure(exc_value=err), reqstatus=iTIPRequestStatus.INVALID_CALENDAR_USER)
-
+
# Now process local recipients
if caldav_recipients:
yield self.generateLocalSchedulingResponses(caldav_recipients, responses, freebusy)
Modified: CalendarServer/branches/new-store/twistedcaldav/sql.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/sql.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/sql.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -88,13 +88,20 @@
self._db_connection = sqlite.connect(db_filename, isolation_level=None)
else:
self._db_connection = sqlite.connect(db_filename)
+
except DatabaseError:
raise DatabaseError("Unable to open database %s" % (self.dbpath,))
+ q = self._db_connection.cursor()
+
#
+ # Set Journal mode to PERSIST to avoid constant unlink calls
+ #
+ q.execute("PRAGMA journal_mode = PERSIST")
+
+ #
# Set up the schema
#
- q = self._db_connection.cursor()
try:
# Create CALDAV table if needed
Modified: CalendarServer/branches/new-store/twistedcaldav/static.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/static.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/static.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -301,13 +301,46 @@
@inlineCallbacks
def iCalendarRolledup(self, request):
if self.isPseudoCalendarCollection():
+
+ # Determine the cache key
+ isvirt = (yield self.isVirtualShare(request))
+ if isvirt:
+ principal = (yield self.resourceOwnerPrincipal(request))
+ if principal:
+ cacheKey = principal.principalUID()
+ else:
+ cacheKey = "unknown"
+ else:
+ isowner = (yield self.isOwner(request, adminprincipals=True, readprincipals=True))
+ cacheKey = "owner" if isowner else "notowner"
+
+ # Now check for a cached .ics
+ rolled = self.fp.child(".subscriptions")
+ if not rolled.exists():
+ try:
+ rolled.makedirs()
+ except IOError, e:
+ log.err("Unable to create internet calendar subscription cache directory: %s because of: %s" % (rolled.path, e,))
+ raise HTTPError(ErrorResponse(responsecode.INTERNAL_SERVER_ERROR))
+ cached = rolled.child(cacheKey)
+ if cached.exists():
+ try:
+ cachedData = cached.open().read()
+ except IOError, e:
+ log.err("Unable to open or read internet calendar subscription cache file: %s because of: %s" % (cached.path, e,))
+ else:
+ # Check the cache token
+ token, data = cachedData.split("\r\n", 1)
+ if token == self.getSyncToken():
+ returnValue(data)
+
# Generate a monolithic calendar
calendar = iComponent("VCALENDAR")
calendar.addProperty(iProperty("VERSION", "2.0"))
# Do some optimisation of access control calculation by determining any inherited ACLs outside of
# the child resource loop and supply those to the checkPrivileges on each child.
- filteredaces = yield self.inheritedACEsforChildren(request)
+ filteredaces = (yield self.inheritedACEsforChildren(request))
tzids = set()
isowner = (yield self.isOwner(request, adminprincipals=True, readprincipals=True))
@@ -346,6 +379,14 @@
calendar.addComponent(component)
+ # Cache the data
+ data = str(calendar)
+ data = self.getSyncToken() + "\r\n" + data
+ try:
+ cached.open(mode='w').write(data)
+ except IOError, e:
+ log.err("Unable to open or write internet calendar subscription cache file: %s because of: %s" % (cached.path, e,))
+
returnValue(calendar)
raise HTTPError(ErrorResponse(responsecode.BAD_REQUEST))
@@ -1890,6 +1931,7 @@
setattr(DropBoxChildFile, "http_MKCALENDAR", None)
# FIXME: Little bit of a circular dependency here...
+twistedcaldav.method.acl.CalDAVFile = CalDAVFile
twistedcaldav.method.copymove.CalDAVFile = CalDAVFile
twistedcaldav.method.delete.CalDAVFile = CalDAVFile
twistedcaldav.method.get.CalDAVFile = CalDAVFile
Modified: CalendarServer/branches/new-store/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/stdconfig.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/stdconfig.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -48,6 +48,8 @@
"twistedcaldav.directory.appleopendirectory.OpenDirectoryService": {
"node": "/Search",
"cacheTimeout": 30,
+ "restrictEnabledRecords": False,
+ "restrictToGroup": "",
"recordTypes": ("users", "groups"),
},
}
@@ -302,7 +304,9 @@
"ProcessType": "Combined",
"MultiProcess": {
"ProcessCount": 0,
- "MinProcessCount": 4,
+ "MinProcessCount": 2,
+ "PerCPU": 1,
+ "PerGB": 2,
"StaggeredStartup": {
"Enabled": False,
"Interval": 15,
@@ -418,6 +422,7 @@
"AllowGroupAsOrganizer" : False, # Allow groups to be Organizers
"AllowLocationAsOrganizer" : False, # Allow locations to be Organizers
"AllowResourceAsOrganizer" : False, # Allow resources to be Organizers
+ "LimitFreeBusyAttendees" : 30, # Maximum number of attendees to request freebusy for
}
},
Modified: CalendarServer/branches/new-store/twistedcaldav/test/util.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/test/util.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/test/util.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -203,7 +203,8 @@
childPath = os.path.join(parent, childName)
- if not os.path.exists(childPath):
+ if (not os.path.exists(childPath) and
+ not childStructure.has_key("@optional")):
print "Missing:", childPath
return False
Modified: CalendarServer/branches/new-store/twistedcaldav/util.py
===================================================================
--- CalendarServer/branches/new-store/twistedcaldav/util.py 2010-06-10 19:37:44 UTC (rev 5718)
+++ CalendarServer/branches/new-store/twistedcaldav/util.py 2010-06-10 20:18:42 UTC (rev 5719)
@@ -28,7 +28,7 @@
from twext.python.log import LoggingMixIn
##
-# getNCPU
+# System Resources (Memory size and processor count)
##
try:
@@ -42,6 +42,9 @@
libc = cdll.LoadLibrary(ctypes.util.find_library("libc"))
def getNCPU():
+ """
+ Returns the number of processors detected
+ """
ncpu = c_int(0)
size = c_size_t(sizeof(ncpu))
@@ -58,12 +61,36 @@
return int(ncpu.value)
+ def getMemorySize():
+ """
+ Returns the physical amount of RAM installed, in bytes
+ """
+ memsize = c_uint64(0)
+ size = c_size_t(sizeof(memsize))
+
+ libc.sysctlbyname.argtypes = [
+ c_char_p, c_void_p, c_void_p, c_void_p, c_ulong
+ ]
+ libc.sysctlbyname(
+ "hw.memsize",
+ c_voidp(addressof(memsize)),
+ c_voidp(addressof(size)),
+ None,
+ 0
+ )
+
+ return int(memsize.value)
+
+
elif sys.platform == "linux2" and hasCtypes:
libc = cdll.LoadLibrary(ctypes.util.find_library("libc"))
def getNCPU():
return libc.get_nprocs()
+ def getMemorySize():
+ return libc.getpagesize() * libc.get_phys_pages()
+
else:
def getNCPU():
if not hasCtypes:
@@ -73,6 +100,9 @@
raise NotImplementedError("getNCPU not supported on %s%s" % (sys.platform, msg))
+ def getMemorySize():
+ raise NotImplementedError("getMemorySize not yet supported on %s" % (sys.platform))
+
##
# Module management
##
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100610/10940c54/attachment-0001.html>
More information about the calendarserver-changes
mailing list