[CalendarServer-changes] [5699] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Tue Jun 8 11:23:42 PDT 2010
Revision: 5699
http://trac.macosforge.org/projects/calendarserver/changeset/5699
Author: sagen at apple.com
Date: 2010-06-08 11:23:38 -0700 (Tue, 08 Jun 2010)
Log Message:
-----------
The number of processes spawned now takes into account both cpu count and amount of RAM installed.
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/twistedcaldav/util.py
Added Paths:
-----------
CalendarServer/trunk/calendarserver/tap/test/test_util.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2010-06-08 18:23:38 UTC (rev 5699)
@@ -77,7 +77,6 @@
from twistedcaldav.static import TimezoneServiceFile
from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twistedcaldav.upgrade import upgradeData
-from twistedcaldav.util import getNCPU
from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
@@ -93,7 +92,7 @@
from calendarserver.provision.root import RootResource
from calendarserver.webadmin.resource import WebAdminResource
from calendarserver.webcal.resource import WebCalendarResource
-from calendarserver.tap.util import getRootResource
+from calendarserver.tap.util import getRootResource, computeProcessCount
log = Logger()
@@ -775,30 +774,16 @@
parentEnv["KRB5_KTNAME"] = os.environ["KRB5_KTNAME"]
#
- # Attempt to calculate the number of processes to use 1 per processor
+ # Calculate the number of processes to spawn
#
if config.MultiProcess.ProcessCount == 0:
- try:
- cpuCount = getNCPU()
- except NotImplementedError, e:
- self.log_error("Unable to detect number of CPUs: %s"
- % (str(e),))
- cpuCount = 0
- else:
- if cpuCount < 1:
- self.log_error(
- "%d processors detected, which is hard to believe."
- % (cpuCount,)
- )
-
- processCount = config.MultiProcess.MinProcessCount
- if 2 * cpuCount > processCount:
- processCount = 2 * cpuCount
-
- self.log_info("%d processors found. Configuring %d processes."
- % (cpuCount, processCount))
-
+ processCount = computeProcessCount(
+ config.MultiProcess.MinProcessCount,
+ config.MultiProcess.PerCPU,
+ config.MultiProcess.PerGB,
+ )
config.MultiProcess.ProcessCount = processCount
+ self.log_info("Configuring %d processes." % (processCount,))
# Open the socket(s) to be inherited by the slaves
Added: CalendarServer/trunk/calendarserver/tap/test/test_util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_util.py (rev 0)
+++ CalendarServer/trunk/calendarserver/tap/test/test_util.py 2010-06-08 18:23:38 UTC (rev 5699)
@@ -0,0 +1,49 @@
+##
+# Copyright (c) 2007-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from calendarserver.tap.util import computeProcessCount
+from twistedcaldav.test.util import TestCase
+
+class ProcessCountTestCase(TestCase):
+
+ def test_count(self):
+
+ data = (
+ # minimum, perCPU, perGB, cpu, memory (in GB), expected:
+ (0, 1, 1, 0, 0, 0),
+ (1, 2, 2, 0, 0, 1),
+ (1, 2, 1, 1, 1, 1),
+ (1, 2, 2, 1, 1, 2),
+ (1, 2, 2, 2, 1, 2),
+ (1, 2, 2, 1, 2, 2),
+ (1, 2, 2, 2, 2, 4),
+ (4, 1, 2, 8, 2, 4),
+ (6, 2, 2, 2, 2, 6),
+ (1, 2, 1, 4, 99, 8),
+
+ (2, 1, 2, 2, 2, 2), # 2 cores, 2GB = 2
+ (2, 1, 2, 2, 4, 2), # 2 cores, 4GB = 2
+ (2, 1, 2, 8, 6, 8), # 8 cores, 6GB = 8
+ (2, 1, 2, 8, 16, 8), # 8 cores, 16GB = 8
+ )
+
+ for min, perCPU, perGB, cpu, mem, expected in data:
+ mem *= (1024 * 1024 * 1024)
+
+ self.assertEquals(
+ expected,
+ computeProcessCount(min, perCPU, perGB, cpuCount=cpu, memSize=mem)
+ )
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2010-06-08 18:23:38 UTC (rev 5699)
@@ -53,6 +53,7 @@
from twistedcaldav.static import TimezoneServiceFile
from twistedcaldav.static import AddressBookHomeProvisioningFile, DirectoryBackedAddressBookFile
from twistedcaldav.timezones import TimezoneCache
+from twistedcaldav.util import getMemorySize, getNCPU
from twisted.internet.defer import inlineCallbacks, returnValue
try:
@@ -463,7 +464,39 @@
+def computeProcessCount(minimum, perCPU, perGB, cpuCount=None, memSize=None):
+ """
+ Determine how many process to spawn based on installed RAM and CPUs,
+ returning at least "mininum"
+ """
+ if cpuCount is None:
+ try:
+ cpuCount = getNCPU()
+ except NotImplementedError, e:
+ log.error("Unable to detect number of CPUs: %s" % (str(e),))
+ return minimum
+
+ if memSize is None:
+ try:
+ memSize = getMemorySize()
+ except NotImplementedError, e:
+ log.error("Unable to detect amount of installed RAM: %s" % (str(e),))
+ return minimum
+
+ countByCore = perCPU * cpuCount
+ countByMemory = perGB * (memSize / (1024 * 1024 * 1024))
+
+ # Pick the smaller of the two:
+ count = min(countByCore, countByMemory)
+
+ # ...but at least "minimum"
+ return max(count, minimum)
+
+
+
+
+
class FakeRequest(object):
def __init__(self, rootResource, method, path):
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2010-06-08 18:23:38 UTC (rev 5699)
@@ -304,7 +304,9 @@
"ProcessType": "Combined",
"MultiProcess": {
"ProcessCount": 0,
- "MinProcessCount": 4,
+ "MinProcessCount": 2,
+ "PerCPU": 1,
+ "PerGB": 2,
"StaggeredStartup": {
"Enabled": False,
"Interval": 15,
Modified: CalendarServer/trunk/twistedcaldav/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/util.py 2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/twistedcaldav/util.py 2010-06-08 18:23:38 UTC (rev 5699)
@@ -28,7 +28,7 @@
from twext.python.log import LoggingMixIn
##
-# getNCPU
+# System Resources (Memory size and processor count)
##
try:
@@ -42,6 +42,9 @@
libc = cdll.LoadLibrary(ctypes.util.find_library("libc"))
def getNCPU():
+ """
+ Returns the number of processors detected
+ """
ncpu = c_int(0)
size = c_size_t(sizeof(ncpu))
@@ -58,12 +61,36 @@
return int(ncpu.value)
+ def getMemorySize():
+ """
+ Returns the physical amount of RAM installed, in bytes
+ """
+ memsize = c_uint64(0)
+ size = c_size_t(sizeof(memsize))
+
+ libc.sysctlbyname.argtypes = [
+ c_char_p, c_void_p, c_void_p, c_void_p, c_ulong
+ ]
+ libc.sysctlbyname(
+ "hw.memsize",
+ c_voidp(addressof(memsize)),
+ c_voidp(addressof(size)),
+ None,
+ 0
+ )
+
+ return int(memsize.value)
+
+
elif sys.platform == "linux2" and hasCtypes:
libc = cdll.LoadLibrary(ctypes.util.find_library("libc"))
def getNCPU():
return libc.get_nprocs()
+ def getMemorySize():
+ return libc.getpagesize() * libc.get_phys_pages()
+
else:
def getNCPU():
if not hasCtypes:
@@ -73,6 +100,9 @@
raise NotImplementedError("getNCPU not supported on %s%s" % (sys.platform, msg))
+ def getMemorySize():
+ raise NotImplementedError("getMemorySize not yet supported on %s" % (sys.platform))
+
##
# Module management
##
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100608/e54b1c36/attachment.html>
More information about the calendarserver-changes
mailing list