[CalendarServer-changes] [5699] CalendarServer/trunk

source_changes at macosforge.org source_changes at macosforge.org
Tue Jun 8 11:23:42 PDT 2010


Revision: 5699
          http://trac.macosforge.org/projects/calendarserver/changeset/5699
Author:   sagen at apple.com
Date:     2010-06-08 11:23:38 -0700 (Tue, 08 Jun 2010)
Log Message:
-----------
The number of processes spawned now takes into account both cpu count and amount of RAM installed.

Modified Paths:
--------------
    CalendarServer/trunk/calendarserver/tap/caldav.py
    CalendarServer/trunk/calendarserver/tap/util.py
    CalendarServer/trunk/twistedcaldav/stdconfig.py
    CalendarServer/trunk/twistedcaldav/util.py

Added Paths:
-----------
    CalendarServer/trunk/calendarserver/tap/test/test_util.py

Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py	2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py	2010-06-08 18:23:38 UTC (rev 5699)
@@ -77,7 +77,6 @@
 from twistedcaldav.static import TimezoneServiceFile
 from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
 from twistedcaldav.upgrade import upgradeData
-from twistedcaldav.util import getNCPU
 
 from twext.web2.metafd import ConnectionLimiter, ReportingHTTPService
 
@@ -93,7 +92,7 @@
 from calendarserver.provision.root import RootResource
 from calendarserver.webadmin.resource import WebAdminResource
 from calendarserver.webcal.resource import WebCalendarResource
-from calendarserver.tap.util import getRootResource
+from calendarserver.tap.util import getRootResource, computeProcessCount
 
 log = Logger()
 
@@ -775,30 +774,16 @@
             parentEnv["KRB5_KTNAME"] = os.environ["KRB5_KTNAME"]
 
         #
-        # Attempt to calculate the number of processes to use 1 per processor
+        # Calculate the number of processes to spawn
         #
         if config.MultiProcess.ProcessCount == 0:
-            try:
-                cpuCount = getNCPU()
-            except NotImplementedError, e:
-                self.log_error("Unable to detect number of CPUs: %s"
-                               % (str(e),))
-                cpuCount = 0
-            else:
-                if cpuCount < 1:
-                    self.log_error(
-                        "%d processors detected, which is hard to believe."
-                        % (cpuCount,)
-                    )
-
-            processCount = config.MultiProcess.MinProcessCount
-            if 2 * cpuCount > processCount:
-                processCount = 2 * cpuCount
-
-            self.log_info("%d processors found. Configuring %d processes."
-                          % (cpuCount, processCount))
-
+            processCount = computeProcessCount(
+                config.MultiProcess.MinProcessCount,
+                config.MultiProcess.PerCPU,
+                config.MultiProcess.PerGB,
+            )
             config.MultiProcess.ProcessCount = processCount
+            self.log_info("Configuring %d processes." % (processCount,))
 
 
         # Open the socket(s) to be inherited by the slaves

Added: CalendarServer/trunk/calendarserver/tap/test/test_util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_util.py	                        (rev 0)
+++ CalendarServer/trunk/calendarserver/tap/test/test_util.py	2010-06-08 18:23:38 UTC (rev 5699)
@@ -0,0 +1,49 @@
+##
+# Copyright (c) 2007-2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from calendarserver.tap.util import computeProcessCount
+from twistedcaldav.test.util import TestCase
+
+class ProcessCountTestCase(TestCase):
+
+    def test_count(self):
+
+        data = (
+            # minimum, perCPU, perGB, cpu, memory (in GB), expected:
+            (0, 1, 1, 0, 0, 0),
+            (1, 2, 2, 0, 0, 1),
+            (1, 2, 1, 1, 1, 1),
+            (1, 2, 2, 1, 1, 2),
+            (1, 2, 2, 2, 1, 2),
+            (1, 2, 2, 1, 2, 2),
+            (1, 2, 2, 2, 2, 4),
+            (4, 1, 2, 8, 2, 4),
+            (6, 2, 2, 2, 2, 6),
+            (1, 2, 1, 4, 99, 8),
+
+            (2, 1, 2, 2, 2, 2), # 2 cores, 2GB = 2
+            (2, 1, 2, 2, 4, 2), # 2 cores, 4GB = 2
+            (2, 1, 2, 8, 6, 8), # 8 cores, 6GB = 8
+            (2, 1, 2, 8, 16, 8), # 8 cores, 16GB = 8
+        )
+
+        for min, perCPU, perGB, cpu, mem, expected in data:
+            mem *= (1024 * 1024 * 1024)
+
+            self.assertEquals(
+                expected,
+                computeProcessCount(min, perCPU, perGB, cpuCount=cpu, memSize=mem)
+            )

Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py	2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/calendarserver/tap/util.py	2010-06-08 18:23:38 UTC (rev 5699)
@@ -53,6 +53,7 @@
 from twistedcaldav.static import TimezoneServiceFile
 from twistedcaldav.static import AddressBookHomeProvisioningFile, DirectoryBackedAddressBookFile
 from twistedcaldav.timezones import TimezoneCache
+from twistedcaldav.util import getMemorySize, getNCPU
 from twisted.internet.defer import inlineCallbacks, returnValue
 
 try:
@@ -463,7 +464,39 @@
 
 
 
+def computeProcessCount(minimum, perCPU, perGB, cpuCount=None, memSize=None):
+    """
+    Determine how many process to spawn based on installed RAM and CPUs,
+    returning at least "mininum"
+    """
 
+    if cpuCount is None:
+        try:
+            cpuCount = getNCPU()
+        except NotImplementedError, e:
+            log.error("Unable to detect number of CPUs: %s" % (str(e),))
+            return minimum
+
+    if memSize is None:
+        try:
+            memSize = getMemorySize()
+        except NotImplementedError, e:
+            log.error("Unable to detect amount of installed RAM: %s" % (str(e),))
+            return minimum
+
+    countByCore = perCPU * cpuCount
+    countByMemory = perGB * (memSize / (1024 * 1024 * 1024))
+
+    # Pick the smaller of the two:
+    count = min(countByCore, countByMemory)
+
+    # ...but at least "minimum"
+    return max(count, minimum)
+
+
+
+
+
 class FakeRequest(object):
 
     def __init__(self, rootResource, method, path):

Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py	2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py	2010-06-08 18:23:38 UTC (rev 5699)
@@ -304,7 +304,9 @@
     "ProcessType": "Combined",
     "MultiProcess": {
         "ProcessCount": 0,
-        "MinProcessCount": 4,
+        "MinProcessCount": 2,
+        "PerCPU": 1,
+        "PerGB": 2,
         "StaggeredStartup": {
             "Enabled": False,
             "Interval": 15,

Modified: CalendarServer/trunk/twistedcaldav/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/util.py	2010-06-07 21:18:44 UTC (rev 5698)
+++ CalendarServer/trunk/twistedcaldav/util.py	2010-06-08 18:23:38 UTC (rev 5699)
@@ -28,7 +28,7 @@
 from twext.python.log import LoggingMixIn
 
 ##
-# getNCPU
+# System Resources (Memory size and processor count)
 ##
 
 try:
@@ -42,6 +42,9 @@
     libc = cdll.LoadLibrary(ctypes.util.find_library("libc"))
 
     def getNCPU():
+        """
+        Returns the number of processors detected
+        """
         ncpu = c_int(0)
         size = c_size_t(sizeof(ncpu))
 
@@ -58,12 +61,36 @@
 
         return int(ncpu.value)
 
+    def getMemorySize():
+        """
+        Returns the physical amount of RAM installed, in bytes
+        """
+        memsize = c_uint64(0)
+        size = c_size_t(sizeof(memsize))
+
+        libc.sysctlbyname.argtypes = [
+            c_char_p, c_void_p, c_void_p, c_void_p, c_ulong
+        ]
+        libc.sysctlbyname(
+            "hw.memsize",
+            c_voidp(addressof(memsize)),
+            c_voidp(addressof(size)),
+            None,
+            0
+        )
+
+        return int(memsize.value)
+
+
 elif sys.platform == "linux2" and hasCtypes:
     libc = cdll.LoadLibrary(ctypes.util.find_library("libc"))
 
     def getNCPU():
         return libc.get_nprocs()
 
+    def getMemorySize():
+        return libc.getpagesize() * libc.get_phys_pages()
+
 else:
     def getNCPU():
         if not hasCtypes:
@@ -73,6 +100,9 @@
 
         raise NotImplementedError("getNCPU not supported on %s%s" % (sys.platform, msg))
 
+    def getMemorySize():
+        raise NotImplementedError("getMemorySize not yet supported on %s" % (sys.platform))
+
 ##
 # Module management
 ##
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100608/e54b1c36/attachment.html>


More information about the calendarserver-changes mailing list