[CalendarServer-changes] [11034] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Apr 12 12:05:39 PDT 2013
Revision: 11034
http://trac.calendarserver.org//changeset/11034
Author: sagen at apple.com
Date: 2013-04-12 12:05:39 -0700 (Fri, 12 Apr 2013)
Log Message:
-----------
Dynamically compute MaxConnections and SharedBuffers for Postgres
Modified Paths:
--------------
CalendarServer/trunk/calendarserver/tap/caldav.py
CalendarServer/trunk/calendarserver/tap/test/test_util.py
CalendarServer/trunk/calendarserver/tap/util.py
CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
CalendarServer/trunk/calendarserver/tools/test/principals/caldavd.plist
CalendarServer/trunk/conf/caldavd-apple.plist
CalendarServer/trunk/twistedcaldav/stdconfig.py
CalendarServer/trunk/twistedcaldav/test/test_config.py
CalendarServer/trunk/twistedcaldav/test/test_stdconfig.py
CalendarServer/trunk/twistedcaldav/util.py
Modified: CalendarServer/trunk/calendarserver/tap/caldav.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/caldav.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/calendarserver/tap/caldav.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -53,6 +53,8 @@
from twisted.application.service import MultiService, IServiceMaker
from twisted.application.service import Service
+from twistedcaldav.config import config, ConfigurationError
+from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twext.web2.server import Site
from twext.python.log import Logger, LoggingMixIn
from twext.python.log import logLevelForNamespace, setLogLevelForNamespace
@@ -68,13 +70,10 @@
)
from txdav.common.datastore.upgrade.migrate import UpgradeToDatabaseService
-from twistedcaldav.config import ConfigurationError
-from twistedcaldav.config import config
from twistedcaldav.directory import calendaruserproxy
from twistedcaldav.directory.directory import GroupMembershipCacheUpdater
from twistedcaldav.localization import processLocalizationFiles
from twistedcaldav import memcachepool
-from twistedcaldav.stdconfig import DEFAULT_CONFIG, DEFAULT_CONFIG_FILE
from twistedcaldav.upgrade import UpgradeFileSystemFormatService, PostDBImportService
from calendarserver.tap.util import pgServiceFromConfig, getDBPool, MemoryLimitService
@@ -102,8 +101,7 @@
from calendarserver.accesslog import AMPCommonAccessLoggingObserver
from calendarserver.accesslog import AMPLoggingFactory
from calendarserver.accesslog import RotatingFileAccessLoggingObserver
-from calendarserver.tap.util import getRootResource, computeProcessCount
-
+from calendarserver.tap.util import getRootResource
from calendarserver.tap.util import storeFromConfig
from calendarserver.tap.util import pgConnectorFromConfig
from calendarserver.tap.util import oracleConnectorFromConfig
@@ -1271,18 +1269,6 @@
monitor.addProcess('memcached-%s' % (name,), memcachedArgv,
env=PARENT_ENVIRONMENT)
- #
- # Calculate the number of processes to spawn
- #
- if config.MultiProcess.ProcessCount == 0:
- # TODO: this should probably be happening in a configuration hook.
- processCount = computeProcessCount(
- config.MultiProcess.MinProcessCount,
- config.MultiProcess.PerCPU,
- config.MultiProcess.PerGB,
- )
- config.MultiProcess.ProcessCount = processCount
- self.log_info("Configuring %d processes." % (processCount,))
# Open the socket(s) to be inherited by the slaves
inheritFDs = []
Modified: CalendarServer/trunk/calendarserver/tap/test/test_util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/test/test_util.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/calendarserver/tap/test/test_util.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -14,7 +14,8 @@
# limitations under the License.
##
-from calendarserver.tap.util import computeProcessCount, directoryFromConfig, MemoryLimitService
+from calendarserver.tap.util import directoryFromConfig, MemoryLimitService
+from twistedcaldav.util import computeProcessCount
from twistedcaldav.test.util import TestCase
from twistedcaldav.config import config
from twistedcaldav.directory.augment import AugmentXMLDB
Modified: CalendarServer/trunk/calendarserver/tap/util.py
===================================================================
--- CalendarServer/trunk/calendarserver/tap/util.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/calendarserver/tap/util.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -765,37 +765,8 @@
-def computeProcessCount(minimum, perCPU, perGB, cpuCount=None, memSize=None):
- """
- Determine how many process to spawn based on installed RAM and CPUs,
- returning at least "mininum"
- """
- if cpuCount is None:
- try:
- cpuCount = getNCPU()
- except NotImplementedError, e:
- log.error("Unable to detect number of CPUs: %s" % (str(e),))
- return minimum
- if memSize is None:
- try:
- memSize = getMemorySize()
- except NotImplementedError, e:
- log.error("Unable to detect amount of installed RAM: %s" % (str(e),))
- return minimum
-
- countByCore = perCPU * cpuCount
- countByMemory = perGB * (memSize / (1024 * 1024 * 1024))
-
- # Pick the smaller of the two:
- count = min(countByCore, countByMemory)
-
- # ...but at least "minimum"
- return max(count, minimum)
-
-
-
class FakeRequest(object):
def __init__(self, rootResource, method, path, uri='/', transaction=None):
Modified: CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/calendarserver/tools/test/gateway/caldavd.plist 2013-04-12 19:05:39 UTC (rev 11034)
@@ -718,6 +718,10 @@
<integer>30</integer> <!-- in minutes -->
+ <!-- For unit tests, enable SharedConnectionPool so we don't use up shared memory -->
+ <key>SharedConnectionPool</key>
+ <true/>
+
<!--
Twisted
-->
Modified: CalendarServer/trunk/calendarserver/tools/test/principals/caldavd.plist
===================================================================
--- CalendarServer/trunk/calendarserver/tools/test/principals/caldavd.plist 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/calendarserver/tools/test/principals/caldavd.plist 2013-04-12 19:05:39 UTC (rev 11034)
@@ -743,6 +743,9 @@
<key>ResponseCacheTimeout</key>
<integer>30</integer> <!-- in minutes -->
+ <!-- For unit tests, enable SharedConnectionPool so we don't use up shared memory -->
+ <key>SharedConnectionPool</key>
+ <true/>
<!--
Twisted
Modified: CalendarServer/trunk/conf/caldavd-apple.plist
===================================================================
--- CalendarServer/trunk/conf/caldavd-apple.plist 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/conf/caldavd-apple.plist 2013-04-12 19:05:39 UTC (rev 11034)
@@ -112,6 +112,8 @@
<string>-c deadlock_timeout=10</string>
<string>-c log_line_prefix='%m [%p] '</string>
</array>
+ <key>ExtraConnections</key>
+ <integer>20</integer>
</dict>
<!-- Data root -->
Modified: CalendarServer/trunk/twistedcaldav/stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/twistedcaldav/stdconfig.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -37,7 +37,9 @@
from twisted.python.runtime import platform
from calendarserver.push.util import getAPNTopicFromCertificate
+from twistedcaldav.util import computeProcessCount
+
log = Logger()
if platform.isMacOSX():
@@ -907,8 +909,14 @@
"DatabaseName": "caldav",
"LogFile": "postgres.log",
"ListenAddresses": [],
- "SharedBuffers": 30,
- "MaxConnections": 20,
+ "SharedBuffers": 0, # BuffersToConnectionsRatio * MaxConnections
+ # Note: don't set this, it will be computed dynamically
+ # See _updateMultiProcess( ) below for details
+ "MaxConnections": 0, # Dynamically computed based on ProcessCount, etc.
+ # Note: don't set this, it will be computed dynamically
+ # See _updateMultiProcess( ) below for details
+ "ExtraConnections": 3, # how many extra connections to leave for utilities
+ "BuffersToConnectionsRatio": 1.5,
"Options": [
"-c standard_conforming_strings=on",
],
@@ -1128,7 +1136,38 @@
configDict.ServerHostName = hostname
+def _updateMultiProcess(configDict, reloading=False):
+ """
+ Dynamically compute ProcessCount if it's set to 0. Always compute
+ MaxConnections and SharedBuffers based on ProcessCount, ExtraConnections,
+ SharedConnectionPool, MaxDBConnectionsPerPool, and BuffersToConnectionsRatio
+ """
+ if configDict.MultiProcess.ProcessCount == 0:
+ processCount = computeProcessCount(
+ configDict.MultiProcess.MinProcessCount,
+ configDict.MultiProcess.PerCPU,
+ configDict.MultiProcess.PerGB,
+ )
+ configDict.MultiProcess.ProcessCount = processCount
+ # Start off with extra connections to be used by command line utilities and
+ # administration/inspection tools
+ maxConnections = configDict.Postgres.ExtraConnections
+
+ if configDict.SharedConnectionPool:
+ # If SharedConnectionPool is enabled, then only the master process will
+ # be connection to the database, therefore use MaxDBConnectionsPerPool
+ maxConnections += configDict.MaxDBConnectionsPerPool
+ else:
+ # Otherwise the master *and* each worker process will be connecting
+ maxConnections += ((configDict.MultiProcess.ProcessCount + 1) *
+ configDict.MaxDBConnectionsPerPool)
+
+ configDict.Postgres.MaxConnections = maxConnections
+ configDict.Postgres.SharedBuffers = int(configDict.Postgres.MaxConnections *
+ configDict.Postgres.BuffersToConnectionsRatio)
+
+
def _preUpdateDirectoryService(configDict, items, reloading=False):
# Special handling for directory services configs
dsType = items.get("DirectoryService", {}).get("type", None)
@@ -1502,6 +1541,7 @@
_preUpdateDirectoryAddressBookBackingDirectoryService,
)
POST_UPDATE_HOOKS = (
+ _updateMultiProcess,
_updateDataStore,
_updateHostName,
_postUpdateDirectoryService,
Modified: CalendarServer/trunk/twistedcaldav/test/test_config.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_config.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/twistedcaldav/test/test_config.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -83,7 +83,8 @@
def testDefaults(self):
for key, value in DEFAULT_CONFIG.iteritems():
- if key in ("ServerHostName", "Notifications"):
+ if key in ("ServerHostName", "Notifications", "MultiProcess",
+ "Postgres"):
# Value is calculated and may vary
continue
for item in RELATIVE_PATHS:
Modified: CalendarServer/trunk/twistedcaldav/test/test_stdconfig.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/test/test_stdconfig.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/twistedcaldav/test/test_stdconfig.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -20,9 +20,10 @@
from twext.python.filepath import CachingFilePath as FilePath
from twisted.trial.unittest import TestCase
-from twistedcaldav.config import Config
+from twistedcaldav.config import Config, ConfigDict
from twistedcaldav.stdconfig import NoUnicodePlistParser, PListConfigProvider,\
- _updateDataStore
+ _updateDataStore, _updateMultiProcess
+import twistedcaldav.stdconfig
nonASCIIValue = "→←"
nonASCIIPlist = "<plist version='1.0'><string>%s</string></plist>" % (
@@ -149,3 +150,25 @@
}
_updateDataStore(configDict)
self.assertEquals(configDict["ServerRoot"], "/a/b/c")
+
+ def test_updateMultiProcess(self):
+ def stubProcessCount(*args):
+ return 3
+ self.patch(twistedcaldav.stdconfig, "computeProcessCount", stubProcessCount)
+ configDict = ConfigDict({
+ "MultiProcess" : {
+ "ProcessCount" : 0,
+ "MinProcessCount" : 2,
+ "PerCPU" : 1,
+ "PerGB" : 1,
+ },
+ "Postgres" : {
+ "ExtraConnections" : 5,
+ "BuffersToConnectionsRatio" : 1.5,
+ },
+ "SharedConnectionPool" : False,
+ "MaxDBConnectionsPerPool" : 10,
+ })
+ _updateMultiProcess(configDict)
+ self.assertEquals(35, configDict.Postgres.MaxConnections)
+ self.assertEquals(52, configDict.Postgres.SharedBuffers)
Modified: CalendarServer/trunk/twistedcaldav/util.py
===================================================================
--- CalendarServer/trunk/twistedcaldav/util.py 2013-04-12 18:59:06 UTC (rev 11033)
+++ CalendarServer/trunk/twistedcaldav/util.py 2013-04-12 19:05:39 UTC (rev 11034)
@@ -106,6 +106,37 @@
def getMemorySize():
raise NotImplementedError("getMemorySize not yet supported on %s" % (sys.platform))
+
+def computeProcessCount(minimum, perCPU, perGB, cpuCount=None, memSize=None):
+ """
+ Determine how many process to spawn based on installed RAM and CPUs,
+ returning at least "mininum"
+ """
+
+ if cpuCount is None:
+ try:
+ cpuCount = getNCPU()
+ except NotImplementedError, e:
+ log.error("Unable to detect number of CPUs: %s" % (str(e),))
+ return minimum
+
+ if memSize is None:
+ try:
+ memSize = getMemorySize()
+ except NotImplementedError, e:
+ log.error("Unable to detect amount of installed RAM: %s" % (str(e),))
+ return minimum
+
+ countByCore = perCPU * cpuCount
+ countByMemory = perGB * (memSize / (1024 * 1024 * 1024))
+
+ # Pick the smaller of the two:
+ count = min(countByCore, countByMemory)
+
+ # ...but at least "minimum"
+ return max(count, minimum)
+
+
##
# Module management
##
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20130412/066c2457/attachment-0001.html>
More information about the calendarserver-changes
mailing list