[CalendarServer-changes] [5360] CalendarServer/trunk
source_changes at macosforge.org
source_changes at macosforge.org
Fri Mar 19 13:33:24 PDT 2010
Revision: 5360
http://trac.macosforge.org/projects/calendarserver/changeset/5360
Author: glyph at apple.com
Date: 2010-03-19 13:33:23 -0700 (Fri, 19 Mar 2010)
Log Message:
-----------
Low-level bindings for sending file descriptors between processes.
Modified Paths:
--------------
CalendarServer/trunk/setup.py
Added Paths:
-----------
CalendarServer/trunk/twext/python/sendfd.py
CalendarServer/trunk/twext/python/sendmsg.c
CalendarServer/trunk/twext/python/test/pullpipe.py
CalendarServer/trunk/twext/python/test/test_sendmsg.py
Modified: CalendarServer/trunk/setup.py
===================================================================
--- CalendarServer/trunk/setup.py 2010-03-19 19:06:03 UTC (rev 5359)
+++ CalendarServer/trunk/setup.py 2010-03-19 20:33:23 UTC (rev 5360)
@@ -68,7 +68,10 @@
from distutils.core import Extension
-extensions = []
+extensions = [
+ Extension("twext.python.sendmsg",
+ sources=["twext/python/sendmsg.c"])
+]
if sys.platform == "darwin":
extensions.append(
Added: CalendarServer/trunk/twext/python/sendfd.py
===================================================================
--- CalendarServer/trunk/twext/python/sendfd.py (rev 0)
+++ CalendarServer/trunk/twext/python/sendfd.py 2010-03-19 20:33:23 UTC (rev 5360)
@@ -0,0 +1,61 @@
+# -*- test-case-name: twext.python.test.test_sendmsg -*-
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+from struct import pack, unpack
+from socket import SOL_SOCKET
+
+from twext.python.sendmsg import sendmsg, recvmsg, SCM_RIGHTS
+
+def sendfd(socketfd, fd):
+ """
+ Send the given FD to another process via L{sendmsg} on the given C{AF_UNIX}
+ socket.
+
+ @param socketfd: An C{AF_UNIX} socket, attached to another process waiting
+ to receive sockets via the ancillary data mechanism in L{sendmsg}.
+
+ @type socketfd: C{int}
+
+ @param fd: A file descriptor to be sent to the other process.
+
+ @type fd: C{int}
+ """
+ args = (socketfd, "", 0, [(SOL_SOCKET, SCM_RIGHTS, pack("i", fd))])
+ sendmsg(*args)
+
+
+def recvfd(socketfd):
+ """
+ Receive a file descriptor from a L{sendmsg} message on the given C{AF_UNIX}
+ socket.
+
+ @param socketfd: An C{AF_UNIX} socket, attached to another process waiting
+ to send sockets via the ancillary data mechanism in L{sendmsg}.
+
+ @param fd: C{int}
+
+ @return: a new file descriptor.
+
+ @rtype: C{int}
+ """
+ data, flags, ancillary = recvmsg(socketfd)
+ [(cmsg_level, cmsg_type, packedFD)] = ancillary
+ # cmsg_level and cmsg_type really need to be SOL_SOCKET / SCM_RIGHTS, but
+ # since those are the *only* standard values, there's not much point in
+ # checking.
+ [unpackedFD] = unpack("i", packedFD)
+ return unpackedFD
Added: CalendarServer/trunk/twext/python/sendmsg.c
===================================================================
--- CalendarServer/trunk/twext/python/sendmsg.c (rev 0)
+++ CalendarServer/trunk/twext/python/sendmsg.c 2010-03-19 20:33:23 UTC (rev 5360)
@@ -0,0 +1,331 @@
+/*
+ * Copyright (c) 2010 Apple Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Python.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <signal.h>
+
+PyObject *sendmsg_socket_error;
+
+static PyObject *sendmsg_sendmsg(PyObject *self, PyObject *args, PyObject *keywds);
+static PyObject *sendmsg_recvmsg(PyObject *self, PyObject *args, PyObject *keywds);
+
+static PyMethodDef sendmsg_methods[] = {
+ {"sendmsg", (PyCFunction) sendmsg_sendmsg, METH_VARARGS | METH_KEYWORDS,
+ NULL},
+ {"recvmsg", (PyCFunction) sendmsg_recvmsg, METH_VARARGS | METH_KEYWORDS,
+ NULL},
+ {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC initsendmsg(void) {
+ PyObject *module;
+
+ sendmsg_socket_error = NULL; /* Make sure that this has a known value
+ before doing anything that might exit. */
+
+ module = Py_InitModule("sendmsg", sendmsg_methods);
+
+ if (!module) {
+ return;
+ }
+
+ /*
+ The following is the only value mentioned by POSIX:
+ http://www.opengroup.org/onlinepubs/9699919799/basedefs/sys_socket.h.html
+ */
+
+ if (-1 == PyModule_AddIntConstant(module, "SCM_RIGHTS", SCM_RIGHTS)) {
+ return;
+ }
+
+
+ /* BSD, Darwin, Hurd */
+#if defined(SCM_CREDS)
+ if (-1 == PyModule_AddIntConstant(module, "SCM_CREDS", SCM_CREDS)) {
+ return;
+ }
+#endif
+
+ /* Linux */
+#if defined(SCM_CREDENTIALS)
+ if (-1 == PyModule_AddIntConstant(module, "SCM_CREDENTIALS", SCM_CREDENTIALS)) {
+ return;
+ }
+#endif
+
+ /* Apparently everywhere, but not standardized. */
+#if defined(SCM_TIMESTAMP)
+ if (-1 == PyModule_AddIntConstant(module, "SCM_TIMESTAMP", SCM_TIMESTAMP)) {
+ return;
+ }
+#endif
+
+ module = PyImport_ImportModule("socket");
+ if (!module) {
+ return;
+ }
+
+ sendmsg_socket_error = PyObject_GetAttrString(module, "error");
+ if (!sendmsg_socket_error) {
+ return;
+ }
+}
+
+static PyObject *sendmsg_sendmsg(PyObject *self, PyObject *args, PyObject *keywds) {
+
+ int fd;
+ int flags = 0;
+ int sendmsg_result;
+ struct msghdr message_header;
+ struct iovec iov[1];
+ PyObject *ancillary = NULL;
+ static char *kwlist[] = {"fd", "data", "flags", "ancillary", NULL};
+
+ if (!PyArg_ParseTupleAndKeywords(
+ args, keywds, "it#|iO:sendmsg", kwlist,
+ &fd,
+ &iov[0].iov_base,
+ &iov[0].iov_len,
+ &flags,
+ &ancillary)) {
+ return NULL;
+ }
+
+ message_header.msg_name = NULL;
+ message_header.msg_namelen = 0;
+
+ message_header.msg_iov = iov;
+ message_header.msg_iovlen = 1;
+
+ message_header.msg_control = NULL;
+ message_header.msg_controllen = 0;
+
+ message_header.msg_flags = 0;
+
+ if (ancillary) {
+
+ if (!PyList_Check(ancillary)) {
+ PyErr_Format(PyExc_TypeError,
+ "sendmsg argument 3 expected list, got %s",
+ ancillary->ob_type->tp_name);
+ return NULL;
+ }
+
+ PyObject *iterator = PyObject_GetIter(ancillary);
+ PyObject *item = NULL;
+
+ if (iterator == NULL) {
+ return NULL;
+ }
+
+ int all_data_len = 0;
+
+ /* First we need to know how big the buffer needs to be in order to
+ have enough space for all of the messages. */
+ while ( (item = PyIter_Next(iterator)) ) {
+ int data_len, type, level;
+ char *data;
+ if (!PyArg_ParseTuple(item, "iit#:sendmsg ancillary data (level, type, data)",
+ &level,
+ &type,
+ &data,
+ &data_len)) {
+ Py_DECREF(item);
+ Py_DECREF(iterator);
+ return NULL;
+ }
+ all_data_len += CMSG_SPACE(data_len);
+
+ Py_DECREF(item);
+ }
+
+ Py_DECREF(iterator);
+ iterator = NULL;
+
+ /* Allocate the buffer for all of the ancillary elements. */
+ message_header.msg_control = malloc(all_data_len);
+ if (!message_header.msg_control) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ message_header.msg_controllen = all_data_len;
+
+ iterator = PyObject_GetIter(ancillary); /* again */
+ item = NULL;
+
+ if (!iterator) {
+ free(message_header.msg_control);
+ return NULL;
+ }
+
+ /* Unpack the tuples into the control message. */
+ struct cmsghdr *control_message = CMSG_FIRSTHDR(&message_header);
+ while ( (item = PyIter_Next(iterator)) ) {
+ int data_len, type, level;
+ unsigned char *data, *cmsg_data;
+
+ if (!PyArg_ParseTuple(item,
+ "iit#:sendmsg ancillary data (level, type, data)",
+ &level,
+ &type,
+ &data,
+ &data_len)) {
+ Py_DECREF(item);
+ Py_DECREF(iterator);
+ free(message_header.msg_control);
+ return NULL;
+ }
+
+ control_message->cmsg_level = level;
+ control_message->cmsg_type = type;
+ control_message->cmsg_len = CMSG_LEN(data_len);
+
+ cmsg_data = CMSG_DATA(control_message);
+ memcpy(cmsg_data, data, data_len);
+
+ Py_DECREF(item);
+
+ control_message = CMSG_NXTHDR(&message_header, control_message);
+
+ /* We explicitly allocated enough space for all ancillary data
+ above; if there isn't enough room, all bets are off. */
+ assert(control_message);
+ }
+
+ Py_DECREF(iterator);
+
+ if (PyErr_Occurred()) {
+ free(message_header.msg_control);
+ return NULL;
+ }
+ }
+
+ sendmsg_result = sendmsg(fd, &message_header, flags);
+
+ if (sendmsg_result < 0) {
+ PyErr_SetFromErrno(sendmsg_socket_error);
+ if (message_header.msg_control) {
+ free(message_header.msg_control);
+ }
+ return NULL;
+ }
+
+ return Py_BuildValue("i", sendmsg_result);
+}
+
+static PyObject *sendmsg_recvmsg(PyObject *self, PyObject *args, PyObject *keywds) {
+ int fd = -1;
+ int flags = 0;
+ size_t maxsize = 8192;
+ size_t cmsg_size = 4*1024;
+ int recvmsg_result;
+ struct msghdr message_header;
+ struct cmsghdr *control_message;
+ struct iovec iov[1];
+ char *cmsgbuf;
+ PyObject *ancillary;
+ PyObject *final_result = NULL;
+
+ static char *kwlist[] = {"fd", "flags", "maxsize", "cmsg_size", NULL};
+
+ if (!PyArg_ParseTupleAndKeywords(args, keywds, "i|iii:recvmsg", kwlist,
+ &fd, &flags, &maxsize, &cmsg_size)) {
+ return NULL;
+ }
+
+ cmsg_size = CMSG_SPACE(cmsg_size);
+
+ message_header.msg_name = NULL;
+ message_header.msg_namelen = 0;
+
+ iov[0].iov_len = maxsize;
+ iov[0].iov_base = malloc(maxsize);
+
+ if (!iov[0].iov_base) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+
+ message_header.msg_iov = iov;
+ message_header.msg_iovlen = 1;
+
+ cmsgbuf = malloc(cmsg_size);
+
+ if (!cmsgbuf) {
+ free(iov[0].iov_base);
+ PyErr_NoMemory();
+ return NULL;
+ }
+
+ memset(cmsgbuf, 0, cmsg_size);
+ message_header.msg_control = cmsgbuf;
+ message_header.msg_controllen = cmsg_size;
+
+ recvmsg_result = recvmsg(fd, &message_header, flags);
+ if (recvmsg_result < 0) {
+ PyErr_SetFromErrno(sendmsg_socket_error);
+ goto finished;
+ }
+
+ ancillary = PyList_New(0);
+ if (!ancillary) {
+ goto finished;
+ }
+
+ for (control_message = CMSG_FIRSTHDR(&message_header);
+ control_message;
+ control_message = CMSG_NXTHDR(&message_header,
+ control_message)) {
+ PyObject *entry;
+ entry = Py_BuildValue(
+ "(iis#)",
+ control_message->cmsg_level,
+ control_message->cmsg_type,
+ CMSG_DATA(control_message),
+ control_message->cmsg_len - sizeof(struct cmsghdr));
+
+ if (!entry) {
+ Py_DECREF(ancillary);
+ goto finished;
+ }
+
+ if (PyList_Append(ancillary, entry) < 0) {
+ Py_DECREF(ancillary);
+ Py_DECREF(entry);
+ goto finished;
+ } else {
+ Py_DECREF(entry);
+ }
+ }
+
+ final_result = Py_BuildValue(
+ "s#iO",
+ iov[0].iov_base,
+ recvmsg_result,
+ message_header.msg_flags,
+ ancillary);
+
+ Py_DECREF(ancillary);
+
+ finished:
+ free(iov[0].iov_base);
+ free(cmsgbuf);
+ return final_result;
+}
+
Added: CalendarServer/trunk/twext/python/test/pullpipe.py
===================================================================
--- CalendarServer/trunk/twext/python/test/pullpipe.py (rev 0)
+++ CalendarServer/trunk/twext/python/test/pullpipe.py 2010-03-19 20:33:23 UTC (rev 5360)
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- test-case-name: twext.python.test.test_sendmsg -*-
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+if __name__ == '__main__':
+ from twext.python.sendfd import recvfd
+ import sys, os
+ fd = recvfd(int(sys.argv[1]))
+ os.write(fd, "Test fixture data.\n")
+ os.close(fd)
+
+
Added: CalendarServer/trunk/twext/python/test/test_sendmsg.py
===================================================================
--- CalendarServer/trunk/twext/python/test/test_sendmsg.py (rev 0)
+++ CalendarServer/trunk/twext/python/test/test_sendmsg.py 2010-03-19 20:33:23 UTC (rev 5360)
@@ -0,0 +1,172 @@
+##
+# Copyright (c) 2010 Apple Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+import socket
+from os import pipe, read, close, environ
+from twext.python.filepath import CachingFilePath as FilePath
+import sys
+
+from twisted.internet.defer import Deferred
+from twisted.internet.error import ProcessDone
+from twisted.trial.unittest import TestCase
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet import reactor
+
+from twext.python.sendmsg import sendmsg, recvmsg
+from twext.python.sendfd import sendfd
+from twisted.internet.protocol import ProcessProtocol
+
+class ExitedWithStderr(Exception):
+ """
+ A process exited with some stderr.
+ """
+
+ def __str__(self):
+ """
+ Dump the errors in a pretty way in the event of a subprocess traceback.
+ """
+ return '\n'.join([''] + list(self.args))
+
+
+class StartStopProcessProtocol(ProcessProtocol):
+ """
+ An L{IProcessProtocol} with a Deferred for events where the subprocess
+ starts and stops.
+ """
+
+ def __init__(self):
+ self.started = Deferred()
+ self.stopped = Deferred()
+ self.output = ''
+ self.errors = ''
+
+ def connectionMade(self):
+ self.started.callback(self.transport)
+
+ def outReceived(self, data):
+ self.output += data
+
+ def errReceived(self, data):
+ self.errors += data
+
+ def processEnded(self, reason):
+ if reason.check(ProcessDone):
+ self.stopped.callback(self.output)
+ else:
+ self.stopped.errback(ExitedWithStderr(
+ self.errors, self.output))
+
+
+
+def bootReactor():
+ """
+ Yield this from a trial test to bootstrap the reactor in order to avoid
+ PotentialZombieWarning, for tests that use subprocesses. This hack will no
+ longer be necessary in Twisted 10.1, since U{the underlying bug was fixed
+ <http://twistedmatrix.com/trac/ticket/2078>}.
+ """
+ d = Deferred()
+ reactor.callLater(0, d.callback, None)
+ return d
+
+
+
+class SendmsgTestCase(TestCase):
+ """
+ Tests for sendmsg extension module and associated file-descriptor sending
+ functionality in L{twext.python.sendfd}.
+ """
+
+ def setUp(self):
+ """
+ Create a pair of UNIX sockets.
+ """
+ self.input, self.output = socket.socketpair(socket.AF_UNIX)
+
+
+ def tearDown(self):
+ """
+ Close the sockets opened by setUp.
+ """
+ self.input.close()
+ self.output.close()
+
+
+ def test_roundtrip(self):
+ """
+ L{recvmsg} will retrieve a message sent via L{sendmsg}.
+ """
+ sendmsg(self.input.fileno(), "hello, world!", 0)
+
+ result = recvmsg(fd=self.output.fileno())
+ self.assertEquals(result, ("hello, world!", 0, []))
+
+
+ def test_wrongTypeAncillary(self):
+ """
+ L{sendmsg} will show a helpful exception message when given the wrong
+ type of object for the 'ancillary' argument.
+ """
+ error = self.assertRaises(TypeError,
+ sendmsg, self.input.fileno(),
+ "hello, world!", 0, 4321)
+ self.assertEquals(str(error),
+ "sendmsg argument 3 expected list, got int")
+
+
+ def spawn(self, script):
+ """
+ Start a script that is a peer of this test as a subprocess.
+
+ @param script: the module name of the script in this directory (no
+ package prefix, no '.py')
+ @type script: C{str}
+
+ @rtype: L{StartStopProcessProtocol}
+ """
+ sspp = StartStopProcessProtocol()
+ reactor.spawnProcess(
+ sspp, sys.executable, [
+ sys.executable,
+ FilePath(__file__).sibling(script + ".py").path,
+ str(self.output.fileno()),
+ ],
+ environ,
+ childFDs={0: "w", 1: "r", 2: "r",
+ self.output.fileno(): self.output.fileno()}
+ )
+ return sspp
+
+
+ @inlineCallbacks
+ def test_sendSubProcessFD(self):
+ """
+ Calling L{sendsmsg} with SOL_SOCKET, SCM_RIGHTS , and a platform-endian
+ packed file descriptor number should send that file descriptor to a
+ different process, where it can be retrieved by using L{recvmsg}.
+ """
+ yield bootReactor()
+ sspp = self.spawn("pullpipe")
+ yield sspp.started
+ pipeOut, pipeIn = pipe()
+ self.addCleanup(close, pipeOut)
+ sendfd(self.input.fileno(), pipeIn)
+ close(pipeIn)
+ yield sspp.stopped
+ self.assertEquals(read(pipeOut, 1024), "Test fixture data.\n")
+ # Make sure that the pipe is actually closed now.
+ self.assertEquals(read(pipeOut, 1024), "")
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.macosforge.org/pipermail/calendarserver-changes/attachments/20100319/c0d23ae4/attachment-0001.html>
More information about the calendarserver-changes
mailing list