From ad24d6ca66aeb1928ac18bbc787402478db0799f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 20 Dec 2023 12:20:33 +0100 Subject: [PATCH 1/7] Fix NetBSD connections memory leak / core dump (#2341) --- HISTORY.rst | 4 ++ psutil/arch/netbsd/socks.c | 131 ++++++++++++++++++++++------------ psutil/tests/test_memleaks.py | 3 - 3 files changed, 88 insertions(+), 50 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 46591705b..c6ad7e144 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,10 @@ 5.9.8 (IN DEVELOPMENT) ====================== +**Bug fixes** + +- 930_, [NetBSD], [critical]: `net_connections()`_ implementation was broken. + It could either leak memory or core dump. - 2340_, [NetBSD]: if process is terminated, `Process.cwd()`_ will return an empty string instead of raising `NoSuchProcess`_. diff --git a/psutil/arch/netbsd/socks.c b/psutil/arch/netbsd/socks.c index 08b0b7e67..5c8ad3bd6 100644 --- a/psutil/arch/netbsd/socks.c +++ b/psutil/arch/netbsd/socks.c @@ -44,6 +44,8 @@ enum af_filter { struct kif { SLIST_ENTRY(kif) kifs; struct kinfo_file *kif; + char *buf; + int has_buf; }; // kinfo_file results list @@ -54,36 +56,31 @@ SLIST_HEAD(kifhead, kif) kihead = SLIST_HEAD_INITIALIZER(kihead); struct kpcb { SLIST_ENTRY(kpcb) kpcbs; struct kinfo_pcb *kpcb; + struct kinfo_pcb *buf; + int has_buf; }; // kinfo_pcb results list SLIST_HEAD(kpcbhead, kpcb) kpcbhead = SLIST_HEAD_INITIALIZER(kpcbhead); -static void psutil_kiflist_init(void); -static void psutil_kiflist_clear(void); -static void psutil_kpcblist_init(void); -static void psutil_kpcblist_clear(void); -static int psutil_get_files(void); -static int psutil_get_sockets(const char *name); -static int psutil_get_info(int aff); - // Initialize kinfo_file results list. static void psutil_kiflist_init(void) { SLIST_INIT(&kihead); - return; } // Clear kinfo_file results list. static void psutil_kiflist_clear(void) { - while (!SLIST_EMPTY(&kihead)) { - SLIST_REMOVE_HEAD(&kihead, kifs); - } - - return; + while (!SLIST_EMPTY(&kihead)) { + struct kif *kif = SLIST_FIRST(&kihead); + if (kif->has_buf == 1) + free(kif->buf); + free(kif); + SLIST_REMOVE_HEAD(&kihead, kifs); + } } @@ -91,18 +88,19 @@ psutil_kiflist_clear(void) { static void psutil_kpcblist_init(void) { SLIST_INIT(&kpcbhead); - return; } // Clear kinof_pcb result list. static void psutil_kpcblist_clear(void) { - while (!SLIST_EMPTY(&kpcbhead)) { - SLIST_REMOVE_HEAD(&kpcbhead, kpcbs); - } - - return; + while (!SLIST_EMPTY(&kpcbhead)) { + struct kpcb *kpcb = SLIST_FIRST(&kpcbhead); + if (kpcb->has_buf == 1) + free(kpcb->buf); + free(kpcb); + SLIST_REMOVE_HEAD(&kpcbhead, kpcbs); + } } @@ -112,7 +110,7 @@ psutil_get_files(void) { size_t len; size_t j; int mib[6]; - char *buf; + char *buf = NULL; off_t offset; mib[0] = CTL_KERN; @@ -124,7 +122,7 @@ psutil_get_files(void) { if (sysctl(mib, 6, NULL, &len, NULL, 0) == -1) { PyErr_SetFromErrno(PyExc_OSError); - return -1; + goto error; } offset = len % sizeof(off_t); @@ -132,33 +130,46 @@ psutil_get_files(void) { if ((buf = malloc(len + offset)) == NULL) { PyErr_NoMemory(); - return -1; + goto error; } if (sysctl(mib, 6, buf + offset, &len, NULL, 0) == -1) { - free(buf); PyErr_SetFromErrno(PyExc_OSError); - return -1; + goto error; } len /= sizeof(struct kinfo_file); struct kinfo_file *ki = (struct kinfo_file *)(buf + offset); - for (j = 0; j < len; j++) { - struct kif *kif = malloc(sizeof(struct kif)); - kif->kif = &ki[j]; - SLIST_INSERT_HEAD(&kihead, kif, kifs); + if (len > 0) { + for (j = 0; j < len; j++) { + struct kif *kif = malloc(sizeof(struct kif)); + if (kif == NULL) { + PyErr_NoMemory(); + goto error; + } + kif->kif = &ki[j]; + if (j == 0) { + kif->has_buf = 1; + kif->buf = buf; + } + else { + kif->has_buf = 0; + kif->buf = NULL; + } + SLIST_INSERT_HEAD(&kihead, kif, kifs); + } } - - /* - // debug - struct kif *k; - SLIST_FOREACH(k, &kihead, kifs) { - printf("%d\n", k->kif->ki_pid); // NOQA + else { + free(buf); } - */ return 0; + +error: + if (buf != NULL) + free(buf); + return -1; } @@ -167,7 +178,7 @@ static int psutil_get_sockets(const char *name) { size_t namelen; int mib[8]; - struct kinfo_pcb *pcb; + struct kinfo_pcb *pcb = NULL; size_t len; size_t j; @@ -175,17 +186,17 @@ psutil_get_sockets(const char *name) { if (sysctlnametomib(name, mib, &namelen) == -1) { PyErr_SetFromErrno(PyExc_OSError); - return -1; + goto error; } if (sysctl(mib, __arraycount(mib), NULL, &len, NULL, 0) == -1) { PyErr_SetFromErrno(PyExc_OSError); - return -1; + goto error; } if ((pcb = malloc(len)) == NULL) { PyErr_NoMemory(); - return -1; + goto error; } memset(pcb, 0, len); @@ -193,20 +204,42 @@ psutil_get_sockets(const char *name) { mib[7] = len / sizeof(*pcb); if (sysctl(mib, __arraycount(mib), pcb, &len, NULL, 0) == -1) { - free(pcb); PyErr_SetFromErrno(PyExc_OSError); - return -1; + goto error; } len /= sizeof(struct kinfo_pcb); struct kinfo_pcb *kp = (struct kinfo_pcb *)pcb; - for (j = 0; j < len; j++) { - struct kpcb *kpcb = malloc(sizeof(struct kpcb)); - kpcb->kpcb = &kp[j]; - SLIST_INSERT_HEAD(&kpcbhead, kpcb, kpcbs); + if (len > 0) { + for (j = 0; j < len; j++) { + struct kpcb *kpcb = malloc(sizeof(struct kpcb)); + if (kpcb == NULL) { + PyErr_NoMemory(); + goto error; + } + kpcb->kpcb = &kp[j]; + if (j == 0) { + kpcb->has_buf = 1; + kpcb->buf = pcb; + } + else { + kpcb->has_buf = 0; + kpcb->buf = NULL; + } + SLIST_INSERT_HEAD(&kpcbhead, kpcb, kpcbs); + } + } + else { + free(pcb); } + return 0; + +error: + if (pcb != NULL) + free(pcb); + return -1; } @@ -318,6 +351,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { psutil_kiflist_init(); psutil_kpcblist_init(); + if (psutil_get_files() != 0) goto error; if (psutil_get_info(ALL) != 0) @@ -429,8 +463,11 @@ psutil_net_connections(PyObject *self, PyObject *args) { return py_retlist; error: + psutil_kiflist_clear(); + psutil_kpcblist_clear(); + Py_DECREF(py_retlist); Py_XDECREF(py_tuple); Py_XDECREF(py_laddr); Py_XDECREF(py_raddr); - return 0; + return NULL; } diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index da3c37932..cd1b3f290 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -25,7 +25,6 @@ import psutil._common from psutil import LINUX from psutil import MACOS -from psutil import NETBSD from psutil import OPENBSD from psutil import POSIX from psutil import SUNOS @@ -249,7 +248,6 @@ def test_rlimit_set(self): # Windows implementation is based on a single system-wide # function (tested later). @unittest.skipIf(WINDOWS, "worthless on WINDOWS") - @unittest.skipIf(NETBSD, "critically broken on NETBSD (#930)") def test_connections(self): # TODO: UNIX sockets are temporarily implemented by parsing # 'pfiles' cmd output; we don't want that part of the code to @@ -422,7 +420,6 @@ def test_net_io_counters(self): @fewtimes_if_linux() @unittest.skipIf(MACOS and os.getuid() != 0, "need root access") - @unittest.skipIf(NETBSD, "critically broken on NETBSD (#930)") def test_net_connections(self): # always opens and handle on Windows() (once) psutil.net_connections(kind='all') From cb3ab3782faaff9b78bcbdc2f47582c8f4ef1c0a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 20 Dec 2023 16:42:41 +0100 Subject: [PATCH 2/7] [NetBSD] speedup net_connections() with proper filtering in C (#2342) Modify psutil.net_connections() to retrieving unnecessary connection types unless explicitly asked. E.g., on an IDLE system with few IPv6 connections this will run around 170% faster: ``` import psutil, time started = time.monotonic() for x in range(1000): psutil.net_connections("tcp6") print(f"completed in {(time.monotonic() - started):.4f} secs") ``` Before all connection types (TCP, UDP, UNIX) were retrived internally, even if they were not returned. --- HISTORY.rst | 14 +++ psutil/_psbsd.py | 12 ++- psutil/arch/netbsd/socks.c | 178 +++++++++++++++++-------------------- 3 files changed, 102 insertions(+), 102 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c6ad7e144..932ac21b9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,20 @@ 5.9.8 (IN DEVELOPMENT) ====================== +**Enhancements** + +- 2342_, [NetBSD]: filter `net_connections()`_ returned list in C instead of + Python, and avoid to retrieve unnecessary connection types unless explicitly + asked. E.g., on an IDLE system with few IPv6 connections this will run around + 170% faster. Before all connection types (TCP, UDP, UNIX) were retrived + internally, even if they were not returned.:: + + import psutil, time + started = time.monotonic() + for x in range(1000): + psutil.net_connections("tcp6") + print(f"completed in {(time.monotonic() - started):.4f} secs") + **Bug fixes** - 930_, [NetBSD], [critical]: `net_connections()`_ implementation was broken. diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 6653dc9e1..04b615ff9 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -426,14 +426,13 @@ def net_connections(kind): if OPENBSD: rawlist = cext.net_connections(-1, families, types) elif NETBSD: - rawlist = cext.net_connections(-1) + rawlist = cext.net_connections(-1, kind) else: # FreeBSD rawlist = cext.net_connections() for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item - if NETBSD or FREEBSD: - # OpenBSD implements filtering in C + if FREEBSD: if (fam not in families) or (type not in types): continue nt = conn_to_ntuple(fd, fam, type, laddr, raddr, @@ -786,16 +785,15 @@ def connections(self, kind='inet'): ret = [] if NETBSD: - rawlist = cext.net_connections(self.pid) + rawlist = cext.net_connections(self.pid, kind) elif OPENBSD: rawlist = cext.net_connections(self.pid, families, types) - else: # FreeBSD + else: rawlist = cext.proc_connections(self.pid, families, types) for item in rawlist: fd, fam, type, laddr, raddr, status = item[:6] - if NETBSD: - # FreeBSD and OpenBSD implement filtering in C + if FREEBSD: if (fam not in families) or (type not in types): continue nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, diff --git a/psutil/arch/netbsd/socks.c b/psutil/arch/netbsd/socks.c index 5c8ad3bd6..b3d6aac51 100644 --- a/psutil/arch/netbsd/socks.c +++ b/psutil/arch/netbsd/socks.c @@ -25,21 +25,6 @@ #include "../../_psutil_posix.h" -// address family filter -enum af_filter { - INET, - INET4, - INET6, - TCP, - TCP4, - TCP6, - UDP, - UDP4, - UDP6, - UNIX, - ALL, -}; - // kinfo_file results struct kif { SLIST_ENTRY(kif) kifs; @@ -245,84 +230,85 @@ psutil_get_sockets(const char *name) { // Collect open file and connections. static int -psutil_get_info(int aff) { - switch (aff) { - case INET: - if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet.udp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) - return -1; - break; - case INET4: - if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet.udp.pcblist") != 0) - return -1; - break; - case INET6: - if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) - return -1; - break; - case TCP: - if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) - return -1; - break; - case TCP4: - if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) - return -1; - break; - case TCP6: - if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) - return -1; - break; - case UDP: - if (psutil_get_sockets("net.inet.udp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) - return -1; - break; - case UDP4: - if (psutil_get_sockets("net.inet.udp.pcblist") != 0) - return -1; - break; - case UDP6: - if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) - return -1; - break; - case UNIX: - if (psutil_get_sockets("net.local.stream.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.local.seqpacket.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.local.dgram.pcblist") != 0) - return -1; - break; - case ALL: - if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet.udp.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.local.stream.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.local.seqpacket.pcblist") != 0) - return -1; - if (psutil_get_sockets("net.local.dgram.pcblist") != 0) - return -1; - break; +psutil_get_info(char *kind) { + if (strcmp(kind, "inet") == 0) { + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "inet4") == 0) { + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "inet6") == 0) { + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "tcp") == 0) { + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "tcp4") == 0) { + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "tcp6") == 0) { + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "udp") == 0) { + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "udp4") == 0) { + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "udp6") == 0) { + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "unix") == 0) { + if (psutil_get_sockets("net.local.stream.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.seqpacket.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.dgram.pcblist") != 0) + return -1; + } + else if (strcmp(kind, "all") == 0) { + if (psutil_get_sockets("net.inet.tcp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet.udp.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.tcp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.inet6.udp6.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.stream.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.seqpacket.pcblist") != 0) + return -1; + if (psutil_get_sockets("net.local.dgram.pcblist") != 0) + return -1; + } + else { + PyErr_SetString(PyExc_ValueError, "invalid kind value"); + return -1; } - return 0; } @@ -334,6 +320,7 @@ PyObject * psutil_net_connections(PyObject *self, PyObject *args) { char laddr[PATH_MAX]; char raddr[PATH_MAX]; + char *kind; int32_t lport; int32_t rport; int32_t status; @@ -346,15 +333,16 @@ psutil_net_connections(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; - if (! PyArg_ParseTuple(args, "l", &pid)) - return NULL; + if (! PyArg_ParseTuple(args, _Py_PARSE_PID "s", &pid, &kind)) { + goto error; + } psutil_kiflist_init(); psutil_kpcblist_init(); if (psutil_get_files() != 0) goto error; - if (psutil_get_info(ALL) != 0) + if (psutil_get_info(kind) != 0) goto error; struct kif *k; From 5003480c03a648f2406e7d962c5b165e29fbfb87 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 20 Dec 2023 07:57:30 +0000 Subject: [PATCH 3/7] NetBSD: use _Py_PARSE_PID instead of long Signed-off-by: Giampaolo Rodola --- psutil/arch/netbsd/proc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/psutil/arch/netbsd/proc.c b/psutil/arch/netbsd/proc.c index 300f81753..c645f301e 100644 --- a/psutil/arch/netbsd/proc.c +++ b/psutil/arch/netbsd/proc.c @@ -93,7 +93,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { char path[MAXPATHLEN]; size_t pathlen = sizeof path; - if (! PyArg_ParseTuple(args, "l", &pid)) + if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) return NULL; #ifdef KERN_PROC_CWD @@ -116,7 +116,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { free(buf); if (len == -1) { if (errno == ENOENT) - NoSuchProcess("sysctl -> ENOENT"); + NoSuchProcess("readlink -> ENOENT"); else PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -143,7 +143,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { int ret; size_t size; - if (! PyArg_ParseTuple(args, "l", &pid)) + if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) return NULL; if (pid == 0) { // else returns ENOENT @@ -190,7 +190,7 @@ psutil_proc_num_threads(PyObject *self, PyObject *args) { // Return number of threads used by process as a Python integer. long pid; kinfo_proc kp; - if (! PyArg_ParseTuple(args, "l", &pid)) + if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) return NULL; if (psutil_kinfo_proc(pid, &kp) == -1) return NULL; @@ -211,7 +211,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; - if (! PyArg_ParseTuple(args, "l", &pid)) + if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) goto error; mib[0] = CTL_KERN; @@ -396,7 +396,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { struct kinfo_file *freep; - if (! PyArg_ParseTuple(args, "l", &pid)) + if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) return NULL; errno = 0; From 3aa5147ea7529e44825fd796d7fd216f0c4b8c46 Mon Sep 17 00:00:00 2001 From: Hang Date: Thu, 21 Dec 2023 02:38:29 +0800 Subject: [PATCH 4/7] Fix Alpine build doc (#2326) Signed-off-by: Hang --- INSTALL.rst | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 2e8a1cd51..e3dab9cb4 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -27,7 +27,7 @@ RedHat / CentOS:: Alpine:: - sudo apk add gcc python3-dev + sudo apk add gcc python3-dev musl-dev linux-headers pip install --no-binary :all: psutil Windows (build) diff --git a/setup.py b/setup.py index 1995fa055..ae25116ae 100755 --- a/setup.py +++ b/setup.py @@ -473,7 +473,7 @@ def main(): elif which('rpm'): missdeps("sudo yum install gcc %s%s-devel" % (pyimpl, py3)) elif which('apk'): - missdeps("sudo apk add gcc %s%s-dev" % (pyimpl, py3)) + missdeps("sudo apk add gcc %s%s-dev musl-dev linux-headers" % (pyimpl, py3)) elif MACOS: print(hilite("XCode (https://developer.apple.com/xcode/) " "is not installed", color="red"), file=sys.stderr) From e0f5b0842fb8a1100ad12ca4fb27350c2886260e Mon Sep 17 00:00:00 2001 From: Marcel Telka Date: Wed, 20 Dec 2023 19:40:30 +0100 Subject: [PATCH 5/7] Remove useless shebang from tests/__main__.py (#2316) --- psutil/tests/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 434515d21..0cf39cc80 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be From 2786c095c7cd0177940789daac3ba25d66efb84f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 20 Dec 2023 19:41:39 +0100 Subject: [PATCH 6/7] remove +x bit from psutil/tests/__main__.py Signed-off-by: Giampaolo Rodola --- psutil/tests/__main__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 psutil/tests/__main__.py diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py old mode 100755 new mode 100644 From 13e1fe7eaef37193f4f2980d3684a06dfcb953a9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 20 Dec 2023 22:43:27 +0100 Subject: [PATCH 7/7] [FreeBSD] speed up net_connections() (#2343) --- HISTORY.rst | 13 ++---- psutil/_psbsd.py | 5 +- psutil/arch/freebsd/sys_socks.c | 83 ++++++++++++++++++++++++++++----- 3 files changed, 77 insertions(+), 24 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 932ac21b9..0f84d59f4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,17 +5,12 @@ **Enhancements** -- 2342_, [NetBSD]: filter `net_connections()`_ returned list in C instead of +- 2343_, [FreeBSD]: filter `net_connections()`_ returned list in C instead of Python, and avoid to retrieve unnecessary connection types unless explicitly asked. E.g., on an IDLE system with few IPv6 connections this will run around - 170% faster. Before all connection types (TCP, UDP, UNIX) were retrived - internally, even if they were not returned.:: - - import psutil, time - started = time.monotonic() - for x in range(1000): - psutil.net_connections("tcp6") - print(f"completed in {(time.monotonic() - started):.4f} secs") + 4 times faster. Before all connection types (TCP, UDP, UNIX) were retrieved + internally, even if only a portion was returned. +- 2342_, [NetBSD]: same as above (#2343) but for NetBSD. **Bug fixes** diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 04b615ff9..fc03d1114 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -428,13 +428,10 @@ def net_connections(kind): elif NETBSD: rawlist = cext.net_connections(-1, kind) else: # FreeBSD - rawlist = cext.net_connections() + rawlist = cext.net_connections(families, types) for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item - if FREEBSD: - if (fam not in families) or (type not in types): - continue nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, TCP_STATUSES, pid) ret.add(nt) diff --git a/psutil/arch/freebsd/sys_socks.c b/psutil/arch/freebsd/sys_socks.c index 48d3fb4f5..3887bd403 100644 --- a/psutil/arch/freebsd/sys_socks.c +++ b/psutil/arch/freebsd/sys_socks.c @@ -33,7 +33,7 @@ static struct xfile *psutil_xfiles; static int psutil_nxfiles; -int +static int psutil_populate_xfiles(void) { size_t len; @@ -61,7 +61,7 @@ psutil_populate_xfiles(void) { } -struct xfile * +static struct xfile * psutil_get_file_from_sock(kvaddr_t sock) { struct xfile *xf; int n; @@ -76,7 +76,10 @@ psutil_get_file_from_sock(kvaddr_t sock) { // Reference: // https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c -int psutil_gather_inet(int proto, PyObject *py_retlist) { +static int +psutil_gather_inet( + int proto, int include_v4, int include_v6, PyObject *py_retlist) +{ struct xinpgen *xig, *exig; struct xinpcb *xip; struct xtcpcb *xtp; @@ -177,6 +180,12 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { goto error; } + // filter + if ((inp->inp_vflag & INP_IPV4) && (include_v4 == 0)) + continue; + if ((inp->inp_vflag & INP_IPV6) && (include_v6 == 0)) + continue; + char lip[200], rip[200]; xf = psutil_get_file_from_sock(so->xso_so); @@ -235,7 +244,8 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { } -int psutil_gather_unix(int proto, PyObject *py_retlist) { +static int +psutil_gather_unix(int proto, PyObject *py_retlist) { struct xunpgen *xug, *exug; struct xunpcb *xup; const char *varname = NULL; @@ -339,23 +349,74 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { } +static int +psutil_int_in_seq(int value, PyObject *py_seq) { + int inseq; + PyObject *py_value; + + py_value = PyLong_FromLong((long)value); + if (py_value == NULL) + return -1; + inseq = PySequence_Contains(py_seq, py_value); // return -1 on failure + Py_DECREF(py_value); + return inseq; +} + + PyObject* psutil_net_connections(PyObject* self, PyObject* args) { - // Return system-wide open connections. + int include_v4, include_v6, include_unix, include_tcp, include_udp; + PyObject *py_af_filter = NULL; + PyObject *py_type_filter = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; - if (psutil_populate_xfiles() != 1) + if (! PyArg_ParseTuple(args, "OO", &py_af_filter, &py_type_filter)) { + goto error; + } + if (!PySequence_Check(py_af_filter) || !PySequence_Check(py_type_filter)) { + PyErr_SetString(PyExc_TypeError, "arg 2 or 3 is not a sequence"); + goto error; + } + + if ((include_v4 = psutil_int_in_seq(AF_INET, py_af_filter)) == -1) goto error; - if (psutil_gather_inet(IPPROTO_TCP, py_retlist) == 0) + if ((include_v6 = psutil_int_in_seq(AF_INET6, py_af_filter)) == -1) goto error; - if (psutil_gather_inet(IPPROTO_UDP, py_retlist) == 0) + if ((include_unix = psutil_int_in_seq(AF_UNIX, py_af_filter)) == -1) goto error; - if (psutil_gather_unix(SOCK_STREAM, py_retlist) == 0) - goto error; - if (psutil_gather_unix(SOCK_DGRAM, py_retlist) == 0) + if ((include_tcp = psutil_int_in_seq(SOCK_STREAM, py_type_filter)) == -1) goto error; + if ((include_udp = psutil_int_in_seq(SOCK_DGRAM, py_type_filter)) == -1) + goto error; + + if (psutil_populate_xfiles() != 1) + goto error; + + // TCP + if (include_tcp == 1) { + if (psutil_gather_inet( + IPPROTO_TCP, include_v4, include_v6, py_retlist) == 0) + { + goto error; + } + } + // UDP + if (include_udp == 1) { + if (psutil_gather_inet( + IPPROTO_UDP, include_v4, include_v6, py_retlist) == 0) + { + goto error; + } + } + // UNIX + if (include_unix == 1) { + if (psutil_gather_unix(SOCK_STREAM, py_retlist) == 0) + goto error; + if (psutil_gather_unix(SOCK_DGRAM, py_retlist) == 0) + goto error; + } free(psutil_xfiles); return py_retlist;