diff --git a/.github/workflows/libtrace.yaml b/.github/workflows/libtrace.yaml index aa546fa3..6756d7bc 100644 --- a/.github/workflows/libtrace.yaml +++ b/.github/workflows/libtrace.yaml @@ -66,7 +66,7 @@ jobs: sudo apt-get -y install pfring sudo ldconfig - name: Checkout libxdp - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: xdp-tools repository: xdp-project/xdp-tools @@ -103,7 +103,7 @@ jobs: sudo make -j 3 libxdp_install sudo ldconfig - name: Checkout Libbpf - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: libbpf repository: libbpf/libbpf @@ -115,7 +115,7 @@ jobs: echo "/usr/lib64" | sudo tee -a /etc/ld.so.conf.d/lib64.conf sudo ldconfig - name: Checkout Libtrace - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: libtrace submodules: recursive @@ -162,31 +162,31 @@ jobs: matrix: os: [macos-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Install Dependencies run: | - brew install flex bison libpcap libtool autoconf automake m4 gcc uthash + brew install openssl flex bison libpcap libtool autoconf automake m4 gcc uthash - name: Build/Install Wandio run: | cd wandio ./bootstrap.sh - ./configure + ./configure CFLAGS="-I/opt/homebrew/include" LDFLAGS="-L/opt/homebrew/lib" make -j 4 sudo make install - name: Build/Install Libwandder run: | cd libwandder ./bootstrap.sh - ./configure + ./configure CFLAGS="-I/opt/homebrew/include" LDFLAGS="-L/opt/homebrew/lib" make -j 3 sudo make install - name: Build/Install Libtrace run: | ./bootstrap.sh - ./configure - make -j 4 + ./configure CPPFLAGS="-I/opt/homebrew/include" CFLAGS="-I/opt/homebrew/include" LDFLAGS="-L/opt/homebrew/lib" + make V=1 -j 4 sudo make install - name: Build Tests run: cd test; make -j 4 diff --git a/.github/workflows/pkg-build.yaml b/.github/workflows/pkg-build.yaml index 08b0f901..31fd4298 100644 --- a/.github/workflows/pkg-build.yaml +++ b/.github/workflows/pkg-build.yaml @@ -19,12 +19,13 @@ jobs: - "debian:bookworm" - "debian:buster" - "debian:bullseye" + - "ubuntu:noble" - "ubuntu:focal" - "ubuntu:jammy" steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install prereq packages run: ./debpkg-setup.sh - name: Build packages @@ -36,7 +37,7 @@ jobs: mkdir -p packages/${DIRNAME} cp ../*.deb packages/${DIRNAME}/ - name: Store packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: packages-${{ env.DIRNAME }} path: packages/${{ env.DIRNAME }}/*.deb @@ -56,13 +57,14 @@ jobs: - "debian:bullseye" - "ubuntu:focal" - "ubuntu:jammy" + - "ubuntu:noble" - "debian:bookworm" needs: build steps: - name: Set environment variables for download run: echo DIRNAME=${{ matrix.target }} | tr ':' '_' >> $GITHUB_ENV - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: packages-${{ env.DIRNAME }} - name: Add repositories @@ -98,12 +100,13 @@ jobs: - "debian:bookworm" - "ubuntu:focal" - "ubuntu:jammy" + - "ubuntu:noble" needs: test steps: - name: Set environment variables for download run: echo DIRNAME=${{ matrix.target }} | tr ':' '_' >> $GITHUB_ENV - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: packages-${{ env.DIRNAME }} - name: Copy packages @@ -111,7 +114,7 @@ jobs: mkdir -p packages/${DIRNAME} find . -name "*.deb" | xargs cp -t packages/${DIRNAME}/ - name: Publish package to cloudsmith - uses: wanduow/action-cloudsmith-upload-packages@v1 + uses: salcock/action-cloudsmith-upload-packages@v1.8 with: path: packages/ repo: ${{ secrets.CLOUDSMITH_OWNER }}/libtrace diff --git a/.github/workflows/rpm-build.yaml b/.github/workflows/rpm-build.yaml index 676dfe5f..98a113c5 100644 --- a/.github/workflows/rpm-build.yaml +++ b/.github/workflows/rpm-build.yaml @@ -20,11 +20,10 @@ jobs: - "rockylinux:9" - "fedora:39" - "fedora:38" - - "centos:7" steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install prereq packages run: ./rpmpkg-setup.sh ${{ matrix.target }} - name: Build packages @@ -36,7 +35,7 @@ jobs: mkdir -p packages/${DIRNAME} cp ~/rpmbuild/RPMS/x86_64/*.rpm packages/${DIRNAME}/ - name: Store packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: packages-${{ env.DIRNAME }} path: packages/${{ env.DIRNAME }}/*.rpm @@ -56,13 +55,12 @@ jobs: - "rockylinux:9" - "fedora:39" - "fedora:38" - - "centos:7" needs: build steps: - name: Set environment variables for download run: echo DIRNAME=${{ matrix.target }} | tr ':' '_' >> $GITHUB_ENV - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: packages-${{ env.DIRNAME }} - name: Add repositories @@ -71,9 +69,6 @@ jobs: run: | curl -1sLf 'https://dl.cloudsmith.io/public/wand/libwandio/cfg/setup/bash.rpm.sh' | bash curl -1sLf 'https://dl.cloudsmith.io/public/wand/libwandder/cfg/setup/bash.rpm.sh' | bash - if [ "${TARGET}" == "centos:7" ]; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm || true - fi if [[ "${TARGET}" =~ rocky* ]]; then dnf install -y dnf-plugins-core epel-release || true dnf config-manager --set-enabled powertools || true @@ -106,13 +101,12 @@ jobs: - "rockylinux:9" - "fedora:39" - "fedora:38" - - "centos:7" needs: test steps: - name: Set environment variables for download run: echo DIRNAME=${{ matrix.target }} | tr ':' '_' >> $GITHUB_ENV - name: Download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: packages-${{ env.DIRNAME }} - name: Copy packages @@ -121,7 +115,7 @@ jobs: yum install -y findutils find . -name "*.rpm" | xargs cp -t packages/${DIRNAME}/ - name: Publish package to cloudsmith - uses: wanduow/action-cloudsmith-upload-packages@v1 + uses: salcock/action-cloudsmith-upload-packages@v1.8 with: path: packages/ repo: ${{ secrets.CLOUDSMITH_OWNER }}/libtrace diff --git a/README b/README index 9df3f106..b96791f1 100644 --- a/README +++ b/README @@ -1,7 +1,7 @@ -libtrace 4.0.24 +libtrace 4.0.25 Code and documentation added since version 4.0.20 is -Copyright (c) 2023 Shane Alcock and has been contributed as per +Copyright (c) 2023-2024 Shane Alcock and has been contributed as per the terms of the GNU Lesser General Public License version 3. All rights reserved. diff --git a/configure.in b/configure.in index 72e56831..7c2a3c88 100644 --- a/configure.in +++ b/configure.in @@ -3,11 +3,11 @@ # Now you only need to update the version number in two places - below, # and in the README -AC_INIT([libtrace],[4.0.24],[shane@alcock.co.nz],[libtrace]) +AC_INIT([libtrace],[4.0.25],[shane@alcock.co.nz],[libtrace]) LIBTRACE_MAJOR=4 LIBTRACE_MID=0 -LIBTRACE_MINOR=24 +LIBTRACE_MINOR=25 # OpenSolaris hides libraries like libncurses in /usr/gnu/lib, which is not # searched by default - add it to LDFLAGS so we at least have a chance of diff --git a/debian/changelog b/debian/changelog index 9748c1aa..ddde44fe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +libtrace4 (4.0.25-1) unstable; urgency=medium + + * ndag: fix issue where the most recently received packets may not + be delivered to libtrace if the multicast source stops sending. + + -- Shane Alcock Thu, 9 May 2024 10:16:36 +1200 + libtrace4 (4.0.24-1) unstable; urgency=medium * ndagtcp: fix miscalculation of ERF header length that would diff --git a/lib/Makefile.am b/lib/Makefile.am index f4f0e296..82881a4e 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -146,7 +146,7 @@ endif AM_CPPFLAGS= @ADD_INCLS@ libtrace_la_LIBADD = @LIBTRACE_LIBS@ @LTLIBOBJS@ $(DPDKLIBS) -libtrace_la_LDFLAGS=-version-info 7:8:0 @ADD_LDFLAGS@ +libtrace_la_LDFLAGS=-version-info 7:9:0 @ADD_LDFLAGS@ dagapi.c: cp @DAG_TOOLS_DIR@/dagapi.c . diff --git a/lib/format_ndag.c b/lib/format_ndag.c index 90a13844..8a0c8202 100644 --- a/lib/format_ndag.c +++ b/lib/format_ndag.c @@ -1607,7 +1607,8 @@ static int receive_from_sockets(recvstream_t *rt, uint8_t socktype) { #if HAVE_DECL_RECVMMSG /* Plenty of full buffers, just use the packets in those */ if (socktype == NDAG_SOCKET_TYPE_MULTICAST) { - if (rt->sources[i].bufavail < RECV_BATCH_SIZE / 2) { + if (rt->sources[i].bufavail < + RECV_BATCH_SIZE / 2) { readybufs ++; continue; } @@ -1639,11 +1640,16 @@ static int receive_from_sockets(recvstream_t *rt, uint8_t socktype) { } for (i = 0; i < rt->sourcecount; i++) { - if (rt->sources[i].sock == -1 || - !FD_ISSET(rt->sources[i].sock, &fds)) { - if (rt->sources[i].bufavail < ENCAP_BUFFERS) { - readybufs ++; - } + streamsock_t *ssock = &(rt->sources[i]); + if (ssock->sock == -1 || !FD_ISSET(ssock->sock, &fds)) { + + if (ssock->nextreadind != ssock->nextwriteind) { + readybufs ++; + } else if (ssock->savedsize[ssock->nextreadind] - + (ssock->nextread - ssock->saved[ssock->nextreadind]) > 0) { + readybufs ++; + } + continue; } #if HAVE_DECL_RECVMMSG @@ -1666,6 +1672,7 @@ static int receive_encap_records_block(libtrace_t *libtrace, recvstream_t *rt, libtrace_packet_t *packet, libtrace_message_queue_t *msg) { int iserr = 0; + int delay = 0; if (packet->buf_control == TRACE_CTRL_PACKET) { free(packet->buffer); @@ -1686,11 +1693,15 @@ static int receive_encap_records_block(libtrace_t *libtrace, recvstream_t *rt, return iserr; } + if (delay > 1000000) { + return rt->sourcecount + 1; + } /* If blocking and no sources, sleep for a bit and then try * checking for messages again. */ if (rt->sourcecount == 0) { usleep(10000); + delay += 10000; continue; } @@ -1714,6 +1725,7 @@ static int receive_encap_records_block(libtrace_t *libtrace, recvstream_t *rt, */ if (iserr == 0) { usleep(100); + delay += 100; } } while (!rt->halted); @@ -1748,57 +1760,54 @@ static int receive_encap_records_nonblock(libtrace_t *libtrace, recvstream_t *rt } static streamsock_t *select_next_packet(recvstream_t *rt) { - int i, r; - streamsock_t *ssock = NULL; - uint64_t earliest = 0; - uint64_t currentts = 0; - - for (i = 0; i < rt->sourcecount; i ++) { - if (!readable_data(&(rt->sources[i]))) { - continue; - } - - if (rt->sources[i].rectype == 0) { - r = process_ndag_encap_headers(&(rt->sources[i]), rt); - if (r < 0) { - return NULL; - } - if (r == 0) { - continue; - } - } - - if (rt->sources[i].nextts == 0) { - if (rt->sources[i].rectype == NDAG_PKT_ENCAPERF) { - r = process_ndag_erf_headers( - &(rt->sources[i]), rt); - } else if (rt->sources[i].rectype == - NDAG_PKT_CORSAROTAG) { - r = process_ndag_corsaro_header( - &(rt->sources[i]), rt); - } else { - r = 0; - } - - if (r < 0) { - return NULL; - } - if (r == 0) { - continue; - } - } - assert(rt->sources[i].nextts != 0); - currentts = rt->sources[i].nextts; - - if (earliest == 0 || earliest > currentts) { - earliest = currentts; - ssock = &(rt->sources[i]); - } - } - if (ssock && got_complete_packet(ssock)) { - return ssock; - } - return NULL; + int i, r; + streamsock_t *ssock = NULL; + uint64_t earliest = 0; + uint64_t currentts = 0; + + for (i = 0; i < rt->sourcecount; i ++) { + if (!readable_data(&(rt->sources[i]))) { + continue; + } + + if (rt->sources[i].rectype == 0) { + r = process_ndag_encap_headers(&(rt->sources[i]), rt); + if (r < 0) { + return NULL; + } + if (r == 0) { + continue; + } + } + + if (rt->sources[i].nextts == 0) { + if (rt->sources[i].rectype == NDAG_PKT_ENCAPERF) { + r = process_ndag_erf_headers(&(rt->sources[i]), rt); + } else if (rt->sources[i].rectype == NDAG_PKT_CORSAROTAG) { + r = process_ndag_corsaro_header(&(rt->sources[i]), rt); + } else { + r = 0; + } + + if (r < 0) { + return NULL; + } + if (r == 0) { + continue; + } + } + assert(rt->sources[i].nextts != 0); + currentts = rt->sources[i].nextts; + + if (earliest == 0 || earliest > currentts) { + earliest = currentts; + ssock = &(rt->sources[i]); + } + } + if (ssock && got_complete_packet(ssock)) { + return ssock; + } + return NULL; } static int ndag_read_packet(libtrace_t *libtrace, libtrace_packet_t *packet) { @@ -1853,12 +1862,23 @@ static int ndag_pread_packets(libtrace_t *libtrace, libtrace_thread_t *t, rem = receive_encap_records_block(libtrace, rt, packets[read_packets], &t->messages); if (rem < 0) { + if (rem == READ_MESSAGE && read_packets == 0) { return rem; + } else { + /* allow us to return the packets we've read, + * then libtrace can check for messages + */ + break; + } } if (rem == 0) { break; } + + if (rem > rt->sourcecount && read_packets > 0) { + break; + } } nextavail = select_next_packet(rt); if (nextavail == NULL) { diff --git a/rpm/libtrace4.spec b/rpm/libtrace4.spec index 08e2074d..876120d1 100644 --- a/rpm/libtrace4.spec +++ b/rpm/libtrace4.spec @@ -1,5 +1,5 @@ Name: libtrace4 -Version: 4.0.24 +Version: 4.0.25 Release: 1%{?dist} Summary: C Library for capturing and analysing network packets @@ -127,6 +127,9 @@ find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' %changelog +* Thu May 9 2024 Shane Alcock - 4.0.25-1 +- Updated for 4.0.25 release + * Wed Jan 24 2024 Shane Alcock - 4.0.24-1 - Updated for 4.0.24 release diff --git a/test/test-datastruct-ringbuffer.c b/test/test-datastruct-ringbuffer.c index 0214c738..50f5667c 100644 --- a/test/test-datastruct-ringbuffer.c +++ b/test/test-datastruct-ringbuffer.c @@ -20,6 +20,9 @@ static void * consumer(void * a) { void *value; for (i = NULL; i < TEST_SIZE; i++) { value = libtrace_ringbuffer_read(rb); + if (value != i) { + fprintf(stderr, "value: %p i: %p\n", value, i); + } assert(value == i); } return 0; @@ -40,6 +43,9 @@ static void * consumer_bulk(void * a) { void *value; for (i = NULL; i < TEST_SIZE; i++) { assert (libtrace_ringbuffer_read_bulk(rb, &value, 1, 1) == 1); + if (value != i) { + fprintf(stderr, "value: %p i: %p\n", value, i); + } assert(value == i); } return 0; diff --git a/tools/Makefile.tools b/tools/Makefile.tools index 693c984c..ef9ca551 100644 --- a/tools/Makefile.tools +++ b/tools/Makefile.tools @@ -1,6 +1,6 @@ -AM_CFLAGS=-I"$(top_srcdir)/lib" -I"$(top_srcdir)/libpacketdump" -AM_CXXFLAGS=-I"$(top_srcdir)/lib" -I"$(top_srcdir)/libpacketdump" +AM_CFLAGS=-I"$(top_srcdir)/lib" -I"$(top_srcdir)/libpacketdump" $(CFLAGS) +AM_CXXFLAGS=-I"$(top_srcdir)/lib" -I"$(top_srcdir)/libpacketdump" $(CPPFLAGS) AM_LDFLAGS=-L"$(top_srcdir)/lib" -L"$(top_srcdir)/libpacketdump" \ - -ltrace @TOOLS_LIBS@ + $(LDFLAGS) -ltrace @TOOLS_LIBS@ diff --git a/tools/tracemcast/traceucast.c b/tools/tracemcast/traceucast.c index 91c6af5e..484978c9 100644 --- a/tools/tracemcast/traceucast.c +++ b/tools/tracemcast/traceucast.c @@ -531,7 +531,7 @@ static libtrace_packet_t *packet_reader_thread(libtrace_t *trace UNUSED, rdata->bufsize + MAX_PACKET_SIZE); rdata->bufsize += MAX_PACKET_SIZE; rdata->writeptr = rdata->pbuffer + writeoff; - rdata->encaphdr = rdata->pbuffer + encapoff; + rdata->encaphdr = (ndag_encap_t *)(rdata->pbuffer + encapoff); } /* append this packet to the buffer */