Release v5.0 (#963)

Huge thanks to everyone who contributed, did some testing and put up with some buggy releases!! It wouldn't be possible with yall! Co-authored-by: Tad Smith <[email protected]> Co-authored-by: Nick <[email protected]> Co-authored-by: ge0metrix <[email protected]> Co-authored-by: decoy4fun <[email protected]> Co-authored-by: tadscottsmith <[email protected]> Co-authored-by: penguinzephyr <[email protected]>
robotastic · Jul 25, 2024 · 2d838f7 · 2d838f7
1 parent f0af22d
commit 2d838f7
Show file tree

Hide file tree

Showing 106 changed files with 5,076 additions and 3,349 deletions.
diff --git a/.gitignore b/.gitignore
@@ -31,3 +31,5 @@ top_block.py
 debug_recorder.py
 /cmake_build/
 /.cache/
+.gitmodules
+user_plugins/*/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -22,7 +22,9 @@
 ########################################################################
 
 cmake_minimum_required(VERSION 3.12)
-project(Trunk-Recorder LANGUAGES CXX C VERSION "4.7.2")
+
+project(Trunk-Recorder LANGUAGES CXX C VERSION "5.0.0")
+
 
 configure_file(cmake.h.in "${PROJECT_BINARY_DIR}/cmake.h" @ONLY)
 add_definitions(-DMANUAL_GITINFO="${MANUAL_GITINFO}")
@@ -219,12 +221,14 @@ link_directories(
     ${OPENSSL_ROOT_DIR}/lib
 )
 
-set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wno-unused-local-typedef -Wno-deprecated-declarations -Wno-error=deprecated-declarations -g3")
+set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wno-deprecated-declarations -Wno-error=deprecated-declarations -g3")
 
 SET(CMAKE_CXX_STANDARD 17)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-narrowing -fvisibility=hidden -fPIC")
+# The D_GLIBCXX_ASSERTIONS flag will cause a crash when an error is detected, making it easier to catch bugs
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-narrowing -fvisibility=hidden -fPIC -D_GLIBCXX_ASSERTIONS")
 
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-narrowing -fvisibility=hidden -fPIC")
 
 
 list(APPEND trunk_recorder_sources
@@ -250,6 +254,8 @@ list(APPEND trunk_recorder_sources
   trunk-recorder/sources/iq_file_source.cc
   trunk-recorder/csv_helper.cc
   trunk-recorder/config.cc
+  trunk-recorder/setup_systems.cc
+  trunk-recorder/monitor_systems.cc
   trunk-recorder/talkgroup.cc
   trunk-recorder/talkgroups.cc
   trunk-recorder/unit_tag.cc
@@ -272,8 +278,13 @@ list(APPEND trunk_recorder_sources
   trunk-recorder/gr_blocks/decoder_wrapper_impl.cc
   trunk-recorder/gr_blocks/plugin_wrapper_impl.cc
   trunk-recorder/gr_blocks/selector_impl.cc
+  trunk-recorder/gr_blocks/pwr_squelch_cc_impl.cc
+  trunk-recorder/gr_blocks/squelch_base_cc_impl.cc
   trunk-recorder/gr_blocks/wavfile_gr3.8.cc
   trunk-recorder/gr_blocks/rms_agc.cc
+  trunk-recorder/gr_blocks/channelizer.cc
+    trunk-recorder/gr_blocks/xlat_channelizer.cc
+    trunk-recorder/gr_blocks/signal_detector_cvf_impl.cc
   )
 
 
@@ -337,19 +348,28 @@ install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/lib/json.hpp" # source file
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/trunk-recorder" # target directory
 )
 
-
+# Add built-in plugins located in /plugins
 add_subdirectory(plugins/openmhz_uploader)
-
 add_subdirectory(plugins/stat_socket)
-
 add_subdirectory(plugins/broadcastify_uploader)
-
 add_subdirectory(plugins/unit_script)
-
 add_subdirectory(plugins/rdioscanner_uploader)
-
 add_subdirectory(plugins/simplestream)
 
+# Add user plugins located in /user_plugins
+# Matching: /user_plugins/${plugin_dir}/CMakeLists.txt
+file(GLOB plugin_dirs "user_plugins/*")
+foreach(plugin_dir ${plugin_dirs})
+  if(IS_DIRECTORY ${plugin_dir})
+    file(GLOB cmakelists "${plugin_dir}/CMakeLists.txt")
+    if(cmakelists)
+      add_subdirectory(${plugin_dir})
+      get_filename_component(dir_name ${plugin_dir} NAME)
+      message(STATUS "Added user plugin: ${dir_name}")
+    endif()
+  endif()
+endforeach()
+
 add_executable(trunk-recorder trunk-recorder/main.cc) # ${trunk_recorder_sources})
 
 target_link_libraries(trunk-recorder git trunk_recorder_library gnuradio-op25_repeater   ${CMAKE_DL_LIBS} ssl crypto ${CURL_LIBRARIES} ${Boost_LIBRARIES} ${GNURADIO_PMT_LIBRARIES} ${GNURADIO_RUNTIME_LIBRARIES} ${GNURADIO_FILTER_LIBRARIES} ${GNURADIO_DIGITAL_LIBRARIES} ${GNURADIO_ANALOG_LIBRARIES} ${GNURADIO_AUDIO_LIBRARIES} ${GNURADIO_UHD_LIBRARIES} ${UHD_LIBRARIES} ${GNURADIO_BLOCKS_LIBRARIES} ${GNURADIO_OSMOSDR_LIBRARIES} ) # gRPC::grpc++_reflection protobuf::libprotobuf)

diff --git a/Dockerfile b/Dockerfile
@@ -50,6 +50,20 @@ RUN cd /tmp && \
   cd /tmp && \
   rm -rf librtlsdr
 
+# Compile librtlsdr-dev 2.0 for SDR-Blog v4 support and other updates
+# Ubuntu 22.04 LTS has librtlsdr 0.6.0
+RUN cd /tmp && \
+  git clone https://github.com/steve-m/librtlsdr.git && \
+  cd librtlsdr && \
+  mkdir build && \
+  cd build && \
+  cmake .. && \
+  make -j$(nproc) && \
+  make install && \
+  ldconfig && \
+  cd /tmp && \
+  rm -rf librtlsdr
+
 # Compile gr-osmosdr ourselves using a fork with various patches included
 RUN cd /tmp && \
   git clone https://github.com/racerxdl/gr-osmosdr.git && \

diff --git a/README.md b/README.md
@@ -1,48 +1,52 @@
-Trunk Recorder - v4.7.2
-=======================
+![Trunk Recorder](./docs/media/trunk-recorder-header.png)
+---
+[![Discord](./docs/media/discord.jpg)](https://discord.gg/btJAhESnks) &nbsp;&nbsp;
+
 
 ## Sponsors
 **Do you find Trunk Recorder and OpenMHz useful?** 
 Become a [Sponsor](https://github.com/sponsors/robotastic) to help support continued development and operation!
 Thank you to everyone who has contributed!
 
+## 🎉 V5.0 Our Best Release Yet!!
+Thanks to everyone who contributed, tested and helped collect cored dumps! 
+
 ## Overview
 Need help? Got something working? Share it!
 
-[Discord Server](https://discord.gg/btJAhESnks) - and don't forget the [Wiki](https://github.com/robotastic/trunk-recorder/wiki)
 
-[Documentation](https://trunkrecorder.com/docs/intro)
+- [Discord Server](https://discord.gg/btJAhESnks) 
+
+- [Documentation](https://trunkrecorder.com/docs/intro)
+
+- ... and don't forget the [Wiki](https://github.com/robotastic/trunk-recorder/wiki)
+
+![screenshot](./docs/media/screenshot.jpg)
 
 Trunk Recorder is able to record the calls on trunked and conventional radio systems. It uses 1 or more Software Defined Radios (SDRs) to do this. The SDRs capture large swathes of RF and then use software to process what was received. [GNU Radio](https://gnuradio.org/) is used to do this processing because it provides lots of convenient RF blocks that can be pieced together to allow for complex RF processing. The libraries from the amazing [OP25](http://op25.osmocom.org/trac/wiki) project are used for a lot of the P25 functionality. Multiple radio systems can be recorded at the same time.
 
 
 Trunk Recorder currently supports the following:
 
  - Trunked P25 & SmartNet Systems
- - Conventional P25 & analog systems, where each group has a dedicated RF channel
+ - Conventional P25, DMR & analog systems, where each talkgroup has a dedicated RF channel
  - P25 Phase 1, P25 Phase 2 & Analog voice channels
 
-Supported platforms:
+### Supported platforms
 
-**Ubuntu** (18.04,  20.04, 21.04, 22.04, 23.04); **Raspberry Pi** (Raspberry OS/Raspbian & Ubuntu 21.04, 22.04); **Arch Linux** (2021.09.20); **Debian** (9.x); **macOS**
+- **Ubuntu** (18.04,  20.04, 21.04, 22.04, 23.04) 
+- **Raspberry Pi** (Raspberry OS/Raspbian & Ubuntu 21.04, 22.04) 
+- **Arch Linux** (2021.09.20)
+- **Debian** (9.x)
+- **macOS**
 
 GNU Radio 3.7 - 3.10
 
-...and SDRs:
+### SDRs
 
 RTL-SDR dongles; HackRF; Ettus USRP B200, B210, B205; BladeRF; Airspy; SDRplay
 
 
-## Version Notes
-### v4.7
-- The talkgroup file format was updated and Column headers are now required. Details are [here](docs/CONFIGURE.md#talkgroupsfile).
-### v4.0
-- The executable generated has changed from `recorder` to `trunk-recorder` to help differentiate it from other applications that maybe installed.
-- A new method is used to detect the end of call. Instead of waiting for a timeout after the last trunking message is received, the voice channel is monitored for messages announcing the end of a transmission. Each transmission is stored in a separate file and then merged together after a talkgroup stops using a frequency.
-- The format for audio filenames has changed slightly. 
-  It is now: [ Talkgroup ID ]\_[ Unix Timestamp ]-[ Frequency ]-call\_[ Call Counter ].wav
-
-
 ## Install
 
 ### Linux

diff --git a/docs/CONFIGURE.md b/docs/CONFIGURE.md
@@ -166,6 +166,7 @@ There is a list of available Plugins [here](./Plugins.md).
 | gain             |    ✓     |               | number                      | The RF gain setting for the SDR. Use a program like GQRX to find a good value. |
 | digitalRecorders |          |               | number                      | The number of Digital Recorders to have attached to this source. This is essentially the number of simultaneous calls you can record at the same time in the frequency range that this Source will be tuned to. It is limited by the CPU power of the machine. Some experimentation might be needed to find the appropriate number. *This is only required for Trunk systems. Channels in Conventional systems have dedicated recorders and do not need to be included here.* |
 | analogRecorders  |          |               | number                      | The number of Analog Recorder to have attached to this source. The same as Digital Recorders except for Analog Voice channels. *This is only required for Trunk systems. Channels in Conventional systems have dedicated recorders and do not need to be included here.* |
+| signalDetectorThreshold |       |           | number                      | If set, a static threshold will be used for the Signal Detector on all conventional recorder. Otherwise, the threshold value for the noise floor will be automatically be determined. Only set this is you are having problems. The value is in dB, but is generally higher than the Squelch value because the power is measured differently |
 | ppm              |          |       0       | number                      | The tuning error for the SDR in ppm (parts per million), as an alternative to `error` above. Use a program like GQRX to find an accurate value. |
 | agc              |          |     false     | **true** / **false**        | Whether or not to enable the SDR's automatic gain control (if supported). This is false by default. It is not recommended to set this as it often yields worse performance compared to a manual gain setting. |
 | gainSettings     |          |               | { "stageName": value}       | Set the gain for any stage. The value for this setting should be passed as an object, where the key specifies the name of the gain stage and the value is the amount of gain, as an int. For example:<br /> ````"gainSettings": { "IF": 10, "BB": 11},```` |
@@ -213,14 +214,15 @@ There is a list of available Plugins [here](./Plugins.md).
 | Key                    | Required | Default Value              | Type                                                                         | Description                                                  |
 | ---------------------- | :------: | -------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------ |
 | shortName              |    ✓     |                            | string                                                                       | This is a nickname for the system. It is used to help name and organize the recordings from this system. It should be 4-6 letters with no spaces. |
-| type                   |    ✓     |                            | **"smartnet"**, **"p25"**, **"conventional"**, **"conventionalDMR"** or **"conventionalP25"** | The type of radio system.                                    |
+| type                   |    ✓     |                            | **"smartnet"**, **"p25"**, **"conventional"**, **"conventionalDMR"** or **"conventionalP25"**, **"conventionalSIGMF"** | The type of radio system.                                    |
 | control_channels       |    ✓     |                            | array of numbers;<br />[496537500, 496437500]                                | *For trunked systems* The control channel frequencies for the system, in Hz. The frequencies will automatically be cycled through if the system moves to an alternate channel. |
 | channels               |    ✓     |                            | array of numbers;<br />[166725000, 166925000, 167075000, 166850000]          | *For conventional systems*  The channel frequencies, in Hz, used for the system. The channels get assigned a virtual talkgroup number based upon their position in the array. Squelch levels need to be specified for the Source(s) being used. |
 | channelFile            |    ✓     |                            | string                                                                       | *For conventional systems* The filename for a CSV file that provides information about the conventional channels. The format for the file is described below. Squelch levels need to be specified for the Source(s) being used. *Use channels or channelFile, not both*. |
 | modulation             |          | "qpsk"                     | **"qpsk"** or  **"fsk4"**                                                    | The type of digital modulation that the system uses. You do not need to specify this with **conventionalDMR** systems.          |
 | squelch                |          | -160                       | number                                                                       | Squelch in DB, this needs to be set for all conventional systems. The squelch setting is also used for analog talkgroups in a SmartNet system. I generally use -60 for my rtl-sdr. The closer the squelch is to 0, the stronger the signal has to be to unmute it. |
 | talkgroupsFile         |          |                            | string                                                                       | The filename for a CSV file that provides information about the talkgroups. It determines whether a talkgroup is analog or digital, and what priority it should have. This file should be located in the same directory as the trunk-recorder executable. |
 | apiKey                 |          |                            | string                                                                       | *if uploadServer is set* System-specific API key for uploading calls to OpenMHz.com. See the Config tab for your system in OpenMHz to find what the value should be. |
+| openmhzSystemId        |          | `shortName`                | string                                                                       | *if uploadServer is set* By default, the plugin will upload calls to the `shortName` OpenMHz system.  Setting this value will allow uploads to any specific OpenMHz system with its valid API key.  This is useful in a multi-site setup where multiple trunk-recorder systems may be aggregating calls to the same OpenMHz feed. | 
 | broadcastifyApiKey     |          |                            | string                                                                       | *if broadcastifyCallsServer is set* System-specific API key for Broadcastify Calls |
 | broadcastifySystemId   |          |                            | number                                                                       | *if broadcastifyCallsServer is set* System ID for Broadcastify Calls <br />(this is an integer, and different from the RadioReference system ID) |
 | uploadScript           |          |                            | string                                                                       | The filename of a script that is called after each recording has finished. Checkout *encode-upload.sh.sample* as an example. Should probably start with `./` ( or `../`). |
@@ -490,29 +492,28 @@ Here are the column headers and some sample data:
 
 ## channelFile
 
-This file allows for you to specify additional information about conventional channels. A recorder is started for each line in the file and set the to frequency specified. The type of recorder is based on the type of System. A **Conventional** system would have Analog Recorders, while **ConventionalP25** or **ConventionalDMR** would have digital recorders.
-
-*Tone based squelch is currently not supported.*
-
+This file allows for you to specify additional information about conventional channels. A recorder is started for each line in the file and set the to frequency specified. The type of recorder is based on the type of System. A **conventional** system would have Analog Recorders, while **conventionalP25** or **conventionalDMR** would have digital recorders. **conventionalSIGMF** is a conventional system with SIGMF Recorders.
 
 | Column Name | Required | Value |
 |-------------|----------|-------|
 | TG Number     | ✔️        | The Talkgroup Number formatted as a decimal number. This has to be the first column |
 | Frequency        |  ✔️       | The frequency in MHz or Hz for the channel (decimal point must be used for MHz) |
-| Tone | ✔️        | The Tone for the talkgroup. This value is not used. *Tone based squelch is currently not supported.* |
+| Tone |        | The CTCSS Tone for the talkgroup. |
 | Alpha Tag |       | A 16 character description that is intended as a shortened display on radio displays |
+| Description |   | A longer description of the talkgroup  |
 | Category |    |  The category for the Talkgroup |
 | Tag       |   |  The Service Tag for the Talkgroup |
 | Comment |        | Use this field to capture comments about a talkgroup. It will be ignored by Trunk Recorder. |
 | Enable |        | Set to 'false' if you do not want this talkgroup/channel to created |
+| Signal Detector |    | Set to `false` if you do not want to use the Signal Detector for this channel. The Signal Detector scans a source's bandwidth and only enables a channel if a signal over a threshold is detected. If it not used, the channel will always be enabled and the Squelch will be running which uses more CPU. Default is `true`|
+| Squelch |    | Value in dB to use for the Squelch for this channel. If this is not set then the System Squelch value will be used instead. |
 
+A **Header Row** is required for the file, with a header provided for each of the columns that will be used. The columns can be in any order. For the Optional columns, if they are left blank for some of the rows, the default value will be used instead.
 
-The **Enable** Column is optional and defaults to *True*. It only needs to be added to rows that you do not want to have recorded. For those rows, set **Enable** to *False*.
-
-| TG Number | Frequency | Tone     | Alpha Tag     | Description            | Tag    | Category  | Enable (*optional*) |
-| --------- | --------- | -------- | ------------- | ---------------------- | ------ | ------ | ------------------- |
-| 300       | 462275000 | 94.8  | Town A Police | Town A Police Dispatch | Police | Town A |                     |
-| 325       | 462275000 | 151.4 | Town B DPW    | Town B Trash Dispatch  | DPW    | Town B | False               |
+| TG Number | Frequency | Tone     | Alpha Tag     | Description            | Tag    | Category  | Enable | Signal Detector | Squelch |
+| --------- | --------- | -------- | ------------- | ---------------------- | ------ | ------ | ------------------- | ---- | ---- |
+| 300       | 462275000 | 94.8  | Town A Police | Town A Police Dispatch | Police | Town A |    |  false |  |
+| 325       | 462275000 | 151.4 | Town B DPW    | Town B Trash Dispatch  | DPW    | Town B | false   |  |  -50 |
 
 
 ## unitTagsFile

diff --git a/docs/Install/INSTALL-DOCKER.md b/docs/Install/INSTALL-DOCKER.md
@@ -6,7 +6,7 @@ sidebar_position: 2
 # Docker
 If you are not going to be modifying the source code, **a [Docker](https://www.docker.com/) based install is the easiest way to get started.** Images are published frequently to  [Docker Hub](https://hub.docker.com/r/robotastic/trunk-recorder). The images have GNURadio 3.8 and all other required dependencies built into it, so it should be ready to go and be a much faster solution than compiling. Images have been built for amd64 (amd64 is used by all modern Intel and AMD CPUs) and most popular flavors of ARM.
 
-To get started, create a directory and place your **config.json** file there and a **talkgroup.csv** file if you are using one. Refer to [Configuring Trunk Recorder](CONFIGURE.md) for instructions on how to create these files.
+To get started, create a directory and place your **config.json** file there and a **talkgroup.csv** file if you are using one. Refer to [Configuring Trunk Recorder](../CONFIGURE.md) for instructions on how to create these files.
 
 ```bash
 docker run -it \