From d4c2def202cf68b117170a52f783f3a7961f314d Mon Sep 17 00:00:00 2001 From: lo-simon Date: Thu, 2 Mar 2023 10:54:38 +0000 Subject: [PATCH 01/35] use conan 1.59.0 rather than using the latest version 2.0.0 (cherry picked from commit 81aa45fa05ddaad94d989facfb1f3fdd1b6a6d45) --- .github/workflows/build-test.yml | 4 ++-- .github/workflows/src/build-setup.yml | 2 +- Documents/Dependencies.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1332081cb..c09f3a195 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -93,7 +93,7 @@ jobs: - name: install conan if: matrix.use_conan == true run: | - pip install conan + pip install conan==1.59.0 conan config set general.revisions_enabled=1 - name: install cmake @@ -600,7 +600,7 @@ jobs: - name: install conan if: matrix.use_conan == true run: | - pip install conan + pip install conan==1.59.0 conan config set general.revisions_enabled=1 - name: install cmake diff --git a/.github/workflows/src/build-setup.yml b/.github/workflows/src/build-setup.yml index 0580afcfc..ff8ad926d 100644 --- a/.github/workflows/src/build-setup.yml +++ b/.github/workflows/src/build-setup.yml @@ -1,7 +1,7 @@ - name: install conan if: matrix.use_conan == true run: | - pip install conan + pip install conan==1.59.0 conan config set general.revisions_enabled=1 - name: install cmake diff --git a/Documents/Dependencies.md b/Documents/Dependencies.md index 58940e7ce..67dea2ee5 100644 --- a/Documents/Dependencies.md +++ b/Documents/Dependencies.md @@ -54,10 +54,10 @@ By default nmos-cpp uses [Conan](https://conan.io) to download most of its depen 1. Install Python 3 if necessary Note: The Python scripts directory needs to be added to the `PATH`, so the Conan executable can be found -2. Install Conan using `pip install conan` +2. Install Conan using `pip install conan==1.59.0` Notes: - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install conan` - - Currently, Conan 1.47 or higher is required; version 1.53.0 (latest release at the time) has been tested + - Currently, Conan 1.47 or up to version 1.59.0 is required; version 1.59.0 has been tested - Conan evolves fairly quickly, so it's worth running `pip install --upgrade conan` regularly - By default [Conan assumes semver compatibility](https://docs.conan.io/en/1.42/creating_packages/define_abi_compatibility.html#versioning-schema). Boost and other C++ libraries do not meet this expectation and break ABI compatibility between e.g. minor versions. From f0642601503f00a55a111660439eca868a661fcf Mon Sep 17 00:00:00 2001 From: lo-simon Date: Thu, 2 Mar 2023 14:09:51 +0000 Subject: [PATCH 02/35] fix to support new AUTH test added to the latest nmos testsuite (cherry picked from commit 15d8cb450a9723e3fead41eafaafafa7750552af) --- Sandbox/run_nmos_testing.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Sandbox/run_nmos_testing.sh b/Sandbox/run_nmos_testing.sh index a2fd75147..d5d2c6169 100755 --- a/Sandbox/run_nmos_testing.sh +++ b/Sandbox/run_nmos_testing.sh @@ -114,17 +114,17 @@ else echo "Running non-Auth tests" auth=false # 6 test cases per API under test - (( expected_disabled_IS_04_01+=6 )) - (( expected_disabled_IS_04_03+=6 )) - (( expected_disabled_IS_05_01+=6 )) - (( expected_disabled_IS_05_02+=12 )) - (( expected_disabled_IS_07_01+=6 )) - (( expected_disabled_IS_07_02+=18 )) - (( expected_disabled_IS_08_01+=6 )) - (( expected_disabled_IS_08_02+=12 )) + (( expected_disabled_IS_04_01+=7 )) + (( expected_disabled_IS_04_03+=7 )) + (( expected_disabled_IS_05_01+=7 )) + (( expected_disabled_IS_05_02+=14 )) + (( expected_disabled_IS_07_01+=7 )) + (( expected_disabled_IS_07_02+=21 )) + (( expected_disabled_IS_08_01+=7 )) + (( expected_disabled_IS_08_02+=14 )) # test_33, test_33_1 - (( expected_disabled_IS_04_02+=14 )) - (( expected_disabled_IS_09_01+=6 )) + (( expected_disabled_IS_04_02+=16 )) + (( expected_disabled_IS_09_01+=7 )) fi "${node_command}" "{\"how_many\":6,\"http_port\":1080 ${common_params}}" > ${results_dir}/nodeoutput 2>&1 & From 55978f4afea48e4afaaa2495944f3d7aefd3203f Mon Sep 17 00:00:00 2001 From: lo-simon Date: Thu, 2 Mar 2023 19:59:17 +0000 Subject: [PATCH 03/35] Update comment (cherry picked from commit ab3aec101d4de5dfcc3dcfe114f790041d2a4d3c) --- Sandbox/run_nmos_testing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sandbox/run_nmos_testing.sh b/Sandbox/run_nmos_testing.sh index d5d2c6169..9e45dcc6a 100755 --- a/Sandbox/run_nmos_testing.sh +++ b/Sandbox/run_nmos_testing.sh @@ -113,7 +113,7 @@ if [[ "${config_auth}" == "True" ]]; then else echo "Running non-Auth tests" auth=false - # 6 test cases per API under test + # 7 test cases per API under test (( expected_disabled_IS_04_01+=7 )) (( expected_disabled_IS_04_03+=7 )) (( expected_disabled_IS_05_01+=7 )) From 6fdaefc917cc352d51ad2f991f1c99b0908acca2 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Fri, 3 Mar 2023 09:49:57 +0000 Subject: [PATCH 04/35] Use the lastest Conan 1.x version (cherry picked from commit 32787cce34ce859f850c4646ea2fb5a99036d37c) --- .github/workflows/build-test.yml | 4 ++-- .github/workflows/src/build-setup.yml | 2 +- Documents/Dependencies.md | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c09f3a195..29cb358ba 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -93,7 +93,7 @@ jobs: - name: install conan if: matrix.use_conan == true run: | - pip install conan==1.59.0 + pip install "conan<2" conan config set general.revisions_enabled=1 - name: install cmake @@ -600,7 +600,7 @@ jobs: - name: install conan if: matrix.use_conan == true run: | - pip install conan==1.59.0 + pip install "conan<2" conan config set general.revisions_enabled=1 - name: install cmake diff --git a/.github/workflows/src/build-setup.yml b/.github/workflows/src/build-setup.yml index ff8ad926d..f8bb80164 100644 --- a/.github/workflows/src/build-setup.yml +++ b/.github/workflows/src/build-setup.yml @@ -1,7 +1,7 @@ - name: install conan if: matrix.use_conan == true run: | - pip install conan==1.59.0 + pip install "conan<2" conan config set general.revisions_enabled=1 - name: install cmake diff --git a/Documents/Dependencies.md b/Documents/Dependencies.md index 67dea2ee5..66f051630 100644 --- a/Documents/Dependencies.md +++ b/Documents/Dependencies.md @@ -54,10 +54,10 @@ By default nmos-cpp uses [Conan](https://conan.io) to download most of its depen 1. Install Python 3 if necessary Note: The Python scripts directory needs to be added to the `PATH`, so the Conan executable can be found -2. Install Conan using `pip install conan==1.59.0` +2. Install Conan using `pip install "conan<2"` Notes: - - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install conan` - - Currently, Conan 1.47 or up to version 1.59.0 is required; version 1.59.0 has been tested + - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install "conan<2"` + - Currently, Conan 1.47 or lower than version 2 is required; version 1.59.0 has been tested - Conan evolves fairly quickly, so it's worth running `pip install --upgrade conan` regularly - By default [Conan assumes semver compatibility](https://docs.conan.io/en/1.42/creating_packages/define_abi_compatibility.html#versioning-schema). Boost and other C++ libraries do not meet this expectation and break ABI compatibility between e.g. minor versions. From b45170ce7b099b8e7f926b99a013e801a2f3df7d Mon Sep 17 00:00:00 2001 From: Simon Lo Date: Wed, 8 Mar 2023 15:44:44 +0000 Subject: [PATCH 05/35] Update build-setup.yml --- .github/workflows/src/build-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/src/build-setup.yml b/.github/workflows/src/build-setup.yml index f8bb80164..2559b3092 100644 --- a/.github/workflows/src/build-setup.yml +++ b/.github/workflows/src/build-setup.yml @@ -1,7 +1,7 @@ - name: install conan if: matrix.use_conan == true run: | - pip install "conan<2" + pip install conan~=1.47 conan config set general.revisions_enabled=1 - name: install cmake From 45965a0f0af4f79e20e075d15e82eb46aa8b7b2b Mon Sep 17 00:00:00 2001 From: Simon Lo Date: Wed, 8 Mar 2023 15:47:27 +0000 Subject: [PATCH 06/35] Update build-test.yml --- .github/workflows/build-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 29cb358ba..2f92f4959 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -93,7 +93,7 @@ jobs: - name: install conan if: matrix.use_conan == true run: | - pip install "conan<2" + pip install conan~=1.47 conan config set general.revisions_enabled=1 - name: install cmake @@ -600,7 +600,7 @@ jobs: - name: install conan if: matrix.use_conan == true run: | - pip install "conan<2" + pip install conan~=1.47 conan config set general.revisions_enabled=1 - name: install cmake @@ -1073,4 +1073,4 @@ jobs: git config --global user.name 'test-results-uploader' git config --global user.email 'test-results-uploader@nmos-cpp.iam.gserviceaccount.com' git commit -qm "Badges for README at ${{ env.GITHUB_COMMIT }}" - git push -f `git remote` badges-${{ env.GITHUB_COMMIT }}:badges \ No newline at end of file + git push -f `git remote` badges-${{ env.GITHUB_COMMIT }}:badges From cfe4de00978bbe82fe68c875b85dc611d5f1933f Mon Sep 17 00:00:00 2001 From: Simon Lo Date: Wed, 8 Mar 2023 15:50:00 +0000 Subject: [PATCH 07/35] Update Dependencies.md --- Documents/Dependencies.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documents/Dependencies.md b/Documents/Dependencies.md index 66f051630..7a8663fdb 100644 --- a/Documents/Dependencies.md +++ b/Documents/Dependencies.md @@ -54,10 +54,10 @@ By default nmos-cpp uses [Conan](https://conan.io) to download most of its depen 1. Install Python 3 if necessary Note: The Python scripts directory needs to be added to the `PATH`, so the Conan executable can be found -2. Install Conan using `pip install "conan<2"` +2. Install Conan using `pip install conan~=1.47` Notes: - - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install "conan<2"` - - Currently, Conan 1.47 or lower than version 2 is required; version 1.59.0 has been tested + - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install conan~=1.47` + - Currently, Conan 1.47 or higher (and lower than version 2.0) is required; version 1.59.0 has been tested - Conan evolves fairly quickly, so it's worth running `pip install --upgrade conan` regularly - By default [Conan assumes semver compatibility](https://docs.conan.io/en/1.42/creating_packages/define_abi_compatibility.html#versioning-schema). Boost and other C++ libraries do not meet this expectation and break ABI compatibility between e.g. minor versions. From c875df01da53e71fe74abce43fbfb519b96cac44 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Mon, 13 Mar 2023 17:14:45 +0000 Subject: [PATCH 08/35] Add config settings in nmos-cpp-node to support development of BCP-002-02 "Distinguishing Information for NMOS Node and Device Resources" --- Development/nmos-cpp-node/config.json | 10 ++++++++++ Development/nmos-cpp-node/node_implementation.cpp | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Development/nmos-cpp-node/config.json b/Development/nmos-cpp-node/config.json index ce83f8ee5..1fc75d4b2 100644 --- a/Development/nmos-cpp-node/config.json +++ b/Development/nmos-cpp-node/config.json @@ -4,6 +4,16 @@ { // Custom settings for the example node implementation + // node_tags, device_tags: used in resource tags fields + // "Each tag has a single key, but MAY have multiple values." + // See https://specs.amwa.tv/is-04/releases/v1.3.2/docs/APIs_-_Common_Keys.html#tags + // { + // "tag_1": [ "tag_1_value_1", "tag_1_value_2" ], + // "tag_2": [ "tag_2_value_1" ] + // } + //"node_tags": {}, + //"device_tags": {}, + // how_many: provides for very basic testing of a node with many sub-resources of each type //"how_many": 4, diff --git a/Development/nmos-cpp-node/node_implementation.cpp b/Development/nmos-cpp-node/node_implementation.cpp index 850549fe2..dc3ee9d26 100644 --- a/Development/nmos-cpp-node/node_implementation.cpp +++ b/Development/nmos-cpp-node/node_implementation.cpp @@ -56,6 +56,16 @@ namespace impl // custom settings for the example node implementation namespace fields { + // node_tags, device_tags: used in resource tags fields + // "Each tag has a single key, but MAY have multiple values." + // See https://specs.amwa.tv/is-04/releases/v1.3.2/docs/APIs_-_Common_Keys.html#tags + // { + // "tag_1": [ "tag_1_value_1", "tag_1_value_2" ], + // "tag_2": [ "tag_2_value_1" ] + // } + const web::json::field_as_value_or node_tags{ U("node_tags"), web::json::value::object() }; + const web::json::field_as_value_or device_tags{ U("device_tags"), web::json::value::object() }; + // how_many: provides for very basic testing of a node with many sub-resources of each type const web::json::field_as_integer_or how_many{ U("how_many"), 1 }; @@ -295,6 +305,7 @@ void node_implementation_init(nmos::node_model& model, slog::base_gate& gate) // example node { auto node = nmos::make_node(node_id, clocks, nmos::make_node_interfaces(interfaces), model.settings); + node.data[nmos::fields::tags] = impl::fields::node_tags(model.settings); if (!insert_resource_after(delay_millis, model.node_resources, std::move(node), gate)) throw node_implementation_init_exception(); } @@ -338,7 +349,9 @@ void node_implementation_init(nmos::node_model& model, slog::base_gate& gate) auto sender_ids = impl::make_ids(seed_id, nmos::types::sender, rtp_sender_ports, how_many); if (0 <= nmos::fields::events_port(model.settings)) boost::range::push_back(sender_ids, impl::make_ids(seed_id, nmos::types::sender, ws_sender_ports, how_many)); auto receiver_ids = impl::make_ids(seed_id, nmos::types::receiver, receiver_ports, how_many); - if (!insert_resource_after(delay_millis, model.node_resources, nmos::make_device(device_id, node_id, sender_ids, receiver_ids, model.settings), gate)) throw node_implementation_init_exception(); + auto device = nmos::make_device(device_id, node_id, sender_ids, receiver_ids, model.settings); + device.data[nmos::fields::tags] = impl::fields::device_tags(model.settings); + if (!insert_resource_after(delay_millis, model.node_resources, std::move(device), gate)) throw node_implementation_init_exception(); } // example sources, flows and senders From 3fd76fcd406b9a2e7211f6d9c0d51b42be41e905 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Wed, 22 Mar 2023 11:50:51 +0000 Subject: [PATCH 09/35] websocket_listener should set SO_REUSEADDR like http_listener thanks to @RichardNutman, Richard Nutman, Grass Valley --- Development/cpprest/ws_listener_impl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Development/cpprest/ws_listener_impl.cpp b/Development/cpprest/ws_listener_impl.cpp index 712611f3b..101a54524 100644 --- a/Development/cpprest/ws_listener_impl.cpp +++ b/Development/cpprest/ws_listener_impl.cpp @@ -313,6 +313,7 @@ namespace web if (!init) { server.init_asio(); + server.set_reuse_addr(true); init = true; } else From d6b2626b3a76fe13d65af0dcb284cbad5d05a337 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 11 Apr 2023 16:29:14 +0100 Subject: [PATCH 10/35] Bind client and server sockets to the specified network interfaces. No websocket client socket binding support, as it is not currently supported by the C++ REST SDK v2.10.18. --- Development/cpprest/uri_schemes.h | 8 ++ Development/nmos-cpp-node/config.json | 7 ++ Development/nmos-cpp-registry/config.json | 7 ++ Development/nmos/client_utils.cpp | 147 ++++++++++++++++++++-- Development/nmos/client_utils.h | 6 + Development/nmos/node_server.cpp | 10 +- Development/nmos/ocsp_behaviour.cpp | 7 +- Development/nmos/registry_server.cpp | 10 +- Development/nmos/settings.cpp | 38 +++++- Development/nmos/settings.h | 10 ++ 10 files changed, 229 insertions(+), 21 deletions(-) diff --git a/Development/cpprest/uri_schemes.h b/Development/cpprest/uri_schemes.h index 8963e5543..c399ccb4e 100644 --- a/Development/cpprest/uri_schemes.h +++ b/Development/cpprest/uri_schemes.h @@ -1,6 +1,7 @@ #ifndef CPPREST_URI_SCHEMES_H #define CPPREST_URI_SCHEMES_H +#include "cpprest/base_uri.h" #include "cpprest/details/basic_types.h" namespace web @@ -22,6 +23,13 @@ namespace web inline utility::string_t http_scheme(bool secure) { return secure ? uri_schemes::https : uri_schemes::http; } inline utility::string_t ws_scheme(bool secure) { return secure ? uri_schemes::wss : uri_schemes::ws; } + + // Check if the URI is secure + inline bool get_secure(const web::uri& uri) + { + const auto scheme = uri.scheme(); + return (scheme == uri_schemes::wss || scheme == uri_schemes::https); + } } #endif diff --git a/Development/nmos-cpp-node/config.json b/Development/nmos-cpp-node/config.json index 1fc75d4b2..ed41ccac8 100644 --- a/Development/nmos-cpp-node/config.json +++ b/Development/nmos-cpp-node/config.json @@ -191,6 +191,13 @@ //"settings_port": 3209, //"logging_port": 5106, + // client_address [registry, node]: IP address of the network interface to bind client connections for Linux + // only supprting HTTP/HTTPS client connections, no ws/wss support yet + //"client_address": "", + + // server_address [registry, node]: IP address of the network interface to bind server connections + //"server_address": "", + // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address //"settings_address": "127.0.0.1", diff --git a/Development/nmos-cpp-registry/config.json b/Development/nmos-cpp-registry/config.json index 48312edeb..9307d18e5 100644 --- a/Development/nmos-cpp-registry/config.json +++ b/Development/nmos-cpp-registry/config.json @@ -109,6 +109,13 @@ //"mdns_port": 3208, //"schemas_port": 3208, + // client_address [registry, node]: IP address of the network interface to bind client connections for Linux + // only supprting HTTP/HTTPS client connections, no ws/wss support yet + //"client_address": "", + + // server_address [registry, node]: IP address of the network interface to bind server connections + //"server_address": "", + // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address //"settings_address": "127.0.0.1", diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index b2b902379..2a6cb2764 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -1,8 +1,13 @@ #include "nmos/client_utils.h" -// cf. preprocessor conditions in nmos::details::make_client_ssl_context_callback +// cf. preprocessor conditions in nmos::details::make_client_ssl_context_callback and nmos::details::make_client_nativehandle_options #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) +#if !defined(_WIN32) +#include "boost/asio.hpp" +#endif #include "boost/asio/ssl/set_cipher_list.hpp" +#include "boost/range/algorithm.hpp" +#include "cpprest/host_utils.h" #endif #include "cpprest/basic_utils.h" #include "cpprest/details/system_error.h" @@ -55,39 +60,165 @@ namespace nmos } }; } +#endif + // get the associated network interface name from the IP address + inline utility::string_t get_interface_name(const utility::string_t& address) + { + utility::string_t interface_name; + if (!address.empty()) + { + const auto interfaces = web::hosts::experimental::host_interfaces(); + + auto find_interface = [&]() + { + return boost::range::find_if(interfaces, [&](const web::hosts::experimental::host_interface& interface) + { + return interface.addresses.end() != boost::range::find(interface.addresses, address); + }); + }; + const auto interface = find_interface(); + if (interfaces.end() != interface) { interface_name = interface->name; } + } + return interface_name; + } + +#if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) + // bind socket to a specific network interface, only supporting Linux and Boost.Asio +#if !defined(_WIN32) + inline bool bind_to_device(const utility::string_t& interface_name, bool secure, void* native_handle) + { + boost::asio::basic_socket::native_handle_type socket_fd; + if (secure) + { + auto socket = (boost::asio::ssl::stream*)native_handle; + if (!socket->lowest_layer().is_open()) + { + // for now, limited to IPv4 + socket->lowest_layer().open(boost::asio::ip::tcp::v4()); + } + socket_fd = socket->lowest_layer().native_handle(); + } + else + { + auto socket = (boost::asio::ip::tcp::socket*)native_handle; + if (!socket->is_open()) + { + // for now, limited to IPv4 + socket->open(boost::asio::ip::tcp::v4()); + } + socket_fd = socket->lowest_layer().native_handle(); + } + // SO_BINDTODEVICE not defined in windows + return (setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, utility::us2s(interface_name).c_str(), interface_name.length()) == 0); + } +#endif + + inline std::function make_client_nativehandle_options(const utility::string_t& client_address, bool secure, slog::base_gate& gate) + { + // get the associated network interface name from IP address + const auto interface_name = get_interface_name(client_address); + if (interface_name.empty()) + { + slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the HTTP client connection"; + } + + return [interface_name, secure, &gate](web::http::client::native_handle native_handle) + { + if (!interface_name.empty()) + { +#if !defined(_WIN32) + if (!bind_to_device(interface_name, secure, native_handle)) + { + char error[1024]; + slog::log(gate, SLOG_FLF) << "Unable to bind HTTP client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); + } +#else + slog::log(gate, SLOG_FLF) << "Bind HTTP client connection to " << interface_name << " not supported"; +#endif + } + }; + } + +#ifdef CPPRESTSDK_ENABLE_BIND_WEBSOCKET_CLIENT + // The current version of the C++ REST SDK 2.10.18 does not provide the callback to enable the custom websocket setting + inline std::function make_ws_client_nativehandle_options(const utility::string_t& client_address, bool secure, slog::base_gate& gate) + { + // get the associated network interface name from IP address + const auto interface_name = get_interface_name(client_address); + if (interface_name.empty()) + { + slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the websocket client connection"; + } + + return [interface_name, secure, &gate](web::websockets::client::native_handle native_handle) + { + if (!interface_name.empty()) + { +#if !defined(_WIN32) + if (!bind_to_device(interface_name, secure, native_handle)) + { + char error[1024]; + slog::log(gate, SLOG_FLF) << "Unable to bind websocket client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); + } +#else + slog::log(gate, SLOG_FLF) << "Bind websocket client connection to " << interface_name << " not supported"; +#endif + } + }; + } +#endif + #endif } - // construct client config based on settings, e.g. using the specified proxy + // construct client config based on settings and specified secure flag, e.g. using the specified proxy, and the OCSP client // with the remaining options defaulted, e.g. request timeout - web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) + web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate) { web::http::client::http_client_config config; const auto proxy = proxy_uri(settings); if (!proxy.is_empty()) config.set_proxy(proxy); - config.set_validate_certificates(nmos::experimental::fields::validate_certificates(settings)); + if (secure) config.set_validate_certificates(nmos::experimental::fields::validate_certificates(settings)); #if !defined(_WIN32) && !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) - config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); + if (secure) config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); + config.set_nativehandle_options(details::make_client_nativehandle_options(nmos::experimental::fields::client_address(settings), secure, gate)); #endif return config; } // construct client config based on settings, e.g. using the specified proxy + // with the remaining options defaulted, e.g. request timeout + web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) + { + return make_http_client_config(settings, load_ca_certificates, nmos::experimental::fields::client_secure(settings), gate); + } + + // construct client config based on settings and specified secure flag, e.g. using the specified proxy // with the remaining options defaulted - web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) + web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate) { web::websockets::client::websocket_client_config config; const auto proxy = proxy_uri(settings); if (!proxy.is_empty()) config.set_proxy(proxy); - config.set_validate_certificates(nmos::experimental::fields::validate_certificates(settings)); + if (secure) config.set_validate_certificates(nmos::experimental::fields::validate_certificates(settings)); #if !defined(_WIN32) || !defined(__cplusplus_winrt) - config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); + if (secure) config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); +#ifdef CPPRESTSDK_BIND_WEBSOCKET_CLIENT + config.set_nativehandle_options(details::make_ws_client_nativehandle_options(nmos::experimental::fields::client_address(settings), secure, gate)); +#endif #endif return config; } + // construct client config based on settings, e.g. using the specified proxy + // with the remaining options defaulted + web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) + { + return make_websocket_client_config(settings, load_ca_certificates, nmos::experimental::fields::client_secure(settings), gate); + } + // make a request with logging pplx::task api_request(web::http::client::http_client client, web::http::http_request request, slog::base_gate& gate, const pplx::cancellation_token& token) { diff --git a/Development/nmos/client_utils.h b/Development/nmos/client_utils.h index 1e76e8bc8..15f942b17 100644 --- a/Development/nmos/client_utils.h +++ b/Development/nmos/client_utils.h @@ -11,10 +11,16 @@ namespace slog { class base_gate; } // Utility types, constants and functions for implementing NMOS REST API clients namespace nmos { + // construct client config based on settings and specified secure flag, e.g. using the specified proxy and the OCSP client + // with the remaining options defaulted, e.g. request timeout + web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate); // construct client config based on settings, e.g. using the specified proxy // with the remaining options defaulted, e.g. request timeout web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate); + // construct client config based on settings and specified secure flag, e.g. using the specified proxy + // with the remaining options defaulted + web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate); // construct client config based on settings, e.g. using the specified proxy // with the remaining options defaulted web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate); diff --git a/Development/nmos/node_server.cpp b/Development/nmos/node_server.cpp index 2e4469788..d86e2f720 100644 --- a/Development/nmos/node_server.cpp +++ b/Development/nmos/node_server.cpp @@ -35,6 +35,8 @@ namespace nmos const auto hsts = nmos::experimental::get_hsts(node_model.settings); + const auto server_address = nmos::experimental::get_server_address(node_model.settings); + // Configure the Settings API const host_port settings_address(nmos::experimental::fields::settings_address(node_model.settings), nmos::experimental::fields::settings_port(node_model.settings)); @@ -70,8 +72,8 @@ namespace nmos for (auto& api_router : node_server.api_routers) { - // default empty string means the wildcard address - const auto& host = !api_router.first.first.empty() ? api_router.first.first : web::http::experimental::listener::host_wildcard; + // empty string and empty server IP address means the wildcard address + const auto& host = !api_router.first.first.empty() ? api_router.first.first : !server_address.empty() ? server_address : web::http::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address node_server.http_listeners.push_back(nmos::make_api_listener(server_secure, host, nmos::experimental::server_port(api_router.first.second, node_model.settings), api_router.second, http_config, hsts, gate)); @@ -84,8 +86,8 @@ namespace nmos for (auto& ws_handler : node_server.ws_handlers) { - // default empty string means the wildcard address - const auto& host = !ws_handler.first.first.empty() ? ws_handler.first.first : web::websockets::experimental::listener::host_wildcard; + // empty string and empty server IP address means the wildcard address + const auto& host = !ws_handler.first.first.empty() ? ws_handler.first.first : !server_address.empty() ? server_address : web::websockets::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address node_server.ws_listeners.push_back(nmos::make_ws_api_listener(server_secure, host, nmos::experimental::server_port(ws_handler.first.second, node_model.settings), ws_handler.second.first, websocket_config, gate)); diff --git a/Development/nmos/ocsp_behaviour.cpp b/Development/nmos/ocsp_behaviour.cpp index cd759b880..68f9347db 100644 --- a/Development/nmos/ocsp_behaviour.cpp +++ b/Development/nmos/ocsp_behaviour.cpp @@ -1,6 +1,7 @@ #include "nmos/ocsp_behaviour.h" #include "pplx/pplx_utils.h" // for pplx::complete_at +#include "cpprest/uri_schemes.h" #include "nmos/client_utils.h" #include "nmos/model.h" #include "nmos/ocsp_state.h" @@ -141,9 +142,9 @@ namespace nmos namespace details { - web::http::client::http_client_config make_ocsp_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) + web::http::client::http_client_config make_ocsp_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate) { - auto config = nmos::make_http_client_config(settings, std::move(load_ca_certificates), gate); + auto config = nmos::make_http_client_config(settings, std::move(load_ca_certificates), secure, gate); config.set_timeout(std::chrono::seconds(nmos::experimental::fields::ocsp_request_max(settings))); return config; } @@ -345,7 +346,7 @@ namespace nmos if (!state.client) { const auto ocsp_uri = ocsp_uris.front(); - state.client.reset(new web::http::client::http_client(ocsp_uri, make_ocsp_client_config(model.settings, state.load_ca_certificates, gate))); + state.client.reset(new web::http::client::http_client(ocsp_uri, make_ocsp_client_config(model.settings, state.load_ca_certificates, get_secure(ocsp_uri), gate))); } auto token = cancellation_source.get_token(); diff --git a/Development/nmos/registry_server.cpp b/Development/nmos/registry_server.cpp index 6fc2f7314..779f52191 100644 --- a/Development/nmos/registry_server.cpp +++ b/Development/nmos/registry_server.cpp @@ -45,6 +45,8 @@ namespace nmos const auto hsts = nmos::experimental::get_hsts(registry_model.settings); + const auto server_address = nmos::experimental::get_server_address(registry_model.settings); + // Configure the DNS-SD Browsing API const host_port mdns_address(nmos::experimental::fields::mdns_address(registry_model.settings), nmos::experimental::fields::mdns_port(registry_model.settings)); @@ -111,8 +113,8 @@ namespace nmos for (auto& api_router : registry_server.api_routers) { - // default empty string means the wildcard address - const auto& host = !api_router.first.first.empty() ? api_router.first.first : web::http::experimental::listener::host_wildcard; + // empty string and empty server IP address means the wildcard address + const auto& host = !api_router.first.first.empty() ? api_router.first.first : !server_address.empty() ? server_address : web::http::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address registry_server.http_listeners.push_back(nmos::make_api_listener(server_secure, host, nmos::experimental::server_port(api_router.first.second, registry_model.settings), api_router.second, http_config, hsts, gate)); @@ -125,8 +127,8 @@ namespace nmos for (auto& ws_handler : registry_server.ws_handlers) { - // default empty string means the wildcard address - const auto& host = !ws_handler.first.first.empty() ? ws_handler.first.first : web::websockets::experimental::listener::host_wildcard; + //empty string and empty server IP address means the wildcard address + const auto& host = !ws_handler.first.first.empty() ? ws_handler.first.first : !server_address.empty() ? server_address : web::websockets::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address registry_server.ws_listeners.push_back(nmos::make_ws_api_listener(server_secure, host, nmos::experimental::server_port(ws_handler.first.second, registry_model.settings), ws_handler.second.first, websocket_config, gate)); diff --git a/Development/nmos/settings.cpp b/Development/nmos/settings.cpp index c8943c93e..7b8f6c4e1 100644 --- a/Development/nmos/settings.cpp +++ b/Development/nmos/settings.cpp @@ -1,6 +1,7 @@ #include "nmos/settings.h" #include +#include #include #include #include @@ -54,8 +55,14 @@ namespace nmos } } + // if the "server_address" settings is presented, use it for the node's "api endpoints" and "href", and device's "controls href" + const auto server_address = experimental::get_server_address(settings); + if (!server_address.empty()) + { + web::json::insert(settings, std::make_pair(nmos::fields::host_address, server_address)); + } // if the "host_address" setting was omitted, use the first of the "host_addresses" - if (settings.has_field(nmos::fields::host_addresses)) + else if (settings.has_field(nmos::fields::host_addresses)) { web::json::insert(settings, std::make_pair(nmos::fields::host_address, nmos::fields::host_addresses(settings)[0])); } @@ -151,7 +158,12 @@ namespace nmos { hosts.push_back(get_host_name(settings)); } - if (0 != (mode & nmos::experimental::href_mode_addresses)) + const auto server_address = experimental::get_server_address(settings); + if (!server_address.empty()) + { + hosts.push_back(server_address); + } + else if (0 != (mode & nmos::experimental::href_mode_addresses)) { const auto at_least_one_host_address = web::json::value_of({ web::json::value::string(nmos::fields::host_address(settings)) }); const auto& host_addresses = settings.has_field(nmos::fields::host_addresses) ? nmos::fields::host_addresses(settings) : at_least_one_host_address.as_array(); @@ -189,6 +201,28 @@ namespace nmos return web::http::experimental::hsts{ (uint32_t)nmos::experimental::fields::hsts_max_age(settings), nmos::experimental::fields::hsts_include_sub_domains(settings) }; return bst::nullopt; } + + // Get server IP address of the interface to bind server connections + utility::string_t get_server_address(const settings& settings) + { + const auto server_address = nmos::experimental::fields::server_address(settings); + if (!server_address.empty()) + { + // verify the server address match one of the interface addresses + const auto interfaces = web::hosts::experimental::host_interfaces(); + + auto find_interface = [&]() + { + return boost::range::find_if(interfaces, [&](const web::hosts::experimental::host_interface& interface) + { + return interface.addresses.end() != boost::range::find(interface.addresses, server_address); + }); + }; + const auto interface = find_interface(); + if (interfaces.end() != interface) { return server_address; } + } + return {}; + } } // Get a summary of the build configuration, including versions of dependencies diff --git a/Development/nmos/settings.h b/Development/nmos/settings.h index 62be9c25a..e7cb1ab13 100644 --- a/Development/nmos/settings.h +++ b/Development/nmos/settings.h @@ -52,6 +52,9 @@ namespace nmos { // Get HTTP Strict-Transport-Security settings bst::optional get_hsts(const settings& settings); + + // Get server IP address of the network interface for binding the server connections + utility::string_t get_server_address(const settings& settings); } // Get a summary of the build configuration, including versions of dependencies @@ -240,6 +243,13 @@ namespace nmos const web::json::field_as_integer_or mdns_port{ U("mdns_port"), 3208 }; const web::json::field_as_integer_or schemas_port{ U("schemas_port"), 3208 }; + // client_address [registry, node]: IP address of the network interface to bind client connections for Linux + // only supprting HTTP/HTTPS client connections, no ws/wss support yet + const web::json::field_as_string_or client_address{ U("client_address"), U("") }; + + // server_address [registry, node]: IP address of the network interface to bind server connections + const web::json::field_as_string_or server_address{ U("server_address"), U("") }; + // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address const web::json::field_as_string_or settings_address{ U("settings_address"), U("") }; From c4800b6960bff0bfe8792cd6d1baa6c56b46a128 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 11 Apr 2023 19:49:02 +0100 Subject: [PATCH 11/35] Fix socket file descriptor for Ubuntu 14.04 default boost version 1.54 --- Development/nmos/client_utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index 2a6cb2764..ba2d6fb4d 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -87,7 +87,7 @@ namespace nmos #if !defined(_WIN32) inline bool bind_to_device(const utility::string_t& interface_name, bool secure, void* native_handle) { - boost::asio::basic_socket::native_handle_type socket_fd; + int socket_fd; if (secure) { auto socket = (boost::asio::ssl::stream*)native_handle; From 4e08be0a22611359dec0b8db844e0708fd3797b7 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 11 Apr 2023 19:51:20 +0100 Subject: [PATCH 12/35] Don't log if client_address is emptied --- Development/nmos/client_utils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index ba2d6fb4d..d43a12f50 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -117,7 +117,7 @@ namespace nmos { // get the associated network interface name from IP address const auto interface_name = get_interface_name(client_address); - if (interface_name.empty()) + if (!client_address.empty() && interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the HTTP client connection"; } @@ -145,7 +145,7 @@ namespace nmos { // get the associated network interface name from IP address const auto interface_name = get_interface_name(client_address); - if (interface_name.empty()) + if (!client_address.empty() && interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the websocket client connection"; } From 43e94545addba26237fcc5421d3b3d2a0989c397 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 11 Apr 2023 23:38:43 +0100 Subject: [PATCH 13/35] SO_BINDTODEVICE defined in Linux only --- Development/nmos/client_utils.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index d43a12f50..d83d2cfe7 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -83,8 +83,8 @@ namespace nmos } #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) - // bind socket to a specific network interface, only supporting Linux and Boost.Asio -#if !defined(_WIN32) + // bind socket to a specific network interface, only supporting Linux +#if defined(linux) || defined(__linux) || defined(__linux__) inline bool bind_to_device(const utility::string_t& interface_name, bool secure, void* native_handle) { int socket_fd; @@ -108,7 +108,7 @@ namespace nmos } socket_fd = socket->lowest_layer().native_handle(); } - // SO_BINDTODEVICE not defined in windows + // SO_BINDTODEVICE not defined in windows & mac return (setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, utility::us2s(interface_name).c_str(), interface_name.length()) == 0); } #endif @@ -126,7 +126,7 @@ namespace nmos { if (!interface_name.empty()) { -#if !defined(_WIN32) +#if defined(linux) || defined(__linux) || defined(__linux__) if (!bind_to_device(interface_name, secure, native_handle)) { char error[1024]; @@ -154,7 +154,7 @@ namespace nmos { if (!interface_name.empty()) { -#if !defined(_WIN32) +#if defined(linux) || defined(__linux) || defined(__linux__) if (!bind_to_device(interface_name, secure, native_handle)) { char error[1024]; From f12762e44b792a0f117a7e03f69bc933c1f830eb Mon Sep 17 00:00:00 2001 From: lo-simon Date: Wed, 12 Apr 2023 10:46:53 +0100 Subject: [PATCH 14/35] Fix typo --- Development/nmos-cpp-node/config.json | 2 +- Development/nmos-cpp-registry/config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Development/nmos-cpp-node/config.json b/Development/nmos-cpp-node/config.json index ed41ccac8..c7deb944b 100644 --- a/Development/nmos-cpp-node/config.json +++ b/Development/nmos-cpp-node/config.json @@ -192,7 +192,7 @@ //"logging_port": 5106, // client_address [registry, node]: IP address of the network interface to bind client connections for Linux - // only supprting HTTP/HTTPS client connections, no ws/wss support yet + // only supporting HTTP/HTTPS client connections, no ws/wss support yet //"client_address": "", // server_address [registry, node]: IP address of the network interface to bind server connections diff --git a/Development/nmos-cpp-registry/config.json b/Development/nmos-cpp-registry/config.json index 9307d18e5..4b2dd6834 100644 --- a/Development/nmos-cpp-registry/config.json +++ b/Development/nmos-cpp-registry/config.json @@ -110,7 +110,7 @@ //"schemas_port": 3208, // client_address [registry, node]: IP address of the network interface to bind client connections for Linux - // only supprting HTTP/HTTPS client connections, no ws/wss support yet + // only supporting HTTP/HTTPS client connections, no ws/wss support yet //"client_address": "", // server_address [registry, node]: IP address of the network interface to bind server connections From 9fe1153607d96acccb6e0b30d36f965f109a6a8f Mon Sep 17 00:00:00 2001 From: lo-simon Date: Mon, 17 Apr 2023 09:24:56 +0100 Subject: [PATCH 15/35] Fix typo --- Development/nmos/settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Development/nmos/settings.h b/Development/nmos/settings.h index e7cb1ab13..9b67e38f8 100644 --- a/Development/nmos/settings.h +++ b/Development/nmos/settings.h @@ -244,7 +244,7 @@ namespace nmos const web::json::field_as_integer_or schemas_port{ U("schemas_port"), 3208 }; // client_address [registry, node]: IP address of the network interface to bind client connections for Linux - // only supprting HTTP/HTTPS client connections, no ws/wss support yet + // only supporting HTTP/HTTPS client connections, no ws/wss support yet const web::json::field_as_string_or client_address{ U("client_address"), U("") }; // server_address [registry, node]: IP address of the network interface to bind server connections From 4c5a0b1aa3e3dac15a7690e0ae10114560de990f Mon Sep 17 00:00:00 2001 From: Simon Lo Date: Fri, 21 Apr 2023 10:32:09 +0100 Subject: [PATCH 16/35] Update Development/cpprest/uri_schemes.h Co-authored-by: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> --- Development/cpprest/uri_schemes.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Development/cpprest/uri_schemes.h b/Development/cpprest/uri_schemes.h index c399ccb4e..6c7c87c8d 100644 --- a/Development/cpprest/uri_schemes.h +++ b/Development/cpprest/uri_schemes.h @@ -27,8 +27,7 @@ namespace web // Check if the URI is secure inline bool get_secure(const web::uri& uri) { - const auto scheme = uri.scheme(); - return (scheme == uri_schemes::wss || scheme == uri_schemes::https); + return uri_schemes::https == uri.scheme() || uri_schemes::wss == uri.scheme(); } } From c487d33584005e61bedb24748ead55bd59e27443 Mon Sep 17 00:00:00 2001 From: Simon Lo Date: Fri, 21 Apr 2023 10:38:48 +0100 Subject: [PATCH 17/35] Update Development/nmos-cpp-registry/config.json Co-authored-by: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> --- Development/nmos-cpp-registry/config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Development/nmos-cpp-registry/config.json b/Development/nmos-cpp-registry/config.json index 4b2dd6834..d42738828 100644 --- a/Development/nmos-cpp-registry/config.json +++ b/Development/nmos-cpp-registry/config.json @@ -109,8 +109,8 @@ //"mdns_port": 3208, //"schemas_port": 3208, - // client_address [registry, node]: IP address of the network interface to bind client connections for Linux - // only supporting HTTP/HTTPS client connections, no ws/wss support yet + // client_address [registry, node]: IP address of the network interface to bind client connections + // for now, only supporting HTTP/HTTPS client connections on Linux //"client_address": "", // server_address [registry, node]: IP address of the network interface to bind server connections From ff2f644183c1432258de9cb9e88ea3932e9e999d Mon Sep 17 00:00:00 2001 From: Simon Lo Date: Fri, 21 Apr 2023 18:03:56 +0100 Subject: [PATCH 18/35] Update Development/nmos/client_utils.cpp Co-authored-by: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> --- Development/nmos/client_utils.cpp | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index d83d2cfe7..126d97465 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -61,25 +61,14 @@ namespace nmos }; } #endif - // get the associated network interface name from the IP address - inline utility::string_t get_interface_name(const utility::string_t& address) + // get the associated network interface name from an IP address + inline utility::string_t get_interface_name(const utility::string_t& address, const std::vector& host_interfaces = web::hosts::experimental::host_interfaces()) { - utility::string_t interface_name; - if (!address.empty()) + const auto interface = boost::range::find_if(interfaces, [&](const web::hosts::experimental::host_interface& interface) { - const auto interfaces = web::hosts::experimental::host_interfaces(); - - auto find_interface = [&]() - { - return boost::range::find_if(interfaces, [&](const web::hosts::experimental::host_interface& interface) - { - return interface.addresses.end() != boost::range::find(interface.addresses, address); - }); - }; - const auto interface = find_interface(); - if (interfaces.end() != interface) { interface_name = interface->name; } - } - return interface_name; + return interface.addresses.end() != boost::range::find(interface.addresses, address); + }); + return interfaces.end() != interface ? interface->name : utility::string_t{}; } #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) From 7cd4d4f12e2e95d5324d918ea52acc4a8d81162c Mon Sep 17 00:00:00 2001 From: lo-simon Date: Mon, 24 Apr 2023 15:05:11 +0100 Subject: [PATCH 19/35] Move get_interface_name() to host_utils --- Development/cpprest/host_utils.cpp | 11 +++++++++++ Development/cpprest/host_utils.h | 3 +++ Development/nmos/client_utils.cpp | 14 ++------------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Development/cpprest/host_utils.cpp b/Development/cpprest/host_utils.cpp index 2d631a151..59a87739b 100644 --- a/Development/cpprest/host_utils.cpp +++ b/Development/cpprest/host_utils.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "cpprest/asyncrt_utils.h" // for utility::conversions #if defined(_WIN32) @@ -328,6 +329,16 @@ namespace web } return addresses; // empty if host_name cannot be resolved } + + // get the associated network interface name from an IP address + utility::string_t get_interface_name(const utility::string_t& address, const std::vector& host_interfaces) + { + const auto interface = boost::range::find_if(host_interfaces, [&](const web::hosts::experimental::host_interface& interface) + { + return interface.addresses.end() != boost::range::find(interface.addresses, address); + }); + return host_interfaces.end() != interface ? interface->name : utility::string_t {}; + } } } } diff --git a/Development/cpprest/host_utils.h b/Development/cpprest/host_utils.h index f9dc6ad95..40ed32e2c 100644 --- a/Development/cpprest/host_utils.h +++ b/Development/cpprest/host_utils.h @@ -30,6 +30,9 @@ namespace web std::vector host_names(const utility::string_t& address); std::vector host_addresses(const utility::string_t& host_name); + + // get the associated network interface name from an IP address + utility::string_t get_interface_name(const utility::string_t& address, const std::vector& host_interfaces = web::hosts::experimental::host_interfaces()); } } } diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index 126d97465..a94f3880b 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -6,7 +6,6 @@ #include "boost/asio.hpp" #endif #include "boost/asio/ssl/set_cipher_list.hpp" -#include "boost/range/algorithm.hpp" #include "cpprest/host_utils.h" #endif #include "cpprest/basic_utils.h" @@ -61,15 +60,6 @@ namespace nmos }; } #endif - // get the associated network interface name from an IP address - inline utility::string_t get_interface_name(const utility::string_t& address, const std::vector& host_interfaces = web::hosts::experimental::host_interfaces()) - { - const auto interface = boost::range::find_if(interfaces, [&](const web::hosts::experimental::host_interface& interface) - { - return interface.addresses.end() != boost::range::find(interface.addresses, address); - }); - return interfaces.end() != interface ? interface->name : utility::string_t{}; - } #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) // bind socket to a specific network interface, only supporting Linux @@ -105,7 +95,7 @@ namespace nmos inline std::function make_client_nativehandle_options(const utility::string_t& client_address, bool secure, slog::base_gate& gate) { // get the associated network interface name from IP address - const auto interface_name = get_interface_name(client_address); + const auto interface_name = web::hosts::experimental::get_interface_name(client_address); if (!client_address.empty() && interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the HTTP client connection"; @@ -133,7 +123,7 @@ namespace nmos inline std::function make_ws_client_nativehandle_options(const utility::string_t& client_address, bool secure, slog::base_gate& gate) { // get the associated network interface name from IP address - const auto interface_name = get_interface_name(client_address); + const auto interface_name = web::hosts::experimental::get_interface_name(client_address); if (!client_address.empty() && interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the websocket client connection"; From 785d871ac3d598026f6a0cef3b132032f0d82b15 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Mon, 24 Apr 2023 15:45:09 +0100 Subject: [PATCH 20/35] Rename get_secure() to is_secure() --- Development/cpprest/uri_schemes.h | 2 +- Development/nmos/ocsp_behaviour.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Development/cpprest/uri_schemes.h b/Development/cpprest/uri_schemes.h index 6c7c87c8d..079d73eb9 100644 --- a/Development/cpprest/uri_schemes.h +++ b/Development/cpprest/uri_schemes.h @@ -25,7 +25,7 @@ namespace web inline utility::string_t ws_scheme(bool secure) { return secure ? uri_schemes::wss : uri_schemes::ws; } // Check if the URI is secure - inline bool get_secure(const web::uri& uri) + inline bool is_secure(const web::uri& uri) { return uri_schemes::https == uri.scheme() || uri_schemes::wss == uri.scheme(); } diff --git a/Development/nmos/ocsp_behaviour.cpp b/Development/nmos/ocsp_behaviour.cpp index 68f9347db..8bd50051a 100644 --- a/Development/nmos/ocsp_behaviour.cpp +++ b/Development/nmos/ocsp_behaviour.cpp @@ -346,7 +346,7 @@ namespace nmos if (!state.client) { const auto ocsp_uri = ocsp_uris.front(); - state.client.reset(new web::http::client::http_client(ocsp_uri, make_ocsp_client_config(model.settings, state.load_ca_certificates, get_secure(ocsp_uri), gate))); + state.client.reset(new web::http::client::http_client(ocsp_uri, make_ocsp_client_config(model.settings, state.load_ca_certificates, is_secure(ocsp_uri), gate))); } auto token = cancellation_source.get_token(); From 024d652c2b2362b9011c5f9b9d1fdeb18bed70ab Mon Sep 17 00:00:00 2001 From: lo-simon Date: Mon, 24 Apr 2023 16:06:06 +0100 Subject: [PATCH 21/35] Correct #ifdef CPPRESTSDK_BIND_WEBSOCKET_CLIENT to CPPRESTSDK_ENABLE_BIND_WEBSOCKET_CLIENT --- Development/nmos/client_utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index a94f3880b..927e38f7b 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -183,7 +183,7 @@ namespace nmos if (secure) config.set_validate_certificates(nmos::experimental::fields::validate_certificates(settings)); #if !defined(_WIN32) || !defined(__cplusplus_winrt) if (secure) config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); -#ifdef CPPRESTSDK_BIND_WEBSOCKET_CLIENT +#ifdef CPPRESTSDK_ENABLE_BIND_WEBSOCKET_CLIENT config.set_nativehandle_options(details::make_ws_client_nativehandle_options(nmos::experimental::fields::client_address(settings), secure, gate)); #endif #endif From 0b34ae37e9324e29d5835069851822bf71a47c8e Mon Sep 17 00:00:00 2001 From: lo-simon Date: Mon, 24 Apr 2023 16:40:09 +0100 Subject: [PATCH 22/35] Move client_address and server_address below the addresses section --- Development/nmos-cpp-node/config.json | 14 +++++++------- Development/nmos-cpp-registry/config.json | 14 +++++++------- Development/nmos/settings.h | 14 +++++++------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Development/nmos-cpp-node/config.json b/Development/nmos-cpp-node/config.json index c7deb944b..186027ad6 100644 --- a/Development/nmos-cpp-node/config.json +++ b/Development/nmos-cpp-node/config.json @@ -191,18 +191,18 @@ //"settings_port": 3209, //"logging_port": 5106, - // client_address [registry, node]: IP address of the network interface to bind client connections for Linux - // only supporting HTTP/HTTPS client connections, no ws/wss support yet - //"client_address": "", - - // server_address [registry, node]: IP address of the network interface to bind server connections - //"server_address": "", - // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address //"settings_address": "127.0.0.1", //"logging_address": "", + // client_address [registry, node]: IP address of the network interface to bind client connections + // for now, only supporting HTTP/HTTPS client connections on Linux + //"client_address": "", + + // server_address [registry, node]: IP address of the network interface to bind server connections + //"server_address": "", + // logging_limit [registry, node]: maximum number of log events cached for the Logging API //"logging_limit": 1234, diff --git a/Development/nmos-cpp-registry/config.json b/Development/nmos-cpp-registry/config.json index d42738828..c9968062d 100644 --- a/Development/nmos-cpp-registry/config.json +++ b/Development/nmos-cpp-registry/config.json @@ -109,13 +109,6 @@ //"mdns_port": 3208, //"schemas_port": 3208, - // client_address [registry, node]: IP address of the network interface to bind client connections - // for now, only supporting HTTP/HTTPS client connections on Linux - //"client_address": "", - - // server_address [registry, node]: IP address of the network interface to bind server connections - //"server_address": "", - // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address //"settings_address": "127.0.0.1", @@ -127,6 +120,13 @@ //"mdns_address": "", //"schemas_address": "", + // client_address [registry, node]: IP address of the network interface to bind client connections + // for now, only supporting HTTP/HTTPS client connections on Linux + //"client_address": "", + + // server_address [registry, node]: IP address of the network interface to bind server connections + //"server_address": "", + // query_ws_paging_default/query_ws_paging_limit [registry]: default/maximum number of events per message when using the Query WebSocket API (a client may request a lower limit) //"query_ws_paging_default": 10, //"query_ws_paging_limit": 100, diff --git a/Development/nmos/settings.h b/Development/nmos/settings.h index 9b67e38f8..9f8a883a1 100644 --- a/Development/nmos/settings.h +++ b/Development/nmos/settings.h @@ -243,13 +243,6 @@ namespace nmos const web::json::field_as_integer_or mdns_port{ U("mdns_port"), 3208 }; const web::json::field_as_integer_or schemas_port{ U("schemas_port"), 3208 }; - // client_address [registry, node]: IP address of the network interface to bind client connections for Linux - // only supporting HTTP/HTTPS client connections, no ws/wss support yet - const web::json::field_as_string_or client_address{ U("client_address"), U("") }; - - // server_address [registry, node]: IP address of the network interface to bind server connections - const web::json::field_as_string_or server_address{ U("server_address"), U("") }; - // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address const web::json::field_as_string_or settings_address{ U("settings_address"), U("") }; @@ -261,6 +254,13 @@ namespace nmos const web::json::field_as_string_or mdns_address{ U("mdns_address"), U("") }; const web::json::field_as_string_or schemas_address{ U("schemas_address"), U("") }; + // client_address [registry, node]: IP address of the network interface to bind client connections + // for now, only supporting HTTP/HTTPS client connections on Linux + const web::json::field_as_string_or client_address{ U("client_address"), U("") }; + + // server_address [registry, node]: IP address of the network interface to bind server connections + const web::json::field_as_string_or server_address{ U("server_address"), U("") }; + // query_ws_paging_default/query_ws_paging_limit [registry]: default/maximum number of events per message when using the Query WebSocket API (a client may request a lower limit) const web::json::field_as_integer_or query_ws_paging_default{ U("query_ws_paging_default"), 10 }; const web::json::field_as_integer_or query_ws_paging_limit{ U("query_ws_paging_limit"), 100 }; From a76b56d7b3c0fa5a3625cafec55a60478b3293a3 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Mon, 24 Apr 2023 17:13:24 +0100 Subject: [PATCH 23/35] Use __linux__ for linux only --- Development/nmos/client_utils.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index 927e38f7b..a4a4a309c 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -63,7 +63,7 @@ namespace nmos #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) // bind socket to a specific network interface, only supporting Linux -#if defined(linux) || defined(__linux) || defined(__linux__) +#if defined(__linux__) inline bool bind_to_device(const utility::string_t& interface_name, bool secure, void* native_handle) { int socket_fd; @@ -105,7 +105,7 @@ namespace nmos { if (!interface_name.empty()) { -#if defined(linux) || defined(__linux) || defined(__linux__) +#if defined(__linux__) if (!bind_to_device(interface_name, secure, native_handle)) { char error[1024]; @@ -133,7 +133,7 @@ namespace nmos { if (!interface_name.empty()) { -#if defined(linux) || defined(__linux) || defined(__linux__) +#if defined(__linux__) if (!bind_to_device(interface_name, secure, native_handle)) { char error[1024]; From c85d4065fff8c18853868ce583ed20e814726e3a Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:23:48 +0100 Subject: [PATCH 24/35] pip install --upgrade conan~=1.47 everywhere --- Documents/Dependencies.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documents/Dependencies.md b/Documents/Dependencies.md index 7a8663fdb..4c31f4243 100644 --- a/Documents/Dependencies.md +++ b/Documents/Dependencies.md @@ -54,11 +54,11 @@ By default nmos-cpp uses [Conan](https://conan.io) to download most of its depen 1. Install Python 3 if necessary Note: The Python scripts directory needs to be added to the `PATH`, so the Conan executable can be found -2. Install Conan using `pip install conan~=1.47` +2. Install or upgrade Conan using `pip install --upgrade conan~=1.47` Notes: - - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install conan~=1.47` - - Currently, Conan 1.47 or higher (and lower than version 2.0) is required; version 1.59.0 has been tested - - Conan evolves fairly quickly, so it's worth running `pip install --upgrade conan` regularly + - On some platforms with Python 2 and Python 3 both installed this may need to be `pip3 install --upgrade conan~=1.47` + - Currently, Conan 1.47 or higher (and lower than version 2.0) is required by the nmos-cpp recipe; dependencies may require a higher version; version 1.59.0 has been tested + - Conan evolves fairly quickly, so it's worth running `pip install --upgrade conan~=1.47` regularly - By default [Conan assumes semver compatibility](https://docs.conan.io/en/1.42/creating_packages/define_abi_compatibility.html#versioning-schema). Boost and other C++ libraries do not meet this expectation and break ABI compatibility between e.g. minor versions. Unfortunately, the recipes in Conan Center Index do not generally customize their `package_id` method to take this into account. From 4a8f265ca4495b9c394ff4bd250a5a487e403475 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:29:21 +0100 Subject: [PATCH 25/35] Update NmosCppConan.cmake to tested Conan 1.59 --- Development/cmake/NmosCppConan.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Development/cmake/NmosCppConan.cmake b/Development/cmake/NmosCppConan.cmake index 977c035ca..e88f61a2a 100644 --- a/Development/cmake/NmosCppConan.cmake +++ b/Development/cmake/NmosCppConan.cmake @@ -17,7 +17,7 @@ include(${CMAKE_CURRENT_BINARY_DIR}/conan.cmake) # it would be nice to output a message if its a more recent version than tested, like: # "Found Conan version 99.99 that is higher than the current tested version: " ${CONAN_VERSION_CUR}) set(CONAN_VERSION_MIN "1.47.0") -set(CONAN_VERSION_CUR "1.53.0") +set(CONAN_VERSION_CUR "1.59.0") conan_check(VERSION ${CONAN_VERSION_MIN} REQUIRED) set(NMOS_CPP_CONAN_BUILD_LIBS "missing" CACHE STRING "Semicolon separated list of libraries to build rather than download") From cfc75aee23c901be0a8d1ab55dfba3cc43f4259d Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 25 Apr 2023 01:03:21 +0100 Subject: [PATCH 26/35] Do no-op in event of the empty interface name --- Development/nmos/client_utils.cpp | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index a4a4a309c..00223e447 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -99,22 +99,22 @@ namespace nmos if (!client_address.empty() && interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the HTTP client connection"; + + // no-op + return [](web::http::client::native_handle) {}; } return [interface_name, secure, &gate](web::http::client::native_handle native_handle) { - if (!interface_name.empty()) - { #if defined(__linux__) - if (!bind_to_device(interface_name, secure, native_handle)) - { - char error[1024]; - slog::log(gate, SLOG_FLF) << "Unable to bind HTTP client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); - } + if (!bind_to_device(interface_name, secure, native_handle)) + { + char error[1024]; + slog::log(gate, SLOG_FLF) << "Unable to bind HTTP client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); + } #else - slog::log(gate, SLOG_FLF) << "Bind HTTP client connection to " << interface_name << " not supported"; + slog::log(gate, SLOG_FLF) << "Bind HTTP client connection to " << interface_name << " not supported"; #endif - } }; } @@ -127,22 +127,22 @@ namespace nmos if (!client_address.empty() && interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the websocket client connection"; + + // no-op + return [](web::websockets::client::native_handle) {}; } return [interface_name, secure, &gate](web::websockets::client::native_handle native_handle) { - if (!interface_name.empty()) - { #if defined(__linux__) - if (!bind_to_device(interface_name, secure, native_handle)) - { - char error[1024]; - slog::log(gate, SLOG_FLF) << "Unable to bind websocket client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); - } + if (!bind_to_device(interface_name, secure, native_handle)) + { + char error[1024]; + slog::log(gate, SLOG_FLF) << "Unable to bind websocket client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); + } #else - slog::log(gate, SLOG_FLF) << "Bind websocket client connection to " << interface_name << " not supported"; + slog::log(gate, SLOG_FLF) << "Bind websocket client connection to " << interface_name << " not supported"; #endif - } }; } #endif From afd640d14ab2d36c264ddf441567d6bcdd151c5e Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 25 Apr 2023 11:41:07 +0100 Subject: [PATCH 27/35] Use interface_name .data() instead of .c_str() in setsockopt for interface binding --- Development/nmos/client_utils.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index 00223e447..b18b2ef27 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -88,7 +88,8 @@ namespace nmos socket_fd = socket->lowest_layer().native_handle(); } // SO_BINDTODEVICE not defined in windows & mac - return (setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, utility::us2s(interface_name).c_str(), interface_name.length()) == 0); + const auto interface_name_ = utility::us2s(interface_name); + return setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, interface_name_.data(), interface_name_.length()) == 0; } #endif From f69a9f833b2d0d8523928b1cdf76c59fa856f8a4 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 25 Apr 2023 15:33:16 +0100 Subject: [PATCH 28/35] server_address is an implementation detail, where the host_address(es) are the public IP addresses, e.g. for the reverse proxy server --- Development/nmos/settings.cpp | 37 ++--------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/Development/nmos/settings.cpp b/Development/nmos/settings.cpp index 7b8f6c4e1..9e6a73dd0 100644 --- a/Development/nmos/settings.cpp +++ b/Development/nmos/settings.cpp @@ -55,14 +55,8 @@ namespace nmos } } - // if the "server_address" settings is presented, use it for the node's "api endpoints" and "href", and device's "controls href" - const auto server_address = experimental::get_server_address(settings); - if (!server_address.empty()) - { - web::json::insert(settings, std::make_pair(nmos::fields::host_address, server_address)); - } // if the "host_address" setting was omitted, use the first of the "host_addresses" - else if (settings.has_field(nmos::fields::host_addresses)) + if (settings.has_field(nmos::fields::host_addresses)) { web::json::insert(settings, std::make_pair(nmos::fields::host_address, nmos::fields::host_addresses(settings)[0])); } @@ -158,12 +152,7 @@ namespace nmos { hosts.push_back(get_host_name(settings)); } - const auto server_address = experimental::get_server_address(settings); - if (!server_address.empty()) - { - hosts.push_back(server_address); - } - else if (0 != (mode & nmos::experimental::href_mode_addresses)) + if (0 != (mode & nmos::experimental::href_mode_addresses)) { const auto at_least_one_host_address = web::json::value_of({ web::json::value::string(nmos::fields::host_address(settings)) }); const auto& host_addresses = settings.has_field(nmos::fields::host_addresses) ? nmos::fields::host_addresses(settings) : at_least_one_host_address.as_array(); @@ -201,28 +190,6 @@ namespace nmos return web::http::experimental::hsts{ (uint32_t)nmos::experimental::fields::hsts_max_age(settings), nmos::experimental::fields::hsts_include_sub_domains(settings) }; return bst::nullopt; } - - // Get server IP address of the interface to bind server connections - utility::string_t get_server_address(const settings& settings) - { - const auto server_address = nmos::experimental::fields::server_address(settings); - if (!server_address.empty()) - { - // verify the server address match one of the interface addresses - const auto interfaces = web::hosts::experimental::host_interfaces(); - - auto find_interface = [&]() - { - return boost::range::find_if(interfaces, [&](const web::hosts::experimental::host_interface& interface) - { - return interface.addresses.end() != boost::range::find(interface.addresses, server_address); - }); - }; - const auto interface = find_interface(); - if (interfaces.end() != interface) { return server_address; } - } - return {}; - } } // Get a summary of the build configuration, including versions of dependencies From a857ce1f1f15e554d81ec40d63825135cbc50c24 Mon Sep 17 00:00:00 2001 From: lo-simon Date: Tue, 25 Apr 2023 15:36:21 +0100 Subject: [PATCH 29/35] Remove server_address verification --- Development/nmos/node_server.cpp | 2 +- Development/nmos/registry_server.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Development/nmos/node_server.cpp b/Development/nmos/node_server.cpp index d86e2f720..e46bec62d 100644 --- a/Development/nmos/node_server.cpp +++ b/Development/nmos/node_server.cpp @@ -35,7 +35,7 @@ namespace nmos const auto hsts = nmos::experimental::get_hsts(node_model.settings); - const auto server_address = nmos::experimental::get_server_address(node_model.settings); + const auto server_address = nmos::experimental::fields::server_address(node_model.settings); // Configure the Settings API diff --git a/Development/nmos/registry_server.cpp b/Development/nmos/registry_server.cpp index 779f52191..58a4ad8be 100644 --- a/Development/nmos/registry_server.cpp +++ b/Development/nmos/registry_server.cpp @@ -45,7 +45,7 @@ namespace nmos const auto hsts = nmos::experimental::get_hsts(registry_model.settings); - const auto server_address = nmos::experimental::get_server_address(registry_model.settings); + const auto server_address = nmos::experimental::fields::server_address(registry_model.settings); // Configure the DNS-SD Browsing API From c6d5161f7e19d03ef01fc45b668ff35b2ed42eba Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Wed, 26 Apr 2023 11:59:04 +0100 Subject: [PATCH 30/35] Just clean up naming, ordering of function parameters and comments --- Development/cpprest/host_utils.cpp | 5 +++-- Development/cpprest/uri_schemes.h | 6 ++---- Development/nmos-cpp-node/config.json | 10 ++++++---- Development/nmos-cpp-registry/config.json | 12 +++++++----- Development/nmos/client_utils.cpp | 22 +++++++++++----------- Development/nmos/client_utils.h | 10 +++++----- Development/nmos/node_server.cpp | 4 ++-- Development/nmos/ocsp_behaviour.cpp | 7 ++++--- Development/nmos/registry_server.cpp | 4 ++-- Development/nmos/settings.cpp | 1 - Development/nmos/settings.h | 15 +++++++-------- 11 files changed, 49 insertions(+), 47 deletions(-) diff --git a/Development/cpprest/host_utils.cpp b/Development/cpprest/host_utils.cpp index 59a87739b..9c9b0b7e0 100644 --- a/Development/cpprest/host_utils.cpp +++ b/Development/cpprest/host_utils.cpp @@ -5,7 +5,8 @@ #include #include #include -#include +#include +#include #include "cpprest/asyncrt_utils.h" // for utility::conversions #if defined(_WIN32) @@ -337,7 +338,7 @@ namespace web { return interface.addresses.end() != boost::range::find(interface.addresses, address); }); - return host_interfaces.end() != interface ? interface->name : utility::string_t {}; + return host_interfaces.end() != interface ? interface->name : utility::string_t{}; } } } diff --git a/Development/cpprest/uri_schemes.h b/Development/cpprest/uri_schemes.h index 079d73eb9..76e63e02b 100644 --- a/Development/cpprest/uri_schemes.h +++ b/Development/cpprest/uri_schemes.h @@ -1,7 +1,6 @@ #ifndef CPPREST_URI_SCHEMES_H #define CPPREST_URI_SCHEMES_H -#include "cpprest/base_uri.h" #include "cpprest/details/basic_types.h" namespace web @@ -24,10 +23,9 @@ namespace web inline utility::string_t http_scheme(bool secure) { return secure ? uri_schemes::https : uri_schemes::http; } inline utility::string_t ws_scheme(bool secure) { return secure ? uri_schemes::wss : uri_schemes::ws; } - // Check if the URI is secure - inline bool is_secure(const web::uri& uri) + inline bool is_secure_uri_scheme(const utility::string_t& scheme) { - return uri_schemes::https == uri.scheme() || uri_schemes::wss == uri.scheme(); + return uri_schemes::https == scheme || uri_schemes::wss == scheme; } } diff --git a/Development/nmos-cpp-node/config.json b/Development/nmos-cpp-node/config.json index 186027ad6..5bf63a9ff 100644 --- a/Development/nmos-cpp-node/config.json +++ b/Development/nmos-cpp-node/config.json @@ -191,7 +191,12 @@ //"settings_port": 3209, //"logging_port": 5106, - // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address + // addresses [registry, node]: IP addresses on which to listen for each API, or empty string for the wildcard address + + // server_address [registry, node]: if specified, this becomes the default address on which to listen for each API instead of the wildcard address + //"server_address": "", + + // addresses [registry, node]: IP addresses on which to listen for specific APIs //"settings_address": "127.0.0.1", //"logging_address": "", @@ -200,9 +205,6 @@ // for now, only supporting HTTP/HTTPS client connections on Linux //"client_address": "", - // server_address [registry, node]: IP address of the network interface to bind server connections - //"server_address": "", - // logging_limit [registry, node]: maximum number of log events cached for the Logging API //"logging_limit": 1234, diff --git a/Development/nmos-cpp-registry/config.json b/Development/nmos-cpp-registry/config.json index c9968062d..940eb0645 100644 --- a/Development/nmos-cpp-registry/config.json +++ b/Development/nmos-cpp-registry/config.json @@ -109,12 +109,17 @@ //"mdns_port": 3208, //"schemas_port": 3208, - // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address + // addresses [registry, node]: IP addresses on which to listen for each API, or empty string for the wildcard address + + // server_address [registry, node]: if specified, this becomes the default address on which to listen for each API instead of the wildcard address + //"server_address": "", + + // addresses [registry, node]: IP addresses on which to listen for specific APIs //"settings_address": "127.0.0.1", //"logging_address": "", - // addresses [registry]: addresses on which to listen for each API, or empty string for the wildcard address + // addresses [registry]: IP addresses on which to listen for specific APIs //"admin_address": "", //"mdns_address": "", @@ -124,9 +129,6 @@ // for now, only supporting HTTP/HTTPS client connections on Linux //"client_address": "", - // server_address [registry, node]: IP address of the network interface to bind server connections - //"server_address": "", - // query_ws_paging_default/query_ws_paging_limit [registry]: default/maximum number of events per message when using the Query WebSocket API (a client may request a lower limit) //"query_ws_paging_default": 10, //"query_ws_paging_limit": 100, diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index b18b2ef27..68c59e738 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -93,7 +93,7 @@ namespace nmos } #endif - inline std::function make_client_nativehandle_options(const utility::string_t& client_address, bool secure, slog::base_gate& gate) + inline std::function make_client_nativehandle_options(bool secure, const utility::string_t& client_address, slog::base_gate& gate) { // get the associated network interface name from IP address const auto interface_name = web::hosts::experimental::get_interface_name(client_address); @@ -121,7 +121,7 @@ namespace nmos #ifdef CPPRESTSDK_ENABLE_BIND_WEBSOCKET_CLIENT // The current version of the C++ REST SDK 2.10.18 does not provide the callback to enable the custom websocket setting - inline std::function make_ws_client_nativehandle_options(const utility::string_t& client_address, bool secure, slog::base_gate& gate) + inline std::function make_ws_client_nativehandle_options(bool secure, const utility::string_t& client_address, slog::base_gate& gate) { // get the associated network interface name from IP address const auto interface_name = web::hosts::experimental::get_interface_name(client_address); @@ -151,9 +151,9 @@ namespace nmos #endif } - // construct client config based on settings and specified secure flag, e.g. using the specified proxy, and the OCSP client + // construct client config based on specified secure flag and settings, e.g. using the specified proxy and OCSP config // with the remaining options defaulted, e.g. request timeout - web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate) + web::http::client::http_client_config make_http_client_config(bool secure, const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) { web::http::client::http_client_config config; const auto proxy = proxy_uri(settings); @@ -161,22 +161,22 @@ namespace nmos if (secure) config.set_validate_certificates(nmos::experimental::fields::validate_certificates(settings)); #if !defined(_WIN32) && !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) if (secure) config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); - config.set_nativehandle_options(details::make_client_nativehandle_options(nmos::experimental::fields::client_address(settings), secure, gate)); + config.set_nativehandle_options(details::make_client_nativehandle_options(secure, nmos::experimental::fields::client_address(settings), gate)); #endif return config; } - // construct client config based on settings, e.g. using the specified proxy + // construct client config based on settings, e.g. using the specified proxy and OCSP config // with the remaining options defaulted, e.g. request timeout web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) { - return make_http_client_config(settings, load_ca_certificates, nmos::experimental::fields::client_secure(settings), gate); + return make_http_client_config(nmos::experimental::fields::client_secure(settings), settings, load_ca_certificates, gate); } - // construct client config based on settings and specified secure flag, e.g. using the specified proxy + // construct client config based on specified secure flag and settings, e.g. using the specified proxy // with the remaining options defaulted - web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate) + web::websockets::client::websocket_client_config make_websocket_client_config(bool secure, const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) { web::websockets::client::websocket_client_config config; const auto proxy = proxy_uri(settings); @@ -185,7 +185,7 @@ namespace nmos #if !defined(_WIN32) || !defined(__cplusplus_winrt) if (secure) config.set_ssl_context_callback(details::make_client_ssl_context_callback(settings, load_ca_certificates, gate)); #ifdef CPPRESTSDK_ENABLE_BIND_WEBSOCKET_CLIENT - config.set_nativehandle_options(details::make_ws_client_nativehandle_options(nmos::experimental::fields::client_address(settings), secure, gate)); + config.set_nativehandle_options(details::make_ws_client_nativehandle_options(secure, nmos::experimental::fields::client_address(settings), gate)); #endif #endif @@ -196,7 +196,7 @@ namespace nmos // with the remaining options defaulted web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) { - return make_websocket_client_config(settings, load_ca_certificates, nmos::experimental::fields::client_secure(settings), gate); + return make_websocket_client_config(nmos::experimental::fields::client_secure(settings), settings, load_ca_certificates, gate); } // make a request with logging diff --git a/Development/nmos/client_utils.h b/Development/nmos/client_utils.h index 15f942b17..826d9e23c 100644 --- a/Development/nmos/client_utils.h +++ b/Development/nmos/client_utils.h @@ -11,16 +11,16 @@ namespace slog { class base_gate; } // Utility types, constants and functions for implementing NMOS REST API clients namespace nmos { - // construct client config based on settings and specified secure flag, e.g. using the specified proxy and the OCSP client + // construct client config based on specified secure flag and settings, e.g. using the specified proxy and OCSP config // with the remaining options defaulted, e.g. request timeout - web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate); - // construct client config based on settings, e.g. using the specified proxy + web::http::client::http_client_config make_http_client_config(bool secure, const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate); + // construct client config based on settings, e.g. using the specified proxy and OCSP config // with the remaining options defaulted, e.g. request timeout web::http::client::http_client_config make_http_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate); - // construct client config based on settings and specified secure flag, e.g. using the specified proxy + // construct client config based on specified secure flag and settings, e.g. using the specified proxy // with the remaining options defaulted - web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate); + web::websockets::client::websocket_client_config make_websocket_client_config(bool secure, const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate); // construct client config based on settings, e.g. using the specified proxy // with the remaining options defaulted web::websockets::client::websocket_client_config make_websocket_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate); diff --git a/Development/nmos/node_server.cpp b/Development/nmos/node_server.cpp index e46bec62d..ecc75c461 100644 --- a/Development/nmos/node_server.cpp +++ b/Development/nmos/node_server.cpp @@ -72,7 +72,7 @@ namespace nmos for (auto& api_router : node_server.api_routers) { - // empty string and empty server IP address means the wildcard address + // if IP address isn't specified for this router, use default server address or wildcard address const auto& host = !api_router.first.first.empty() ? api_router.first.first : !server_address.empty() ? server_address : web::http::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address @@ -86,7 +86,7 @@ namespace nmos for (auto& ws_handler : node_server.ws_handlers) { - // empty string and empty server IP address means the wildcard address + // if IP address isn't specified for this router, use default server address or wildcard address const auto& host = !ws_handler.first.first.empty() ? ws_handler.first.first : !server_address.empty() ? server_address : web::websockets::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address diff --git a/Development/nmos/ocsp_behaviour.cpp b/Development/nmos/ocsp_behaviour.cpp index 8bd50051a..f16140be6 100644 --- a/Development/nmos/ocsp_behaviour.cpp +++ b/Development/nmos/ocsp_behaviour.cpp @@ -142,9 +142,9 @@ namespace nmos namespace details { - web::http::client::http_client_config make_ocsp_client_config(const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, bool secure, slog::base_gate& gate) + web::http::client::http_client_config make_ocsp_client_config(bool secure, const nmos::settings& settings, load_ca_certificates_handler load_ca_certificates, slog::base_gate& gate) { - auto config = nmos::make_http_client_config(settings, std::move(load_ca_certificates), secure, gate); + auto config = nmos::make_http_client_config(secure, settings, std::move(load_ca_certificates), gate); config.set_timeout(std::chrono::seconds(nmos::experimental::fields::ocsp_request_max(settings))); return config; } @@ -346,7 +346,8 @@ namespace nmos if (!state.client) { const auto ocsp_uri = ocsp_uris.front(); - state.client.reset(new web::http::client::http_client(ocsp_uri, make_ocsp_client_config(model.settings, state.load_ca_certificates, is_secure(ocsp_uri), gate))); + const auto secure = web::is_secure_uri_scheme(ocsp_uri.scheme()); + state.client.reset(new web::http::client::http_client(ocsp_uri, make_ocsp_client_config(secure, model.settings, state.load_ca_certificates, gate))); } auto token = cancellation_source.get_token(); diff --git a/Development/nmos/registry_server.cpp b/Development/nmos/registry_server.cpp index 58a4ad8be..bcd3d024a 100644 --- a/Development/nmos/registry_server.cpp +++ b/Development/nmos/registry_server.cpp @@ -113,7 +113,7 @@ namespace nmos for (auto& api_router : registry_server.api_routers) { - // empty string and empty server IP address means the wildcard address + // if IP address isn't specified for this router, use default server address or wildcard address const auto& host = !api_router.first.first.empty() ? api_router.first.first : !server_address.empty() ? server_address : web::http::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address @@ -127,7 +127,7 @@ namespace nmos for (auto& ws_handler : registry_server.ws_handlers) { - //empty string and empty server IP address means the wildcard address + // if IP address isn't specified for this router, use default server address or wildcard address const auto& host = !ws_handler.first.first.empty() ? ws_handler.first.first : !server_address.empty() ? server_address : web::websockets::experimental::listener::host_wildcard; // map the configured client port to the server port on which to listen // hmm, this should probably also take account of the address diff --git a/Development/nmos/settings.cpp b/Development/nmos/settings.cpp index 9e6a73dd0..c8943c93e 100644 --- a/Development/nmos/settings.cpp +++ b/Development/nmos/settings.cpp @@ -1,7 +1,6 @@ #include "nmos/settings.h" #include -#include #include #include #include diff --git a/Development/nmos/settings.h b/Development/nmos/settings.h index 9f8a883a1..cf55267fd 100644 --- a/Development/nmos/settings.h +++ b/Development/nmos/settings.h @@ -52,9 +52,6 @@ namespace nmos { // Get HTTP Strict-Transport-Security settings bst::optional get_hsts(const settings& settings); - - // Get server IP address of the network interface for binding the server connections - utility::string_t get_server_address(const settings& settings); } // Get a summary of the build configuration, including versions of dependencies @@ -243,12 +240,17 @@ namespace nmos const web::json::field_as_integer_or mdns_port{ U("mdns_port"), 3208 }; const web::json::field_as_integer_or schemas_port{ U("schemas_port"), 3208 }; - // addresses [registry, node]: addresses on which to listen for each API, or empty string for the wildcard address + // addresses [registry, node]: IP addresses on which to listen for each API, or empty string for the wildcard address + + // server_address [registry, node]: if specified, this becomes the default address on which to listen for each API instead of the wildcard address + const web::json::field_as_string_or server_address{ U("server_address"), U("") }; + + // addresses [registry, node]: IP addresses on which to listen for specific APIs const web::json::field_as_string_or settings_address{ U("settings_address"), U("") }; const web::json::field_as_string_or logging_address{ U("logging_address"), U("") }; - // addresses [registry]: addresses on which to listen for each API, or empty string for the wildcard address + // addresses [registry]: IP addresses on which to listen for specific APIs const web::json::field_as_string_or admin_address{ U("admin_address"), U("") }; const web::json::field_as_string_or mdns_address{ U("mdns_address"), U("") }; @@ -258,9 +260,6 @@ namespace nmos // for now, only supporting HTTP/HTTPS client connections on Linux const web::json::field_as_string_or client_address{ U("client_address"), U("") }; - // server_address [registry, node]: IP address of the network interface to bind server connections - const web::json::field_as_string_or server_address{ U("server_address"), U("") }; - // query_ws_paging_default/query_ws_paging_limit [registry]: default/maximum number of events per message when using the Query WebSocket API (a client may request a lower limit) const web::json::field_as_integer_or query_ws_paging_default{ U("query_ws_paging_default"), 10 }; const web::json::field_as_integer_or query_ws_paging_limit{ U("query_ws_paging_limit"), 100 }; From 6c39cf09601fdd61179f04e72d933232e88e729d Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Wed, 26 Apr 2023 13:54:24 +0100 Subject: [PATCH 31/35] Just clean up client_utils.cpp, include specific header, separate logging from platform-specific implementation, use empty std::function rather than no-op --- Development/nmos/client_utils.cpp | 63 ++++++++++++++++--------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/Development/nmos/client_utils.cpp b/Development/nmos/client_utils.cpp index 68c59e738..784a5183b 100644 --- a/Development/nmos/client_utils.cpp +++ b/Development/nmos/client_utils.cpp @@ -2,8 +2,8 @@ // cf. preprocessor conditions in nmos::details::make_client_ssl_context_callback and nmos::details::make_client_nativehandle_options #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) -#if !defined(_WIN32) -#include "boost/asio.hpp" +#if defined(__linux__) +#include #endif #include "boost/asio/ssl/set_cipher_list.hpp" #include "cpprest/host_utils.h" @@ -62,11 +62,13 @@ namespace nmos #endif #if !defined(_WIN32) || !defined(__cplusplus_winrt) || defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) - // bind socket to a specific network interface, only supporting Linux -#if defined(__linux__) - inline bool bind_to_device(const utility::string_t& interface_name, bool secure, void* native_handle) + // bind socket to a specific network interface + // for now, only supporting Linux because SO_BINDTODEVICE is not defined on Windows and Mac + inline void bind_to_device(const utility::string_t& interface_name, bool secure, void* native_handle) { +#if defined(__linux__) int socket_fd; + // hmm, frustrating that native_handle type has been erased so we need secure flag if (secure) { auto socket = (boost::asio::ssl::stream*)native_handle; @@ -87,35 +89,38 @@ namespace nmos } socket_fd = socket->lowest_layer().native_handle(); } - // SO_BINDTODEVICE not defined in windows & mac const auto interface_name_ = utility::us2s(interface_name); - return setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, interface_name_.data(), interface_name_.length()) == 0; - } + if (0 != setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, interface_name_.data(), interface_name_.length())) + { + char error[1024]; + throw std::runtime_error(strerror_r(errno, error, sizeof(error))); + } +#else + throw std::logic_error("unsupported"); #endif + } inline std::function make_client_nativehandle_options(bool secure, const utility::string_t& client_address, slog::base_gate& gate) { + if (client_address.empty()) return {}; // get the associated network interface name from IP address const auto interface_name = web::hosts::experimental::get_interface_name(client_address); - if (!client_address.empty() && interface_name.empty()) + if (interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the HTTP client connection"; - - // no-op - return [](web::http::client::native_handle) {}; + return {}; } return [interface_name, secure, &gate](web::http::client::native_handle native_handle) { -#if defined(__linux__) - if (!bind_to_device(interface_name, secure, native_handle)) + try { - char error[1024]; - slog::log(gate, SLOG_FLF) << "Unable to bind HTTP client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); + bind_to_device(interface_name, secure, native_handle); + } + catch (const std::exception& e) + { + slog::log(gate, SLOG_FLF) << "Unable to bind HTTP client connection to " << interface_name << ": " << e.what(); } -#else - slog::log(gate, SLOG_FLF) << "Bind HTTP client connection to " << interface_name << " not supported"; -#endif }; } @@ -123,27 +128,25 @@ namespace nmos // The current version of the C++ REST SDK 2.10.18 does not provide the callback to enable the custom websocket setting inline std::function make_ws_client_nativehandle_options(bool secure, const utility::string_t& client_address, slog::base_gate& gate) { + if (client_address.empty()) return {}; // get the associated network interface name from IP address const auto interface_name = web::hosts::experimental::get_interface_name(client_address); - if (!client_address.empty() && interface_name.empty()) + if (interface_name.empty()) { slog::log(gate, SLOG_FLF) << "No network interface found for " << client_address << " to bind for the websocket client connection"; - - // no-op - return [](web::websockets::client::native_handle) {}; + return {}; } return [interface_name, secure, &gate](web::websockets::client::native_handle native_handle) { -#if defined(__linux__) - if (!bind_to_device(interface_name, secure, native_handle)) + try { - char error[1024]; - slog::log(gate, SLOG_FLF) << "Unable to bind websocket client connection to " << interface_name << ", bind_to_device reported error : " << strerror_r(errno, error, sizeof(error)); + bind_to_device(interface_name, secure, native_handle); + } + catch (const std::exception& e) + { + slog::log(gate, SLOG_FLF) << "Unable to bind websocket client connection to " << interface_name << ": " << e.what(); } -#else - slog::log(gate, SLOG_FLF) << "Bind websocket client connection to " << interface_name << " not supported"; -#endif }; } #endif From ea977153ced03a74a8cd6d0f74e6a3337787ce0f Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Wed, 26 Apr 2023 13:02:57 +0100 Subject: [PATCH 32/35] Disable git SSL certificate verification on Ubuntu 14.04 --- .github/workflows/build-test.yml | 2 ++ .github/workflows/src/build-test.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2f92f4959..ed8737899 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -577,6 +577,8 @@ jobs: apt-get install -y software-properties-common apt-get --allow-unauthenticated update -q apt-get --allow-unauthenticated install -y curl g++ git make patch zlib1g-dev libssl-dev bsdmainutils dnsutils unzip + # ubuntu-14.04 ca-certificates are out of date + git config --global http.sslVerify false curl -sS https://www.python.org/ftp/python/3.6.9/Python-3.6.9.tar.xz | tar -xJ cd Python-3.6.9 ./configure diff --git a/.github/workflows/src/build-test.yml b/.github/workflows/src/build-test.yml index e9095e815..0b5663ff3 100644 --- a/.github/workflows/src/build-test.yml +++ b/.github/workflows/src/build-test.yml @@ -127,6 +127,8 @@ jobs: apt-get install -y software-properties-common apt-get --allow-unauthenticated update -q apt-get --allow-unauthenticated install -y curl g++ git make patch zlib1g-dev libssl-dev bsdmainutils dnsutils unzip + # ubuntu-14.04 ca-certificates are out of date + git config --global http.sslVerify false curl -sS https://www.python.org/ftp/python/3.6.9/Python-3.6.9.tar.xz | tar -xJ cd Python-3.6.9 ./configure From 772feb6051978427ada1ebd8c5b3592bc737159f Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Thu, 4 May 2023 22:10:42 +0100 Subject: [PATCH 33/35] Add nmos::parse_fmtp_channel_order --- Development/nmos/channels.cpp | 48 +++++++++++++++++++++++++ Development/nmos/channels.h | 1 + Development/nmos/test/channels_test.cpp | 37 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/Development/nmos/channels.cpp b/Development/nmos/channels.cpp index 611e22f3d..e9f596099 100644 --- a/Development/nmos/channels.cpp +++ b/Development/nmos/channels.cpp @@ -155,4 +155,52 @@ namespace nmos return channel_order.str(); } + + // See SMPTE ST 2110-30:2017 Section 6.2.2 Channel Order Convention + std::vector parse_fmtp_channel_order(const utility::string_t& channel_order) + { + std::vector channels; + + const auto first = channel_order.data(); + const auto last = first + channel_order.size(); + auto it = first; + + // check prefix + + static const auto prefix = U("SMPTE2110.("); + auto pit = &prefix[0]; + while (it != last && *pit != U('\0') && *it == *pit) ++it, ++pit; + if (*pit != U('\0')) return {}; + + // parse comma-separated channel group symbols + + while (true) + { + const auto git = it; + while (it != last && *it != U(')') && *it != U(',')) ++it; + if (it == last) return {}; + + const channel_group_symbol symbol(utility::string_t(git, it)); + + // hm, does not handle 22.2 Surround ('222'), SDI audio group ('SGRP') or Undefined ('U01' to 'U64') + auto group = std::find_if(details::channel_groups.begin(), details::channel_groups.end(), + [&](const std::pair, channel_group_symbol>& group) + { + return symbol == group.second; + }); + if (details::channel_groups.end() == group) return {}; + channels.insert(channels.end(), group->first.begin(), group->first.end()); + + if (*it == U(')')) break; + ++it; + } + + // check suffix + + if (it == last) return {}; + ++it; + if (it != last) return {}; + + return channels; + } } diff --git a/Development/nmos/channels.h b/Development/nmos/channels.h index 77e12a945..df0522b12 100644 --- a/Development/nmos/channels.h +++ b/Development/nmos/channels.h @@ -113,6 +113,7 @@ namespace nmos // See SMPTE ST 2110-30:2017 Section 6.2.2 Channel Order Convention utility::string_t make_fmtp_channel_order(const std::vector& channels); + std::vector parse_fmtp_channel_order(const utility::string_t& channel_order); } #endif diff --git a/Development/nmos/test/channels_test.cpp b/Development/nmos/test/channels_test.cpp index 1868ee853..ce316e592 100644 --- a/Development/nmos/test/channels_test.cpp +++ b/Development/nmos/test/channels_test.cpp @@ -28,3 +28,40 @@ BST_TEST_CASE(testMakeFmtpChannelOrder) const std::vector example_3{ M1, M1, M1, M1, L, R, C, LFE }; BST_REQUIRE_EQUAL(U("SMPTE2110.(M,M,M,M,ST,U02)"), nmos::make_fmtp_channel_order(example_3)); } + +//////////////////////////////////////////////////////////////////////////////////////////// +BST_TEST_CASE(testParseFmtpChannelOrder) +{ + using namespace nmos::channel_symbols; + + // two simple examples + + const std::vector stereo{ L, R }; + BST_REQUIRE_EQUAL(stereo, nmos::parse_fmtp_channel_order(U("SMPTE2110.(ST)"))); + + const std::vector dual_mono{ M1, M2 }; + BST_REQUIRE_EQUAL(dual_mono, nmos::parse_fmtp_channel_order(U("SMPTE2110.(DM)"))); + + // two examples from ST 2110-30:2017 Section 6.2.2 Channel Order Convention + + const std::vector example_1{ L, R, C, LFE, Ls, Rs, L, R }; + BST_REQUIRE_EQUAL(example_1, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,ST)"))); + + //const std::vector example_2{ M1, M1, M1, M1, L, R, Undefined(1), Undefined(2) }; + //BST_REQUIRE_EQUAL(example_2, nmos::parse_fmtp_channel_order(U("SMPTE2110.(M,M,M,M,ST,U02)"))); + + // bad examples + + const std::vector empty; + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,ST)BAD"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,ST,)"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,,ST)"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,BAD)"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,ST"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51,"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.(51"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110.("))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("SMPTE2110."))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U("BAD"))); + BST_REQUIRE_EQUAL(empty, nmos::parse_fmtp_channel_order(U(""))); +} From 508908d76313b7d1c6e3e956eeddbfce426a525a Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Thu, 4 May 2023 22:41:15 +0100 Subject: [PATCH 34/35] Add tags for BCP-002-02 NMOS Asset Distinguishing Information --- Development/cmake/NmosCppLibraries.cmake | 1 + Development/nmos/asset.h | 20 ++++++++++++++++++++ README.md | 1 + 3 files changed, 22 insertions(+) create mode 100644 Development/nmos/asset.h diff --git a/Development/cmake/NmosCppLibraries.cmake b/Development/cmake/NmosCppLibraries.cmake index b6d6d972a..944eb9902 100644 --- a/Development/cmake/NmosCppLibraries.cmake +++ b/Development/cmake/NmosCppLibraries.cmake @@ -811,6 +811,7 @@ set(NMOS_CPP_NMOS_HEADERS nmos/api_downgrade.h nmos/api_utils.h nmos/api_version.h + nmos/asset.h nmos/capabilities.h nmos/certificate_handlers.h nmos/certificate_settings.h diff --git a/Development/nmos/asset.h b/Development/nmos/asset.h new file mode 100644 index 000000000..fafd2c834 --- /dev/null +++ b/Development/nmos/asset.h @@ -0,0 +1,20 @@ +#ifndef NMOS_ASSET_H +#define NMOS_ASSET_H + +#include "cpprest/json_utils.h" + +// Asset Distinguishing Information +// See https://specs.amwa.tv/bcp-002-02/ +// and https://specs.amwa.tv/nmos-parameter-registers/branches/main/tags/ +namespace nmos +{ + namespace fields + { + const web::json::field_as_value_or asset_manufacturer{ U("urn:x-nmos:tag:asset:manufacturer/v1.0"), web::json::value::array() }; + const web::json::field_as_value_or asset_product_name{ U("urn:x-nmos:tag:asset:product/v1.0"), web::json::value::array() }; + const web::json::field_as_value_or asset_instance_id{ U("urn:x-nmos:tag:asset:instance-id/v1.0"), web::json::value::array() }; + const web::json::field_as_value_or asset_function{ U("urn:x-nmos:tag:asset:function/v1.0"), web::json::value::array() }; + } +} + +#endif diff --git a/README.md b/README.md index db34aad60..050394924 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ This repository contains an implementation of the [AMWA Networked Media Open Spe - [AMWA IS-08 NMOS Audio Channel Mapping Specification](https://specs.amwa.tv/is-08/) - [AMWA IS-09 NMOS System Parameters Specification](https://specs.amwa.tv/is-09/) (originally defined in JT-NM TR-1001-1:2018 Annex A) - [AMWA BCP-002-01 NMOS Grouping Recommendations - Natural Grouping](https://specs.amwa.tv/bcp-002-01/) +- [AMWA BCP-002-02 NMOS Asset Distinguishing Information](https://specs.amwa.tv/bcp-002-02/) - [AMWA BCP-003-01 Secure Communication in NMOS Systems](https://specs.amwa.tv/bcp-003-01/) - [AMWA BCP-004-01 NMOS Receiver Capabilities](https://specs.amwa.tv/bcp-004-01/) - [AMWA BCP-006-01 NMOS With JPEG XS](https://specs.amwa.tv/bcp-006-01/) From f56f8b3f3a26cc6653c82ee5b4ac24ea52d4251c Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Mon, 30 Jan 2023 11:01:15 +0000 Subject: [PATCH 35/35] Workaround for registration DELETE issue --- Development/nmos/api_utils.cpp | 3 --- Development/nmos/api_utils.h | 3 +++ Development/nmos/registration_api.cpp | 31 +++++++++++++++++++++++---- Development/nmos/resource.h | 2 ++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Development/nmos/api_utils.cpp b/Development/nmos/api_utils.cpp index 33c385d6a..bf352c77c 100644 --- a/Development/nmos/api_utils.cpp +++ b/Development/nmos/api_utils.cpp @@ -467,9 +467,6 @@ namespace nmos }; } - static const utility::string_t received_time{ U("X-Received-Time") }; - static const utility::string_t actual_method{ U("X-Actual-Method") }; - // make handler to set appropriate response headers, and error response body if indicated web::http::experimental::listener::route_handler make_api_finally_handler(slog::base_gate& gate) { diff --git a/Development/nmos/api_utils.h b/Development/nmos/api_utils.h index 2dda425a6..0db8898bb 100644 --- a/Development/nmos/api_utils.h +++ b/Development/nmos/api_utils.h @@ -161,6 +161,9 @@ namespace nmos namespace details { + const utility::string_t received_time{ U("X-Received-Time") }; + const utility::string_t actual_method{ U("X-Actual-Method") }; + // exception to skip other route handlers and then send the response (see add_api_finally_handler) struct to_api_finally_handler {}; diff --git a/Development/nmos/registration_api.cpp b/Development/nmos/registration_api.cpp index 9d174eb75..1f7f704ab 100644 --- a/Development/nmos/registration_api.cpp +++ b/Development/nmos/registration_api.cpp @@ -282,7 +282,15 @@ namespace nmos const bool valid_version = creating || unchanged || nmos::fields::version(data) > nmos::fields::version(resource->data); valid = valid && valid_version; - if (!valid_type) + // check received request isn't being processed out of order + const auto received_time = req.headers().find(details::received_time); + const auto received = req.headers().end() != received_time ? nmos::parse_version(received_time->second) : nmos::tai{}; + const bool valid_received = creating || received == nmos::tai{} || received > resource->received; + valid = valid && valid_received; + + if (!valid_received) + slog::log(gate, SLOG_FLF) << "Registration requested for " << id_type << " at " << nmos::make_version(resource->received) << " processed before request received at " << nmos::make_version(received); + else if (!valid_type) slog::log(gate, SLOG_FLF) << "Registration requested for " << id_type << " would modify type from " << resource->type.name; else if (!valid_api_version) slog::log(gate, SLOG_FLF) << "Registration requested for " << id_type << " would modify API version from " << nmos::make_api_version(resource->version); @@ -399,6 +407,7 @@ namespace nmos if (creating) { nmos::resource created_resource{ version, type, data, false }; + created_resource.received = received; set_reply(res, status_codes::Created, data); res.headers().add(web::http::header_names::location, make_registration_api_resource_location(created_resource)); @@ -410,8 +419,9 @@ namespace nmos set_reply(res, status_codes::OK, data); res.headers().add(web::http::header_names::location, make_registration_api_resource_location(*resource)); - modify_resource(resources, id, [&data](nmos::resource& resource) + modify_resource(resources, id, [&received, &data](nmos::resource& resource) { + resource.received = received; resource.data = data; }); } @@ -424,6 +434,10 @@ namespace nmos slog::log(gate, SLOG_FLF) << "Notifying query websockets thread"; // and anyone else who cares... model.notify(); } + else if (!valid_received) + { + set_reply(res, status_codes::InternalError); + } else if (!valid_api_version) { // experimental extension, proposed for v1.3, using a more specific status code to distinguish conflicts from validation errors @@ -585,10 +599,19 @@ namespace nmos const string_t resourceType = parameters.at(nmos::patterns::resourceType.name); const string_t resourceId = parameters.at(nmos::patterns::resourceId.name); - auto resource = find_resource(resources, { resourceId, nmos::type_from_resourceType(resourceType) }); + const std::pair id_type{ resourceId, nmos::type_from_resourceType(resourceType) }; + auto resource = find_resource(resources, id_type); if (resources.end() != resource) { - if (resource->version == version) + // check received request isn't being processed out of order + const auto received_time = req.headers().find(details::received_time); + const auto received = req.headers().end() != received_time ? nmos::parse_version(received_time->second) : nmos::tai{}; + if (received != nmos::tai{} && received < resource->received) + { + slog::log(gate, SLOG_FLF) << "Registration deletion requested for " << id_type << " at " << nmos::make_version(resource->received) << " processed before request received at " << nmos::make_version(received); + set_reply(res, status_codes::InternalError); + } + else if (resource->version == version) { slog::log(gate, SLOG_FLF) << "Deleting resource: " << resourceId; diff --git a/Development/nmos/resource.h b/Development/nmos/resource.h index 81eb0f558..fedfde3a4 100644 --- a/Development/nmos/resource.h +++ b/Development/nmos/resource.h @@ -66,6 +66,8 @@ namespace nmos // see https://specs.amwa.tv/is-04/releases/v1.2.0/docs/2.5._APIs_-_Query_Parameters.html#pagination tai created; tai updated; + // when the most recently applied request was received + tai received; // see https://specs.amwa.tv/is-04/releases/v1.2.0/docs/4.1._Behaviour_-_Registration.html#heartbeating mutable details::copyable_atomic health;