From 1c4838ebe6425611e9e37d9450caf9e936800750 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 24 Aug 2023 11:10:53 +0100 Subject: [PATCH 1/4] ECKIT-619 Enable PUT method on EasyCURL --- src/eckit/io/EasyCURL.cc | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/eckit/io/EasyCURL.cc b/src/eckit/io/EasyCURL.cc index 4a47e2408..73858f087 100644 --- a/src/eckit/io/EasyCURL.cc +++ b/src/eckit/io/EasyCURL.cc @@ -933,14 +933,31 @@ EasyCURLResponse EasyCURL::POST(const std::string& url, const std::string& data) return request(url); } +namespace { + +/** + * Copies the content from input `userdata` into output `buffer` + * + * Returns the number of characters copied + */ +size_t readCallback(char* buffer, size_t size [[maybe_unused]], size_t nitems [[maybe_unused]], void* userdata) { + auto data = static_cast(userdata); + + data->copy(buffer, data->size()); + return data->size(); +} + +} // namespace + EasyCURLResponse EasyCURL::PUT(const std::string& url, const std::string& data) { - NOTIMP; - // Disable unreachable code -#if 0 - _(curl_easy_setopt(ch_->curl_, CURLOPT_CUSTOMREQUEST, NULL)); - _(curl_easy_setopt(ch_->curl_, CURLOPT_PUT, 1L)); + _(curl_easy_setopt(ch_->curl_, CURLOPT_UPLOAD, 1L)); + + // Setup callback to read PUT request body (i.e. copy the given 'data' into an internal 'buffer') + _(curl_easy_setopt(ch_->curl_, CURLOPT_READFUNCTION, readCallback)); + _(curl_easy_setopt(ch_->curl_, CURLOPT_READDATA, &data)); + _(curl_easy_setopt(ch_->curl_, CURLOPT_INFILESIZE, data.size())); + return request(url); -#endif } EasyCURLResponse EasyCURL::DELETE(const std::string& url) { From 80258be6974536d0ec3c497af90c6c2e71af083d Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 19 Sep 2023 11:59:45 +0100 Subject: [PATCH 2/4] ECKIT-619 Setup EasyCURL specific test A specific test file was created containing the existing EasyCURL test. The existing EasyCURL test itself was updated to correct the URL used to access the test data at ECMWF's Nexus Repository. --- tests/io/CMakeLists.txt | 5 +++++ tests/io/test_easycurl.cc | 42 ++++++++++++++++++++++++++++++++++++++ tests/io/test_urlhandle.cc | 11 ---------- 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 tests/io/test_easycurl.cc diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index 540f62f57..d22fdcd60 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -13,6 +13,11 @@ ecbuild_add_test( TARGET eckit_test_filepool SOURCES test_filepool.cc LIBS eckit ) +ecbuild_add_test( TARGET eckit_test_easycurl + SOURCES test_easycurl.cc + CONDITION HAVE_EXTRA_TESTS AND eckit_HAVE_CURL + LIBS eckit ) + ecbuild_add_test( TARGET eckit_test_urlhandle SOURCES test_urlhandle.cc CONDITION HAVE_EXTRA_TESTS AND eckit_HAVE_CURL diff --git a/tests/io/test_easycurl.cc b/tests/io/test_easycurl.cc new file mode 100644 index 000000000..b95e8b86d --- /dev/null +++ b/tests/io/test_easycurl.cc @@ -0,0 +1,42 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/io/EasyCURL.h" +#include "eckit/value/Value.h" + +#include "eckit/testing/Test.h" + +using namespace std; +using namespace eckit; +using namespace eckit::testing; + +namespace eckit { +namespace test { + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("EasyCURL GET") { + auto curl = EasyCURL(); + + auto response = curl.GET("https://get.ecmwf.int/repository/test-data/eckit/tests/io/t.grib.md5"); + EXPECT(response.code() == 200); + EXPECT(response.body() == "f59fdc6a09c1d11b0e567309ef541bef t.grib\n"); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace test +} // namespace eckit + +int main(int argc, char** argv) { + return run_tests(argc, argv); +} diff --git a/tests/io/test_urlhandle.cc b/tests/io/test_urlhandle.cc index a5bc342ec..63b4e3287 100644 --- a/tests/io/test_urlhandle.cc +++ b/tests/io/test_urlhandle.cc @@ -8,14 +8,10 @@ * does it submit to any jurisdiction. */ -#include #include #include -#include "eckit/eckit_config.h" -#include "eckit/io/EasyCURL.h" #include "eckit/io/URLHandle.h" -#include "eckit/value/Value.h" #include "eckit/testing/Test.h" @@ -96,13 +92,6 @@ CASE("No use of SSL") { out.unlink(); } - -CASE("EasyCURL GET") { - auto r = EasyCURL().GET("http://get.ecmwf.int/test-data/eckit/tests/io/t.grib.md5"); - // Log::info() << "[" << r.body() << "]" << std::endl; - EXPECT(r.body() == "f59fdc6a09c1d11b0e567309ef541bef t.grib\n"); -} - //---------------------------------------------------------------------------------------------------------------------- } // namespace test From 87facb53d03bb359664d102d0fd31a0e9b986469 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 19 Sep 2023 14:53:24 +0100 Subject: [PATCH 3/4] ECKIT-619 Add more EasyCURL tests --- tests/io/test_easycurl.cc | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/io/test_easycurl.cc b/tests/io/test_easycurl.cc index b95e8b86d..ca4091dda 100644 --- a/tests/io/test_easycurl.cc +++ b/tests/io/test_easycurl.cc @@ -19,6 +19,25 @@ using namespace std; using namespace eckit; using namespace eckit::testing; +/* + * + * The following tests access the ECMWF's Nexus Repository, namely + * the repository 'test-data' which holds public ancillary data to + * support tests available at: + * + * https://get.ecmwf.int/#browse/browse:test-data:eckit + * + * The tests also use the Nexus' REST API, which has its base path at: + * + * https://get.ecmwf.int/service/rest/ + * + * The description of the end-points made available by Nexus, is + * available at: + * + * https://get.ecmwf.int/service/rest/swagger.json + * + */ + namespace eckit { namespace test { @@ -32,6 +51,42 @@ CASE("EasyCURL GET") { EXPECT(response.body() == "f59fdc6a09c1d11b0e567309ef541bef t.grib\n"); } +CASE("EasyCURL GET (REST API, Successful operation)") { + EasyCURLHeaders headers; + headers["content-type"] = "application/json"; + + auto curl = EasyCURL(); + curl.headers(headers); + + auto response = curl.GET("https://get.ecmwf.int/service/rest/v1/assets?repository=test-data"); + EXPECT(response.code() == 200); + EXPECT(response.body().find(R"("items")") != std::string::npos); +} + +CASE("EasyCURL GET (REST API, Insufficient permissions)") { + EasyCURLHeaders headers; + headers["content-type"] = "application/json"; + + auto curl = EasyCURL(); + curl.headers(headers); + + auto response = curl.GET("https://get.ecmwf.int/service/rest/v1/security/anonymous"); + EXPECT(response.code() == 403); + EXPECT(response.body() == ""); +} + +CASE("EasyCURL PUT (REST API, Insufficient permissions)") { + EasyCURLHeaders headers; + headers["content-type"] = "application/json"; + + auto curl = EasyCURL(); + curl.headers(headers); + + auto response = curl.PUT("https://get.ecmwf.int/service/rest/v1/security/anonymous", ""); + EXPECT(response.code() == 403); + EXPECT(response.body() == ""); +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace test From 9fc8e83c4d0539012fec7cfe98470476e8076fcc Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Wed, 20 Sep 2023 13:31:27 +0100 Subject: [PATCH 4/4] ECKIT-619 Add even mode EasyCURL tests The tests exercise EasyCURL calls for GET, POST, PUT and DELETE. Existing and new tests are supported by a Python-based Mock REST API that stores key+value pairs. A Python environment with Flask is necessary to run the Mock REST API. --- tests/io/CMakeLists.txt | 5 + tests/io/mock/MockREST.launcher.sh | 8 ++ tests/io/mock/MockREST.py | 148 +++++++++++++++++++++++ tests/io/mock/requirements.txt | 1 + tests/io/test_easycurl.cc | 185 +++++++++++++++++++++++------ 5 files changed, 312 insertions(+), 35 deletions(-) create mode 100755 tests/io/mock/MockREST.launcher.sh create mode 100644 tests/io/mock/MockREST.py create mode 100644 tests/io/mock/requirements.txt diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index d22fdcd60..b2a35896e 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -18,6 +18,11 @@ ecbuild_add_test( TARGET eckit_test_easycurl CONDITION HAVE_EXTRA_TESTS AND eckit_HAVE_CURL LIBS eckit ) +file( + COPY mock/MockREST.py mock/MockREST.launcher.sh + DESTINATION ${CMAKE_CURRENT_BINARY_DIR} +) + ecbuild_add_test( TARGET eckit_test_urlhandle SOURCES test_urlhandle.cc CONDITION HAVE_EXTRA_TESTS AND eckit_HAVE_CURL diff --git a/tests/io/mock/MockREST.launcher.sh b/tests/io/mock/MockREST.launcher.sh new file mode 100755 index 000000000..072d87b55 --- /dev/null +++ b/tests/io/mock/MockREST.launcher.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -ex + +BASE_DIR="$( cd "$( dirname "$0" )" >/dev/null 2>&1 && pwd )" + +export FLASK_APP=${BASE_DIR}/MockREST.py +flask run -h localhost -p 49111 diff --git a/tests/io/mock/MockREST.py b/tests/io/mock/MockREST.py new file mode 100644 index 000000000..af85accda --- /dev/null +++ b/tests/io/mock/MockREST.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +import os +import signal +from flask import Flask, request, jsonify + +""" + +MockREST provides a simple REST API over HTTP to support tests. + +The API requires a Python environment with `flask` installed to execute -- see requirements.txt. + +The API consists of the following end-points: + + * /ping [GET] + - always replies "Hi!" (200) + * /insufficient-permissions [GET] + - always replies "Forbidden" (403) + * /shutdown [GET] + - shuts down the running server + * /blobs [GET] + - replies with the list of currently available blobs (in JSON) + + * /blob/ [GET] + - replies with the blob associated with (in JSON) (200) + - if not found, replies with "Not found" (404) + + * /blob [POST] + - creates a new blob, based on payload '{"key": "", "value": ""}' + - replies with "Request must be JSON" (415) is payload is not JSON + - replies with "Bad Request" (400) if payload doesn't include "key" or "value" + - replies with "Bad Request" (400) if a blob is already associated with + - replies with "Created" (201), in case of success + + * /blob [PUT] + - updates an existing blob, based on payload '{"key": "", "value": ""}' + - replies with "Request must be JSON" (415) is payload is not JSON + - replies with "Bad Request" (400) if payload doesn't include "key" or "value" + - replies with "Bad Request" (400) if no blob is associated with + - replies with "OK" (200), in case of success + + * /blob/ [DELETE] + - deletes an existing blob, associated with + - if not found, replies with "Not found" (404) + - replies with "" (204), in case of success + + * /blob//content [GET] + - replies with the blob value (in Plain Text) (200) + - if not found, replies with "Not found" (404) + +""" + +app = Flask(__name__) + + +blobs = {} + + +def _blob_has_all_of(blob, fields): + for field in fields: + if field not in blob: + return False, field + return True, None + + +@app.get("/ping") +def ping(): + return "Hi!", 200 + + +@app.get("/insufficient-permissions") +def no_permissions(): + return "Forbidden", 403 + + +@app.get("/shutdown") +def shutdown(): + # terminate own running process!... + os.kill(os.getpid(), signal.SIGINT) + return jsonify(blobs) + + +@app.get("/blobs") +def get_blobs(): + return jsonify(blobs) + + +@app.get("/blob/") +def get_blob(key): + if key not in blobs: + return {"error": f"Not Found: Unable to find blob with key '{key}'"}, 404 + blob = {"key": key, "value": blobs[key]} + return jsonify(blob) + + +@app.get("/blob//content") +def get_blob_content(key): + if key not in blobs: + return {"error": f"Not Found: Unable to find blob with key '{key}'"}, 404 + return blobs[key] + + +@app.post("/blob") +def create_blob(): + if not request.is_json: + return {"error": "Request must be JSON"}, 415 + + blob = request.get_json() + + found, field = _blob_has_all_of(blob, ["key", "value"]) + if not found: + return {"error": f"Bad Request: New blob must contain field '{field}'"}, 400 + + key = blob["key"] + if key in blobs: + return {"error": f"Bad Request: Key {blob['key']} already exists"}, 400 + + value = blob["value"] + blobs[key] = value + return f"Created: {blob}", 201 + + +@app.put("/blob") +def update_blob(): + if not request.is_json: + return {"error": "Request must be JSON"}, 415 + + blob = request.get_json() + + found, field = _blob_has_all_of(blob, ["key", "value"]) + if not found: + return {"error": f"Bad Request: Update blob must contain field '{field}'"}, 400 + + key = blob["key"] + if key not in blobs: + return {"error": f"Bad Request: Key {blob['key']} does not exist"}, 400 + + value = blob["value"] + blobs[key] = value + return f"OK: {blob}", 200 + + +@app.delete("/blob/") +def delete_blob(key): + if key not in blobs: + return {"error": f"Not Found (Unable to find blob with key '{key}')"}, 404 + del blobs[key] + return "", 204 diff --git a/tests/io/mock/requirements.txt b/tests/io/mock/requirements.txt new file mode 100644 index 000000000..047e9501a --- /dev/null +++ b/tests/io/mock/requirements.txt @@ -0,0 +1 @@ +Flask==3.0.0 diff --git a/tests/io/test_easycurl.cc b/tests/io/test_easycurl.cc index ca4091dda..85b811ae0 100644 --- a/tests/io/test_easycurl.cc +++ b/tests/io/test_easycurl.cc @@ -8,7 +8,10 @@ * does it submit to any jurisdiction. */ +#include +#include #include +#include #include "eckit/io/EasyCURL.h" #include "eckit/value/Value.h" @@ -21,70 +24,180 @@ using namespace eckit::testing; /* * - * The following tests access the ECMWF's Nexus Repository, namely - * the repository 'test-data' which holds public ancillary data to - * support tests available at: - * - * https://get.ecmwf.int/#browse/browse:test-data:eckit - * - * The tests also use the Nexus' REST API, which has its base path at: - * - * https://get.ecmwf.int/service/rest/ - * - * The description of the end-points made available by Nexus, is - * available at: - * - * https://get.ecmwf.int/service/rest/swagger.json + * The following tests take advantage of the Python-based Mock REST API -- see mock/MockREST.py. + * The Mock API is launched during test setup and serves as target for EasyCURL GET/POST/PUT/DELETE calls. * */ +namespace { + +const std::string BASE_URL = "http://127.0.0.1:49111"; + +class MockREST { +public: + MockREST() { + // Launch REST API in a (detached) thread + std::thread api(MockREST::launch_rest_api); + api.detach(); + + // ... give the service some lead time + std::this_thread::sleep_for(std::chrono::seconds(2)); + // ... and way for the service initialization + MockREST::wait_for_running_api(); + } + + ~MockREST() { + // Gracefully, request REST API to shut down + MockREST::shutdown_running_api(); + } + +private: + static void launch_rest_api() { + std::system("MockREST.launcher.sh"); + } + + static void wait_for_running_api() { + auto curl = EasyCURL(); + curl.verbose(true); + for (size_t attempt = 0; attempt < 30; ++attempt) { + try { + auto response = curl.GET(BASE_URL + "/ping"); + if (response.code() == 200) { + // Got a ping! We're done... + return; + } + } + catch (const SeriousBug& error) { + // Unable to connect to server! Must retry... + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + // Enough! Let's give up... + throw std::runtime_error("Unable to start Mock REST API"); + } + + static void shutdown_running_api() { + auto curl = EasyCURL(); + try { + auto response = curl.GET(BASE_URL + "/shutdown"); + } + catch (const SeriousBug& error) { + // Nothing to do... + } + } +}; + +namespace impl { + +template +constexpr std::array, N> make_array(T (&a)[N], std::index_sequence) { + return {{std::move(a[I])...}}; +} + +template +std::array make_array(const T (&values)[N]) { + return make_array(values, std::make_index_sequence{}); +} + +} // namespace impl + +std::string make_random_token(size_t length) { + auto randomize = []() -> char { + static std::array selected = impl::make_array("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + return selected[rand() % (selected.size() - 1)]; + }; + std::string str(length, 0); + std::generate_n(str.begin(), length, randomize); + return str; +} + +std::string make_blob(std::string_view key, std::string_view value) { + std::ostringstream oss; + oss << R"({"key": ")" << key << R"(", "value": ")" << value << R"("})"; + return oss.str(); +} + +} // namespace + namespace eckit { namespace test { //---------------------------------------------------------------------------------------------------------------------- -CASE("EasyCURL GET") { +CASE("EasyCURL GET (Successful operation)") { auto curl = EasyCURL(); - auto response = curl.GET("https://get.ecmwf.int/repository/test-data/eckit/tests/io/t.grib.md5"); + auto response = curl.GET(BASE_URL + "/ping"); EXPECT(response.code() == 200); - EXPECT(response.body() == "f59fdc6a09c1d11b0e567309ef541bef t.grib\n"); + EXPECT(response.body() == "Hi!"); } -CASE("EasyCURL GET (REST API, Successful operation)") { - EasyCURLHeaders headers; - headers["content-type"] = "application/json"; +CASE("EasyCURL GET (Not Found)") { + auto curl = EasyCURL(); + + auto response = curl.GET(BASE_URL + "/ping__NOT-FOUND"); + EXPECT(response.code() == 404); + EXPECT(response.body().find("Not Found") != std::string::npos); +} +CASE("EasyCURL GET (REST API, Successful operation)") { auto curl = EasyCURL(); - curl.headers(headers); - auto response = curl.GET("https://get.ecmwf.int/service/rest/v1/assets?repository=test-data"); + auto response = curl.GET(BASE_URL + "/blobs"); EXPECT(response.code() == 200); - EXPECT(response.body().find(R"("items")") != std::string::npos); + EXPECT(response.body() == "{}\n"); } CASE("EasyCURL GET (REST API, Insufficient permissions)") { - EasyCURLHeaders headers; - headers["content-type"] = "application/json"; - - auto curl = EasyCURL(); - curl.headers(headers); - - auto response = curl.GET("https://get.ecmwf.int/service/rest/v1/security/anonymous"); + auto curl = EasyCURL(); + auto response = curl.GET(BASE_URL + "/insufficient-permissions"); EXPECT(response.code() == 403); - EXPECT(response.body() == ""); + EXPECT(response.body() == "Forbidden"); } -CASE("EasyCURL PUT (REST API, Insufficient permissions)") { +CASE("EasyCURL PUT (REST API, Create+Update+Delete new entity)") { EasyCURLHeaders headers; headers["content-type"] = "application/json"; auto curl = EasyCURL(); curl.headers(headers); - auto response = curl.PUT("https://get.ecmwf.int/service/rest/v1/security/anonymous", ""); - EXPECT(response.code() == 403); - EXPECT(response.body() == ""); + std::string key = make_random_token(8); + std::string original_value = make_random_token(64); + { + auto blob = make_blob(key, original_value); + auto response = curl.POST(BASE_URL + "/blob", blob); + EXPECT(response.code() == 201); + EXPECT(response.body().find(original_value) != std::string::npos); + } + { + auto response = curl.GET(BASE_URL + "/blob/" + key + "/content"); + EXPECT(response.code() == 200); + EXPECT(response.body() == original_value); + } + std::string updated_value = make_random_token(64); + { + auto blob = make_blob(key, updated_value); + auto response = curl.PUT(BASE_URL + "/blob", blob); + EXPECT(response.code() == 200); + EXPECT(response.body().find(updated_value) != std::string::npos); + } + { + auto response = curl.GET(BASE_URL + "/blob/" + key + "/content"); + EXPECT(response.code() == 200); + EXPECT(response.body() == updated_value); + } + { + auto response = curl.DELETE(BASE_URL + "/blob/" + key); + EXPECT(response.code() == 204); + EXPECT(response.body().empty()); + } + { + auto response = curl.GET(BASE_URL + "/blob/" + key + "/content"); + EXPECT(response.code() == 404); + EXPECT(response.body().find("Not Found") != std::string::npos); + } } //---------------------------------------------------------------------------------------------------------------------- @@ -93,5 +206,7 @@ CASE("EasyCURL PUT (REST API, Insufficient permissions)") { } // namespace eckit int main(int argc, char** argv) { + MockREST mock_rest_api_running_in_background; + return run_tests(argc, argv); }