From 6fb6a1ea11fb3b8f8da39914591a0d7c1749757e Mon Sep 17 00:00:00 2001 From: vsian Date: Fri, 11 Oct 2024 16:06:01 +0800 Subject: [PATCH] new HTTP APIs : SHOW OBJECTS, SHOW OBJECT, SHOW FILES (#2013) ### What problem does this PR solve? new HTTP APIs : - SHOW OBJECTS (Documented) - SHOW OBJECT (Documented) - SHOW FILES (Documented) Add docs: - SHOW LOGS - SHOW DELTA CHECKPOINTS - SHOW FULL CHECKPOINTS Fix: infinity::ShowObject will cause segfault. Issue link: #1937 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [x] Documentation Update --- docs/references/http_api_reference.mdx | 323 ++++++++++++++++++++++++ src/executor/operator/physical_show.cpp | 4 +- src/network/http_server.cpp | 128 +++++++++- 3 files changed, 445 insertions(+), 10 deletions(-) diff --git a/docs/references/http_api_reference.mdx b/docs/references/http_api_reference.mdx index 436f76baf1..31d8bbbd54 100644 --- a/docs/references/http_api_reference.mdx +++ b/docs/references/http_api_reference.mdx @@ -2780,6 +2780,329 @@ A `500` HTTP status code indicates an error condition. The response includes a J --- +## Show logs + +**GET** `/instance/logs` + +Shows logs. + +### Request + +- Method: GET +- URL: `/instance/logs` +- Headers: `accept: application/json` + +#### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/logs \ + --header 'accept: application/json' +``` + +### Response + +#### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "logs":[ + { + "command_type":"CHECKPOINT", + "commit_ts":"1", + "text":"catalog path: catalog/FULL.82192.json\nmax commit ts: 82192\nis full checkpoint: 1\n", + "transaction_id":"82195" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +## Show delta checkpoints + +**GET** `/instance/delta_checkpoint` + +Shows delta logs. + +### Request + +- Method: GET +- URL: `/instance/delta_checkpoint` +- Headers: `accept: application/json` + +#### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/delta_checkpoint \ + --header 'accept: application/json' +``` + +### Response + +#### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +## Show full checkpoints + +**GET** `/instance/global_checkpoint` + +Shows delta logs. + +### Request + +- Method: GET +- URL: `/instance/global_checkpoint` +- Headers: `accept: application/json` + +#### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/global_checkpoint \ + --header 'accept: application/json' +``` + +### Response + +#### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "full_checkpoint":[ + { + "file_path":"/var/infinity/data/catalog/FULL.82192.json", + "max_commit_timestamp":"82192" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +## Show objects + +**GET** `/instance/objects` + +Shows persistence objects. + +### Request + +- Method: GET +- URL: `/instance/objects` +- Headers: `accept: application/json` + +#### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/objects \ + --header 'accept: application/json' +``` + +### Response + +#### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "objects": [ + { + "deleted_ranges": "", + "name": "01927642-6491-79d8-8891-5f91d7960cdf", + "parts": "4", + "reference_count": "0", + "size": "262236" + }, + { + "deleted_ranges": "[0, 426120) ", + "name": "019275ce-9eaf-7950-8bf0-b451cb15e6e1", + "parts": "0", + "reference_count": "0", + "size": "0" + }, + { + "deleted_ranges": "", + "name": "01927641-793c-7a30-b20c-b9089b6c0a9b", + "parts": "9", + "reference_count": "0", + "size": "426188" + }, + { + "deleted_ranges": "[0, 65722552) ", + "name": "019275cd-7310-7b40-a46a-cb12db4a391b", + "parts": "0", + "reference_count": "0", + "size": "0" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +## Show object + +**GET** `/instance/objects/{object_name}` + +Shows persistence object by its name. + +#### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/objects/{object_name} \ + --header 'accept: application/json' +``` + +#### Request parameters + +- `object_name`: Required (*Path parameter*) + The object name. + +### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "object": [ + { + "end": "426120", + "start": "0" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 7008, + "error_message": "File: {object_name} isn't found@src/executor/operator/physical_show.cpp:5631@src/scheduler/fragment_context.cpp:1417" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_message"`: `string` + When `error_code` is non-zero, `"error_message"` provides additional details about the error. + + + + +--- + +## Show files + +**GET** `/instance/files` + +Shows files in persistent manager. + +### Request + +- Method: GET +- URL: `/instance/files` +- Headers: `accept: application/json` + +#### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/files \ + --header 'accept: application/json' +``` + +### Response + +#### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "files": [ + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_4/0.col", + "object_name": "01927a51-068a-7ba8-8d80-4d89bd6391de", + "offset": "78660", + "size": "131096" + }, + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_0/col_1_out_0", + "object_name": "01927a4f-c82c-7ba8-b135-80dd3cfb2d87", + "offset": "340852", + "size": "11435869" + }, + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_3/1.col", + "object_name": "01927a50-dd6d-7138-9665-9e21502390a6", + "offset": "209756", + "size": "131096" + }, + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_0/0.col", + "object_name": "01927a4f-c82c-7ba8-b135-80dd3cfb2d87", + "offset": "78660", + "size": "131096" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + ## Admin set node role **POST** `/admin/node/current` diff --git a/src/executor/operator/physical_show.cpp b/src/executor/operator/physical_show.cpp index 0c37177331..9d787831e6 100644 --- a/src/executor/operator/physical_show.cpp +++ b/src/executor/operator/physical_show.cpp @@ -5641,13 +5641,13 @@ void PhysicalShow::ExecuteShowPersistenceObject(QueryContext *query_context, Sho // start Value value = Value::MakeBigInt(range.start_); ValueExpression value_expr(value); - value_expr.AppendToChunk(output_block_ptr->column_vectors[2]); + value_expr.AppendToChunk(output_block_ptr->column_vectors[0]); } { // end Value value = Value::MakeBigInt(range.end_); ValueExpression value_expr(value); - value_expr.AppendToChunk(output_block_ptr->column_vectors[3]); + value_expr.AppendToChunk(output_block_ptr->column_vectors[1]); } ++row_count; diff --git a/src/network/http_server.cpp b/src/network/http_server.cpp index 0b02e6251a..3c6f2ea0c5 100644 --- a/src/network/http_server.cpp +++ b/src/network/http_server.cpp @@ -3575,7 +3575,7 @@ class ShowFullCheckpointHandler final : public HttpRequestHandler { const String &column_value = value.ToString(); json_table[column_name] = column_value; } - json_response["global_checkpoint"].push_back(json_table); + json_response["full_checkpoint"].push_back(json_table); } } json_response["error_code"] = 0; @@ -3590,10 +3590,9 @@ class ShowFullCheckpointHandler final : public HttpRequestHandler { } }; - class ShowQueryHandler final : public HttpRequestHandler { public: -SharedPtr handle(const SharedPtr &request) final { + SharedPtr handle(const SharedPtr &request) final { auto infinity = Infinity::RemoteConnect(); DeferFn defer_fn([&]() { infinity->RemoteDisconnect(); }); @@ -3601,7 +3600,7 @@ SharedPtr handle(const SharedPtr &request) fi nlohmann::json json_table; HTTPStatus http_status; String query_id = request->getPathVariable("query_id"); - QueryResult result = infinity->Query(fmt::format("show query {}", query_id)); + QueryResult result = infinity->Query(fmt::format("show query {}", query_id)); if (result.IsOk()) { DataBlock *data_block = result.result_table_->GetDataBlockById(0).get(); @@ -3627,7 +3626,7 @@ SharedPtr handle(const SharedPtr &request) fi class ShowTransactionsHandler final : public HttpRequestHandler { public: -SharedPtr handle(const SharedPtr &request) final { + SharedPtr handle(const SharedPtr &request) final { auto infinity = Infinity::RemoteConnect(); DeferFn defer_fn([&]() { infinity->RemoteDisconnect(); }); @@ -3666,7 +3665,7 @@ SharedPtr handle(const SharedPtr &request) fi class ShowTransactionHandler final : public HttpRequestHandler { public: -SharedPtr handle(const SharedPtr &request) final { + SharedPtr handle(const SharedPtr &request) final { auto infinity = Infinity::RemoteConnect(); DeferFn defer_fn([&]() { infinity->RemoteDisconnect(); }); @@ -3686,7 +3685,80 @@ SharedPtr handle(const SharedPtr &request) fi json_table[column_name] = column_value; } json_response["error_code"] = 0; - json_response["transaction"]= json_table; + json_response["transaction"] = json_table; + http_status = HTTPStatus::CODE_200; + } else { + json_response["error_code"] = result.ErrorCode(); + json_response["error_message"] = result.ErrorMsg(); + http_status = HTTPStatus::CODE_500; + } + + return ResponseFactory::createResponse(http_status, json_response.dump()); + } +}; + +class ShowObjectsHandler final : public HttpRequestHandler { +public: + SharedPtr handle(const SharedPtr &request) final { + auto infinity = Infinity::RemoteConnect(); + DeferFn defer_fn([&]() { infinity->RemoteDisconnect(); }); + + nlohmann::json json_response; + HTTPStatus http_status; + QueryResult result = infinity->ShowObjects(); + + if (result.IsOk()) { + SizeT block_rows = result.result_table_->DataBlockCount(); + for (SizeT block_id = 0; block_id < block_rows; ++block_id) { + DataBlock *data_block = result.result_table_->GetDataBlockById(block_id).get(); + auto row_count = data_block->row_count(); + auto column_cnt = result.result_table_->ColumnCount(); + for (int row = 0; row < row_count; ++row) { + nlohmann::json json_table; + for (SizeT col = 0; col < column_cnt; ++col) { + const String &column_name = result.result_table_->GetColumnNameById(col); + Value value = data_block->GetValue(col, row); + const String &column_value = value.ToString(); + json_table[column_name] = column_value; + } + json_response["objects"].push_back(json_table); + } + } + json_response["error_code"] = 0; + http_status = HTTPStatus::CODE_200; + } else { + json_response["error_code"] = result.ErrorCode(); + json_response["error_message"] = result.ErrorMsg(); + http_status = HTTPStatus::CODE_500; + } + + return ResponseFactory::createResponse(http_status, json_response.dump()); + } +}; + +class ShowObjectHandler final : public HttpRequestHandler { +public: + SharedPtr handle(const SharedPtr &request) final { + auto infinity = Infinity::RemoteConnect(); + DeferFn defer_fn([&]() { infinity->RemoteDisconnect(); }); + + nlohmann::json json_response; + nlohmann::json json_table; + HTTPStatus http_status; + String object_name = request->getPathVariable("object_name"); + QueryResult result = infinity->ShowObject(object_name); + + if (result.IsOk()) { + DataBlock *data_block = result.result_table_->GetDataBlockById(0).get(); + auto column_cnt = result.result_table_->ColumnCount(); + for (SizeT col = 0; col < column_cnt; ++col) { + const String &column_name = result.result_table_->GetColumnNameById(col); + Value value = data_block->GetValue(col, 0); + const String &column_value = value.ToString(); + json_table[column_name] = column_value; + } + json_response["error_code"] = 0; + json_response["object"] = json_table; http_status = HTTPStatus::CODE_200; } else { json_response["error_code"] = result.ErrorCode(); @@ -3698,7 +3770,44 @@ SharedPtr handle(const SharedPtr &request) fi } }; +class ShowFilesHandler final : public HttpRequestHandler { +public: + SharedPtr handle(const SharedPtr &request) final { + auto infinity = Infinity::RemoteConnect(); + DeferFn defer_fn([&]() { infinity->RemoteDisconnect(); }); + + nlohmann::json json_response; + HTTPStatus http_status; + QueryResult result = infinity->ShowFilesInObject(); + + if (result.IsOk()) { + SizeT block_rows = result.result_table_->DataBlockCount(); + for (SizeT block_id = 0; block_id < block_rows; ++block_id) { + DataBlock *data_block = result.result_table_->GetDataBlockById(block_id).get(); + auto row_count = data_block->row_count(); + auto column_cnt = result.result_table_->ColumnCount(); + for (int row = 0; row < row_count; ++row) { + nlohmann::json json_table; + for (SizeT col = 0; col < column_cnt; ++col) { + const String &column_name = result.result_table_->GetColumnNameById(col); + Value value = data_block->GetValue(col, row); + const String &column_value = value.ToString(); + json_table[column_name] = column_value; + } + json_response["files"].push_back(json_table); + } + } + json_response["error_code"] = 0; + http_status = HTTPStatus::CODE_200; + } else { + json_response["error_code"] = result.ErrorCode(); + json_response["error_message"] = result.ErrorMsg(); + http_status = HTTPStatus::CODE_500; + } + return ResponseFactory::createResponse(http_status, json_response.dump()); + } +}; class AdminShowCurrentNodeHandler final : public HttpRequestHandler { public: @@ -4035,10 +4144,13 @@ void HTTPServer::Start(const String &ip_address, u16 port) { router->route("GET", "/instance/queries", MakeShared()); router->route("GET", "/instance/logs", MakeShared()); router->route("GET", "/instance/delta_checkpoint", MakeShared()); - router->route("GET", "/instance/global_checkpoint", MakeShared()); + router->route("GET", "/instance/global_checkpoint", MakeShared()); router->route("GET", "/instance/queries/{query_id}", MakeShared()); router->route("GET", "/instance/transactions", MakeShared()); router->route("GET", "/instance/transactions/{transaction_id}", MakeShared()); + router->route("GET", "/instance/objects", MakeShared()); + router->route("GET", "/instance/objects/{object_name}", MakeShared()); + router->route("GET", "/instance/files", MakeShared()); // variable router->route("GET", "/variables/global", MakeShared());