From e2919d8d9ad72db1e0c42ff1d8260aeb523f1e54 Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Sat, 21 Dec 2024 01:24:18 +0000 Subject: [PATCH] Add JSON storage class & API access Add a generic JSON-based storage class and an extension of it that will be used in the API to store blobs from clients. This is a similar structure to the Settings class / user_settings table but enforces a JSON value. This creates a shortcut in the API to allow JSON input to bypass being deserialized and the output to bypass being serialized and allow the routing function to manage that. --- SETUP/API.md | 22 +++++ SETUP/configuration.sh | 1 + SETUP/db_schema.sql | 12 +++ SETUP/tests/ci_configuration.sh | 1 + SETUP/tests/unittests/ApiTest.php | 47 +++++++++- SETUP/tests/unittests/StorageTest.php | 50 ++++++++++ .../22/20241219_create_json_storage.php | 23 +++++ api/ApiRouter.inc | 35 ++++++- api/dp-openapi.yaml | 52 +++++++++++ api/index.php | 24 ++--- api/v1.inc | 6 ++ api/v1_storage.inc | 40 ++++++++ api/v1_validators.inc | 10 ++ pinc/ApiStorage.inc | 38 ++++++++ pinc/DPDatabase.inc | 2 +- pinc/JsonStorage.inc | 91 +++++++++++++++++++ pinc/site_vars.php.template | 1 + 17 files changed, 440 insertions(+), 15 deletions(-) create mode 100644 SETUP/tests/unittests/StorageTest.php create mode 100644 SETUP/upgrade/22/20241219_create_json_storage.php create mode 100644 api/v1_storage.inc create mode 100644 pinc/ApiStorage.inc create mode 100644 pinc/JsonStorage.inc diff --git a/SETUP/API.md b/SETUP/API.md index ab5b2c445f..5ae35c22f0 100644 --- a/SETUP/API.md +++ b/SETUP/API.md @@ -66,3 +66,25 @@ Three settings in `configuration.sh` control limiting: allowed per given window. * `_API_RATE_LIMIT_SECONDS_IN_WINDOW` - the number of seconds within a given window. + +## Storage + +To facilitate javascript UI clients persisting data across browsers and devices, +the API includes an optional endpoint for clients to store and fetch JSON blobs. +To enable this feature, add a storage key to the `_API_STORAGE_KEYS` +configuration setting and have the client use that string with the endpoint +as the `storagekey`. + +Some important notes about this feature: +* API storage is one blob per user per storage key. Said another way: API users are + only able to store one blob per `storagekey` and that blob can only be + set and retrieved by the user authenticated with the API. +* Beyond validating these are valid JSON objects, they are treated as opaque + blobs server-side. It is up to the client to manage the object, including + the schema and the possibility that the object will not match an expected + schema. +* When used inside javascript in the browser, the `storagekey` is visible to + the browser user and is therefore not a secret. Nothing prevents users with + API keys (or valid PHP session keys) from using this endpoint with a valid + storage key to change the contents of this blob for their user. API users + should treat this blob as unvalidated user input and act accordingly. diff --git a/SETUP/configuration.sh b/SETUP/configuration.sh index 2f9444081e..87e4c525ac 100644 --- a/SETUP/configuration.sh +++ b/SETUP/configuration.sh @@ -361,6 +361,7 @@ _API_ENABLED=true _API_RATE_LIMIT=false _API_RATE_LIMIT_REQUESTS_PER_WINDOW=3600 _API_RATE_LIMIT_SECONDS_IN_WINDOW=3600 +_API_STORAGE_KEYS='[]' # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/SETUP/db_schema.sql b/SETUP/db_schema.sql index 16cf81e542..7ae4ecca0d 100644 --- a/SETUP/db_schema.sql +++ b/SETUP/db_schema.sql @@ -122,6 +122,18 @@ CREATE TABLE `job_logs` ( KEY `timestamp` (`tracetime`, `succeeded`) ); +-- +-- Table structure for table `json_storage` +-- + +CREATE TABLE `json_storage` ( + `username` varchar(25) NOT NULL, + `setting` varchar(32) NOT NULL, + `value` json NOT NULL, + `timestamp` int NOT NULL DEFAULT '0', + PRIMARY KEY (`username`,`setting`) +); + -- -- Table structure for table `news_items` -- diff --git a/SETUP/tests/ci_configuration.sh b/SETUP/tests/ci_configuration.sh index a2734445fc..fa4979339a 100644 --- a/SETUP/tests/ci_configuration.sh +++ b/SETUP/tests/ci_configuration.sh @@ -47,6 +47,7 @@ _API_ENABLED=true _API_RATE_LIMIT=false _API_RATE_LIMIT_REQUESTS_PER_WINDOW=3600 _API_RATE_LIMIT_SECONDS_IN_WINDOW=3600 +_API_STORAGE_KEYS='[]' _EXTERNAL_CATALOG_LOCATOR='z3950.loc.gov:7090/Voyager' diff --git a/SETUP/tests/unittests/ApiTest.php b/SETUP/tests/unittests/ApiTest.php index 4f3bf64a14..f9dbfd86d4 100644 --- a/SETUP/tests/unittests/ApiTest.php +++ b/SETUP/tests/unittests/ApiTest.php @@ -787,6 +787,9 @@ public function test_pickersets(): void $this->assertEquals(["¿", "INVERTED QUESTION MARK"], $pickerset["subsets"][3]["rows"][1][1]); } + //--------------------------------------------------------------------------- + // tests for documents + public function test_available_italian_documents(): void { $path = "v1/documents"; @@ -823,11 +826,53 @@ public function test_unavailable_document(): void $_SERVER["REQUEST_METHOD"] = "GET"; $router->route($path, ['language_code' => 'de']); } + + //--------------------------------------------------------------------------- + // tests for storage + + public function test_storage_valid(): void + { + global $pguser; + global $api_storage_keys; + global $request_body; + + $pguser = $this->TEST_USERNAME_PM; + array_push($api_storage_keys, "valid"); + + $path = "v1/storage/valid"; + $query_params = []; + $request_body = json_encode(["key" => 1]); + $router = ApiRouter::get_router(); + + $_SERVER["REQUEST_METHOD"] = "PUT"; + $response = $router->route($path, $query_params); + $this->assertEquals(json_decode($request_body), json_decode($response)); + + $_SERVER["REQUEST_METHOD"] = "GET"; + $response = $router->route($path, $query_params); + $this->assertEquals(json_decode($request_body), json_decode($response)); + + $_SERVER["REQUEST_METHOD"] = "DELETE"; + $response = $router->route($path, $query_params); + $this->assertEquals(null, $response); + } + + public function test_storage_invalid(): void + { + $this->expectExceptionCode(4); + + $query_params = []; + + $path = "v1/storage/invalid"; + $_SERVER["REQUEST_METHOD"] = "GET"; + $router = ApiRouter::get_router(); + $router->route($path, $query_params); + } } // this mocks the function in index.php /** @return string|array */ -function api_get_request_body() +function api_get_request_body(bool $raw = false) { global $request_body; return $request_body; diff --git a/SETUP/tests/unittests/StorageTest.php b/SETUP/tests/unittests/StorageTest.php new file mode 100644 index 0000000000..276a194a99 --- /dev/null +++ b/SETUP/tests/unittests/StorageTest.php @@ -0,0 +1,50 @@ +set("setting", "{}"); + $value = $storage->get("setting"); + $this->assertEquals("{}", $value); + $value = $storage->delete("setting"); + $this->assertEquals(null, $value); + } + + public function test_invalid_json(): void + { + $this->expectException(ValueError::class); + $storage = new JsonStorage("username"); + $storage->set("setting", "blearg"); + } + + //------------------------------------------------------------------------ + // API storage test + + public function test_valid_storagekey(): void + { + global $api_storage_keys; + $api_storage_keys = ["valid"]; + + $storage = new ApiStorage("valid", "username"); + $storage->set("{}"); + $value = $storage->get(); + $this->assertEquals("{}", $value); + $value = $storage->delete(); + $this->assertEquals(null, $value); + } + + public function test_invalid_storagekey(): void + { + global $api_storage_keys; + $api_storage_keys = []; + + $this->expectException(ValueError::class); + + $storage = new ApiStorage("invalid", "username"); + } +} diff --git a/SETUP/upgrade/22/20241219_create_json_storage.php b/SETUP/upgrade/22/20241219_create_json_storage.php new file mode 100644 index 0000000000..531c3580a9 --- /dev/null +++ b/SETUP/upgrade/22/20241219_create_json_storage.php @@ -0,0 +1,23 @@ + */ private $_validators; + /** @var mixed */ + private $_response; + private bool $_raw_response = false; public function __construct() { @@ -74,7 +77,8 @@ class ApiRouter if (!$handler) { throw new MethodNotAllowed(); } - return $handler($method, $data, $query_params); + $this->_response = $handler($method, $data, $query_params); + return $this->_response; } public function add_validator(string $label, callable $function): void @@ -93,6 +97,35 @@ class ApiRouter throw new InvalidAPI(); } + /** @return mixed */ + public function request(bool $raw = false) + { + if ($raw) { + return file_get_contents('php://input'); + } else { + $json_object = json_decode(file_get_contents('php://input'), true); + if ($json_object === null) { + throw new InvalidValue("Content was not valid JSON"); + } + return $json_object; + } + } + + public function response(bool $raw = false): string + { + if ($raw || $this->_raw_response) { + return $this->_response; + } else { + return json_encode($this->_response, JSON_PRETTY_PRINT | + JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + } + } + + public function set_raw_response(): void + { + $this->_raw_response = true; + } + public static function get_router(): ApiRouter { /** @var ?ApiRouter */ diff --git a/api/dp-openapi.yaml b/api/dp-openapi.yaml index c69ae9873f..a0f479c294 100644 --- a/api/dp-openapi.yaml +++ b/api/dp-openapi.yaml @@ -1338,6 +1338,58 @@ paths: 404: $ref: '#/components/responses/NotFound' + /storage/{storagekey}: + get: + tags: + - storage + description: Get JSON blob stored for this storage key + parameters: + - name: storagekey + in: path + description: Storage key + required: true + schema: + type: string + responses: + 200: + description: JSON blob + content: + application/json: + schema: + type: object + put: + tags: + - storage + description: Save JSON blob for this storage key + parameters: + - name: storagekey + in: path + description: Storage key + required: true + schema: + type: string + responses: + 200: + description: JSON blob that was persisted + content: + application/json: + schema: + type: object + delete: + tags: + - storage + description: Delete JSON blob for this storage key + parameters: + - name: storagekey + in: path + description: Storage key + required: true + schema: + type: string + responses: + 200: + description: JSON blob was deleted + components: securitySchemes: ApiKeyAuth: diff --git a/api/index.php b/api/index.php index 7b7eaee8ef..83e86f0ea9 100644 --- a/api/index.php +++ b/api/index.php @@ -39,8 +39,8 @@ function api() unset($query_params["url"]); $router = ApiRouter::get_router(); - - api_output_response($router->route($path, $query_params)); + $router->route($path, $query_params); + api_output_response($router->response()); } function api_authenticate() @@ -127,20 +127,16 @@ function api_rate_limit($key) header("X-Rate-Limit-Reset: $seconds_before_reset"); } -function api_get_request_body() +function api_get_request_body(bool $raw = false) { - $json = json_decode(file_get_contents('php://input'), true); - if ($json === null) { - throw new InvalidValue("Content was not valid JSON"); - } - return $json; + $router = ApiRouter::get_router(); + return $router->request($raw); } -function api_output_response($data, $response_code = 200) +function api_output_response(string $data, int $response_code = 200) { http_response_code($response_code); - echo json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | - JSON_UNESCAPED_SLASHES); + echo $data; // output the output buffer we've been storing to ensure we could // send the right HTTP response code @@ -250,7 +246,11 @@ function production_exception_handler($exception) $response_code = 500; } - api_output_response(["error" => $exception->getMessage(), "code" => $exception->getCode()], $response_code); + $response = json_encode( + ["error" => $exception->getMessage(), "code" => $exception->getCode()], + JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES + ); + api_output_response($response, $response_code); } function test_exception_handler($exception) diff --git a/api/v1.inc b/api/v1.inc index 8607632abc..d7a1344a2b 100644 --- a/api/v1.inc +++ b/api/v1.inc @@ -7,6 +7,7 @@ include_once("v1_projects.inc"); include_once("v1_queues.inc"); include_once("v1_stats.inc"); include_once("v1_docs.inc"); +include_once("v1_storage.inc"); $router = ApiRouter::get_router(); @@ -18,6 +19,7 @@ $router->add_validator(":pagename", "validate_page_name"); $router->add_validator(":pageroundid", "validate_page_round"); $router->add_validator(":queueid", "validate_release_queue"); $router->add_validator(":document", "validate_document"); +$router->add_validator(":storagekey", "validate_storage_key"); // Add routes $router->add_route("GET", "v1/documents", "api_v1_documents"); @@ -62,3 +64,7 @@ $router->add_route("GET", "v1/stats/site/projects/stages", "api_v1_stats_site_pr $router->add_route("GET", "v1/stats/site/projects/states", "api_v1_stats_site_projects_states"); $router->add_route("GET", "v1/stats/site/rounds", "api_v1_stats_site_rounds"); $router->add_route("GET", "v1/stats/site/rounds/:roundid", "api_v1_stats_site_round"); + +$router->add_route("GET", "v1/storage/:storagekey", "api_v1_storage"); +$router->add_route("PUT", "v1/storage/:storagekey", "api_v1_storage"); +$router->add_route("DELETE", "v1/storage/:storagekey", "api_v1_storage_delete"); diff --git a/api/v1_storage.inc b/api/v1_storage.inc new file mode 100644 index 0000000000..7c27a6c03d --- /dev/null +++ b/api/v1_storage.inc @@ -0,0 +1,40 @@ +set_raw_response(); + return $storage->get(); + } elseif ($method == "PUT") { + // we store the raw JSON input we were passed in the request + $storage->set(api_get_request_body(true)); + + // and return it without conversion + $router->set_raw_response(); + return $storage->get(); + } +} + +function api_v1_storage_delete(string $method, array $data, array $query_params): void +{ + global $pguser; + + $storage_key = $data[":storagekey"]; + + $storage = new ApiStorage($storage_key, $pguser); + $storage->delete(); +} diff --git a/api/v1_validators.inc b/api/v1_validators.inc index ba18ca5a6b..bf80c4d7d4 100644 --- a/api/v1_validators.inc +++ b/api/v1_validators.inc @@ -77,3 +77,13 @@ function validate_document(string $document): string } return $document; } + +function validate_storage_key(string $storage_key, array $data): string +{ + global $api_storage_keys; + + if (!in_array($storage_key, $api_storage_keys)) { + throw new NotFoundError("$storage_key is not a valid storage key"); + } + return $storage_key; +} diff --git a/pinc/ApiStorage.inc b/pinc/ApiStorage.inc new file mode 100644 index 0000000000..38951bfb02 --- /dev/null +++ b/pinc/ApiStorage.inc @@ -0,0 +1,38 @@ +setting = "api:$storage_key"; + $this->storage = new JsonStorage($username); + } + + // ------------------------------------------------------------------------- + + public function set(string $value): void + { + $this->storage->set($this->setting, $value); + } + + public function get(): string + { + // if the setting isn't set, JsonStorage will return null but we want + // to always return valid JSON for the API so we return an empty JSON + // doc. + return $this->storage->get($this->setting) ?? "{}"; + } + + public function delete(): void + { + $this->storage->delete($this->setting); + } +} diff --git a/pinc/DPDatabase.inc b/pinc/DPDatabase.inc index 8048ba4497..8cbeb66131 100644 --- a/pinc/DPDatabase.inc +++ b/pinc/DPDatabase.inc @@ -117,7 +117,7 @@ final class DPDatabase $error = _("An error occurred during a database query."); } if ($throw_on_failure) { - throw new DBQueryError($error); + throw new DBQueryError($error, 0, $e); } return false; } diff --git a/pinc/JsonStorage.inc b/pinc/JsonStorage.inc new file mode 100644 index 0000000000..8413a241f3 --- /dev/null +++ b/pinc/JsonStorage.inc @@ -0,0 +1,91 @@ +username = $username; + } + + // ------------------------------------------------------------------------- + + /** + * Set or update a json_storage object. + */ + public function set(string $setting, string $value): void + { + // It's possible $value could be very large and including it twice + // in the query will cause it to be too big. The default max query + // size is 64MB which means this will fail if the JSON blob is + // just shy of 32MB. That seems plenty big. + $sql = sprintf( + " + INSERT INTO json_storage + SET + username = '%s', + setting = '%s', + value = '%s', + timestamp = UNIX_TIMESTAMP() + ON DUPLICATE KEY UPDATE + value = '%s', + timestamp = UNIX_TIMESTAMP() + ", + DPDatabase::escape($this->username), + DPDatabase::escape($setting), + DPDatabase::escape($value), + DPDatabase::escape($value) + ); + // We rely on MySQL to validate the JSON is valid or throw an exception. + // It's going to do the check anyway and saves us from de-serializing + // it here. + try { + DPDatabase::query($sql, true, false); + } catch (DBQueryError $e) { + if (startswith($e->getPrevious()->getMessage(), "Invalid JSON")) { + throw new ValueError("Error persisting data, invalid JSON"); + } + throw $e; + } + } + + /** + * Get a json_storage object. + */ + public function get(string $setting): ?string + { + $sql = sprintf( + " + SELECT value + FROM json_storage + WHERE username = '%s' AND setting = '%s' + ", + DPDatabase::escape($this->username), + DPDatabase::escape($setting) + ); + $result = DPDatabase::query($sql); + $value = null; + while ($row = mysqli_fetch_assoc($result)) { + $value = $row['value']; + } + mysqli_free_result($result); + return $value; + } + + /** + * Delete a json_storage object. + */ + public function delete(string $setting): void + { + $sql = sprintf( + " + DELETE FROM json_storage + WHERE username = '%s' AND setting = '%s' + ", + DPDatabase::escape($this->username), + DPDatabase::escape($setting) + ); + DPDatabase::query($sql); + } +} diff --git a/pinc/site_vars.php.template b/pinc/site_vars.php.template index a5a5378f65..0e7dd38bee 100644 --- a/pinc/site_vars.php.template +++ b/pinc/site_vars.php.template @@ -100,6 +100,7 @@ $api_enabled = <>; $api_rate_limit = <>; $api_rate_limit_requests_per_window = <>; $api_rate_limit_seconds_in_window = <>; +$api_storage_keys = <>; // -----------------------------------------------------------------------------