From 20289862db9c20d264ffb58695b568e46ef45287 Mon Sep 17 00:00:00 2001 From: Stefano Scafiti Date: Wed, 24 Jul 2024 12:34:46 +0200 Subject: [PATCH] Update documentation. - Add request metadata section - Add JSON type documentation - Add SQL privileges section Signed-off-by: Stefano Scafiti --- src/.vuepress/sidebar_master.js | 10 +- src/.vuepress/theme/util/index.js | 1 + src/1.9.4/README.md | 26 + src/1.9.4/connecting/authentication.md | 278 ++++ src/1.9.4/connecting/clitools.md | 367 ++++++ src/1.9.4/connecting/commands.md | 245 ++++ src/1.9.4/connecting/healthcheck.md | 102 ++ src/1.9.4/connecting/immugw.md | 127 ++ src/1.9.4/connecting/sdks.md | 55 + src/1.9.4/connecting/webconsole.md | 38 + src/1.9.4/develop/deleting.md | 137 ++ src/1.9.4/develop/document/api.md | 338 +++++ src/1.9.4/develop/document/datamodel.md | 16 + src/1.9.4/develop/expiration.md | 78 ++ src/1.9.4/develop/indexes.md | 282 ++++ src/1.9.4/develop/queries-history.md | 1159 +++++++++++++++++ src/1.9.4/develop/reading.md | 815 ++++++++++++ src/1.9.4/develop/sql/catalog.md | 96 ++ src/1.9.4/develop/sql/datatypes.md | 57 + src/1.9.4/develop/sql/indexes.md | 74 ++ src/1.9.4/develop/sql/insertupdate.md | 107 ++ src/1.9.4/develop/sql/pg.md | 253 ++++ src/1.9.4/develop/sql/privileges.md | 42 + src/1.9.4/develop/sql/querying.md | 425 ++++++ src/1.9.4/develop/sql/sqlstdlib.md | 19 + src/1.9.4/develop/sql/tablesalter.md | 80 ++ src/1.9.4/develop/sql/tablescreate.md | 114 ++ src/1.9.4/develop/sql/tablesdrop.md | 19 + src/1.9.4/develop/sql/transactions.md | 288 ++++ src/1.9.4/develop/sql/users.md | 42 + src/1.9.4/develop/streams.md | 159 +++ src/1.9.4/develop/transactions.md | 566 ++++++++ src/1.9.4/embedded/embedding.md | 16 + src/1.9.4/embedded/embeddingSQL.md | 19 + src/1.9.4/immudb.md | 126 ++ src/1.9.4/management/database.md | 351 +++++ src/1.9.4/management/state.md | 426 ++++++ src/1.9.4/management/user.md | 280 ++++ src/1.9.4/playground.md | 9 + src/1.9.4/production/auditor.md | 152 +++ src/1.9.4/production/backup.md | 125 ++ .../production/backwards-compatibility.md | 67 + src/1.9.4/production/fips.md | 43 + src/1.9.4/production/index-maintenance.md | 140 ++ src/1.9.4/production/monitoring.md | 517 ++++++++ src/1.9.4/production/performance-guide.md | 198 +++ src/1.9.4/production/planning.md | 35 + src/1.9.4/production/replication.md | 169 +++ src/1.9.4/production/request-metadata.md | 26 + src/1.9.4/production/retention.md | 118 ++ src/1.9.4/production/s3-storage.md | 18 + src/1.9.4/production/sync-replication.md | 429 ++++++ src/1.9.4/releasenotes.md | 8 + src/1.9.4/running/build.md | 46 + src/1.9.4/running/configuration.md | 167 +++ src/1.9.4/running/download.md | 29 + src/1.9.4/running/service.md | 34 + src/1.9.4/samples/go.md | 129 ++ src/master/develop/sql/datatypes.md | 38 +- src/master/develop/sql/indexes.md | 12 +- src/master/develop/sql/privileges.md | 42 + src/master/develop/sql/querying.md | 76 +- src/master/production/auditor.md | 24 +- src/master/production/monitoring.md | 20 +- src/master/production/request-metadata.md | 26 + src/master/running/configuration.md | 123 +- 66 files changed, 10334 insertions(+), 119 deletions(-) create mode 100644 src/1.9.4/README.md create mode 100644 src/1.9.4/connecting/authentication.md create mode 100644 src/1.9.4/connecting/clitools.md create mode 100644 src/1.9.4/connecting/commands.md create mode 100644 src/1.9.4/connecting/healthcheck.md create mode 100644 src/1.9.4/connecting/immugw.md create mode 100644 src/1.9.4/connecting/sdks.md create mode 100644 src/1.9.4/connecting/webconsole.md create mode 100644 src/1.9.4/develop/deleting.md create mode 100644 src/1.9.4/develop/document/api.md create mode 100644 src/1.9.4/develop/document/datamodel.md create mode 100644 src/1.9.4/develop/expiration.md create mode 100644 src/1.9.4/develop/indexes.md create mode 100644 src/1.9.4/develop/queries-history.md create mode 100644 src/1.9.4/develop/reading.md create mode 100644 src/1.9.4/develop/sql/catalog.md create mode 100644 src/1.9.4/develop/sql/datatypes.md create mode 100644 src/1.9.4/develop/sql/indexes.md create mode 100644 src/1.9.4/develop/sql/insertupdate.md create mode 100644 src/1.9.4/develop/sql/pg.md create mode 100644 src/1.9.4/develop/sql/privileges.md create mode 100644 src/1.9.4/develop/sql/querying.md create mode 100644 src/1.9.4/develop/sql/sqlstdlib.md create mode 100644 src/1.9.4/develop/sql/tablesalter.md create mode 100644 src/1.9.4/develop/sql/tablescreate.md create mode 100644 src/1.9.4/develop/sql/tablesdrop.md create mode 100644 src/1.9.4/develop/sql/transactions.md create mode 100644 src/1.9.4/develop/sql/users.md create mode 100644 src/1.9.4/develop/streams.md create mode 100644 src/1.9.4/develop/transactions.md create mode 100644 src/1.9.4/embedded/embedding.md create mode 100644 src/1.9.4/embedded/embeddingSQL.md create mode 100644 src/1.9.4/immudb.md create mode 100644 src/1.9.4/management/database.md create mode 100644 src/1.9.4/management/state.md create mode 100644 src/1.9.4/management/user.md create mode 100644 src/1.9.4/playground.md create mode 100644 src/1.9.4/production/auditor.md create mode 100644 src/1.9.4/production/backup.md create mode 100644 src/1.9.4/production/backwards-compatibility.md create mode 100644 src/1.9.4/production/fips.md create mode 100644 src/1.9.4/production/index-maintenance.md create mode 100644 src/1.9.4/production/monitoring.md create mode 100644 src/1.9.4/production/performance-guide.md create mode 100644 src/1.9.4/production/planning.md create mode 100644 src/1.9.4/production/replication.md create mode 100644 src/1.9.4/production/request-metadata.md create mode 100644 src/1.9.4/production/retention.md create mode 100644 src/1.9.4/production/s3-storage.md create mode 100644 src/1.9.4/production/sync-replication.md create mode 100644 src/1.9.4/releasenotes.md create mode 100644 src/1.9.4/running/build.md create mode 100644 src/1.9.4/running/configuration.md create mode 100644 src/1.9.4/running/download.md create mode 100644 src/1.9.4/running/service.md create mode 100644 src/1.9.4/samples/go.md create mode 100644 src/master/develop/sql/privileges.md create mode 100644 src/master/production/request-metadata.md diff --git a/src/.vuepress/sidebar_master.js b/src/.vuepress/sidebar_master.js index 1e40200a4a..2705da4e26 100644 --- a/src/.vuepress/sidebar_master.js +++ b/src/.vuepress/sidebar_master.js @@ -60,6 +60,7 @@ export default version => { immudbInProduction.children.push(`${version}/production/backwards-compatibility`); immudbInProduction.children.push(`${version}/production/performance-guide`); immudbInProduction.children.push(`${version}/production/retention`); + immudbInProduction.children.push(`${version}/production/request-metadata`); /* RUNNING IMMUDB IN PRODUCTION SECTION END */ /* CONNECTING SECTION START */ @@ -126,10 +127,11 @@ export default version => { developSQL.children.push(`${version}/develop/sql/indexes`); developSQL.children.push(`${version}/develop/sql/querying`); developSQL.children.push(`${version}/develop/sql/users`); + developSQL.children.push(`${version}/develop/sql/privileges`); developSQL.children.push(`${version}/develop/sql/catalog`); developSQL.children.push(`${version}/develop/sql/sqlstdlib`); developSQL.children.push(`${version}/develop/sql/pg`); - + /* DEVELOP SQL SECTION END */ /* DEVELOP DOCUMENT SECTION START */ @@ -165,7 +167,7 @@ export default version => { }; releaseNotes.children.push(`${version}/releasenotes`); - + /* RELEASE NOTES SECTION END */ sidebar.push(introduction); @@ -179,6 +181,6 @@ export default version => { sidebar.push(developDocument); sidebar.push(embedded); sidebar.push(releaseNotes); - + return sidebar; -} \ No newline at end of file +} diff --git a/src/.vuepress/theme/util/index.js b/src/.vuepress/theme/util/index.js index 3617ba7073..b69aa33b36 100644 --- a/src/.vuepress/theme/util/index.js +++ b/src/.vuepress/theme/util/index.js @@ -257,6 +257,7 @@ export const versions = [ '1.9DOM.0', '1.9DOM.1', '1.9.3', + '1.9.4', 'master' ] diff --git a/src/1.9.4/README.md b/src/1.9.4/README.md new file mode 100644 index 0000000000..543664743b --- /dev/null +++ b/src/1.9.4/README.md @@ -0,0 +1,26 @@ +# Welcome + + + + + +Welcome to the immudb documentation. Great to see you here! + +immudb is a database written in Go, but unlike other databases, it is immutable: history is preserved and can't be changed without clients noticing. + +immudb can operate as a key-value, relational (SQL) or document database, making it a truly no-SQL database. + +immudb can be run as full database server with replicas or easily embedded as a lightweight database into application. + + + + + +### Help and Support +Join our [Discord community](https://discord.gg/ThSJxNEHhZ) + + + + + + \ No newline at end of file diff --git a/src/1.9.4/connecting/authentication.md b/src/1.9.4/connecting/authentication.md new file mode 100644 index 0000000000..b024f8c46a --- /dev/null +++ b/src/1.9.4/connecting/authentication.md @@ -0,0 +1,278 @@ +# Authentication + + + +## With credentials + +The immudb server runs on port 3322 as the default. The code examples below illustrate how to connect your client to the server and authenticate using default options and the default username and password. +You can modify defaults on the immudb server in [immudb.toml](https://github.com/codenotary/immudb/blob/master/configs/immudb.toml) in the config folder. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/connect-with-auth/main.go +::: + +::: tab Python + +```python +from grpc import RpcError +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + + +def main(): + client = ImmudbClient(URL) + # database parameter is optional + client.login(LOGIN, PASSWORD, database=DB) + client.logout() + + # Bad login + try: + client.login("verybadlogin", "verybadpassword") + except RpcError as exception: + print(exception.debug_error_string()) + print(exception.details()) + + +if __name__ == "__main__": + main() + +``` +::: + +::: tab Java + +Under the hood, during `login`, a token is being retrieved from the server, +stored in memory and reused for subsequent operations. + +The state is internally used for doing _verified_ operations (such as verifiedSet or verifiedGet). + +```java +// Setting the "store" where the internal states are being persisted. +FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("immu_states") + .build(); + +// Creating an new ImmuClient instance. +ImmuClient immuClient = ImmuClient.newBuilder() + .withStateHolder(stateHolder) + .withServerUrl("localhost") + .withServerPort(3322) + .build(); + +// Login with default credentials. +immuClient.login("immudb", "immudb"); +``` + +::: + +::: tab .NET + +The following code snippets show how to create a client. + +Using default configuration: + +``` csharp + ImmuClient immuClient = ImmuClient.NewBuilder().Build(); + + // or + + Immuclient immuClient = new ImmuClient(); + Immuclient immuClient = new ImmuClient("localhost", 3322); + +``` + +Setting `immudb` url and port: + +``` csharp + ImmuClient immuClient = ImmuClient.NewBuilder() + .WithServerUrl("localhost") + .WithServerPort(3322) + .Build(); + + ImmuClient immuClient = ImmuClient.NewBuilder() + .WithServerUrl("localhost") + .WithServerPort(3322) + .Build(); + +``` + +Customizing the `State Holder`: + +``` csharp + FileImmuStateHolder stateHolder = FileImmuStateHolder.NewBuilder() + .WithStatesFolder("./my_immuapp_states") + .Build(); + + ImmuClient immuClient = ImmuClient.NewBuilder() + .WithStateHolder(stateHolder) + .Build(); +``` + +Use `Open` and `Close` methods to initiate and terminate user sessions: + +``` csharp + await immuClient.Open("usr1", "pwd1", "defaultdb"); + + // Interact with immudb using logged-in user. + //... + + await immuClient.Close(); + + // or one liner open the session right + client = await ImmuClient.NewBuilder().Open(); + + //then close it + await immuClient.Close(); + +``` + +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](immugw.md) option. +::: + +:::: + + + +## With Mutual TLS + +To enable mutual authentication, a certificate chain must be provided to both the server and client. +That will cause each to authenticate with the other simultaneously. +In order to generate certs, use the [generate.sh](https://github.com/codenotary/immudb/tree/master/tools/mtls) tool from immudb repository. It generates a list of folders containing certificates and private keys to set up a mTLS connection. + + + + + +```bash +./generate.sh localhost mysecretpassword +``` + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/connect-with-mtls/main.go +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Java +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab .NET +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [.NET SDK github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + const loginReq: Parameters.Login = { user: IMMUDB_USER, password: IMMUDB_PWD } + const loginRes = await cl.login(loginReq) + console.log('success: login:', loginRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](immugw.md) option. +::: + +:::: + + + +## No Auth + +You also have the option to run immudb with authentication disabled. This method is depreciated and not recommended. + +A server configured with databases and user permissions can't be instantiated without authentication enabled. If a valid token is present, authentication is enabled by default. + + + + + +```bash +$ ./immudb --auth=false +``` + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/connect-with-no-auth/main.go +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Java + +```java +FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("immu_states") + .build(); + +ImmuClient immuClient = ImmuClient.newBuilder() + .withStateHolder(stateHolder) + .withServerUrl("localhost") + .withServerPort(3322) + .withAuth(false) // No authentication is needed. + .build(); +try { + immuClient.set(key, val); +} catch (CorruptedDataException e) { + // ... +} +``` + +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](immugw.md) option. +::: + +:::: + diff --git a/src/1.9.4/connecting/clitools.md b/src/1.9.4/connecting/clitools.md new file mode 100644 index 0000000000..6c13e9c520 --- /dev/null +++ b/src/1.9.4/connecting/clitools.md @@ -0,0 +1,367 @@ +# CLI tools + + + +Before any operations can be run by immuadmin or immuclient, it is necessary to authenticate against the running immudb server. + +When immudb is first run, it is ready to use immediately with the default database and credentials: + +- Database name: defaultdb +- User: immudb +- Password: immudb +- Address: 127.0.0.1 +- Port: 3322 + + + + + +## immuadmin + +immuadmin is the admin client for immudb. It is used for a variety of tasks such as creating and updating databases and users. Creating backups, restoring from backups etc. + +You may download the immuadmin binary from [the latest releases on Github](https://github.com/codenotary/immudb/releases/latest). Once you have downloaded immuadmin, rename it to `immuadmin`, make sure to mark it as executable, then run it. The following example shows how to obtain v1.3.0 for Linux amd64: + +```bash +$ wget https://github.com/vchain-us/immudb/releases/download/v1.3.0/immuadmin-v1.3.0-linux-amd64 +$ mv immuadmin-v1.3.0-linux-amd64 immuadmin +$ chmod +x immuadmin +``` + +Alternatively, you may [pull immuadmin docker image from DockerHub](https://hub.docker.com/r/codenotary/immuadmin) and run it in a ready-to-use container: + +```bash +$ docker run -it --rm --name immuadmin codenotary/immuadmin:latest status +``` + + + + + +### Basic operations + +To get started we need to login to `immuadmin` first. The `admin` user is the similar to the `root` user in MySQL etc. + +```bash +$ ./immuadmin login immudb +Password: immudb +``` + +Once logged in we can create a new database using + +```bash +$ ./immuadmin database create mydatabase +database 'mydatabase' {replica: false} successfully created +``` + +To switch to our newly created database + +```bash +$ ./immuclient use mydatabase +Now using mydatabase +``` + +To create new user with read/write access to just created database + +```bash +$ ./immuadmin user create user1 readwrite mydatabase +Choose a password for user1: +Confirm password: +``` + +For detailed description of immuadmin command arguments use help + +```bash +$ ./immuadmin help +``` + + + + + +## immuclient + +immuclient is used for interacting with databases, like reading, writing and querying for data or invoking SQL. + +You may download the immuclient binary from [the latest releases on Github](https://github.com/codenotary/immudb/releases/latest). Once you have downloaded immuclient, rename it to `immuclient`, make sure to mark it as executable, then run it. The following example shows how to obtain v1.3.0 for Linux amd64: + +```bash +$ wget https://github.com/vchain-us/immudb/releases/download/v1.3.0/immuclient-v1.3.0-linux-amd64 +$ mv immuclient-v1.3.0-linux-amd64 immuclient +$ chmod +x immuclient +``` + +Alternatively, you may [pull immuclient docker image from DockerHub](https://hub.docker.com/r/codenotary/immuclient) and run it in a ready-to-use container: + +```bash +$ docker run -it --rm --net host --name immuclient codenotary/immuclient:latest +``` + + + + + +### Basic operations + +To display all available options and their description run: + +```bash +$ ./immuclient help +``` + +Running `login immudb` from within immuclient will use the default database name and port. All you need to supply is the user and password: + +```bash +$ ./immuclient login immudb +Password: immudb +``` + +While immudb supports `set` and `get` for key-value storing and retrieving, its immutability means that we can verify the integrity of the underlying Merkle tree. To do this, we use the `safeset` and `safeget` commands. Let's try setting a value of `100` for the key `balance`: + +```bash +$ ./immuclient safeset balance 100 +tx: 2 +key: balance +value: 100 +verified: true +``` + +Then, we can immediately overwrite the key `balance` with a value of `9001` instead: + +```bash +$ ./immuclient safeset balance 9001 +tx: 3 +key: balance +value: 9001 +verified: true +``` + +If we try to retrieve the current value of key `balance`, we should get `9001`: + +```bash +$ ./immuclient safeget balance +tx: 3 +key: balance +value: 9001 +verified: true +``` + +Note that at each step so far, the `verified` flag is set to true. This ensures that the Merkle tree remains consistent for each transaction. + +We can show the history of transactions for key `balance` using the `history` command: + +```bash +$ ./immuclient history balance +tx: 2 +key: balance +value: 100 + +tx: 3 +key: balance +value: 9001 +``` + + + + + +### SQL operations + +In addition to a key-value store, immudb supports the relational model (SQL). For example, to create a table: + +```bash +$ ./immuclient exec "CREATE TABLE people(id INTEGER, name VARCHAR, salary INTEGER, PRIMARY KEY id);" +Updated rows: 0 +``` + +To insert data, use `UPSERT` (insert or update), which will add an entry, or overwrite it if already exists (based on the primary key): + +```bash +$ ./immuclient exec "UPSERT INTO people(id, name, salary) VALUES (1, 'Joe', 10000);" +Updated rows: 1 +$ ./immuclient exec "UPSERT INTO people(id, name, salary) VALUES (2, 'Bob', 30000);" +Updated rows: 1 +``` + +To query the data you can use the traditional `SELECT`: + +```bash +$ ./immuclient query "SELECT id, name, salary FROM people;" ++------------------------+--------------------------+----------------------------+ +| (MYDATABASE PEOPLE ID) | (MYDATABASE PEOPLE NAME) | (MYDATABASE PEOPLE SALARY) | ++------------------------+--------------------------+----------------------------+ +| 1 | "Joe" | 10000 | +| 2 | "Bob" | 30000 | ++------------------------+--------------------------+----------------------------+ +``` + +If we upsert again on the primary key "1", the value for "Joe" will be overwritten: + +```bash +$ ./immuclient exec "UPSERT INTO people(id, name, salary) VALUES (1, 'Joe', 20000);" +Updated rows: 1 + +$ ./immuclient query "SELECT id, name, salary FROM people;" ++------------------------+--------------------------+----------------------------+ +| (MYDATABASE PEOPLE ID) | (MYDATABASE PEOPLE NAME) | (MYDATABASE PEOPLE SALARY) | ++------------------------+--------------------------+----------------------------+ +| 1 | "Joe" | 20000 | +| 2 | "Bob" | 30000 | ++------------------------+--------------------------+----------------------------+ +``` + + + + + +### Time travel + +immudb is a immutable database. History is always preserved. With immudb you can travel in time! + +```bash +$ ./immuclient query "SELECT id, name, salary FROM people WHERE name='Joe';" ++------------------------+--------------------------+----------------------------+ +| (MYDATABASE PEOPLE ID) | (MYDATABASE PEOPLE NAME) | (MYDATABASE PEOPLE SALARY) | ++------------------------+--------------------------+----------------------------+ +| 1 | "Joe" | 20000 | ++------------------------+--------------------------+----------------------------+ +``` + +We can see the current transaction id using 'current': + +```bash +$ ./immuclient current +database: mydatabase +txID: 5 +hash: 2986dfeb2d15e55d8189f08c2508318addabe9e773e0b6e329cf23b654cc22e7 +``` + +This is the transaction id we will be using for the subsequent queries. + +Eg. before the update: + +```bash +$ ./immuclient query "SELECT id, name, salary FROM people BEFORE TX 5 WHERE name='Joe';" ++------------------------+--------------------------+----------------------------+ +| (MYDATABASE PEOPLE ID) | (MYDATABASE PEOPLE NAME) | (MYDATABASE PEOPLE SALARY) | ++------------------------+--------------------------+----------------------------+ +| 1 | "Joe" | 10000 | ++------------------------+--------------------------+----------------------------+ +``` + +or even before the first time insert (guess what, it is empty!): + +```bash +$ ./immuclient query "SELECT id, name, salary FROM people BEFORE TX 1 WHERE name='Joe';" ++------------------------+--------------------------+----------------------------+ +| (MYDATABASE PEOPLE ID) | (MYDATABASE PEOPLE NAME) | (MYDATABASE PEOPLE SALARY) | ++------------------------+--------------------------+----------------------------+ ++------------------------+--------------------------+----------------------------+ +``` + +You can even `TABLE` a table with itself in the past. Imagine you want to see how people salary changed between two points in time: + +```bash +$ ./immuclient query "SELECT peoplenow.id, peoplenow.name, peoplethen.salary, peoplenow.salary FROM people BEFORE TX 5 AS peoplethen INNER JOIN people AS peoplenow ON peoplenow.id=peoplethen.id;" ++---------------------------+-----------------------------+--------------------------------+-------------------------------+ +| (MYDATABASE PEOPLENOW ID) | (MYDATABASE PEOPLENOW NAME) | (MYDATABASE PEOPLETHEN SALARY) | (MYDATABASE PEOPLENOW SALARY) | ++---------------------------+-----------------------------+--------------------------------+-------------------------------+ +| 1 | "Joe" | 10000 | 20000 | +| 2 | "Bob" | 30000 | 30000 | ++---------------------------+-----------------------------+--------------------------------+-------------------------------+ +``` + + + + + +### KV Data revisions + +Whenever a new value is stored under given key, immudb saves a new revision of that data. +Revision numbers start with 1 - the first value ever written to the database will have +a revision number 1, the second will have 2 and so on. + +When reading a value from immudb, an explicit revision number can be specified. +If the provided number is greater than 0, a value for given revision is retrieved. +If the provided number is less than 0, the nth previous value is retrieved. + +```bash +$ ./immuclient set key value1 +tx: 2 +rev: 1 +key: key +value: value1 + +$ ./immuclient set key value2 +tx: 3 +rev: 2 +key: key +value: value2 + +$ ./immuclient set key value3 +tx: 4 +rev: 3 +key: key +value: value3 + +$ ./immuclient get key@1 # Get the key at the first revision +tx: 2 +rev: 1 +key: key +value: value1 + +$ ./immuclient get key@-1 # Get the key at the previous revision +tx: 3 +rev: 2 +key: key +value: value2 +``` + +The immuclient tool has also the possibility to restore a previous revision for given key. + +```bash +$ ./immuclient restore key@-2 +tx: 5 +rev: 4 +key: key +value: value1 +``` + +### Querying for keys containing revision separator + +In some cases, the key can already contain the `@` character reserved for key separator. +In such case there are few options to read such key. The revision separator can be changed +to any other string that is not part of the key. Also because immuclient will only scan +the last occurrence of the revision separator, an explicit 0th revision can be set to read +the current value behind such key. + +```bash +$ ./immuclient set some@email.address active +tx: 2 +rev: 1 +key: some@email.address +value: active + +# Change the revision separator with environment variable +$ IMMUCLIENT_REVISION_SEPARATOR="###" ./immuclient get some@email.address +tx: 2 +key: some@email.address +value: active +hash: 138033b5a89438758fdb3481ba0dc44816d550749f799223587cb30cd7eadf5a + +# Disable / change the revision separator through command-line argument +$ ./immuclient get --revision-separator="" some@email.address +tx: 2 +rev: 1 +key: some@email.address +value: active + +# Always use the revision number, use 0 for the current value +$ ./immuclient get some@email.address@0 +tx: 2 +rev: 1 +key: some@email.address +value: active +``` + + \ No newline at end of file diff --git a/src/1.9.4/connecting/commands.md b/src/1.9.4/connecting/commands.md new file mode 100644 index 0000000000..3f9409b1d5 --- /dev/null +++ b/src/1.9.4/connecting/commands.md @@ -0,0 +1,245 @@ +# Command reference + +## immudb + +Simply run ```./immudb -d``` to start immudb locally in the background. + +If you want to stop immudb în that case you need to find the process `ps -ax | grep immudb` and then `kill -15 `. Windows PowerShell would be `Get-Process immudb* | Stop-Process`. + +```bash +immudb - the lightweight, high-speed immutable database for systems and applications. + +Environment variables: + IMMUDB_DIR=. + IMMUDB_NETWORK=tcp + IMMUDB_ADDRESS=0.0.0.0 + IMMUDB_PORT=3322 + IMMUDB_DBNAME=immudb + IMMUDB_PIDFILE= + IMMUDB_LOGFILE= + IMMUDB_MTLS=false + IMMUDB_AUTH=true + IMMUDB_DETACHED=false + IMMUDB_CONSISTENCY_CHECK=true + IMMUDB_PKEY=./tools/mtls/3_application/private/localhost.key.pem + IMMUDB_CERTIFICATE=./tools/mtls/3_application/certs/localhost.cert.pem + IMMUDB_CLIENTCAS=./tools/mtls/2_intermediate/certs/ca-chain.cert.pem + IMMUDB_DEVMODE=true + IMMUDB_MAINTENANCE=false + IMMUDB_ADMIN_PASSWORD=immudb + +Usage: + immudb [flags] + immudb [command] + +Available Commands: + help Help about any command + version Show the immudb version + +Flags: + -a, --address string bind address (default "0.0.0.0") + --admin-password string admin password (default is 'immu') as plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) (default "immudb") + -s, --auth enable auth + --certificate string server certificate file path (default "./tools/mtls/3_application/certs/localhost.cert.pem") + --clientcas string clients certificates list. Aka certificate authority (default "./tools/mtls/2_intermediate/certs/ca-chain.cert.pem") + --config string config file (default path are configs or $HOME. Default filename is immudb.ini) + --consistency-check enable consistency check monitor routine. To disable: --consistency-check=false (default true) + -d, --detached run immudb in background + --devmode enable dev mode: accept remote connections without auth + --dir string data folder (default "./data") + -h, --help help for immudb + --logfile string log path with filename. E.g. /tmp/immudb/immudb.log + --maintenance override the authentication flag + -m, --mtls enable mutual tls + --no-histograms disable collection of histogram metrics like query durations + --pidfile string pid path with filename. E.g. /var/run/immudb.pid + --pkey string server private key path (default "./tools/mtls/3_application/private/localhost.key.pem") + -p, --port int port number (default 3322) + +Use "immudb [command] --help" for more information about a command. +``` + +## immugw + +immugw can be found in a different [repository](https://github.com/codenotary/immugw) + +Simply run `./immugw -d` to start immugw on the same machine as immudb (test or dev environment) or point to the remote immudb system `./immugw --immudbaddress "immudb-server"`. + +If you want to stop immugw în that case you need to find the process `ps -ax | grep immugw` and then `kill -15 `. Windows PowerShell would be `Get-Process immugw* | Stop-Process`. + +```bash +immu gateway: a smart REST proxy for immudb - the lightweight, high-speed immutable database for systems and applications. +It exposes all gRPC methods with a REST interface while wrapping all SAFE endpoints with a verification service. + +Environment variables: + IMMUGW_ADDRESS=0.0.0.0 + IMMUGW_PORT=3323 + IMMUGW_IMMUDB_ADDRESS=127.0.0.1 + IMMUGW_IMMUDB_PORT=3322 + IMMUGW_DIR=. + IMMUGW_PIDFILE= + IMMUGW_LOGFILE= + IMMUGW_DETACHED=false + IMMUGW_MTLS=false + IMMUGW_SERVERNAME=localhost + IMMUGW_PKEY=./tools/mtls/4_client/private/localhost.key.pem + IMMUGW_CERTIFICATE=./tools/mtls/4_client/certs/localhost.cert.pem + IMMUGW_CLIENTCAS=./tools/mtls/2_intermediate/certs/ca-chain.cert.pem + +Usage: + immugw [flags] + immugw [command] + +Available Commands: + help Help about any command + version Show the immugw version + +Flags: + -a, --address string immugw host address (default "0.0.0.0") + --audit enable audit mode (continuously fetches latest root from server, checks consistency against a local root and saves the latest root locally) + --audit-interval duration interval at which audit should run (default 5m0s) + --audit-password string immudb password used to login during audit; can be plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) + --audit-username string immudb username used to login during audit (default "immugwauditor") + --certificate string server certificate file path (default "./tools/mtls/4_client/certs/localhost.cert.pem") + --clientcas string clients certificates list. Aka certificate authority (default "./tools/mtls/2_intermediate/certs/ca-chain.cert.pem") + --config string config file (default path are configs or $HOME. Default filename is immugw.toml) + -d, --detached run immudb in background + --dir string program files folder (default ".") + -h, --help help for immugw + -k, --immudb-address string immudb host address (default "127.0.0.1") + -j, --immudb-port int immudb port number (default 3322) + --logfile string log path with filename. E.g. /tmp/immugw/immugw.log + -m, --mtls enable mutual tls + --pidfile string pid path with filename. E.g. /var/run/immugw.pid + --pkey string server private key path (default "./tools/mtls/4_client/private/localhost.key.pem") + -p, --port int immugw port number (default 3323) + --servername string used to verify the hostname on the returned certificates (default "localhost") + +Use "immugw [command] --help" for more information about a command. +``` + +## immuadmin + +For security reasons we recommend using immuadmin only on the same system as immudb. User management is restricted to localhost usage. Simply run ```./immuadmin``` on the same machine. + +```bash +CLI admin client for immudb - the lightweight, high-speed immutable database for systems and applications. + +Environment variables: + IMMUADMIN_IMMUDB_ADDRESS=127.0.0.1 + IMMUADMIN_IMMUDB_PORT=3322 + IMMUADMIN_MTLS=true + IMMUADMIN_SERVERNAME=localhost + IMMUADMIN_PKEY=./tools/mtls/4_client/private/localhost.key.pem + IMMUADMIN_CERTIFICATE=./tools/mtls/4_client/certs/localhost.cert.pem + IMMUADMIN_CLIENTCAS=./tools/mtls/2_intermediate/certs/ca-chain.cert.pem + +Usage: + immuadmin [command] + +Available Commands: + backup Make a copy of the database files and folders + database Issue all database commands + dump Dump database content to a file + help Help about any command + login Login using the specified username and password (admin username is immudb) + logout + print Print merkle tree + restore Restore the database from a snapshot archive or folder + service Manage immu services + set Update server config items: auth (none|password|cryptosig), mtls (true|false) + stats Show statistics as text or visually with the '-v' option. Run 'immuadmin stats -h' for details. + status Show heartbeat status + user Issue all user commands + version Show the immuadmin version + +Flags: + --certificate string server certificate file path (default "./tools/mtls/4_client/certs/localhost.cert.pem") + --clientcas string clients certificates list. Aka certificate authority (default "./tools/mtls/2_intermediate/certs/ca-chain.cert.pem") + --config string config file (default path is configs or $HOME; default filename is immuadmin.toml) + -h, --help help for immuadmin + -a, --immudb-address string immudb host address (default "127.0.0.1") + -p, --immudb-port int immudb port number (default 3322) + -m, --mtls enable mutual tls + --pkey string server private key path (default "./tools/mtls/4_client/private/localhost.key.pem") + --servername string used to verify the hostname on the returned certificates (default "localhost") + --tokenfile string authentication token file (default path is $HOME or binary location; the supplied value will be automatically suffixed with _admin; default filename is token-0.7.0_admin) (default "token-0.7.0") + +Use "immuadmin [command] --help" for more information about a command. +``` + +## immuclient + +Simply run `./immuclient` on the same machine or connect to a remote immudb `./immuclient -a ` + +```bash +CLI client for immudb - the lightweight, high-speed immutable database for systems and applications. +Environment variables: + IMMUCLIENT_IMMUDB_ADDRESS=127.0.0.1 + IMMUCLIENT_IMMUDB_PORT=3322 + IMMUCLIENT_AUTH=true + IMMUCLIENT_MTLS=false + IMMUCLIENT_SERVERNAME=localhost + IMMUCLIENT_PKEY=./tools/mtls/4_client/private/localhost.key.pem + IMMUCLIENT_CERTIFICATE=./tools/mtls/4_client/certs/localhost.cert.pem + IMMUCLIENT_CLIENTCAS=./tools/mtls/2_intermediate/certs/ca-chain.cert.pem + +IMPORTANT: All get and safeget functions return base64-encoded keys and values, while all set and safeset functions expect base64-encoded inputs. + +Usage: + immuclient [flags] + immuclient [command] + +Available Commands: + audit-mode Starts immuclient as daemon in auditor mode. Run 'immuclient audit-mode help' or use -h flag for details + check-consistency Check consistency for the specified index and hash + count Count keys having the specified prefix + current Return the last merkle tree root and index stored locally + get Get item having the specified key + getByIndex Return an element by index + getRawBySafeIndex Return an element by index + help Help about any command + history Fetch history for the item having the specified key + inclusion Check if specified index is included in the current tree + iscan Iterate over all elements by insertion order + it Starts immuclient in CLI mode. Use 'help' or -h flag on the shell for details + login Login using the specified username and password + logout + rawsafeget Get item having the specified key, without parsing structured values + rawsafeset Set a value for the item having the specified key, without setup structured values + reference Add new reference to an existing key + safeget Get and verify item having the specified key + safereference Add and verify new reference to an existing key + safeset Add and verify new item having the specified key and value + safezadd Add and verify new key with score to a new or existing sorted set + scan Iterate over keys having the specified prefix + set Add new item having the specified key and value + status Ping to check if server connection is alive + use Select database + user Issue all user commands + version Show the immuclient version + zadd Add new key with score to a new or existing sorted set + zscan Iterate over a sorted set + +Flags: + --audit-password string immudb password used to login during audit; can be plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) + --audit-username string immudb username used to login during audit + --certificate string server certificate file path (default "./tools/mtls/4_client/certs/localhost.cert.pem") + --clientcas string clients certificates list. Aka certificate authority (default "./tools/mtls/2_intermediate/certs/ca-chain.cert.pem") + --config string config file (default path are configs or $HOME. Default filename is immuclient.toml) + --dir string Main directory for audit process tool to initialize (default "/tmp") + -h, --help help for immuclient + -a, --immudb-address string immudb host address (default "127.0.0.1") + -p, --immudb-port int immudb port number (default 3322) + -m, --mtls enable mutual tls + --pkey string server private key path (default "./tools/mtls/4_client/private/localhost.key.pem") + --prometheus-host string Launch host of the Prometheus exporter. (default "0.0.0.0") + --prometheus-port string Launch port of the Prometheus exporter. (default "9477") + --roots-filepath string Filepath for storing root hashes after every successful audit loop. Default is tempdir of every OS. (default "/tmp/") + --servername string used to verify the hostname on the returned certificates (default "localhost") + --tokenfile string authentication token file (default path is $HOME or binary location; default filename is token-0.7.0) (default "token-0.7.0") + --value-only returning only values for get operations + +Use "immuclient [command] --help" for more information about a command. + +``` diff --git a/src/1.9.4/connecting/healthcheck.md b/src/1.9.4/connecting/healthcheck.md new file mode 100644 index 0000000000..3509bafda0 --- /dev/null +++ b/src/1.9.4/connecting/healthcheck.md @@ -0,0 +1,102 @@ +# HealthCheck + + + +HealthCheck return an error if `immudb` status is not ok. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/connect-healthcheck/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.*; + +public class App { + + public static void main(String[] args) { + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + ImmuClient client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.login("immudb", "immudb"); + + boolean isHealthy = immuClient.healthCheck(); + } + +} +``` + +::: + +::: tab .NET + +The following code snippets show how to invoke the healthcheck endpoint: + +``` csharp + Immuclient immuClient = new ImmuClient("localhost", 3322); + await immuClient.Open("immudb", "immudb", "defaultdb"); + + bool result = await immuClient.HealthCheck(); + Console.WriteLine(result); +``` + +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + check = client.healthCheck() # Returns bool + print(check) # True + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + await cl.health() +})() +``` +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](immugw.md) option. +::: + +:::: diff --git a/src/1.9.4/connecting/immugw.md b/src/1.9.4/connecting/immugw.md new file mode 100644 index 0000000000..123a133b54 --- /dev/null +++ b/src/1.9.4/connecting/immugw.md @@ -0,0 +1,127 @@ +# immugw + + + +immugw is a REST proxy that connects to immudb and provides a RESTful interface for applications. We recommend running immudb and immugw on separate machines to enhance security. + +You may download the immugw binary from [the latest releases on Github](https://github.com/codenotary/immugw/releases/latest). Once you have downloaded immugw, rename it to `immugw`, make sure to mark it as executable, then run it. The following example shows how to obtain v1.3.0 for linux amd64: + +```bash +$ wget https://github.com/codenotary/immugw/releases/download/v1.3.0/immugw-v1.3.0-linux-amd64 +$ mv immugw-v1.3.0-linux-amd64 immugw +$ chmod +x immugw + +# run help to find out about possible arguments +$ ./immugw help + +# and run immugw in the background +$ ./immugw -d --immudb-address {immudb-server-address} +``` + +Alternatively, you may [pull immudb docker image from DockerHub](https://hub.docker.com/r/codenotary/immudb) and run it in a ready-to-use container: + +```bash +$ docker run -it -d -p 3323:3323 --name immugw --env IMMUGW_IMMUDB_ADDRESS={immudb-server-address} codenotary/immugw:latest +``` + + + + + +### Build from sources + +Building binaries requires a Linux operating system. + +To build the binaries yourself, simply clone [immugw repository](https://github.com/codenotary/immugw) and run: + +```bash +$ make all +``` + +immugw can be cross compiled for different systems and architectures by setting `GOOS` and `GOARCH` variables, i.e.: + +```bash +GOOS=windows GOARCH=amd64 make all +``` + +To build immugw docker container locally: + +```bash +docker build -t myown/immugw:latest -f Dockerfile . +``` + + + + + +### Installing with immuadmin + +```bash +# install immugw service +$ ./immugw service install + +# check current immugw service status +$ ./immugw service status + +# stop immugw service +$ ./immugw service stop + +# start immugw service +$ ./immugw service start +``` + +The Linux service is using the following defaults: + +| File or configuration | location | +| ----------------------- | ------------------ | +| all configuration files | /etc/immudb | +| pid file | /var/lib/immudb/immugw.pid | +| log files | /var/log/immudb | + + + + + +### Configuration + +immugw can be configured using environment variables, flags or a config file. + +* `immugw --help` shows you all available flags and environment variables. +* `/etc/immudb/immugw.toml` is used as a default configuration file + + + + +### RESTful API reference + +You can find the swagger schema here: + +[swagger immugw](https://github.com/codenotary/immugw/blob/master/swagger.json) + +If you want to run the Swagger UI, simply run the following Docker command after you cloned this repo: + +```bash +$ wget https://github.com/codenotary/immugw/blob/master/swagger.json +$ docker run -d -it -p 8081:8080 --name swagger-immugw -v ${PWD}/swagger.json:/openapi.json -e SWAGGER_JSON=/openapi.json swaggerapi/swagger-ui +``` + + + + +### immugw as auditor +immugw can be also run as auditor. + +```bash +$ ./immugw --audit +``` + +If you are running immugw as a service, you need to edit /etc/immudb/immugw.toml and add the following section: + +```bash +audit = true # false is default +audit-interval = "5m" # suffixes: "s", "m", "h", examples: 10s, 5m 1h +audit-username = "" # when immudb authentication is enabled, use read-only user credentials here +audit-password = "" # and the password +``` + + \ No newline at end of file diff --git a/src/1.9.4/connecting/sdks.md b/src/1.9.4/connecting/sdks.md new file mode 100644 index 0000000000..f5c63336a8 --- /dev/null +++ b/src/1.9.4/connecting/sdks.md @@ -0,0 +1,55 @@ + +# SDKs + +::: tip +Examples used in this documentation can be found in [immudb examples repository](https://github.com/codenotary/immudb-client-examples). +::: + + + +In the most common scenario, you would perform write and read operations on the database talking to the server. In this case your application will be a client to immudb. + +SDKs make it comfortable to talk to the server from your favorite language, without having to deal with details about how to talk to it. + +The most well-known and recommended immudb SDK is written in [Golang](https://golang.org/), but there are other SDKs available, both maintained by the internal team and by the community. + +| Language | Maintainer | Latest version | link | Notes | +|----------|----------------------|----------------|---------------------|------------------------------| +| `go` | immudb team | 1.4.0 | [link][sdk-go] | | +| `python` | immudb team | 1.4.0 | [link][sdk-python] | | +| `JAVA` | immudb team | 1.0.0 | [link][sdk-java] | | +| `.NET` | immudb team | 1.0.5 | [link][sdk-dotnet] | | +| `NODE` | immudb team | 1.0.4 | [link][sdk-node] | Verification is not working | +| `ruby` | Community ([Ankane]) | 0.1.1 | [link][sdk-ruby] | | + +[sdk-go]: https://pkg.go.dev/github.com/codenotary/immudb/pkg/client +[sdk-python]: https://github.com/codenotary/immudb-py +[sdk-java]: https://github.com/codenotary/immudb4j +[sdk-dotnet]: https://github.com/codenotary/immudb4net +[sdk-node]: https://github.com/codenotary/immudb-node +[sdk-ruby]: https://github.com/ankane/immudb-ruby +[Ankane]: https://github.com/ankane + +The immudb server manages the requests from the outside world to the store. In order to insert or retrieve data, you need to talk with the server. + +
+ +![SDK Architecture](/immudb/immudb-server.svg) + +
+ + + + + +### immugw communication + +For other unsupported programming languages, [immugw](immugw.md) provides a REST gateway that can be used to talk to the server via generic HTTP. + +immugw can be found in its [own repository](https://github.com/codenotary/immugw) + +immugw proxies REST client communication and gRPC server interface. For security reasons, immugw should not run on the same server as immudb. The following diagram shows how the communication works: + +![immugw communication explained](/diagram-immugw.svg) + + diff --git a/src/1.9.4/connecting/webconsole.md b/src/1.9.4/connecting/webconsole.md new file mode 100644 index 0000000000..59ab6895ca --- /dev/null +++ b/src/1.9.4/connecting/webconsole.md @@ -0,0 +1,38 @@ + +# Web Console + + + +immudb includes an embedded web console which can be accessed via the default port 8080 (`web-server-port` option). + +![image](/webconsole.jpg) + +The console allows you to: + +* Metrics for the default database +* Execute SQL queries +* Configure users +* Create databases +* Manage permissions for users + + + + + +### Accessing the Web Console + +Once immudb has started, it will tell you if the web console is enabled and where it is listening: + +``` +immudb 2021/05/17 21:38:30 INFO: Webconsole enabled: 0.0.0.0:8080 +immudb 2021/05/17 21:38:30 INFO: Web API server enabled on 0.0.0.0:8080/api (http) +``` + +Just navigating to that address in your web browser will bring you to the login screen: + +![image](/browser.png) + + + + + diff --git a/src/1.9.4/develop/deleting.md b/src/1.9.4/develop/deleting.md new file mode 100644 index 0000000000..14cfcd6d12 --- /dev/null +++ b/src/1.9.4/develop/deleting.md @@ -0,0 +1,137 @@ +# Deleting + + + +It's possible to achieve deletion by using the `Delete` function. It provides logical deletion, it means that it is not physically deleted from db, but it's not possible to query it anymore after deletion. When immudb is used as an embedded store, it's possible to retrieve deleted entries. It's also possible to see deleted entries from the sdks using History endpoint, it will display if the entry was deleted. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-deleting/main.go +::: + +::: tab Java +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.exceptions.KeyNotFoundException; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] value1 = { 0, 1, 2, 3 }; + + client.set("key1", value1); + + client.delete("key1"); + + try { + client.get("key1"); + throw new RuntimeException("key not found expected"); + } catch (KeyNotFoundException e) { + // exception is expected + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` +::: + +::: tab .NET + +``` csharp + +var client = ImmuClient.NewBuilder().WithServerUrl(immudbServerAddress).Build(); +await client.Open("immudb", "immudb", "defaultdb"); + +string key = "hello"; + +try +{ + await client.VerifiedSet(key, "immutable world!"); + await client.Delete(key); +} +catch (VerificationException e) +{ + // VerificationException means Data Tampering detected! + // This means the history of changes has been tampered. + Console.WriteLine(e.ToString()); +} +await client.Close(); +``` + +::: + +::: tab Python +```python +from immudb import ImmudbClient +from immudb.datatypes import DeleteKeysRequest + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + client.set(b"immu", b"immudb-not-rulezz") + print(client.get(b"immu")) # b"immudb-not-rulezz" + + deleteRequest = DeleteKeysRequest(keys = [b"immu"]) + client.delete(deleteRequest) + print(client.get(b"immu")) # None + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + diff --git a/src/1.9.4/develop/document/api.md b/src/1.9.4/develop/document/api.md new file mode 100644 index 0000000000..947e56d957 --- /dev/null +++ b/src/1.9.4/develop/document/api.md @@ -0,0 +1,338 @@ +# API + +::: tip +By default, Swagger UI is enabled and can be accessed at `http://localhost:8080/api/docs/` +::: + + + +## Authentication + +A session must be active in order for you to be able to access collection and document endpoints. + +### Open session + +In the following script, the default credentials are used to open a session in the defaultdb database. + +A sessionID is assigned by immudb, and this value must be included in all subsequent requests. + +```bash +sessionid=$( +curl -X 'POST' \ +'http://localhost:8080/api/v2/authorization/session/open' \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ + "username": "immudb", + "password":"immudb", + "database":"defaultdb" +}' | jq -r .sessionID) +``` + + +### Close session + +Although immudb automatically closes inactive sessions, it is a good practice to explicitly close sessions when they are not needed anymore in order to free up resources immediately. + +```bash +curl -X 'POST' \ +'http://localhost:8080/api/v2/authorization/session/close' -H "sessionID: $sessionid" +``` + + + + + +## Collections + +Collections allow you to store and manage related documents together, making it easier to search and retrieve relevant data. + +### Create collection + +Any json object can be stored in a collection, but declared fields enable indexes to be created. + +Here is the script that creates a collection with two fields of type `STRING` and a non-unique index over one of them. + +```bash +curl -X 'POST' \ +'http://localhost:8080/api/v2/collection/mycollection' \ +-H "sessionID: $sessionid" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ + "fields": [ + {"name": "name", "type": "STRING"}, + {"name": "surname", "type": "STRING"} + ], + "indexes": [ + {"fields": ["name"], "unique": "false"} + ] +}' +``` + +The available types of fields are: +- STRING +- INTEGER +- BOOLEAN +- DOUBLE +- UUID + +### Add field + +A new field can be added to an existing collection. + +```bash +curl -X 'POST' \ +'http://localhost:8080/api/v2/collection/mycollection/field' \ +-H "sessionID: $sessionid" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ + "field": { + "name": "active", + "type": "BOOLEAN" + } +}' +``` + +### Remove field + +An existing field can be deleted. Prior to removing the field, it is necessary to remove any associated indexes. + +```bash +curl -X 'DELETE' \ +'http://localhost:8080/api/v2/collection/mycollection/field/active' \ +-H "sessionID: $sessionid" +``` + +### Delete collection + +It is possible to delete collections, and the physical removal of any declared index will be carried out. The raw data in the transaction commit log have not been altered, but this operation cannot be reversed. + +```bash +curl -X 'DELETE' \ +'http://localhost:8080/api/v2/collection/mycollection' \ +-H "sessionID: $sessionid" +``` + + + +::: tip +If you create lots of indexes, you may want to adjust default settings to reduce your memory footprint. + +Indexing parameters, including cache-size, flush-thresholds, and max-active-snapshots, can be lowered as needed, but take into account more IO reads and writes, which may lead to poor indexing performance. +::: + + + +## Indexes + +Collections allow you to store and manage related documents together, making it easier to search and retrieve relevant data. + +### Create index + +It is possible to create indexes over the declared fields in the collection. + +Creating non-unique indexes is possible at any time, while creating unique ones is only possible when no documents have been stored. + +```bash +curl -X 'POST' \ + 'http://localhost:8080/api/v2/collection/mycollection/index' \ + -H 'accept: application/json' \ + -H "sessionID: $sessionid" \ + -H 'Content-Type: application/json' \ + -d '{ + "fields": [ + "surname" + ] +}' +``` + +### Delete index + +It is possible to delete collections, and the physical removal of any declared index will be carried out. The raw data in the transaction commit log have not been altered, but this operation cannot be reversed. + +```bash +curl -X 'DELETE' \ +'http://localhost:8080/api/v2/collection/mycollection/index?fields=surname' \ +-H "sessionID: $sessionid" +``` + + + + + +## Documents + +Collections allow you to store and manage related documents together, making it easier to search and retrieve relevant data. + +### Insert document + +Single or multiple documents can be inserted in a single request + +```bash +curl -X 'POST' \ +'http://localhost:8080/api/v2/collection/mycollection/documents' \ +-H "sessionID: $sessionid" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ + "documents": [ + {"name":"John", "surname":"Doe"}, + {"name":"Jane", "surname":"Smith"} + ] +}' +``` + +### Search documents + +It is possible to delete collections, and the physical removal of any declared index will be carried out. The raw data in the transaction commit log have not been altered, but this operation cannot be reversed. + +```bash +curl -X 'POST' \ +'http://localhost:8080/api/v2/collection/mycollection/documents/search' \ +-H "sessionID: $sessionid" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ + "query": { + "expressions": [ + { + "fieldComparisons": [ + { + "field": "name", + "operator": "EQ", + "value": "John" + } + ] + } + ] + }, + "page": 1, + "pageSize": 10 +}' +``` + +The supported operators are: + +- EQ: equals to +- NE: not equals to +- LT: less than +- LE: less than or equal to +- GT: greater than +- GE: greater than or equal to +- LIKE: search using regular expressions, for example "value":"(doc)|(flick)" would allow searching for either values containing "doc" or "flick". The syntax of golang regexp is described in [this GitHub repo](https://github.com/google/re2/wiki/Syntax). + + +### Replace documents + +A single or multiple documents can be atomically replaced. + +```bash +curl -X 'PUT' \ + 'http://localhost:8080/api/v2/collection/mycollection/documents/replace' \ + -H 'accept: application/json' \ + -H "sessionID: $sessionid" \ + -H 'Content-Type: application/json' \ + -d '{ + "query": { + "expressions": [ + { + "fieldComparisons": [ + { + "field": "_id", + "operator": "EQ", + "value": "6530f0fa000000000000001f86853b05" + } + ] + } + ], + "limit": 1 + }, + "document": { + "first_name": "John", + "last_name": "Doe", + "age": 40 + } +}' +``` + +### Delete documents + +Documents can be deleted. A document audit preserves document history and allows for retrieval of all revisions, even deleted ones. + +```bash +curl -X 'POST' \ + 'http://localhost:8080/api/v2/collection/mycollection/documents/delete' \ + -H 'accept: application/json' \ + -H "sessionID: $sessionid" \ + -H 'Content-Type: application/json' \ + -d '{ + "query": { + "expressions": [ + { + "fieldComparisons": [ + { + "field": "first_name", + "operator": "EQ", + "value": "John" + }, + { + "field": "last_name", + "operator": "EQ", + "value": "Doe" + } + ] + } + ], + "limit": 1 + } +}' +``` + +### Count documents + +It is possible to retrieve the number of documents meeting a given criteria by using the document count endpoint. + +```bash +curl -X 'POST' \ + 'http://localhost:8080/api/v2/collection/mycollection/documents/count' \ + -H 'accept: application/json' \ + -H "sessionID: $sessionid" \ + -H 'Content-Type: application/json' \ + -d '{ + "query": { + "expressions": [ + { + "fieldComparisons": [ + { + "field": "first_name", + "operator": "EQ", + "value": "Jane" + } + ] + } + ] + } +}' +``` + +### Audit documents + +Document revisions can be retrieved through a document audit. In auditing, all revisions are tracked and retrievable, even those that have been deleted. + +In order to audit a document, it is necessary to know its unique identifier, which can be obtained by inserting or querying the document. + +```bash +curl -X 'POST' \ + 'http://localhost:8080/api/v2/collection/mycollection/document/6530f0fa000000000000001f86853b05/audit' \ + -H 'accept: application/json' \ + -H "sessionID: $sessionid" \ + -H 'Content-Type: application/json' \ + -d '{ + "page": 1, + "pageSize": 10 +}' +``` + + \ No newline at end of file diff --git a/src/1.9.4/develop/document/datamodel.md b/src/1.9.4/develop/document/datamodel.md new file mode 100644 index 0000000000..761d20066d --- /dev/null +++ b/src/1.9.4/develop/document/datamodel.md @@ -0,0 +1,16 @@ +# Data Model + +::: tip +By default, Swagger UI is enabled and can be accessed at `http://localhost:8080/api/docs/` +::: + + + +### Collection + +A collection is a container for documents of a similar type. Each database can have one or many collections. The documents within a collection can have various structures, but they must all be JSON files. Collections allow you to store and manage related documents together, making it easier to search and retrieve relevant data. + +### Documents +Documents are JSON files containing structured data that are stored within a collection. Each document can have multiple versions, allowing you to track changes and maintain a history of modifications. + + \ No newline at end of file diff --git a/src/1.9.4/develop/expiration.md b/src/1.9.4/develop/expiration.md new file mode 100644 index 0000000000..1a5b91826c --- /dev/null +++ b/src/1.9.4/develop/expiration.md @@ -0,0 +1,78 @@ +# Data Expiration + + + +It's possible to achieve data expiration by using the `ExpirableSet` function. It provides logical deletion, it means that it is not physically deleted from db, but it's not possible to query it anymore after deletion. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-expiration/main.go +::: + +::: tab Java + +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab .NET + +``` csharp + +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +await client.ExpirableSet("key1", "value1", DateTime.Now.AddDays(1)); + +Entry entry = await client.VerifiedGet("key1"); +Console.WriteLine(entry.ToString()); +await client.Close(); + +``` + +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab Python +```python +from immudb import ImmudbClient +from datetime import datetime, timedelta +import time + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + client.expireableSet(b"TEST", b"test", datetime.now() + timedelta(seconds=3)) + print(client.get(b"TEST")) # b"test" + time.sleep(4) + try: + print(client.get(b"TEST")) + except: + pass # Key not found, because it expires, raises Exception + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + diff --git a/src/1.9.4/develop/indexes.md b/src/1.9.4/develop/indexes.md new file mode 100644 index 0000000000..d8c3644143 --- /dev/null +++ b/src/1.9.4/develop/indexes.md @@ -0,0 +1,282 @@ +# Secondary indexes + + + +## Sorted sets + +On top of the key value store immudb provides secondary indexes to help developers to handle complex queries. + +The `sorted set` data type provides a simple secondary index that can be created with immudb. +This data structure contains a set of references to other key-value entries. +Elements of this set are ordered using a floating-point `score` specified for each element upon insertion. +Entries having equal score will have the order in which they were inserted into the set. + +> Note: The score type is a 64-bit floating point number to support a large number of uses cases. +> 64-bit floating point gives a lot of flexibility and dynamic range, at the expense of having only 53-bits of integer. +> When a 64-bit integer is cast to a float value there _could_ be a loss of precision, +> in which case the order of entries having same float64 score value will be determined by the insertion order. + +The KV entry referenced in the set can be bound to a specific transaction id - such entry is called a `bound` reference. +A `bound` reference will always get the value for the key at a specific transaction instead of the most recent value, +including a case where one set contains multiple values for the same key but for different transactions. +That way, sets allow optimal access to historical data using a single immudb read operation. + +> Note: If a compound operation is executed with the `ExecAll` call, +> a bound entry added to the set can reference a key created/updated in the same `ExecAll` call. +> To make such an operation, set the `BoundRef` value to `true` and the `AtTx` value to `0`. + +Inserting entries into sets can be done using the following operations: +`ZAdd`, `VerifiedZAdd`, `ZAddAt`, `VerifiedZAddAt`, `ExecAll`. +Those operations accept the following parameters: + +* `Set`: the name of the collection +* `Score`: entry score used to order items within the set +* `Key`: the key of entry to be added to the set +* `AtTx`: for bound references, a transaction id at which the value will be read, + if set to `0` for `ExecAll` operation, current transaction id will be used. Optional +* `BoundRef`: if set to true, this will be a reference bound to a specific transaction. Optional +* `NoWait`: if set to true, don't wait for indexing to be finished after adding this entry + +Reading data from the set can be done using the following operations: +`ZScan`, `StreamZScan`. Those operations accept the following parameters: + +* `Set`: the name of the collection +* `SeekKey`: initial key for the first entry in the iteration. Optional +* `SeekScore`: the min or max score for the first entry in the iteration, depending on Desc value. Optional +* `SeekAtTx`: the tx id for the first entry in the iteration. Optional +* `InclusiveSeek`: the element resulting from the combination of the `SeekKey` `SeekScore` and `SeekAtTx` + is returned with the result. Optional +* `Desc`: If set to true, entries will be returned in an descending (reversed) order. Optional +* `SinceTx`: immudb will wait that the transaction provided by SinceTx be processed. Optional +* `NoWait`: when true scan doesn't wait that txSinceTx is processed. Optional +* `MinScore`: minimum score filter. Optional +* `MaxScore`: maximum score filter. Optional +* `Limit`: maximum number of returned items. Optional + +> Note: issuing a `ZScan` or `StreamZScan` operation will by default wait for the index to be up-to-date. +> To avoid waiting for the index (and thus to allow reading the data from some older state), +> set the `SinceTx` to a very high value exceeding the most recent transaction id +> (e.g. maximum int value) and set `NoWait` to `true`. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-indexes/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.util.Iterator; + +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.ZEntry; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] value1 = { 0, 1, 2, 3 }; + byte[] value2 = { 4, 5, 6, 7 }; + + client.set("key1", value1); + client.set("key2", value2); + + client.zAdd("set1", "key1", 1); + + client.zAdd("set1", "key2", 2); + + Iterator it = client.zScan("set1"); + + while (it.hasNext()) { + ZEntry zentry = it.next(); + System.out.format("('%s', '%f')\n", new String(zentry.getKey()), zentry.getScore()); + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} + +``` + +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +byte[] value1 = { 0, 1, 2, 3 }; +byte[] value2 = { 4, 5, 6, 7 }; + +try +{ + await client.Set("zadd1", value1); + await client.Set("zadd2", value2); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("A CorruptedDataException occurred."); +} + +try +{ + await client.ZAdd("set1", "zadd1", 1); + await client.ZAdd("set1", "zadd2", 2); + + await client.ZAdd("set2", "zadd1", 2); + await client.ZAdd("set2", "zadd2", 1); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("A CorruptedDataException occurred"); +} + +List zScan1 = await client.ZScan("set1", 5, false); + +Console.WriteLine(zScan1.Count); + +await client.Close(); +``` + +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + client.set(b"user1", b"user1@mail.com") + client.set(b"user2", b"user2@mail.com") + client.set(b"user3", b"user3@mail.com") + client.set(b"user4", b"user3@mail.com") + + client.zAdd(b"age", 100, b"user1") + client.zAdd(b"age", 101, b"user2") + client.zAdd(b"age", 99, b"user3") + client.zAdd(b"age", 100, b"user4") + + scanResult = client.zScan(b"age", b"", 0, 0, True, 50, False, 100, 101) + print(scanResult) # Shows records with 'age' 100 <= score < 101 + # with descending order and limit = 50 + + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const { id: id1 } = await cl.set({ key: 'user1', value: 'user1@mail.com' }) + const { id: id2 } = await cl.set({ key: 'user2', value: 'user2@mail.com' }) + const { id: id3 } = await cl.set({ key: 'user3', value: 'user3@mail.com' }) + const { id: id4 } = await cl.set({ key: 'user3', value: 'another-user3@mail.com' }) + + const zAddAtReq1: Parameters.ZAddAt = { + set: 'age', + score: 25, + key: 'user1', + attx: id1 + } + const zAddAtRes1 = await cl.zAddAt(zAddAtReq1) + const zAddAtReq2: Parameters.ZAddAt = { + set: 'age', + score: 36, + key: 'user2', + attx: id2 + } + const zAddAtRes2 = await cl.zAddAt(zAddAtReq2) + const zAddAtReq3: Parameters.ZAddAt = { + set: 'age', + score: 36, + key: 'user3', + attx: id3 + } + const zAddAtRes3 = await cl.zAddAt(zAddAtReq3) + const zAddAtReq4: Parameters.ZAddAt = { + set: 'age', + score: 54, + key: 'user4', + attx: id4 + } + const zAddAtRes4 = await cl.zAddAt(zAddAtReq4) + + const zScanReq: Parameters.ZScan = { + set: 'age', + sincetx: 0, + nowait: true, + minscore: { + score: 36 + } + } + const zScanRes = await cl.zScan(zScanReq) + console.log('success: zScan all 36-years-old users', zScanRes) +})() +``` + +::: + + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: diff --git a/src/1.9.4/develop/queries-history.md b/src/1.9.4/develop/queries-history.md new file mode 100644 index 0000000000..59250c7f4d --- /dev/null +++ b/src/1.9.4/develop/queries-history.md @@ -0,0 +1,1159 @@ +# Queries and history + + + +The fundamental property of immudb is that it's an append-only database. +This means that an _update_ does not change an existing record. Instead, it is a new insert of the **same key** with a **new value**. +It's possible to retrieve all the values for a particular key with the history command. + +`History` accepts the following parameters: + +* `Key`: a key of an item +* `Offset`: the starting index (excluded from the search). Optional +* `Limit`: maximum returned items. Optional +* `Desc`: items are returned in reverse order. Optional +* `SinceTx`: immudb will wait that the transaction specified by SinceTx is processed. Optional + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-history/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key1 = "myKey1".getBytes(StandardCharsets.UTF_8); + byte[] value1 = new byte[] { 1, 2, 3 }; + byte[] value2 = new byte[] { 4, 5, 6 }; + + client.set(key1, value1); + client.set(key1, value2); + + Iterator it = client.history(key1, false, 0, 0); + + while (it.hasNext()) { + Entry entry = it.next(); + + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} + +``` +Note that, similar with many other methods, `history` method is overloaded to allow different kinds/set of parameters. + +::: + +::: tab .NET + +```csharp + +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +try +{ + await client.Set("history1", value1); + await client.Set("history1", value2); + await client.Set("history2", value1); + await client.Set("history2", value2); + await client.Set("history2", value3); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at set.", e); +} + +List historyResponse1 = await client.History("history1", 10, 0, false); + +await client.Close(); +``` +Note that, similar with many other methods, `history` method is overloaded to allow different kinds/set of parameters. + +::: + +::: tab Python + +Python immudb sdk currently doesn't support `SinceTx` parameter + +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + client.set(b'test', b'1') + client.set(b'test', b'2') + client.set(b'test', b'3') + + history = client.history(b'test', 0, 100, True) # List[immudb.datatypes.historyResponseItem] + responseItemFirst = history[0] + print(responseItemFirst.key) # Entry key (b'test') + print(responseItemFirst.value) # Entry value (b'3') + print(responseItemFirst.tx) # Transaction id + + responseItemThird = history[2] + print(responseItemThird.key) # Entry key (b'test') + print(responseItemThird.value) # Entry value (b'1') + print(responseItemThird.tx) # Transaction id + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const key = 'hello' + + await cl.set({ key, value: 'immutable world' }) + await cl.set({ key, value: 'immudb' }) + + const historyReq: Parameters.History = { + key + } + const historyRes = cl.history(historyReq) + console.log('success: history', historyRes) +})() +``` +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + +
+ + + +## Counting + +Counting entries is not supported at the moment. + + + +:::: tabs + +::: tab Go +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Go sdk github project](https://github.com/codenotary/immudb/issues/new) +::: + +::: tab Java +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [.Net sdk github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + +
+ + + +## Scan + +The `scan` command is used to iterate over the collection of elements present in the currently selected database. +`Scan` accepts the following parameters: + +* `Prefix`: prefix. If not provided all keys will be involved. Optional +* `SeekKey`: initial key for the first entry in the iteration. Optional +* `Desc`: DESC or ASC sorting order. Optional +* `Limit`: maximum returned items. Optional +* `SinceTx`: immudb will wait that the transaction provided by SinceTx be processed. Optional +* `NoWait`: Default false. When true scan doesn't wait for the index to be fully generated and returns the last indexed value. Optional + +To gain speed it's possible to specify `noWait=true`. The control will be returned to the caller immediately, without waiting for the indexing to complete. When `noWait` is used, keep in mind that the returned data may not be yet up to date with the inserted data, as the indexing might not have completed. + + + +:::: tabs + +::: tab Go + +An ordinary `scan` command and a reversed one. + +<<< @/src/code-examples/go/develop-kv-scan/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key1 = "myKey1".getBytes(StandardCharsets.UTF_8); + byte[] value1 = new byte[] { 1, 2, 3 }; + + byte[] key2 = "myKey2".getBytes(StandardCharsets.UTF_8); + byte[] value2 = new byte[] { 4, 5, 6 }; + + client.set(key1, value1); + client.set(key2, value2); + + Iterator it = client.scan("myKey"); + + while (it.hasNext()) { + Entry entry = it.next(); + + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} + +``` + +`scan` is an overloaded method, therefore multiple flavours of it with different parameter options exist. + +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +byte[] value1 = { 0, 1, 2, 3 }; +byte[] value2 = { 4, 5, 6, 7 }; + +try +{ + await client.Set("scan1", value1); + await client.Set("scan2", value2); +} +catch (CorruptedDataException e) +{ + Assert.Fail("Failed at set.", e); +} + +List scanResult = await client.Scan("scan", 5, false); +Console.WriteLine(scanResult.Count); + +await client.Close(); +``` + +`scan` is an overloaded method, therefore multiple flavours of it with different parameter options exist. + +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + toSet = { + b"aaa": b'1', + b'bbb': b'2', + b'ccc': b'3', + b'acc': b'1', + b'aac': b'2', + b'aac:test1': b'3', + b'aac:test2': b'1', + b'aac:xxx:test': b'2' + } + client.setAll(toSet) + + result = client.scan(b'', b'', True, 100) # All entries + print(result) + result = client.scan(b'', b'aac', True, 100) # All entries with prefix 'aac' including 'aac' + print(result) + + # Seek key example (allows retrieve entries in proper chunks): + result = client.scan(b'', b'', False, 3) + while result: + for item, value in result.items(): + print("SEEK", item, value) + lastKey = list(result.keys())[-1] + result = client.scan(lastKey, b'', False, 3) + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + await cl.set({ key: 'aaa', value: 'item1' }) + await cl.set({ key: 'bbb', value: 'item2' }) + await cl.set({ key: 'abc', value: 'item3' }) + + const scanReq: Parameters.Scan = { + prefix: 'a', + desc: false, + limit: 0, + sincetx: 0 + } + const scanRes = await cl.scan(scanReq) + console.log('success: scan', scanRes) + + const scanReq1: Parameters.Scan = { + prefix: 'a', + desc: true, + limit: 0, + sincetx: 0 + } + const scanRes1 = await cl.scan(scanReq1) + console.log('success: scan', scanRes1) +})() +``` + +Example with an offset: + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + await cl.set({ key: 'aaa', value: 'item1' }) + await cl.set({ key: 'bbb', value: 'item2' }) + await cl.set({ key: 'abc', value: 'item3' }) + + const scanReq: Parameters.Scan = { + seekkey: '', + prefix: '', + desc: true, + limit: 0, + sincetx: 0 + } + const scanRes = await cl.scan(scanReq) + console.log('success: scan', scanRes) + + const scanReq1: Parameters.Scan = { + seekkey: 'bbb', + prefix: '', + desc: true, + limit: 0, + sincetx: 0 + } + const scanRes1 = await cl.scan(scanReq1) + console.log('success: scan', scanRes1) + + const scanReq2: Parameters.Scan = { + seekkey: 'b', + prefix: 'b', + desc: true, + limit: 0, + sincetx: 0 + } + const scanRes2 = await cl.scan(scanReq2) + console.log('success: scan', scanRes2) +})() +``` +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + +
+ + + +## References + +`SetReference` is like a "tag" operation. It appends a reference on a key/value element. +As a consequence, when we retrieve that reference with a `Get` or `VerifiedGet` the value retrieved will be the original value associated with the original key. +Its ```VerifiedReference``` counterpart is the same except that it also produces the inclusion and consistency proofs. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-references/main.go + +Example with verifications + +<<< @/src/code-examples/go/develop-kv-references-verified/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key = "myKey".getBytes(StandardCharsets.UTF_8); + byte[] value = new byte[] { 1, 2, 3 }; + + byte[] myRef = "myRef".getBytes(StandardCharsets.UTF_8); + + client.set(key, value); + + client.setReference(myRef, key); + + Entry entry = client.get(myRef); + + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} + +``` +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +byte[] key = Encoding.UTF8.GetBytes("testRef"); +byte[] val = Encoding.UTF8.GetBytes("abc"); + +TxHeader? setTxHdr = null; +try +{ + setTxHdr = await client.Set(key, val); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at set.", e); + return; +} + +byte[] ref1Key = Encoding.UTF8.GetBytes("ref1_to_testRef"); +byte[] ref2Key = Encoding.UTF8.GetBytes("ref2_to_testRef"); + +TxHeader? ref1TxHdr = null; +try +{ + ref1TxHdr = await client.SetReference(ref1Key, key); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at setReference", e); + return; +} + +TxHeader? ref2TxHdr = null; +try +{ + ref2TxHdr = await client.SetReference(ref2Key, key, setTxHdr.Id); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at setReferenceAt.", e); +} + +await client.Close(); +``` + +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + client.verifiedSet(b'x', b'1') + client.verifiedSet(b'y', b'1') + retrieved = client.verifiedGet(b'x') + print(retrieved.refkey) # Entry reference key (None) + + client.verifiedSetReference(b'x', b'reference1') + client.setReference(b'x', b'reference2') + client.setReference(b'y', b'reference2') + client.verifiedSet(b'y', b'2') + + retrieved = client.verifiedGet(b'reference1') + print(retrieved.key) # Entry key (b'x') + print(retrieved.refkey) # Entry reference key (b'reference1') + print(retrieved.verified) # Entry verification status (True) + + retrieved = client.verifiedGet(b'reference2') + print(retrieved.key) # Entry key (b'y') + print(retrieved.refkey) # Entry reference key (b'reference2') + print(retrieved.verified) # Entry verification status (True) + print(retrieved.value) # Entry value (b'3') + + retrieved = client.verifiedGet(b'x') + print(retrieved.key) # Entry key (b'x') + print(retrieved.refkey) # Entry reference key (None) + print(retrieved.verified) # Entry verification status (True) + + retrieved = client.get(b'reference2') + print(retrieved.key) # Entry key (b'y') + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const setReq: Parameters.Set = { + key: 'firstKey', + value: 'firstValue' + } + await cl.set(setReq) + + const referenceReq: Parameters.SetReference = { + key: 'myTag', + referencedKey: 'firstKey' + } + const referenceRes = await cl.setReference(referenceReq) + console.log('success: setReference', referenceRes) + + const getReq: Parameters.Get = { + key: 'myTag' + } + const getRes = await cl.get(getReq) + console.log('success: get by reference', getRes) +})() +``` + +Example with verifications + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const setReq: Parameters.Set = { + key: 'firstKey', + value: 'firstValue' + } + await cl.set(setReq) + + const verifiedReferenceReq: Parameters.SetReference = { + key: 'myTag', + referencedKey: 'firstKey' + } + const verifiedReferenceRes = await cl.verifiedSetReference(verifiedReferenceReq) + console.log('success: verifiedSetReference', verifiedReferenceRes) + + const getReq: Parameters.Get = { + key: 'myTag' + } + const getRes = await cl.get(getReq) + console.log('success: get by reference', getRes) +})() +``` +::: + + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +### GetReference and VerifiedGetReference + +When reference is resolved with get or verifiedGet in case of multiples equals references the last reference is returned. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-references-get/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key = "myKey".getBytes(StandardCharsets.UTF_8); + byte[] value1 = new byte[] { 1, 2, 3 }; + byte[] value2 = new byte[] { 4, 5, 6 }; + + client.set(key, value1); + + byte[] myRef = "myRef".getBytes(StandardCharsets.UTF_8); + client.setReference(myRef, key); + + client.set(key, value2); + + Entry entry = client.get(myRef); + + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` + +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + client.verifiedSet(b'x', b'1') + client.verifiedSet(b'y', b'1') + retrieved = client.verifiedGet(b'x') + print(retrieved.refkey) # Entry reference key (None) + + client.verifiedSetReference(b'x', b'reference1') + client.setReference(b'x', b'reference2') + client.setReference(b'y', b'reference2') + client.verifiedSet(b'y', b'2') + + retrieved = client.verifiedGet(b'reference1') + print(retrieved.key) # Entry key (b'x') + print(retrieved.refkey) # Entry reference key (b'reference1') + print(retrieved.verified) # Entry verification status (True) + + retrieved = client.verifiedGet(b'reference2') + print(retrieved.key) # Entry key (b'y') + print(retrieved.refkey) # Entry reference key (b'reference2') + print(retrieved.verified) # Entry verification status (True) + print(retrieved.value) # Entry value (b'3') + + retrieved = client.verifiedGet(b'x') + print(retrieved.key) # Entry key (b'x') + print(retrieved.refkey) # Entry reference key (None) + print(retrieved.verified) # Entry verification status (True) + + retrieved = client.get(b'reference2') + print(retrieved.key) # Entry key (b'y') + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const setReq: Parameters.Set = { + key: 'firstKey', + value: 'firstValue' + } + await cl.set(setReq) + const setReq1: Parameters.Set = { + key: 'secondKey', + value: 'secondValue' + } + await cl.set(setReq1) + + const verifiedReferenceReq: Parameters.SetReference = { + key: 'myTag', + referencedKey: 'firstKey' + } + await cl.verifiedSetReference(verifiedReferenceReq) + const verifiedReferenceReq1: Parameters.SetReference = { + key: 'myTag', + referencedKey: 'secondKey' + } + await cl.verifiedSetReference(verifiedReferenceReq1) + + const getReq: Parameters.Get = { + key: 'myTag' + } + const getSecondItemRes = await cl.get(getReq) + console.log('success: get by second reference', getSecondItemRes) +})() +``` +::: + + +::: tab .NET + +```csharp + +using ImmuDB; +using ImmuDB.Exceptions; +using ImmuDB.SQL; + +namespace simple_app; + +class Program +{ + public static async Task Main(string[] args) + { + var client = new ImmuClient(); + + await client.Open("immudb", "immudb","defaultdb"); + await client.VerifiedSet("mykey", "myvalue"); + await client.VerifiedSetReference("myreference", "mykey"); + Entry result = await client.Get("myreference"); + + System.Console.WriteLine(result.ToString()); + } +} + +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +### Resolving reference with transaction id + +It's possible to bind a reference to a key on a specific transaction using `SetReferenceAt` and `VerifiedSetReferenceAt` + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-references-txid/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.TxHeader; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key = "myKey".getBytes(StandardCharsets.UTF_8); + byte[] value1 = new byte[] { 1, 2, 3 }; + byte[] value2 = new byte[] { 4, 5, 6 }; + + TxHeader hdr1 = client.set(key, value1); + + byte[] myRef = "myRef".getBytes(StandardCharsets.UTF_8); + client.setReference(myRef, key, hdr1.getId()); + + client.set(key, value2); + + Entry entry = client.get(myRef); + + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} + +``` + +::: + +::: tab .NET + +```csharp + +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +byte[] key = Encoding.UTF8.GetBytes("testRef"); +byte[] val = Encoding.UTF8.GetBytes("abc"); + +TxHeader? setTxHdr = null; +try +{ + setTxHdr = await client.Set(key, val); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at set.", e); + return; +} + +byte[] ref1Key = Encoding.UTF8.GetBytes("ref1_to_testRef"); +byte[] ref2Key = Encoding.UTF8.GetBytes("ref2_to_testRef"); + +TxHeader? ref1TxHdr = null; +try +{ + ref1TxHdr = await client.SetReference(ref1Key, key); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at SetReference", e); + return; +} + +TxHeader? ref2TxHdr = null; +try +{ + ref2TxHdr = await client.SetReference(ref2Key, key, setTxHdr.Id); +} +catch (CorruptedDataException e) +{ + Console.WriteLine("Failed at SetReference.", e); +} + +await client.Close(); +``` + +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const { id } = await cl.set({ key: 'firstKey', value: 'firstValue' }) + await cl.set({ key: 'firstKey', value: 'secondValue' }) + + const verifiedSetReferenceAtReq: Parameters.VerifiedSetReferenceAt = { + key: 'myFirstTag', + referencedKey: 'firstKey', + attx: id + } + const verifiedSetReferenceAtRes = await cl.verifiedSetReferenceAt(verifiedSetReferenceAtReq) + console.log('success: verifiedSetReferenceAt', verifiedSetReferenceAtRes) + + const getSecondItemRes = await cl.get({ key: 'myFirstTag' }) + console.log('success: get second item by reference', getSecondItemRes) +})() +``` +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: diff --git a/src/1.9.4/develop/reading.md b/src/1.9.4/develop/reading.md new file mode 100644 index 0000000000..ec98aa2a35 --- /dev/null +++ b/src/1.9.4/develop/reading.md @@ -0,0 +1,815 @@ +# Reads And Writes + +::: tip +Examples in multiple languages can be found at following links: [immudb SDKs examples](https://github.com/codenotary/immudb-client-examples) +::: + + + +Most of the methods in SDKs have `Verified` equivalent, i.e. `Get` and `VerifiedGet`. The only difference is that with `Verified` methods proofs needed to mathematically verify that the data was not tampered are returned by the server and the verification is done automatically by SDKs. +Note that generating that proof has a slight performance impact, so primitives are allowed without the proof. +It is still possible to get the proofs for a specific item at any time, so the decision about when or how frequently to do checks (with the Verify version of a method) is completely up to the user. +It's possible also to use dedicated [auditors](../production/auditor.md) to ensure the database consistency, but the pattern in which every client is also an auditor is the more interesting one. + + + + + +## Get and Set + +`Get`/`VerifiedGet` and `Set`/`VerifiedSet` methods allow for basic operations on a Key Value level. In addition, `GetAll` and `SetAll` methods allow for adding and reading in a single transaction. See [transactions chapter](transactions.md) for more details. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-get-set/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient +import json + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def encode(what: str): + return what.encode("utf-8") + +def decode(what: bytes): + return what.decode("utf-8") + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + # You have to operate on bytes + setResult = client.set(b'x', b'y') + print(setResult) # immudb.datatypes.SetResponse + print(setResult.id) # id of transaction + print(setResult.verified) # in this case verified = False + # see Tamperproof reading and writing + + # Also you get response in bytes + retrieved = client.get(b'x') + print(retrieved) # immudb.datatypes.GetResponse + print(retrieved.key) # Value is b'x' + print(retrieved.value) # Value is b'y' + print(retrieved.tx) # Transaction number + + print(type(retrieved.key)) # + print(type(retrieved.value)) # + + # Operating with strings + encodedHello = encode("Hello") + encodedImmutable = encode("Immutable") + client.set(encodedHello, encodedImmutable) + retrieved = client.get(encodedHello) + + print(decode(retrieved.value) == "Immutable") # Value is True + + notExisting = client.get(b'asdasd') + print(notExisting) # Value is None + + + # JSON example + toSet = {"hello": "immutable"} + encodedToSet = encode(json.dumps(toSet)) + client.set(encodedHello, encodedToSet) + + retrieved = json.loads(decode(client.get(encodedHello).value)) + print(retrieved) # Value is {"hello": "immutable"} + + # setAll example - sets all keys to value from dictionary + toSet = { + b'1': b'test1', + b'2': b'test2', + b'3': b'test3' + } + + client.setAll(toSet) + retrieved = client.getAll(list(toSet.keys())) + print(retrieved) + # Value is {b'1': b'test1', b'2': b'test2', b'3': b'test3'} + + +if __name__ == "__main__": + main() +``` + +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + client.set("myKey", "myValue".getBytes()); + + Entry entry = client.get("myKey"); + + byte[] value = entry.getValue(); + + System.out.format("('%s', '%s')\n", "myKey", new String(value)); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` + +Note that `value` is a primitive byte array. You can set the value of a String using:
+`"some string".getBytes(StandardCharsets.UTF_8)` + +Also, `set` method is overloaded to allow receiving the `key` parameter as a `byte[]` data type. + +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +await client.Set("k123", "v123"); +string v = await client.Get("k123").ToString(); + +await client.Close(); +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const setReq: Parameters.Set = { key: 'hello', value: 'world' } + const setRes = await cl.set(setReq) + console.log('success: set', setRes) + + const getReq: Parameters.Get = { key: 'hello' } + const getRes = await cl.get(getReq) + console.log('success: get', getRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Get at and since a transaction + +You can retrieve a key on a specific transaction with `GetAt`/`VerifiedGetAt`. If you need to check the last value of a key after given transaction (which represent state of the indexer), you can use `GetSince`/`VerifiedGetSince`. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-get-at-since/main.go +::: + +::: tab Python + +```python +from grpc import RpcError +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + first = client.set(b'justfirsttransaction', b'justfirsttransaction') + + key = b'123123' + + first = client.set(key, b'111') + firstTransaction = first.id + + second = client.set(key, b'222') + secondTransaction = second.id + + third = client.set(key, b'333') + thirdTransaction = third.id + + print(client.verifiedGetSince(key, firstTransaction)) # b"111" + print(client.verifiedGetSince(key, firstTransaction + 1)) # b"222" + + try: + # This key wasn't set on this transaction + print(client.verifiedGetAt(key, firstTransaction - 1)) + except RpcError as exception: + print(exception.debug_error_string()) + print(exception.details()) + + verifiedFirst = client.verifiedGetAt(key, firstTransaction) + # immudb.datatypes.SafeGetResponse + print(verifiedFirst.id) # id of transaction + print(verifiedFirst.key) # Key that was modified + print(verifiedFirst.value) # Value after this transaction + print(verifiedFirst.refkey) # Reference key + # (Queries And History -> setReference) + print(verifiedFirst.verified) # Response is verified or not + print(verifiedFirst.timestamp) # Time of this transaction + + print(client.verifiedGetAt(key, secondTransaction)) + print(client.verifiedGetAt(key, thirdTransaction)) + + try: + # Transaction doesn't exists yet + print(client.verifiedGetAt(key, thirdTransaction + 1)) + except RpcError as exception: + print(exception.debug_error_string()) + print(exception.details()) + +if __name__ == "__main__": + main() + +``` +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.TxHeader; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key = "key1".getBytes(StandardCharsets.UTF_8); + byte[] value = new byte[]{1, 2, 3, 4, 5}; + + TxHeader hdr = client.set(key, value); + + Entry entry = client.getAtTx(key, hdr.getId()); + + System.out.format("('%s', '%s')\n", new String(key), Arrays.toString(entry.getValue())); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` + +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +byte[] v2 = new byte[] { 0, 1, 2, 3 }; + +TxHeader hdr2 = await client.VerifiedSet("k2", v2); +Entry ventry2 = await client.VerifiedGet("k2"); +Entry e = await client.GetSinceTx("k2", hdr2.Id); +Console.WriteLine(e.ToString()); + +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + const { id } = await cl.set({ key: 'key', value: 'value' }) + + const verifiedGetAtReq: Parameters.VerifiedGetAt = { + key: 'key', + attx: id + } + const verifiedGetAtRes = await cl.verifiedGetAt(verifiedGetAtReq) + console.log('success: verifiedGetAt', verifiedGetAtRes) + + for (let i = 0; i < 4; i++) { + await cl.set({ key: 'key', value: `value-${i}` }) + } + + const verifiedGetSinceReq: Parameters.VerifiedGetSince = { + key: 'key', + sincetx: 2 + } + const verifiedGetSinceRes = await cl.verifiedGetSince(verifiedGetSinceReq) + console.log('success: verifiedGetSince', verifiedGetSinceRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Get at revision + +Each historical value for a single key is attached a revision number. +Revision numbers start with 1 and each overwrite of the same key results in +a new sequential revision number assignment. + +A negative revision number can also be specified which means the nth historical value, +e.g. -1 is the previous value, -2 is the one before and so on. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-get-at-revision/main.go +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Java +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.TxHeader; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key = "myKey1".getBytes(StandardCharsets.UTF_8); + byte[] value1 = new byte[]{1, 2, 3, 4, 5}; + byte[] value2 = new byte[]{5, 4, 3, 2, 1}; + + client.set(key, value1); + client.set(key, value2); + + Entry entry1 = client.getAtRevision(key, 1); + Entry entry2 = client.getAtRevision(key, 2); + + System.out.format("('%s', '%s')@rev%d\n", new String(key), Arrays.toString(entry1.getValue()), 1); + System.out.format("('%s', '%s')@rev%d\n", new String(key), Arrays.toString(entry2.getValue()), 2); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` +::: + +::: tab .NET + +``` csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +string key = "hello"; + +try +{ + await client.VerifiedSet(key, "immutable world!"); + Entry entry1 = await client.VerifiedGetAtRevision(key, 0); + Console.WriteLine(entry1.ToString()); + await client.VerifiedSet(key, "immutable world again!"); + Entry entry2 = await client.VerifiedGetAtRevision(key, -1); + Console.WriteLine(entry2.ToString()); +} +catch (VerificationException e) +{ + // VerificationException means Data Tampering detected! + // This means the history of changes has been tampered. + Console.WriteLine(e.ToString()); +} +await client.Close(); +``` + +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Get at TXID + +It's possible to retrieve all the keys inside a specific transaction. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-get-at-txid/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + keyFirst = b'333' + keySecond = b'555' + + first = client.set(keyFirst, b'111') + firstTransaction = first.id + + second = client.set(keySecond, b'222') + secondTransaction = second.id + + toSet = { + b'1': b'test1', + b'2': b'test2', + b'3': b'test3' + } + + third = client.setAll(toSet) + thirdTransaction = third.id + + keysAtFirst = client.txById(firstTransaction) + keysAtSecond = client.txById(secondTransaction) + keysAtThird = client.txById(thirdTransaction) + + print(keysAtFirst) # [b'333'] + print(keysAtSecond) # [b'555'] + print(keysAtThird) # [b'1', b'2', b'3'] + + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import io.codenotary.immudb4j.TxEntry; +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.KVListBuilder; +import io.codenotary.immudb4j.Tx; +import io.codenotary.immudb4j.TxHeader; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] key1 = "myKey1".getBytes(StandardCharsets.UTF_8); + byte[] value1 = new byte[]{1, 2, 3}; + + byte[] key2 = "myKey2".getBytes(StandardCharsets.UTF_8); + byte[] value2 = new byte[]{4, 5, 6}; + + KVListBuilder kvListBuilder = KVListBuilder.newBuilder(). + add(key1, value1). + add(key2, value2); + + TxHeader hdr = client.setAll(kvListBuilder.entries()); + + Tx tx = client.txById(hdr.getId()); + + for (TxEntry txEntry : tx.getEntries()) { + System.out.format("'%s'\n", new String(txEntry.getKey())); + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} + +``` + +::: + +::: tab .NET + +```csharp + +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +TxMetadata txMd = null; +try +{ + txMd = immuClient.VerifiedSet(key, val); +} +catch (VerificationException e) +{ + Console.WriteLine("A VerificationException occurred.") +} +try +{ + Tx tx = immuClient.TxById(txMd.id); +} +catch (Exception e) +{ + Console.WriteLine("An exception occurred.") +} + +await client.Close(); +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + const { id } = await cl.set({ key: 'key', value: 'value' }) + + const txByIdReq: Parameters.TxById = { tx: id } + const txByIdRes = await cl.txById(txByIdReq) + console.log('success: txById', txByIdRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Conditional writes + +immudb can check additional preconditions before the write operation is made. +Precondition is checked atomically with the write operation. +It can be then used to ensure consistent state of data inside the database. + +Following preconditions are supported: + +* MustExist - precondition checks if given key exists in the database, + this precondition takes into consideration logical deletion and data expiration, + if the entry was logically deleted or has expired, MustExist precondition for + such entry will fail +* MustNotExist - precondition checks if given key does not exist in the database, + this precondition also takes into consideration logical deletion and data expiration, + if the entry was logically deleted or has expired, MustNotExist precondition for + such entry will succeed +* NotModifiedAfterTX - precondition checks if given key was not modified after given transaction id, + local deletion and setting entry with expiration data is also considered modification of the + entry + +In many cases, keys used for constraints will be the same as keys for written entries. +A good example here is a situation when a value is set only if that key does not exist. +This is not strictly required - keys used in constraints do not have to be the same +or even overlap with keys for modified entries. An example would be if only one of +two keys should exist in the database. In such case, the first key will be modified +and the second key will be used for MustNotExist constraint. + +A write operation using precondition can not be done in an asynchronous way. +Preconditions are checked twice when processing such requests - first check is done +against the current state of internal index, the second check is done just before +persisting the write and requires up-to-date index. + +Preconditions are available on `SetAll`, `Reference` and `ExecAll` operations. + + + +:::: tabs + +::: tab Go + +In go sdk, the `schema` package contains convenient wrappers for creating constraint objects, +such as `schema.PreconditionKeyMustNotExist`. + +<<< @/src/code-examples/go/develop-kv-preconditions/main.go +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Java +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j) +::: + +::: tab .NET +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: diff --git a/src/1.9.4/develop/sql/catalog.md b/src/1.9.4/develop/sql/catalog.md new file mode 100644 index 0000000000..3e17b62ee7 --- /dev/null +++ b/src/1.9.4/develop/sql/catalog.md @@ -0,0 +1,96 @@ +# Catalog queries + + + +immudb provides a set of useful built-in functions that can be used to query the catalog. + + + + + +## Listing databases + +The `DATABASES()` function can be used as a source of data returning the list of databases +that can be accessed by the user running the query. + +```sql +SELECT * FROM DATABASES(); +``` + +This source can also be constrained using the `WHERE` clause and the set of columns to retrieve. + +```sql +SELECT name FROM DATABASES() WHERE name LIKE '.*db1.*'; +``` + +Alternatively, the `SHOW DATABASES` statement returns the list of databases that can be accessed by the current user. + + + + + +## Listing tables + +The `TABLES()` function can be used as a source of data returning the list of tables in the +currently selected database. + +```sql +SELECT * FROM TABLES(); +``` + +This source can also be constrained using the `WHERE` clause and the set of columns to retrieve. + +```sql +SELECT name FROM TABLES() +WHERE name like '.*est.*' +``` + +Alternatively, the `SHOW TABLES` statement returns the list of tables in the currently selected database. + + + + + +## Listing columns of a table + +The `COLUMNS()` function returns the list of columns for a table. It takes a single argument which is the name of the table. +The table will be looked up in the currently selected database. + +```sql +SELECT * FROM COLUMNS('mytable'); +``` + +This source can also be constrained with the WHERE clause and set of columns to retrieve. + +Note: because colum names can use reserved identifiers such as `table`, make sure to enclose those in double-quotes. + +```sql +SELECT "table", "name", "type" FROM COLUMNS('mytable'); +SELECT name FROM COLUMNS('mytable') WHERE type = 'VARCHAR'; +``` + +Alternatively, the `SHOW TABLE mytable` statement will returns the list of columns for the specified table. + + + + + +## Listing indexes of a table + +The `INDEXES()` function returns a list of indexes for a table. It takes a single argument which is the name of the table. +The table will be looked up in the currently selected database. + +```sql +SELECT * FROM INDEXES('mytable'); +``` + +This source can also be constrained with the WHERE clause and set of columns to retrieve. + +Note: because colum names can use reserved identifiers such as `table`, make sure to enclose those in double-quotes. + +```sql +SELECT "table", "name", "unique", "primary" FROM INDEXES('mytable'); +SELECT name FROM INDEXES('mytable') WHERE "unique"; +``` + + diff --git a/src/1.9.4/develop/sql/datatypes.md b/src/1.9.4/develop/sql/datatypes.md new file mode 100644 index 0000000000..fe55554aba --- /dev/null +++ b/src/1.9.4/develop/sql/datatypes.md @@ -0,0 +1,57 @@ +# Data types + +| Name | Description | Length constraints | +| --------- | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| INTEGER | Signed 64-bit integer value. Usually referred to as `BIGINT` in other databases. | - | +| BOOLEAN | A boolean value, either `TRUE` or `FALSE` | - | +| VARCHAR | UTF8-encoded text | Maximum number of bytes in the UTF-8 encoded representation of the string | +| BLOB | Sequence of bytes | Maximum number of bytes in the sequence | +| TIMESTAMP | Datetime value with microsecond precision | - | +| FLOAT | IEEE-754 64-bit floating-point number | - | +| UUID | Universally Unique Identifier (UUID), 128-bit value | - | +| JSON | RFC 8259 textual JSON data | Maximum number of bytes in the textual representation of the data | + +

+ + + +### Size constraints + +Size constraint is specified with a `[MAX_SIZE]` suffix on the type, +e.g. `BLOB[16]` represents a sequence of up to 16 bytes. + + + + + +### NULL values + +`NULL` values in immudb are not unique - two `NULL` values are considered equal on comparisons. + + + + + +### Timestamp values + +Timestamp values are internally stored as a 64-bit signed integer being a number of microseconds since the epoch time. +Those values are not associated with any timezone, whenever a conversion is needed, it is considered to be in UTC. + + + + + +### JSON primitive types and internal SQL types + +Except for `lists` and `objects`, each JSON type is backed by a specific immudb internal SQL type. The mapping between JSON primitive types and SQL internal types is reported in the following table: + +
+ +| JSON primitive type | Internal SQL type | Description | +| ------------------- | ----------------- | ------------------------------------------------------------------- | +| STRING | VARCHAR | control characters, quotation marks and backslashes must be escaped | +| NUMBER | FLOAT | `NaN`, `Infinity` and `-Infinity` are not supported | +| BOOLEAN | BOOLEAN | only lowercase `true` and `false` are valid | +| NULL | NULL | - | + + diff --git a/src/1.9.4/develop/sql/indexes.md b/src/1.9.4/develop/sql/indexes.md new file mode 100644 index 0000000000..4b605a9524 --- /dev/null +++ b/src/1.9.4/develop/sql/indexes.md @@ -0,0 +1,74 @@ +# Indexes + +::: tip +If you create lots of indexes, you may want to adjust default settings to reduce your memory footprint. + +Indexing parameters, including cache-size, flush-thresholds, and max-active-snapshots, can be lowered as needed, but take into account more IO reads and writes, which may lead to poor indexing performance. +::: + + + +immudb indexes can be used for a quick search of rows +with columns having specific values. + +Certain operations such as ordering values with `ORDER BY` clause +require columns to be indexed. + +```sql +CREATE INDEX ON customers(customer_name); +CREATE INDEX ON customers(country, ip); +CREATE INDEX IF NOT EXISTS ON customers(active); +CREATE UNIQUE INDEX ON customers(email); +``` + +Currently, indexes are subject to the following limitations: + +- Index can only be added to an empty table; + +- Indexes on JSON columns or subfields of them are not supported; + +- Each index doesn't have an explicit name and is referenced by the ordered list of indexed columns. + +### Column value constraints + +Columns of `BLOB` or `VARCHAR` type must have a size limit set on them. +The maximum allowed value size for one indexed column is 256 bytes. + +### Unique indexes + +Index can be marked as unique with extra `UNIQUE` keyword. +Unique index will prevent insertion of new data into the table +that would violate uniqueness of indexed columns within the table. + +### Multi-column indexes + +Index can be set on up to 8 columns. +The order of columns is important when doing range scans, +iterating over such index will first sort by the value of the first column, +then by the second and so on. + +Note: +Large indexes will increase the storage requirement and will reduce the performance of data insertion. +Iterating using small indexes will also be faster than with the large ones. + +### IF NOT EXISTS + +With this clause the `CREATE INDEX` statement will not fail if an index with same type and list of columns already exists. +This includes a use case where the table is not empty which can be used to simplify database schema initialization. + +Note: If the index already exists, it is not compared against the provided index definition neither it is +updated to match it. + + + + + +### DROP INDEX + +An index can be physically deleted. Table data is not deleted and can be queried using either the primary index or any other declared index. Non-unique indexes can be created at any time. + +```sql +DROP INDEX ON customers(surname); +``` + + diff --git a/src/1.9.4/develop/sql/insertupdate.md b/src/1.9.4/develop/sql/insertupdate.md new file mode 100644 index 0000000000..bf98675a5c --- /dev/null +++ b/src/1.9.4/develop/sql/insertupdate.md @@ -0,0 +1,107 @@ +# Inserting or updating data + + + +### INSERT + +immudb supports standard `INSERT` sql statement. +It can be used to add one or multiple values within the same transaction. + +```sql +INSERT INTO customers ( + id, customer_name, email, address, + city, ip, country, age, active, created_at +) +VALUES ( + 1, + 'Isidro Behnen', + 'ibehnen0@mail.ru', + 'ibehnen0@chronoengine.com', + 'Arvika', + '127.0.0.15', + 'SE', + 24, + true, + NOW() +); + +INSERT INTO products (id, product, price, created_at) +VALUES + ( 1, 'Juice - V8, Tomato', '$4.04', NOW() ), + ( 2, 'Milk', '$3.24', NOW() ); + +INSERT INTO orders (customerid, productid, created_at) +VALUES (1, 1, NOW()), (1, 2, NOW()); + +INSERT INTO customer_review (customerid, productid, review, created_at) +VALUES + (1, 1, 'Nice Juice!', NOW()); +``` + + + + + +### UPSERT + +`UPSERT` is an operation with a syntax similar to `INSERT`, +the difference between those two is that `UPSERT` either creates a new or replaces an existing row. +A new row is created if an entry with the same primary key does not yet exist in the table, +otherwise the current row is replaced with the new one. + +If a table contains an `AUTO_INCREMENT` primary key, +the value for that key must be provided +and the `UPSERT` operation will only update the existing row. + +```sql +UPSERT INTO products (id, product, price) +VALUES +( 2, 'Milk', '$3.17' ), +( 3, 'Grapes - Red', '$5.03' ); + +UPSERT INTO orders (id, customerid, productid) +VALUES (1, 1, 3); +``` + + + + + +### ON CONFLICT + +The optional `ON CONFLICT` clause specifies an alternative action to raising a unique violation or constraint error. +`ON CONFLICT DO NOTHING` simply avoids inserting a row as its alternative action. In this case the primary key of the row is returned. + + + + + +### Built-in functions: now(), random_uuid() and cast() + +The built-in `now()` function returns the transaction creation time as seen on the server. +In the scope of a single transaction, it always returns the same result time. + +The built-in `random_uuid()` function returns the new UUID value each time. + +The `cast` function can be used to convert a string or an integer to a timestamp value. +The casting expression can be shortened by using `::` +e.g. the expression `100`::INTEGER converts the string `100` into an integer value. + +The integer value is interpreted as a Unix timestamp (number of seconds since the epoch time). + +The string value passed to the `cast` function must be in one of the following formats: +`2021-12-08`, `2021-12-08 17:21`, `2021-12-08 17:21:59`, `2021-12-08 17:21:59.342516`. +Time components not specified in the string are set to 0. + +```sql +UPSERT INTO products (xid, product, price, created_at) +VALUES +( random_uuid(), 'Bread', '$1.50', now() ), +( 'c698d13a-68cd-11ee-8c99-0242ac120002'::UUID, 'Spinach', '$0.99', cast('2021-02-01' as TIMESTAMP) ) +``` + +```sql +SELECT * FROM products WHERE created_at < NOW() +``` + + \ No newline at end of file diff --git a/src/1.9.4/develop/sql/pg.md b/src/1.9.4/develop/sql/pg.md new file mode 100644 index 0000000000..5dc53948ef --- /dev/null +++ b/src/1.9.4/develop/sql/pg.md @@ -0,0 +1,253 @@ +# Pgsql protocol compatibility + + + +immudb can talk the [pgsql wire protocol](https://www.postgresql.org/docs/9.3/protocol.html) which makes it compatible with a widely available set of clients and drivers. + +Note: immudb supports the pgsql wire protocol. It is *not* compatible with the SQL dialect. Check other topics in the `Develop with SQL` section to see what queries and operations are supported. + +Some pgsql clients and browser application execute incompatible statements in the background or directly query the pgsql catalog. Those may not work with immudb. + +immudb needs to be started with the `pgsql-server` option enabled (`IMMUDB_PGSQL_SERVER=true`). + +SSL is supported, if you configured immudb with a certificate. + + + +:::: tabs + +::: tab CLI +Use the [psql client](https://www.postgresql.org/docs/13/app-psql.html) included with PostgreSQL. +::: + +::: tab C + +You can use a subset of the [libpq](https://www.postgresql.org/docs/9.5/libpq.html) API. You will need to include: + +``` C + #include +``` + +and compile with `gcc -o main $(pkg-config libpq --cflags --libs) main.c`. + +::: + +::: tab Ruby + +You can use the [pg](https://rubygems.org/gems/pg) gem: + +```ruby +require 'pg' +``` +::: + +::: tab Java + +Download the official [JDBC driver](https://jdbc.postgresql.org/) jar artifact for PostgreSQL. + +You can then compile your program: + +``` +$ javac -cp .:./postgresql-42.2.20.jar MyProgram.java +``` + +::: + +::: tab PHP + +Please refer to the [PHP pgsql module](https://www.php.net/manual/en/book.pgsql.php) documentation for instructions on how to enable it in your server. + +::: + +:::: + +### To connect to the database: + +:::: tabs + +::: tab CLI + +``` +psql "host=localhost dbname=defaultdb user=immudb password=immudb sslmode=disable" +psql (13.2, server 0.0.0) +Type "help" for help. +``` + +::: + +::: tab C + +```C +PGconn *conn = PQconnectdb("host=localhost user=immudb password=immudb dbname=defaultdb sslmode=disable"); + +if (PQstatus(conn) == CONNECTION_BAD) { + fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); + PQfinish(conn); + exit(1); +} +``` + +::: + +::: tab Ruby +```ruby +conn = PG::Connection.open("sslmode=allow dbname=defaultdb user=immudb password=immudb host=127.0.0.1 port=5432") +``` +::: + +::: tab Java + +It is important to pass the `preferQueryMode=simple` option, as immudb pgsql server only support simple query mode. + +```java +Connection conn = + DriverManager.getConnection("jdbc:postgresql://127.0.0.1:5432/defaultdb?sslmode=allow&preferQueryMode=simple", + "immudb", "immudb"); +System.out.println("Opened database successfully"); +``` +::: + + +::: tab PHP +```php + +``` +::: + +:::: + +### Execute statements: + +:::: tabs + +::: tab CLI +``` +defaultdb=> CREATE TABLE Orders(id INTEGER, amount INTEGER, title VARCHAR, PRIMARY KEY id); +SELECT 1 +defaultdb=> UPSERT INTO Orders (id, amount, title) VALUES (1, 200, 'title1'); +SELECT 1 +``` +::: + +::: tab C +```C +PGresult *res = PQexec(conn, "CREATE TABLE Orders (id INTEGER, amount INTEGER, title VARCHAR, PRIMARY KEY id)"); +if (PQresultStatus(res) != PGRES_COMMAND_OK) { + do_exit(conn, res); +} +PQclear(res); + +res = PQexec(conn, "UPSERT INTO Orders (id, amount, title) VALUES (1, 200, 'title 1')"); +if (PQresultStatus(res) != PGRES_COMMAND_OK) { + do_exit(conn, res); +} +PQclear(res); +``` +::: + +::: tab Ruby +```ruby +conn.exec( "CREATE TABLE Orders (id INTEGER, amount INTEGER, title VARCHAR, PRIMARY KEY id)" ) +conn.exec( "UPSERT INTO Orders (id, amount, title) VALUES (1, 200, 'title 1')" ) +conn.exec( "UPSERT INTO Orders (id, amount, title) VALUES (2, 400, 'title 2')" ) +``` +::: + +::: tab Java + +```java +Statement stmt = conn.createStatement(); + +stmt.executeUpdate("CREATE TABLE people(id INTEGER, name VARCHAR, salary INTEGER, PRIMARY KEY id);"); + +stmt.executeUpdate("INSERT INTO people(id, name, salary) VALUES (1, 'Joe', 20000);"); +stmt.executeUpdate("INSERT INTO people(id, name, salary) VALUES (2, 'Bob', 30000);"); +``` +::: + +::: tab PHP + +```php +$stmt = 'CREATE TABLE people(id INTEGER, name VARCHAR, salary INTEGER, PRIMARY KEY id);'; +$result = pg_query($stmt) or die('Error message: ' . pg_last_error()); +$stmt = 'INSERT INTO people(id, name, salary) VALUES (1, 'Joe', 20000);'; +$result = pg_query($stmt) or die('Error message: ' . pg_last_error()); +$stmt = 'INSERT INTO people(id, name, salary) VALUES (2, 'Bob', 30000);'; +``` +::: + +:::: + + +### Query and iterate over results: + +:::: tabs + +::: tab CLI +``` +defaultdb=> SELECT id, amount, title FROM Orders; + (defaultdb.Orders.id) | (defaultdb.Orders.amount) | (defaultdb.Orders.title) +-----------------------+---------------------------+-------------------------- + 1 | 200 | "title1" +(1 row) +``` +::: + +::: tab C +```C +res = PQexec(conn, "SELECT id, amount, title FROM Orders"); +if (PQresultStatus(res) != PGRES_TUPLES_OK) { + printf("No data retrieved\n"); + PQclear(res); + do_exit(conn, res); +} + +int rows = PQntuples(res); +for(int i=0; i
+ + + +### Show privileges + +To show the SQL privileges associated to users, you can use the following statements: + +```sql +SHOW GRANTS; -- show all users privileges + +SHOW GRANTS FOR user; -- show the list of privileges connected to a specific user +``` + + + + + +### Alter privileges + +Users with `admin` permission can grant and revoke SQL privileges to other users: + +```sql +GRANT ALL PRIVILEGES TO myuser; -- grant all privileges to user + +REVOKE SELECT, UPDATE ON DATABASE mydb TO USER myuser; -- revoke specific privileges to user +``` + + diff --git a/src/1.9.4/develop/sql/querying.md b/src/1.9.4/develop/sql/querying.md new file mode 100644 index 0000000000..6e33b49532 --- /dev/null +++ b/src/1.9.4/develop/sql/querying.md @@ -0,0 +1,425 @@ +# Querying + + + +### Selecting all columns + +All columns from all joined tables can be queried with `SELECT *` statement. + +```sql +SELECT * +FROM products; +``` + + + + + +### Selecting specific columns + +```sql +SELECT id, customer_name, ip +FROM customers; +``` + + + + + +### Filtering entries + +```sql +SELECT id, customer_name, email +FROM customers +WHERE country = 'SE' AND city = 'Arvika'; +``` + + + + + +### Ordering by column value + +```sql +SELECT id, customer_name +FROM customers +ORDER BY customer_name ASC, id DESC; +``` + +The order may be either ascending (`ASC` suffix, default) or descending (`DESC` suffix). + +Although not required, adding an index on the columns specified in the clause can drastically reduce query times. + + + + + +### INNER JOIN + +immudb supports standard SQL `INNER JOIN` syntax. +The `INNER` join type is optional. + +```sql +SELECT * +FROM orders +INNER JOIN customers ON orders.customerid = customers.id; + +SELECT * +FROM orders +JOIN customers ON orders.customerid = customers.id +WHERE orders.productid = 2; + +SELECT * FROM orders +JOIN customers ON customers.id = orders.customerid +JOIN products ON products.id = orders.productid; +``` + + + + + +### LIKE operator + +immudb supports the `LIKE` operator. +Unlike in other SQL engines though, the pattern use a regexp syntax +supported by the [regexp library in the go language](https://pkg.go.dev/regexp). + +A `NOT` prefix negates the value of the `LIKE` operator. + +```sql +SELECT product +FROM products +WHERE product LIKE '(J.*ce|Red)'; + +SELECT product +FROM products +WHERE product NOT LIKE '(J.*ce|Red)'; + +SELECT id, product +FROM products +WHERE (id > 0 AND NOT products.id >= 10) + AND (products.product LIKE 'J'); +``` + + + + + +### IN operator + +immudb has a basic supports for the `IN` operator. + +A `NOT` prefix negates the value of the `IN` operator. + +Note: Currently the list for the `IN` operator can not be +calculated using a sub-query. + +```sql +SELECT product +FROM products +WHERE product IN ('Milk', 'Grapes - Red'); + +SELECT product +FROM products +WHERE product NOT IN ('Milk', 'Grapes - Red'); + +SELECT id, product +FROM products +WHERE (id > 0 AND NOT products.id >= 10) + AND (product IN ('Milk', 'Grapes - Red')); +``` + + + + + +### Column and table aliasing + +```sql +SELECT c.id, c.customer_name AS name, active +FROM customers AS c +WHERE c.id <= 3 AND c.active = true; + +SELECT c.id, c.customer_name AS name, active +FROM customers c +WHERE c.id <= 3 AND c.active = true; +``` + +Table name aliasing is necessary when using more than one join with the same table. + + + + + +### Aggregations + +Available aggregation functions: + + + +- COUNT +- SUM +- MAX +- MIN +- AVG + + + +```sql +SELECT + COUNT(*) AS c, + SUM(age), + MIN(age), + MAX(age), + AVG(age) +FROM customers; +``` + + + + + +### Grouping results with GROUP BY + +Results can be grouped by the value of one or more columns. + +```sql +SELECT COUNT(*) as customer_count, country +FROM customers +GROUP BY country +ORDER BY country; +``` + + + + + +### Filtering grouped results with HAVING + +```sql +SELECT + active, + COUNT(*) as c, + MIN(age), + MAX(age) +FROM customers +GROUP BY active +HAVING COUNT(*) > 0 +ORDER BY active DESC; +``` + + + + + +### Sub-queries + +The table in the `SELECT` or `JOIN` clauses can be replaced with a sub-query. + +```sql +SELECT * FROM ( + SELECT id, customer_name + FROM customers + WHERE age < 30 +) +INNER JOIN customer_review + ON customer_review.customerid = customers.id; + +SELECT * FROM ( + SELECT id, customer_name + FROM customers + WHERE age < 30 +) AS c +INNER JOIN ( + SELECT * FROM customer_review +) AS r + ON r.customerid = c.id; +``` + +Note: the context of a sub-query does not propagate outside, +e.g. it is not possible to reference a table from a sub-query +in the `WHERE` clause outside of the sub-query. + + + + + +### Combining query results with UNION + +It is possible to combine multiple query results with the `UNION` operator. + +Subqueries must select the same number and type of columns. +The final return will assign the same naming as in the first subquery, even if names differ. + +```sql +SELECT customer_name as name +FROM customers +WHERE age < 30 +UNION +SELECT seller_name +FROM sellers +WHERE age < 30 +``` + +Subqueries are not constrained in any way, they can contain aggregations or joins. + +Duplicate rows are excluded by default. Using `UNION ALL` will leave duplicate rows in place. + +```sql +SELECT AVG(age) FROM customers +UNION ALL +SELECT AVG(age) FROM sellers +``` + + + + + +### Transactions + +The ACID (Atomicity, Consistency, Isolation, and Durability) compliance is complete. + +Handling read-write conflicts may be necessary when dealing with concurrent transactions. Getting the error `ErrTxReadConflict` ("tx read conflict") means there was another transaction committed before the current one, and the data it read may have been invalidated. +[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) validations have not yet been implemented, therefore there may be false positives generated. In case of conflict, a new attempt may be required. + +```sql +BEGIN TRANSACTION; + UPSERT INTO products (id, price, product) + VALUES (4, '$5.76', 'Bread'); + + INSERT INTO orders(productid, customerid) + VALUES(4, 1); +COMMIT; +``` + + + + + +### Time travel + +Time travel allows you to read data from SQL as if it were in a previous state or from a specific time range. +Initial and final points are optional and can be specified using either a transaction ID or a timestamp. + +The temporal range can be used to filter out rows from the specified (physical) table, but it is not supported in subqueries. + +The initial point can be inclusive (`SINCE`) or exclusive (`AFTER`). +The final point can be inclusive (`UNTIL`) or exclusive (`BEFORE`). + +```sql +SELECT id, product, price +FROM products BEFORE TX 13 +WHERE id = 2; +``` + +```sql +SELECT * FROM sales SINCE '2022-01-06 11:38' UNTIL '2022-01-06 12:00' +``` + +Temporal ranges can be specified using functions and parameters + +```sql +SELECT * FROM mytable SINCE TX @initialTx BEFORE now() +``` + + + + + +### Row History + +Historical queries over physical tables (row revisions) is also supported. +Result sets over `history of ` will include the additional `_rev` column, denoting the row revision number, + +Historical queries can use the additional `_rev` column as usual: + +```sql +SELECT _rev, price +FROM (HISTORY OF products) +WHERE id = 2; +``` + +```sql +SELECT * FROM (HISTORY OF mytable) WHERE _rev = @rev +``` + + + + + +### Transaction metadata + +immudb can be configured to inject request information (**user**, **ip address**, etc...) to transaction metadata (see [Request Metadata](../../production/request-metadata.md)). When this functionality is enabled, +such information can be retrieved by querying the special `_tx_metadata` column, whose type is `JSON` (refer to [Querying JSON columns](#querying-json-columns) section). + +```sql +SELECT _tx_metadata->'ip' as ip_addr FROM customers +WHERE _tx_metadata->'usr' = 'username'; +``` + + + + + +### Skipping entries with OFFSET clause + +Using `OFFSET` clause in SQL queries can be used to skip an initial list of entries from the result set. +Internally offsets are implemented by skipping entries from the result on the server side thus it may come with performance penalty when the value of such offset is large. + +```sql +SELECT * +FROM products; +LIMIT 10 OFFSET 30 +``` + +```sql +SELECT id, customer_name, email +FROM customers +WHERE country = 'SE' AND city = 'Arvika'; +ORDER BY customer_name +LIMIT 50 OFFSET 100 +``` + + + + + +## Querying JSON columns + +immudb supports a `JSON` data type designed to store data in the `JavaScript Object Notation` format. It allows for flexible and efficient storage and querying of JSON objects, which can be particularly useful in applications that require a combination of structured and semi-structured data. + +```sql +CREATE TABLE items ( + id INTEGER AUTO_INCREMENT, + details JSON, + + PRIMARY KEY id +); +``` + +Internally, JSON data are represented in their textual format. Columns can be then populated by any valid JSON string: + +```sql +INSERT INTO items(id, details) + VALUES + (1, '{"name": "Alice", "age": 25}'), + (2, '{"name": "Bob", "address": {"city": "Los Angeles", "state": "CA"}}'), + (3, '["apple", "banana", "cherry"]'), + (4, '[{"product": "book", "price": 12.99}, {"product": "pen", "price": 1.49}]'), + (5, '"N/A"') +``` + +The `ARROW (->)` operator, used to access nested fields, is crucial when working with the JSON type. The following shows valid examples of queries over a JSON field: + +```sql +SELECT * FROM items +WHERE details->'name' IS NOT NULL +ORDER BY details->'name'; + +SELECT * FROM items +WHERE json_typeof(details->'1') != 'OBJECT' AND details->'1' = 'banana'; + +SELECT details->'1'->'price' as price FROM items; +``` + + diff --git a/src/1.9.4/develop/sql/sqlstdlib.md b/src/1.9.4/develop/sql/sqlstdlib.md new file mode 100644 index 0000000000..5e534c2e6b --- /dev/null +++ b/src/1.9.4/develop/sql/sqlstdlib.md @@ -0,0 +1,19 @@ +# GO SQL std library + + + +From immudb `v1.1.0` is possible to use go standard library sql interface to query data. + +<<< @/src/code-examples/go/develop-sql-stdlib/main.go + +In alternative is possible to open immudb with a connection string: + +<<< @/src/code-examples/go/develop-sql-stdlib-connstring/main.go + +Available SSL modes are: + +* **disable**. SSL is off +* **insecure-verify**. SSL is on but client will not check the server name. +* **require**. SSL is on. + + \ No newline at end of file diff --git a/src/1.9.4/develop/sql/tablesalter.md b/src/1.9.4/develop/sql/tablesalter.md new file mode 100644 index 0000000000..e811e393e2 --- /dev/null +++ b/src/1.9.4/develop/sql/tablesalter.md @@ -0,0 +1,80 @@ +# Altering tables + + + +immudb supports limited table altering. +The supported operations are lightweight. +They do not require any changes to already written row data +and there is no performance penalty on read/write operations +in such altered tables. + + + + + +### RENAME TABLE + +An existing table can be renamed. +The table with the new name must not exist in the database +when performing the alter operation. + +```sql +ALTER TABLE customer RENAME TO customers; +``` + + + + + +### ADD COLUMN + +A new column can be added to an existing table. +Such column must be nullable. +For rows that already existed in the table before the alter operation, +the value of the newly added column will be read as `NULL`. +New column can not be set as `AUTO_INCREMENT` which is only allowed for the primary key. + +```sql +ALTER TABLE customers +ADD COLUMN created_time TIMESTAMP; + +SELECT customer_name, created_time +FROM customers; +``` + + + + + +### RENAME COLUMN + +An existing column can be renamed. +The column with the new name must not exist in the table +when performing the alter operation. +If the column was previously part of an index, +such index will continue working with the new column name. +Renaming a column does not change column's type. + +```sql +ALTER TABLE customers +RENAME COLUMN created_time TO created_at; + +SELECT customer_name, created_at +FROM customers; +``` + + + + + +### DROP COLUMN + +An existing column can be deleted. +Prior to removing the column, it is necessary to remove any associated indexes. + +```sql +ALTER TABLE customers +DROP COLUMN created_at; +``` + + \ No newline at end of file diff --git a/src/1.9.4/develop/sql/tablescreate.md b/src/1.9.4/develop/sql/tablescreate.md new file mode 100644 index 0000000000..e22ecf9b40 --- /dev/null +++ b/src/1.9.4/develop/sql/tablescreate.md @@ -0,0 +1,114 @@ +# Creating tables + + + +Common examples of `CREATE TABLE` statements are presented below. + +```sql +CREATE TABLE IF NOT EXISTS customers ( + id INTEGER, + customer_name VARCHAR[60], + email VARCHAR[150], + address VARCHAR, + city VARCHAR, + ip VARCHAR[40], + country VARCHAR[15], + age INTEGER, + active BOOLEAN, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS products ( + id INTEGER, + product VARCHAR NOT NULL, + price VARCHAR NOT NULL, + created_at TIMESTAMP, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS orders ( + id INTEGER AUTO_INCREMENT, + customerid INTEGER, + productid INTEGER, + created_at TIMESTAMP, + PRIMARY KEY id +); + +CREATE TABLE customer_review( + customerid INTEGER, + productid INTEGER, + review VARCHAR, + created_at TIMESTAMP, + PRIMARY KEY (customerid, productid) +); +``` + + + + + +### IF NOT EXISTS + +With this clause the `CREATE TABLE` statement will not fail if a table with same name already exists. + +Note: If the table already exists, it is not compared against the provided table definition neither it is + updated to match it. + + + + + +### NOT NULL + +Columns marked as not null can not have a null value assigned. + + + + + +### PRIMARY KEY + +Every table in immudb must have a primary key. +Primary key can use at least 1 and up to 8 columns. + +Columns used in a primary key can not have `NULL` values assigned, +even if those columns are not explicitly marked as `NOT NULL`. + +Primary key creates an implicit unique index on all contained columns. + + + + + +### AUTO_INCREMENT + +A single-column `PRIMARY KEY` can be marked as `AUTO_INCREMENT`. +immudb will automatically set a unique value of this column for new rows. + +When inserting data into a table with an `INSERT` statement, +the value for such primary key must be omitted. +When updating data in such table with `UPSERT` statement, +the value for such primary key is obligatory +and the `UPSERT` statement can only update existing rows. + +The type of an `AUTO_INCREMENT` column must be `INTEGER`. +Internally immudb will assign sequentially increasing values for new rows +ensuring this value is unique within a single table. + + + + + +### Foreign keys + +Explicit support for relations to foreign tables is not currently supported in immudb. +It is possible however to create ordinary columns containing foreign key values that can be used in `JOIN` statements. +Application logic is responsible for ensuring data consistency and foreign key constraints. + +```sql +SELECT * FROM orders +INNER JOIN customers ON customers.id = orders.customerid +INNER JOIN products ON products.id = orders.productid; +``` + + \ No newline at end of file diff --git a/src/1.9.4/develop/sql/tablesdrop.md b/src/1.9.4/develop/sql/tablesdrop.md new file mode 100644 index 0000000000..b02ff7fffb --- /dev/null +++ b/src/1.9.4/develop/sql/tablesdrop.md @@ -0,0 +1,19 @@ +# Dropping tables + + + +immudb supports table deletion. It is important to note, however, that while indexing data is physically removed, transactions from the commit log are not erased. + + + + + +### DROP TABLE + +A table is deleted along with its declared indexes. Data can no longer be queried from SQL and the transaction commit log still preserves raw transaction information, but the operation can't be undone. + +```sql +DROP TABLE customers; +``` + + \ No newline at end of file diff --git a/src/1.9.4/develop/sql/transactions.md b/src/1.9.4/develop/sql/transactions.md new file mode 100644 index 0000000000..003dc41f2b --- /dev/null +++ b/src/1.9.4/develop/sql/transactions.md @@ -0,0 +1,288 @@ +# Transactions + +::: tip +Examples in multiple languages can be found at following links: [immudb SDKs examples](https://github.com/codenotary/immudb-client-examples) +::: + + + +immudb supports transactions both on key-value and SQL level, but interactive transactions are supported only on SQL with the exception of `execAll` method, that provides some additional properties. + +Interactive transactions are a way to execute multiple SQL statements in a single transaction. This makes possible to delegate application logic to SQL statements - a very common use case is for example checking if the balance > 0 before making a purchase. +In order to create a transaction, you must call the `NewTx()` method on the client instance. The resulting object is a transaction object that can be used to execute multiple SQL statements, queries, commit or rollback. + +Following methods are exposed by the transaction object: + +```go +Commit() CommittedSQLTx, error +Rollback() error +SQLExec(sql, params) error +SQLQuery(sql, params) SQLQueryResult, error +``` + +It's possible to rollback a transaction by calling the `Rollback()` method. In this case, the transaction object is no longer valid and should not be used anymore. +To commit a transaction, you must call the `Commit()` method. + +**Note**: immudb implements multi-version concurrency control. Thus multiple read-write transactions may be concurrently processed. It's up the application to handle read conflict errors. In case a read-write conflict is detected, the sdk will return the `25P02` **CodInFailedSqlTransaction** error code. + + + +:::: tabs + +::: tab Go + +```go +package main + +import ( + "context" + "fmt" + "log" + + immudb "github.com/codenotary/immudb/pkg/client" +) + +func handleErr(err error) { + if err != nil { + log.Fatal(err) + } +} + +func main() { + opts := immudb.DefaultOptions(). + WithAddress("localhost"). + WithPort(3322) + + client := immudb.NewClient().WithOptions(opts) + err := immudb.OpenSession( + context.TODO(), + []byte(`immudb`), + []byte(`immudb`), + "defaultdb", + ) + handleErr(err) + + defer client.CloseSession(context.TODO()) + + tx, err := client.NewTx(context.TODO()) + handleErr(err) + + err = tx.SQLExec( + context.TODO(), + `CREATE TABLE IF NOT EXISTS mytable(id INTEGER AUTO_INCREMENT, title VARCHAR[256], active BOOLEAN, PRIMARY KEY id);`, + nil, + ) + handleErr(err) + + nRows := 10 + for i := 0; i < nRows; i++ { + err := tx.SQLExec( + context.TODO(), + "INSERT INTO mytable(title, active) VALUES (@title, @active)", + map[string]interface{}{ + "title": fmt.Sprintf("title%d", i), + "active": i%2 == 0, + }, + ) + handleErr(err) + } + + txh, err := tx.Commit(context.TODO()) + handleErr(err) + + fmt.Printf("Successfully committed rows %d\n", txh.UpdatedRows) + + reader, err := client.SQLQueryReader(context.TODO(), "SELECT * FROM mytable", nil) + handleErr(err) + + for reader.Next() { + row, err := reader.Read() + handleErr(err) + + fmt.Println(row[0], row[1]) + } +} +``` +::: + + +::: tab Java +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.sql.SQLQueryResult; +import io.codenotary.immudb4j.sql.SQLValue; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + client.beginTransaction(); + + client.sqlExec( + "CREATE TABLE IF NOT EXISTS mytable(id INTEGER, title VARCHAR[256], active BOOLEAN, PRIMARY KEY id)"); + + final int rows = 10; + + for (int i = 0; i < rows; i++) { + client.sqlExec("UPSERT INTO mytable(id, title, active) VALUES (?, ?, ?)", + new SQLValue(i), + new SQLValue(String.format("title%d", i)), + new SQLValue(i % 2 == 0)); + } + + SQLQueryResult res = client.sqlQuery("SELECT id, title, active FROM mytable"); + + while (res.next()) { + System.out.format("('%s', '%s')\n", res.getInt(0), res.getString(1), res.getBoolean(2)); + + } + + client.commitTransaction(); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` +::: + +::: tab Python + +Currently immudb Python sdk doesn't support interactive transactions. + +However you can still use non-interactive SQL Transactions. + +```python +from immudb import ImmudbClient +from uuid import uuid4 + +URL = "localhost:3322" # ImmuDB running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + client.sqlExec(""" + CREATE TABLE IF NOT EXISTS example ( + uniqueID VARCHAR[64], + value VARCHAR[32], + created TIMESTAMP, + PRIMARY KEY(uniqueID) + );""") + + client.sqlExec(""" + CREATE TABLE IF NOT EXISTS related ( + id INTEGER AUTO_INCREMENT, + uniqueID VARCHAR[64], + relatedValue VARCHAR[32], + PRIMARY KEY(id) + );""") + + uid1 = str(uuid4()) + uid2 = str(uuid4()) + params = { + "uid1": uid1, + "uid2": uid2 + } + + resp = client.sqlExec(""" + BEGIN TRANSACTION; + + INSERT INTO example (uniqueID, value, created) + VALUES (@uid1, 'test1', NOW()), (@uid2, 'test2', NOW()); + INSERT INTO related (uniqueID, relatedValue) + VALUES (@uid1, 'related1'), (@uid2, 'related2'); + INSERT INTO related (uniqueID, relatedValue) + VALUES (@uid1, 'related3'), (@uid2, 'related4'); + + COMMIT; + """, params) + + transactionId = resp.txs[0].header.id + + result = client.sqlQuery(""" + SELECT + related.id, + related.uniqueID, + example.value, + related.relatedValue, + example.created + FROM related + JOIN example + ON example.uniqueID = related.uniqueID; + """) + for item in result: + id, uid, value, relatedValue, created = item + print("ITEM", id, uid, value, relatedValue, created.isoformat()) + + + result = client.sqlQuery(f""" + SELECT + related.id, + related.uniqueID, + example.value, + related.relatedValue, + example.created + FROM related BEFORE TX {transactionId} + JOIN example BEFORE TX {transactionId} + ON example.uniqueID = related.uniqueID; + """) + print(result) # You can't see just added entries, + # my fellow time traveller + + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [.Net sdk github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../../connecting/immugw.md) option. +::: + +:::: diff --git a/src/1.9.4/develop/sql/users.md b/src/1.9.4/develop/sql/users.md new file mode 100644 index 0000000000..69f527335c --- /dev/null +++ b/src/1.9.4/develop/sql/users.md @@ -0,0 +1,42 @@ +# User Management + + + +### CREATE USER + +Users can be created with the `CREATE USER` statement as follows: + +```sql +CREATE USER user1 WITH PASSWORD 'user1Password!' READWRITE; +``` + +Possible permissions are: `READ`, `READWRITE` and `ADMIN`. + +An admin user is able to fully manage the current database. + + + + + +### ALTER USER + +User password and permissions can be modified with the `ALTER USER` statement as follows: + +```sql +ALTER USER user1 WITH PASSWORD 'newUser1Password!' ADMIN; +``` + + + + + +### DROP USER + +An existing user can be deleted. +Deletion is logically done and the user can be reactivated by executing an `ALTER USER` statement. + +```sql +DROP USER user1; +``` + + \ No newline at end of file diff --git a/src/1.9.4/develop/streams.md b/src/1.9.4/develop/streams.md new file mode 100644 index 0000000000..5124cf9161 --- /dev/null +++ b/src/1.9.4/develop/streams.md @@ -0,0 +1,159 @@ +# Streams + + + +immudb provides stream capabilities. +Internally it uses “delimited” messages technique, every chunk has a trailer that describe the length of the message. In this way the receiver can recompose chunk by chunk the original payload. +Stream methods accepts a `readers` as a part of input and output arguments. In this way the large value is decomposed in small chunks that are streamed over the wire. +Client don't need to allocate the entire value when sending and can read the received one progressively. +For example a client could send a large file much greater than available ram memory. +> At the moment `immudb` is not yet able to write the data without allocating the entire received object, but in the next release it will be possible a complete communication without allocations. +The maximum size of a transaction sent with streams is temporarily limited to a payload of 32M. + +Supported stream method now available in the SDK are: + + + +* StreamSet +* StreamGet +* StreamVerifiedSet +* StreamVerifiedGet +* StreamScan +* StreamZScan +* StreamHistory +* StreamExecAll + + + + + +:::: tabs + +::: tab Go +Here an example on how to send a large file and a regular key value to immudb. + +It's possible to specify the chunk size of the stream with `WithStreamChunkSize()` method. + +<<< @/src/code-examples/go/develop-kv-streams/main.go +::: + +::: tab Java +```java +package io.codenotary.immudb.helloworld; + +import java.util.Arrays; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] value1 = { 0, 1, 2, 3 }; + + client.streamSet("key1", value1); + + Entry entry = client.streamGet("key1"); + + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` +::: + +::: tab Python +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [.Net sdk github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + +### Chunked reading + +It's possible to read returned value chunk by chunk if needed. This grant to the clients capabilities to handle data coming from immudb chunk by chunk + +:::: tabs + +::: tab Go +To read chunk by chunk the inner gRPC protobuffer client is needed. +Then it's possible to use `kvStreamReceiver` to retrieve the key and a value reader. Such reader will fill provided byte array with received data and will return the number of read bytes or error. +If no message is present it returns 0 and io.EOF. If the message is complete it returns 0 and nil, in that case successive calls to Read will returns a new message. + +> There are several receivers available (zStreamReceiver, vEntryStreamReceiver, execAllStreamReceiver) and also a primitive receiver MsgReceiver. The last one can be used to receive a simple row []byte message without additional logics. + +<<< @/src/code-examples/go/develop-kv-streams-chunked-reading/main.go +::: + +::: tab Java +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab Python +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET +Streams is not supported yet in this language SDK. +Do you want to make a feature request or help out? Open an issue on [.Net sdk github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: diff --git a/src/1.9.4/develop/transactions.md b/src/1.9.4/develop/transactions.md new file mode 100644 index 0000000000..7d05788a0c --- /dev/null +++ b/src/1.9.4/develop/transactions.md @@ -0,0 +1,566 @@ +# Transactions + + + +`GetAll`, `SetAll` and `ExecAll` are the foundation of transactions at key value level in immudb. They allow the execution of a group of commands in a single step, with two important guarantees: + +* All the commands in a transaction are serialized and executed sequentially. No request issued by another client can ever interrupt the execution of a transaction. This guarantees that the commands are executed as a single isolated operation. +* Either all of the commands are processed, or none are, so the transaction is also atomic. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-getall/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + client.set(b'key1', b'value1') + client.set(b'key2', b'value2') + client.set(b'key3', b'value3') + + response = client.getAll([b'key1', b'key2', b'key3']) + print(response) # The same as dictToSetGet, retrieved in one step + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.util.Arrays; +import java.util.List; + +import io.codenotary.immudb4j.Entry; +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] value1 = { 0, 1, 2, 3 }; + byte[] value2 = { 4, 5, 6, 7 }; + + client.set("key1", value1); + client.set("key2", value2); + + List keys = Arrays.asList("key1", "key2"); + List entries = client.getAll(keys); + + for (Entry entry : entries) { + System.out.format("('%s', '%s')\n", new String(entry.getKey()), Arrays.toString(entry.getValue())); + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const getAllReq: Parameters.GetAll = { + keysList: ['key1', 'key2', 'key3'], + sincetx: 0 + } + const getAllRes = await cl.getAll(getAllReq) + console.log('success: getAll', getAllRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## SetAll + +A more versatile atomic multi set operation + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-setall/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + dictToSetGet = { + b'key1': b'value1', + b'key2': b'value2', + b'key3': b'value3' + } + response = client.setAll(dictToSetGet) + print(response.id) # All in one transaction + + response = client.getAll([b'key1', b'key2', b'key3']) + print(response) # The same as dictToSetGet, retrieved in one step + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.KVListBuilder; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] value1 = { 0, 1, 2, 3 }; + byte[] value2 = { 4, 5, 6, 7 }; + + KVListBuilder kvListBuilder = KVListBuilder.newBuilder(). + add("key1", value1). + add("key2", value2); + + client.setAll(kvListBuilder.entries()); + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const setAllReq: Parameters.SetAll = { + kvsList: [ + { key: '1,2,3', value: '3,2,1' }, + { key: '4,5,6', value: '6,5,4' }, + ] + } + const setAllRes = await cl.setAll(setAllReq) + console.log('success: setAll', setAllRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## ExecAll + +`ExecAll` allows multiple insertions at once. The difference is that it is possible to specify a list of mixes of key/value sets, references and `zAdd` insertions. +The argument of a `ExecAll` is an array of the following types: + +* `Op_Kv`: ordinary key value item +* `Op_ZAdd`: [ZAdd](indexes.md#sorted-sets) option element +* `Op_Ref`: [Reference](queries-history.md#references) option element + +It's possible to persist and reference items that are already persisted on disk. In that case is mandatory to provide the index of the referenced item. This has to be done for: + +* `Op_ZAdd` +* `Op_Ref` + If `zAdd` or `reference` is not yet persisted on disk it's possible to add it as a regular key value and the reference is done only. In that case if `BoundRef` is true the reference is bounded to the current transaction values. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-execall/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient +from immudb.datatypes import KeyValue, ZAddRequest, ReferenceRequest + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + toExecute = [ + KeyValue(b'key', b'value'), + ZAddRequest(b'testscore', 100, b'key'), + KeyValue(b'key2', b'value2'), + ZAddRequest(b'testscore', 150, b'key2'), + ReferenceRequest(b'reference1', b'key') + ] + info = client.execAll(toExecute) + print(info.id) # All in one transaction + + print(client.zScan(b'testscore', b'', 0, 0, True, 10, True, 0, 200)) # Shows these entries + print(client.get(b'reference1')) + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const { id } = await cl.set({ key: 'persistedKey', value: 'persistedVal' }) + + const setOperation = { kv: { key: 'notPersistedKey', value: 'notPersistedVal' } } + const zAddOperation = { + zadd: { + set: 'mySet', + score: 0.6, + key: 'notPersistedKey', + attx: 0, + boundref: true + } + } + const zAddOperation1 = { + zadd: { + set: 'mySet', + score: 0.6, + key: 'persistedKey', + attx: id, + boundref: true + } + } + const execAllReq: Parameters.ExecAll = { + operationsList: [ + setOperation, + zAddOperation, + zAddOperation1, + ] + } + const execAllRes = await cl.execAll(execAllReq) + console.log('success: execAll', execAllRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## TxScan + +`TxScan` permits iterating over transactions. + +The argument of a `TxScan` is an array of the following types: + +* `InitialTx`: initial transaction id +* `Limit`: number of transactions returned +* `Desc`: order of returned transacations + + + +:::: tabs + +::: tab Go + +<<< @/src/code-examples/go/develop-kv-txscan/main.go + +> Remember to strip the first byte in the key (key prefix). +> Remember that a transaction could contain sorted sets keys that should not be skipped. +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import java.util.List; + +import io.codenotary.immudb4j.FileImmuStateHolder; +import io.codenotary.immudb4j.ImmuClient; +import io.codenotary.immudb4j.Tx; +import io.codenotary.immudb4j.TxHeader; + +public class App { + + public static void main(String[] args) { + + ImmuClient client = null; + + try { + + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.openSession("defaultdb", "immudb", "immudb"); + + byte[] value1 = { 0, 1, 2, 3 }; + byte[] value2 = { 4, 5, 6, 7 }; + + TxHeader hdr = client.set("key1", value1); + client.set("key2", value2); + + List txs = client.txScanAll(hdr.getId()); + + for (Tx tx : txs) { + System.out.format("tx '%d'\n", tx.getHeader().getId()); + } + + client.closeSession(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (client != null) { + try { + client.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + } + +} +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + for (let i = 0; i < 3; i++) { + await cl.set({ key: `key${i}`, value: `val${i}` }) + } + + const txScanReq: Parameters.TxScan = { + initialtx: 2, + limit: 3, + desc: false + } + const txScanRes = await cl.txScan(txScanReq) + console.log('success: txScan', txScanRes) +})() +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Filter Transactions + +The transaction entries are generated by writing key-value pairs, referencing keys, associating scores to key-value pairs (with `ZAdd` operation), and by mapping SQL data model into key-value model. + +With `TxScan` or `TxByIDWithSpec` operations it's possible to retrieve entries of certain types, either retrieving the digest of the value assigned to the key (`EntryTypeAction_ONLY_DIGEST`), the raw value (`EntryTypeAction_RAW_VALUE`) or the structured value (`EntryTypeAction_RESOLVE`). + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/develop-kv-txfilter/main.go +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Java +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: \ No newline at end of file diff --git a/src/1.9.4/embedded/embedding.md b/src/1.9.4/embedded/embedding.md new file mode 100644 index 0000000000..6f4f4f7b43 --- /dev/null +++ b/src/1.9.4/embedded/embedding.md @@ -0,0 +1,16 @@ +# Key Value Store + + + +There are cases where you don't want a separate server but embed immudb directly in the same application process, as a library. + +immudb already provides an embeddable key-value store in the [embedded](https://github.com/codenotary/immudb/tree/master/embedded) package. +The following example shows how to create or open a store, write some data and read it back. + + + + + +<<< @/src/code-examples/go/embedded-kv-v1.5.0/main.go + + diff --git a/src/1.9.4/embedded/embeddingSQL.md b/src/1.9.4/embedded/embeddingSQL.md new file mode 100644 index 0000000000..c1db2a9ee2 --- /dev/null +++ b/src/1.9.4/embedded/embeddingSQL.md @@ -0,0 +1,19 @@ +# SQL Engine + + + +There are cases where you don't want a separate server but embed immudb directly in the same application process, as a library. + +immudb provides you a immutable embedded SQL engine which keeps all history, is tamper-proof and can travel in time. +The SQL engine is mounted on top of the embedded key value store. +The following illustrative example showcase how to initialize the SQL engine, write and read data in the scope of a SQL transaction. + + + + + +<<< @/src/code-examples/go/embedded-sql/main.go + +If you need to change options like where things get stored by default, you can do that in the underlying store objects that the SQL engine is using. + + diff --git a/src/1.9.4/immudb.md b/src/1.9.4/immudb.md new file mode 100644 index 0000000000..a0de5b4d5a --- /dev/null +++ b/src/1.9.4/immudb.md @@ -0,0 +1,126 @@ +# immudb explained + + + +## What is immudb? + +immudb is database with built-in cryptographic proof and verification. It can track changes in sensitive data and the integrity of the history will be protected by the clients, without the need to trust the server. + +immudb can operate as a key-value, relational (SQL) or document database, making it a truly no-SQL database. You can add new transactions, but deletion or modification of older transactions isn’t allowed, thus making your data immutable. When a key record's value changes over time (such as a bank balance), you can get multiple instances with different timestamps to give you the complete history of that record's changes. You can store a variety of common data types, verification checksums, or JSON objects. + + + + + + +## What makes immudb special? + + + +- **Keep track of changes and audit them.** Traditional database transactions and logs are hard to scale and are mutable, so there is no way to know for sure if your data has been compromised. immudb is immutable. You can add new versions of existing records, but never change or delete records. This lets you store critical data without fear of it being changed silently. + +- **Verify your data without sacrificing performance.** Data stored in immudb is cryptographically coherent and verifiable, just like blockchains, but without all the complexity. Unlike blockchains, immudb can handle millions of transactions per second, and can be used both as a lightweight service or embedded in your application as a library. + +- **Protect yourself from cyber attacks.** While Cyber Security is an important part of your organization’s business plan, immudb provides another layer of security to ensure data integrity even in the event your perimeter is breached during an attack. Data cannot be deleted or modified once stored into immudb. Additions of new data are logged and auditable, enabling you to view any suspect additions made during the intrusion. + + + + + +## How can I use immudb? + +Depending on your use case, immudb might function as your application's primary or as a secondary database. As a secondary, complimentary database, use immudb to cross-check the integrity of your important data (by verifying checksums or comparing stored data values). A secondary database enables you to quickly use immudb without completely re-engineering your existing application. + +### Use cases + +- Integration with your DevOps ensures code security throughout the development and deployment process. Embed immudb into your [Azure DevOps](https://codenotary.io/blog/securing-your-azure-devops-ecosystem-jenkins-and-kubernetes-aks/) with Jenkins and Kubernetes. Use just [Jenkins](https://codenotary.io/blog/jenkins-build-deployment-pipeline-a-how-to-for-ensuring-integrity/). Alternatively, integrate with [GitLab](https://codenotary.io/blog/fully-trusted-gitlab-pipeline/) or [GitHub](https://codenotary.io/blog/use-github-actions-for-validated-builds/). +- Guarantee File Integrity of your critical data. Examples include storing your organization's sensitive financial, credit card transactional, invoices, contracts, educational transcripts, and other important data. +- Ensure integrity of your legal Documents and Invoices, contracts, forms, and your downloads and emails. +- Save your Internet of Things (IoT) sensor data as a failsafe plan for loss of data. +- Keep your investment guidelines or stock market data tamperproof for your investment bank or client financial portfolios. +- Store important log files to keep them tamperproof to meet regulations like PCI compliance. +- Protect medical data, test results, or recipes from alteration. +- Companies use immudb to protect credit card transactions and to secure processes by storing digital certificates and checksums. + + + + + +## Key value, Document and SQL + +immudb can be used as a tamper-proof key value, document or SQL database, with audit history capabilities. Within single immudb instance a user can have multiple databases of both types, it is even possible to have KV, Document and SQL withing single database. + +Key value is a foundation layer for Document and SQL, meaning that both Document and SQL are using key value store capabilities underneath. + + + + + +## Standalone and Embeddable + +immudb can be run as full database server with [replicas](production/replication.md) or easily [embedded](embedded/embedding.md) as a lightweight database into application. + + + + + +## Running platforms + +immudb server runs in most operating systems: BSD, Linux, OS X, Solaris, Windows, IBM z/OS. + + + + + +## S3 Storage Backend + +immudb can store its data in the Amazon S3 service (or a compatible alternative). + + + + + +## How is immutability ensured? + +immudb consistency can be verified by any external client or auditor by calculating transaction state and comparing it with one returned from immudb. The state is represented by a digest, which is calculated as part of the transaction commit phase. Calculating such a value in an accumulative manner ensures that any change already made cannot be reversed. + +To increase security, immudb can be provisioned with a signing key to ensure non-repudiation of database states. The immudb server will subsequently sign the state on request from a client application. +That means that an auditor or a third party client, for instance, could verify the authenticity of the returned current state. + +Check [auditor](production/auditor.md) section for additional details. + + + + + +## Theoretical limits + +immudb is an append-only, tamperproof database with key/value and SQL (Structured Query Language) application programming interfaces. + +The immudb core persistence layer consists of a cryptographically-immutable log. Transactions are sequentially stored and uniquely identified by unsigned 64-bit integer values, thus setting a theoretical limit of 18446744073709551615 transactions (1^64 - 1). + +In order to provide manageable limits, immudb is designed to set an upper bound to the number of key-value pairs included in a single transaction. The default value being 1024, so using default settings, the theoretical number of key-value pairs that can be stored in immudb is: 1024 * (1^64 - 1). + +We designed immudb to require stable resources but not imposing strong limitations as most of the settings can be adjusted based on the use-case requirements. + +While key-value pairs consist of arbitrary byte arrays, a maximum length may be set at database creation time. Both parameters can be increased as needed, considering: + +- long keys may degrade performance (when querying data through the index) +- longer values requires more memory at runtime + +Note: The cryptographic linking does not impose a practical restriction. + +immudb relies on a file abstraction layer and does not directly interact with the underlying file-system, if any. But default storage layer implementation writes data into fixed-size files, default size being 512MB. The current theoretical maximum number of files is 100 millions. + +Theoretical limits may be determined by a couple of elements: + +- max number transactions: 2^64-1 (uint64) +- max number of files: 2^63-1 (max file size 2^56-1) +- max value length: 32 MB (max size: 2^56-1 bytes) +- max key length: 1024 Bytes (max length: 2^31-1 bytes) + + + +::: tip +Download the [immudb short research paper](https://codenotary.s3.amazonaws.com/Research-Paper-immudb-CodeNotary_v3.0.pdf) to learn about the technical foundations of immudb. +::: \ No newline at end of file diff --git a/src/1.9.4/management/database.md b/src/1.9.4/management/database.md new file mode 100644 index 0000000000..770ed395f8 --- /dev/null +++ b/src/1.9.4/management/database.md @@ -0,0 +1,351 @@ +# Database management + + + +Multi-database support is included in immudb server. immudb automatically creates an initial database named `defaultdb`. + +Managing users and databases requires the appropriate privileges. A user with `PermissionAdmin` rights can manage everything. Non-admin users have restricted access and can only read or write databases to which they have been granted permission. + +Each database can be configured with a variety of settings. While some values can be changed at any time (though it may require a database reload to take effect), following ones are fixed and cannot be changed: FileSize, MaxKeyLen, MaxValueLen, MaxTxEntries and IndexOptions.MaxNodeSize. + + + + + +## Database creation + +This example shows how to create a new database and how to write records to it. +To create a new database, use `CreateDatabaseV2` method. + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/database-create/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.*; + +public class App { + + public static void main(String[] args) { + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + ImmuClient client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.login("immudb", "immudb"); + + client.createDatabase("db1"); + } + +} +``` + +::: + +::: tab Python + +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + client.createDatabase("db1") + +if __name__ == "__main__": + main() +``` + +::: + +::: tab Node.js + +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const createDatabaseReq: Parameters.CreateDatabase = { + databasename: 'myimmutabledb' + } + + const createDatabaseRes = await cl.createDatabase(createDatabaseReq) + console.log('success: createDatabase', createDatabaseRes) +})() +``` + +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +var dbName = "mydatabase"; +await client.CreateDatabase(dbName); +await client.UseDatabase(dbName); + +await client.Close(); +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + + + +## Listing databases + +This example shows how to list existent databases using `DatabaseListV2` method. + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/database-list/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.*; + +public class App { + + public static void main(String[] args) { + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + ImmuClient client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.login("immudb", "immudb"); + + List dbs = client.databases(); + // List of database names + } + +} +``` + +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +var databases = await client.Databases(); +foreach(var database in databases) +{ + Console.WriteLine(database); +} + +await client.Close(); +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + + + +## Database loading/unloading + +Databases can be dynamically loaded and unloaded without having to restart the server. After the database is unloaded, all its resources are released. Unloaded databases cannot be queried or written to, but their settings can still be changed. +Upon startup, the immudb server will automatically load databases with the attribute `Autoload` set to true. If a user-created database cannot be loaded successfully, it remains closed, but the server continues to run normally. +As a default, autoloading is enabled when creating a database, but it can be disabled during creation or turned on/off at any time thereafter. + +Following example shows how to load and unload a database using `LoadDatabase` and `UnloadDatabase` methods. + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/database-load-unload/main.go +::: + +::: tab Java +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [.Net sdk github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + + + +## Database settings + +Database settings can be individually changed using `UpdateDatabaseV2` method. + +Each database can be configured with a variety of settings. While some values can be changed at any time (though it may require a database reload to take effect), following ones are fixed and cannot be changed: FileSize, MaxKeyLen, MaxValueLen, MaxTxEntries and IndexOptions.MaxNodeSize. + +Note: Replication settings take effect without the need of reloading the database. + +Following example shows how to update database using `UpdateDatabaseV2` method. + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/database-settings/main.go +::: + +::: tab Java +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Java sdk github project](https://github.com/codenotary/immudb4j/issues/new) +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab .NET +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [.Net sdk github project](https://github.com/codenotary/immudb4net/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + + + +## Configuration options + +Following main database options are available: + +| Name | Type | Description | +|------|------|-------------| +| replicationSettings | Replication Setings | Repliation settings are described below | +| indexSettings | Index Settings | Index settings are described below | +| fileSize | Uint32 | maximum file size of files on disk generated by immudb | +| maxKeyLen | Uint32 | maximum length of keys for entries stored in the database | +| maxValueLen | Uint32 | maximum length of values for entries stored in the database | +| maxTxEntries | Uint32 | maximum number of entries inside one transaction | +| excludeCommitTime | Bool | if set to true, commit time is not added to transaction headers allowing reproducible database state | +| maxConcurrency | Uint32 | max number of concurrent operations on the db | +| maxIOConcurrency | Uint32 | max number of concurrent IO operations on the db | +| txLogCacheSize | Uint32 | size of transaction log cache | +| vLogMaxOpenedFiles | Uint32 | maximum number of open files for payload data | +| txLogMaxOpenedFiles | Uint32 | maximum number of open files for transaction log | +| commitLogMaxOpenedFiles | Uint32 | maximum number of open files for commit log | +| syncFrequency | duration | set the fsync frequency during commit process (default 20ms) +| write-buffer-size | uint32 | set the size of in-memory buffers for file abstractions (default 4194304) +| writeTxHeaderVersion | Uint32 | transaction header version, used for backwards compatibility | +| autoload | Bool | if set to false, do not load database on startup | + +Replication settings: + +| Name | Type | Description | +|------|------|-------------| +| replica | Bool | if true, the database is a replica of another one | +| primaryDatabase | String | name of the database to replicate | +| primaryHost | String | hostname of the primary immudb instance | +| primaryPort | Uint32 | tcp port of the primary immudb instance | +| primaryUsername | String | username used to connect to the primary immudb instance | +| primaryPassword | String | password used to connect to the primary immudb instance | + +Additional index options: + +| Name | Type | Description | +|------|------|-------------| +| flushThreshold | Uint32 | threshold in number of entries between automatic flushes | +| syncThreshold | Uint32 | threshold in number of entries between flushes with sync | +| cacheSize | Uint32 | size of btree node cache | +| maxNodeSize | Uint32 | max size of btree node in bytes | +| maxActiveSnapshots | Uint32 | max number of active in-memory btree snapshots | +| renewSnapRootAfter | Uint64 | threshold in time for automated snapshot renewal during data scans | +| compactionThld | Uint32 | minimum number of flushed snapshots to enable full compaction of the index | +| delayDuringCompaction | Uint32 | extra delay added during indexing when full compaction is in progress | +| nodesLogMaxOpenedFiles | Uint32 | maximum number of files opened for nodes data | +| historyLogMaxOpenedFiles | Uint32 | maximum number of files opened for nodes history | +| commitLogMaxOpenedFiles | Uint32 | maximum number of files opened for commit log | +| flushBufferSize | Uint32 | in-memory buffer size when doing flush operation | +| cleanupPercentage | Float | % of data to be cleaned up from during next automatic flush operation | + + diff --git a/src/1.9.4/management/state.md b/src/1.9.4/management/state.md new file mode 100644 index 0000000000..0a1e698970 --- /dev/null +++ b/src/1.9.4/management/state.md @@ -0,0 +1,426 @@ +# State management + + + +## Current State + +Current state of immudb provides proof that clients can use to verify immudb: + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/state-current/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + + state = client.currentState() # immudb.rootService.State + print(state.db) # Current selected DB + print(state.txId) # Current transaction ID + print(state.txHash) # Current transaction hash + print(state.signature) # Current signature + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +```java +ImmuState currState = immuClient.currentState(); + +System.out.printf("The current state is " + currState.toString()); +``` + +::: + +::: tab .NET + +```csharp +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +var state = client.State; +System.Console.WriteLine($"The current state is: {state}"); + +await client.Close(); +``` + +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const currentStateRes = await cl.currentState() + console.log('success: currentState', currentStateRes) +})() +``` +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Automated verification of state by immudb SDK + +It's the responsibility of the immudb client to track the server state. That way it can check each verified read or write operation against a trusted state. + + + +:::: tabs + +::: tab Go +The component in charge of state handling is the `StateService`. +To set up the `stateService` 3 interfaces need to be implemented and provided to the `StateService` constructor: +* `Cache` interface in the `cache` package. Standard cache.NewFileCache provides a file state store solution. +* `StateProvider` in the `stateService` package. It provides a fresh state from immudb server when the client is being initialized for the first time. Standard StateProvider provides a service that retrieve immudb first state hash from a gRPC endpoint. +* `UUIDProvider` in the `stateService` package. It provides the immudb identifier. This is needed to allow the client to safely connect to multiple immudb instances. Standard UUIDProvider provides the immudb server identifier from a gRPC endpoint. + +Following an example how to obtain a client instance with a custom state service. +```go +func MyCustomImmuClient(options *c.Options) (cli c.ImmuClient, err error) { + ctx := context.Background() + + cli = c.DefaultClient() + + options.DialOptions = cli.SetupDialOptions(options) + + cli.WithOptions(options) + + var clientConn *grpc.ClientConn + if clientConn, err = cli.Connect(ctx); err != nil { + return nil, err + } + + cli.WithClientConn(clientConn) + + serviceClient := schema.NewImmuServiceClient(clientConn) + cli.WithServiceClient(serviceClient) + + if err = cli.WaitForHealthCheck(ctx); err != nil { + return nil, err + } + + immudbStateProvider := stateService.NewImmudbStateProvider(serviceClient) + immudbUUIDProvider := stateService.NewImmudbUUIDProvider(serviceClient) + + customDir := "custom_state_dir" + os.Mkdir(customDir, os.ModePerm) + stateService, err := stateService.NewStateService( + cache.NewFileCache(customDir), + logger.NewSimpleLogger("immuclient", os.Stderr), + immudbStateProvider, + immudbUUIDProvider) + if err != nil { + return nil, err + } + + dt, err := timestamp.NewDefaultTimestamp() + if err != nil { + return nil, err + } + + ts := c.NewTimestampService(dt) + cli.WithTimestampService(ts).WithStateService(stateService) + + return cli, nil +} +``` +::: + +::: tab Python +```python +from immudb import ImmudbClient +from immudb.client import PersistentRootService + +# By default RootService is writing state to RAM +# You can choose different implementation of RootService + +# Persistent root service will save to the disk after every verified transaction + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) +PERSISTENT_ROOT_SERVICE_PATH = "/tmp/psr.db" + +def main(): + client = ImmudbClient(URL, rs = PersistentRootService(PERSISTENT_ROOT_SERVICE_PATH)) + client.login(LOGIN, PASSWORD, database = DB) + client.verifiedSet(b'x', b'1') + client.verifiedGet(b'x') + client.verifiedSet(b'x', b'2') + client.verifiedGet(b'x') + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +Any immudb server has its own UUID. This is exposed as part of the login response. +Java SDK can use any implementation of the `ImmuStateHolder` interface, which specifies two methods: +- `ImmuState getState(String serverUuid, String database)` for getting a state. +- `void setState(String serverUuid, ImmuState state)` for setting a state. + +Note that a state is related to a specific database (identified by its name) and a server (identified by the UUID). +Currently, Java SDK offers two implementations of this interface for storing and retriving a state: +- `FileImmuStateHolder` that uses a disk file based store. +- `SerializableImmuStateHolder` that uses an in-memory store. + +As most of the code snippets include `FileImmuStateHolder`, please find below an example using the in-memory alternative: +```java +SerializableImmuStateHolder stateHolder = new SerializableImmuStateHolder(); + +ImmuClient immuClient = ImmuClient.newBuilder() + .withStateHolder(stateHolder) + .withServerUrl("localhost") + .withServerPort(3322) + .build(); + +immuClient.login("immudb", "immudb"); +immuClient.useDatabase("defaultdb"); +// ... +immuClient.logout(); +``` + +::: + +::: tab .NET + +Any immudb server has its own UUID. This is exposed as part of the login response. +.NET SDK can use any implementation of the `ImmuStateHolder` interface, which specifies two methods: + +- `ImmuState GetState(Sstring serverUuid, string database)` for getting a state. +- `void SetState(string serverUuid, ImmuState state)` for setting a state. + +Note that a state is related to a specific database (identified by its name) and a server (identified by the UUID). +Currently, .NET SDK offers one implementations of this interface for storing and retriving a state, `FileImmuStateHolder`, +that uses a disk file based store. + +As most of the code snippets include `FileImmuStateHolder`, please find below an example using the in-memory alternative: + +```csharp + FileImmuStateHolder stateHolder = FileImmuStateHolder.NewBuilder() + .WithStatesFolder("./my_immuapp_states") + .Build(); + +ImmuClient immuClient = ImmuClient.NewBuilder() + .WithStateHolder(stateHolder) + .Build(); + +await client.Open("immudb", "immudb", "defaultdb"); +await client.Close(); + +``` + +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## Verify state signature + +If `immudb` is launched with a private signing key, each signed request can be verified with the public key. +This ensures the server identity. + + + + + +To run immudb server with state signature use: + +```bash +./immudb --signingKey test/signer/ec1.key +``` + +To generate an elliptic curve private key use: + +```bash +openssl ecparam -name prime256v1 -genkey -noout -out private.key +``` + +To generate the public key from the previous one: + +```bash +openssl ec -in private.key -pubout -out public.key +``` + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/state-verify-signature/main.go +::: + +::: tab Python +```python +from immudb import ImmudbClient + +# All operations are checked against public/private key pair + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) +KEYFILE = "public_signing_key.pem" # Public key path + # needs immudb server with --signingKey option enabled + # pointing to corresponding private key + +def main(): + client = ImmudbClient(URL, publicKeyFile = KEYFILE) + client.login(LOGIN, PASSWORD, database = DB) + client.set(b'x', b'1') + client.verifiedGet(b'x') # This operation will also fail if public key + # is not paired with private one used in immudb + + state = client.currentState() # immudb.rootService.State + print(state.db) # Current selected DB + print(state.txId) # Current transaction ID + print(state.txHash) # Current transaction hash + print(state.signature) # Current signature + +if __name__ == "__main__": + main() +``` +::: + +::: tab Java + +```java +// Having immudb server running with state signature enabled +// (by starting it, for example using `immudb --signingKey private_key.pem`) +// we provision the client with the public key file, and this implies that +// state signature verification is done on the client side +// each time the state is retrieved from the server. + +File publicKeyFile = new File("path/to/public_key.pem"); + +immuClient = ImmuClient.newBuilder() + .withServerUrl("localhost") + .withServerPort(3322) + .withServerSigningKey(publicKeyFile.getAbsolutePath()) + .build(); + +try { + ImmuState state = immuClient.currentState(); + // It should all be ok as long as the immudb server has been started with + // state signature feature enabled, otherwise, this verification will fail. + +} catch (RuntimeException e) { + // State signature failed. +} +``` + +::: + +::: tab .NET + +```csharp +// Having immudb server running with state signature enabled +// (by starting it, for example using `immudb --signingKey private_key.pem`) +// we provision the client with the public key file, and this implies that +// state signature verification is done on the client side +// each time the state is retrieved from the server. + +Assembly asm = Assembly.GetExecutingAssembly(); +string resourceName = "public_key.pem"; +AsymmetricKeyParameter assymKey; +ImmuClient client; +try +{ + using (Stream? stream = asm.GetManifestResourceStream(resourceName)) + { + if (stream == null) + { + Assert.Fail("Could not read resource"); + } + using (TextReader tr = new StreamReader(stream)) + { + PemReader pemReader = new PemReader(tr); + assymKey = (AsymmetricKeyParameter)pemReader.ReadObject(); + } + } + client = ImmuClient.NewBuilder() + .WithServerUrl("localhost") + .WithServerSigningKey(assymKey) + .Build(); +} +catch (Exception e) +{ + Console.WriteLine($"An exception occurred: {e}"); + return; +} +``` + +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + await cl.set({ key: 'immudb', value: 'hello world' }) + + const currentStateRes = await cl.currentState(); + console.log('success: currentState', currentStateRes) +})() +``` +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + +
diff --git a/src/1.9.4/management/user.md b/src/1.9.4/management/user.md new file mode 100644 index 0000000000..51e638961b --- /dev/null +++ b/src/1.9.4/management/user.md @@ -0,0 +1,280 @@ +# User management + + + +## Creating users and assigning permissions + +User management is exposed with following methods: + +* CreateUser +* ChangePermission +* ChangePassword + +Password must have between 8 and 32 letters, digits and special characters of which at least 1 uppercase letter, 1 digit and 1 special character. + +Admin permissions are: + +* PermissionSysAdmin = 255 +* PermissionAdmin = 254 + +Non-admin permissions are: + +* PermissionNone = 0 +* PermissionR = 1 +* PermissionRW = 2 + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/user-create/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.*; + +public class App { + + public static void main(String[] args) { + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + ImmuClient client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.login("immudb", "immudb"); + + String database = "defaultdb"; + String username = "testCreateUser"; + String password = "testTest123!"; + Permission permission = Permission.PERMISSION_RW; + + client.createUser(username, password, permission, database); + + System.out.println("listUsers:"); + List users = client.listUsers(); + users.forEach(user -> System.out.println("\t- " + user)); + + // Changing the user password. + client.changePassword(username, password, "newTestTest123!"); + } + +} +``` + +::: + + +::: tab Python +```python +from grpc import RpcError +from immudb import ImmudbClient +from immudb.constants import PERMISSION_ADMIN, PERMISSION_R, PERMISSION_RW +from immudb.grpc.schema_pb2 import GRANT, REVOKE +from enum import IntEnum + +URL = "localhost:3322" # immudb running on your machine +LOGIN = "immudb" # Default username +PASSWORD = "immudb" # Default password +DB = b"defaultdb" # Default database name (must be in bytes) + +def main(): + client = ImmudbClient(URL) + client.login(LOGIN, PASSWORD, database = DB) + passwordForNewUsers = "Te1st!@#Test" + try: + client.createUser("tester1", passwordForNewUsers, PERMISSION_R, DB) + client.createUser("tester2", passwordForNewUsers, PERMISSION_RW, DB) + client.createUser("tester3", passwordForNewUsers, PERMISSION_ADMIN, DB) + except RpcError as exception: + print(exception.details()) + + users = client.listUsers().userlist.users # immudb.handler.listUsers.listUsersResponse + for user in users: + print("User", user.user) + print("Created by", user.createdby) + print("Creation date", user.createdat) + print("Is active", user.active) + for permission in user.permissions: + print("Permission", permission.database, permission.permission) + print("---") + + client.login("tester3", passwordForNewUsers, DB) + client.changePermission(GRANT, "tester2", DB, PERMISSION_ADMIN) + client.changePermission(REVOKE, "tester2", DB, PERMISSION_ADMIN) + + client.login(LOGIN, PASSWORD, database = DB) + # Changing password + client.changePassword("tester1", "N1ewpassword!", passwordForNewUsers) + + # User logs with new password + client.login("tester1", "N1ewpassword!") + + client.login(LOGIN, PASSWORD, database = DB) + client.changePassword("tester1", passwordForNewUsers, "N1ewpassword!") + + + client.login("tester1", passwordForNewUsers, DB) + + # No permissions to write + try: + client.set(b"test", b"test") + except RpcError as exception: + print(exception.details()) + + # But has permissions to read + result = client.get(b"test") + + client.login("tester3", passwordForNewUsers, DB) + + # Now will have permissions to write + client.changePermission(GRANT, "tester1", DB, PERMISSION_RW) + client.login("tester1", passwordForNewUsers, DB) + client.set(b"test", b"test") + result = client.get(b"test") + + client.login("tester3", passwordForNewUsers, DB) + + # Now will have permissions to nothing + client.changePermission(REVOKE, "tester1", DB, PERMISSION_RW) + + try: + client.login("tester1", passwordForNewUsers, DB) + except RpcError as exception: + print(exception.details()) + + client.login("tester3", passwordForNewUsers, DB) + client.changePermission(GRANT, "tester1", DB, PERMISSION_RW) + + +if __name__ == "__main__": + main() +``` +::: + +::: tab Node.js +```ts +import ImmudbClient from 'immudb-node' +import Parameters from 'immudb-node/types/parameters' +import { USER_ACTION, USER_PERMISSION } from 'immudb-node/dist/types/user' + +const IMMUDB_HOST = '127.0.0.1' +const IMMUDB_PORT = '3322' +const IMMUDB_USER = 'immudb' +const IMMUDB_PWD = 'immudb' + +const cl = new ImmudbClient({ host: IMMUDB_HOST, port: IMMUDB_PORT }); + +(async () => { + await cl.login({ user: IMMUDB_USER, password: IMMUDB_PWD }) + + const createUserRequest: Parameters.CreateUser = { + user: 'myNewUser1', + password: 'myS3cretPassword!', + permission: USER_PERMISSION.READ_ONLY, + database: 'defaultdb', + }; + const createUserRes = cl.createUser(createUserRequest) + console.log('success: createUser', createUserRes) + + const changePermissionReq: Parameters.ChangePermission = { + action: USER_ACTION.GRANT, + username: 'myNewUser1', + database: 'defaultDB', + permission: USER_PERMISSION.READ_WRITE + } + const changePermissionRes = await cl.changePermission(changePermissionReq) + console.log('success: changePermission', changePermissionRes) + + const changePasswordReq: Parameters.ChangePassword = { + user: 'myNewUser1', + oldpassword: 'myS3cretPassword!', + newpassword: 'myNewS3cretPassword!' + } + const changePasswordRes = await cl.changePassword(changePasswordReq) + console.log('success: changePassword', changePermissionRes) +})() +``` +::: + +::: tab .NET + +```csharp + +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +try +{ + await client.CreateUser("testUser", "testTest123!", Permission.PERMISSION_ADMIN, "defaultdb"); +} +catch (Exception e) +{ + Console.WriteLine($"An exception in create user: {e}"); + return; +} + +await client.ChangePassword("testUser", "testTest123!", "newTestTest123!"); +await client.Close(); + +``` + +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + +
+ + + +## Changing user's password + +Changing user's password can be done by either the sysadmin user (which can be done for any user in the database) +or by user with Admin permission to at least one database that has created the user whose password is changed: + +```bash +$ immuadmin user changepassword user +Choose a password for user: +Confirm password: +user's password has been changed +``` + +The sysadmin user can change his own password the same way but it requires confirming the existing password: + +```bash +$ immuadmin user changepassword immudb +Old password: +Choose a password for immudb: +Confirm password: +immudb's password has been changed +``` + + + + + +## Emergency reset of sysadmin user password + +If the current password of the sysadmin user is not known, it can be reset by running the immudb server with password reset option: + +```bash +immudb --force-admin-password --admin-password +``` + +The `--force-admin-password` flag ensures that the sysadmin user will have its password reset to the one given through +the `--admin-password` option. + + diff --git a/src/1.9.4/playground.md b/src/1.9.4/playground.md new file mode 100644 index 0000000000..6164ab63b2 --- /dev/null +++ b/src/1.9.4/playground.md @@ -0,0 +1,9 @@ +# Playground + +[**immudb Playground**](https://play.codenotary.com) is an interactive environment for learning about immudb: + + + +[![image](/playground.jpg)](https://play.codenotary.com) + + \ No newline at end of file diff --git a/src/1.9.4/production/auditor.md b/src/1.9.4/production/auditor.md new file mode 100644 index 0000000000..db1c869e36 --- /dev/null +++ b/src/1.9.4/production/auditor.md @@ -0,0 +1,152 @@ +# Tampering detection + +The Auditor is a component for checking if immudb was tampered, it's a good practice to run the auditor as a separate and independent component. + +immuclient and [immugw](https://github.com/codenotary/immugw) are shipped with auditor capabilities. + + + +## Running an Auditor with immuclient + +immuclient can act as Auditor by running the following command: + +```bash +$ ./immuclient audit-mode +1m0s +immuclientd 2022/05/22 12:34:11 INFO: starting auditor with a 1m0s interval ... +immuclientd 2022/05/22 12:34:11 INFO: auditor monitoring HTTP server starting on 0.0.0.0:9477 ... +immuclientd 2022/05/22 12:34:11 INFO: audit #1 started @ 2022-05-22 12:34:11.543823286 +0200 CEST m=+0.153679785 +immuclientd 2022/05/22 12:34:11 INFO: audit #1 - list of databases to audit has been (re)loaded - 2 database(s) found: [defaultdb mydatabase] +immuclientd 2022/05/22 12:34:11 INFO: audit #1 - auditing database defaultdb +immuclientd 2022/05/22 12:34:11 INFO: audit #1 finished in 55.295777ms @ 2022-05-22T12:34:11.599119184+02:00 +``` + +immuclient is now running on the following address: 0.0.0.0:9477/metrics + +example output: + +```bash +# HELP immuclient_audit_curr_root_per_server Current root index used for the latest audit. +# TYPE immuclient_audit_curr_root_per_server gauge +immuclient_audit_curr_root_per_server{server_address="127.0.0.1:3322",server_id="br8eugq036tfln0ct6o0"} 2 +# HELP immuclient_audit_prev_root_per_server Previous root index used for the latest audit. +# TYPE immuclient_audit_prev_root_per_server gauge +immuclient_audit_prev_root_per_server{server_address="127.0.0.1:3322",server_id="br8eugq036tfln0ct6o0"} -1 +# HELP immuclient_audit_result_per_server Latest audit result (1 = ok, 0 = tampered). +# TYPE immuclient_audit_result_per_server gauge +immuclient_audit_result_per_server{server_address="127.0.0.1:3322",server_id="br8eugq036tfln0ct6o0"} -1 +# HELP immuclient_audit_run_at_per_server Timestamp in unix seconds at which latest audit run. +# TYPE immuclient_audit_run_at_per_server gauge +immuclient_audit_run_at_per_server{server_address="127.0.0.1:3322",server_id="br8eugq036tfln0ct6o0"} 1.5907565337454605e+09 +``` + +immuclient looks for immudb at 127.0.0.1:3322 by default with the default username and password. Nevertheless a number of parameters can be defined: + +``` +immuclient audit-mode - Run a foreground auditor +immuclient audit-mode install - Install and runs daemon +immuclient audit-mode stop - Stops the daemon +immuclient audit-mode start - Starts initialized daemon +immuclient audit-mode restart - Restarts daemon +immuclient audit-mode uninstall - Removes daemon and its setup + +Flags: + -h, --help help for audit-mode + +Global Flags: + --audit-databases string Optional comma-separated list of databases (names) to be audited. Can be full name(s) or just name prefix(es). + --audit-monitoring-host string Host for the monitoring HTTP server when running in audit mode (serves endpoints like metrics, health and version). (default "0.0.0.0") + --audit-monitoring-port int Port for the monitoring HTTP server when running in audit mode (serves endpoints like metrics, health and version). (default 9477) + --audit-notification-password string Password used to authenticate when publishing audit result to 'audit-notification-url'. + --audit-notification-url string If set, auditor will send a POST request at this URL with audit result details. + --audit-notification-username string Username used to authenticate when publishing audit result to 'audit-notification-url'. + --audit-password string immudb password used to login during audit; can be plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) + --audit-username string immudb username used to login during audit + --certificate string server certificate file path (default "./tools/mtls/4_client/certs/localhost.cert.pem") + --clientcas string clients certificates list. Aka certificate authority (default "./tools/mtls/2_intermediate/certs/ca-chain.cert.pem") + --config string config file (default path are configs or $HOME. Default filename is immuclient.toml) + --database string immudb database to be used + --dir string Main directory for audit process tool to initialize (default "/var/folders/0z/wk6v4sjd31qbvt7l75t_z_v00000gn/T/") + -a, --immudb-address string immudb host address (default "127.0.0.1") + -p, --immudb-port int immudb port number (default 3322) + --max-recv-msg-size int max message size in bytes the client can receive (default 4194304) + -m, --mtls enable mutual tls + --password string immudb password used to login; can be plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) + --pkey string server private key path (default "./tools/mtls/4_client/private/localhost.key.pem") + --roots-filepath string Filepath for storing root hashes after every successful audit loop. Default is tempdir of every OS. (default "/tmp/") + --server-signing-pub-key string Path to the public key to verify signatures when presents + --servername string used to verify the hostname on the returned certificates (default "localhost") + --tokenfile string authentication token file (default path is $HOME or binary location; default filename is ) + --username string immudb username used to login + --value-only returning only values for get operations +``` + +To get the (signed) state in combination with the immuclient with auditor capabilities: + +```bash +immuclient audit-mode --audit-username {immudb-username} --audit-password {immudb-pw} --server-signing-pub-key {state-public-key} +``` + + + + + +## Running immuclient Auditor as a service + +immuclient as Auditor can be installed in the system with the following command: + +Install service: + +```bash +immuclient audit-mode install +``` + +In this case, all parameters are written into the `immuclient` configuration file: + +- Linux: `/etc/immudb/immuclient.toml` +- Windows: `C:\ProgramData\ImmuClient\config\immuclient.toml` + + + + + +## Auditor best practices + +### How can I be notified if my immudb instance was tampered? + +It's possible to provide an external url that will be triggered in case a tamper is detected. +By configuring `IMMUCLIENT_AUDIT_NOTIFICATION_URL`, a POST request will be sent with the following body: + +``` +{ + "current_state": { + "hash": "string", + "signature": { + "public_key": "string", + "signature": "string" + }, + "tx": 0 + }, + "db": "string", + "password": "string", + "previous_state": { + "hash": "string", + "signature": { + "public_key": "string", + "signature": "string" + }, + "tx": 0 + }, + "run_at": "2020-11-13T00:53:42+01:00", + "tampered": true, + "username": "string" +} +``` + +NOTE: it's not possible to know at which transaction the database was tampered. The Auditor checks every second if the data was tampered - so it's only possible to know at which time frame the tampering was detected. + +### How many Auditors should I run to secure my immudb instance? + +A proper setup of one immuclient instance can fit most of cases, but there are ways to increase the security on detecting tampering. A single instance can go offline for any reason: network problems, hardware failures or attacks. Therefore a good practice can be to have multiple Auditor instances running in different zones. + + diff --git a/src/1.9.4/production/backup.md b/src/1.9.4/production/backup.md new file mode 100644 index 0000000000..19b7777a96 --- /dev/null +++ b/src/1.9.4/production/backup.md @@ -0,0 +1,125 @@ + +# Hot backup and restore + + + +Hot backup/restore feature allows to backup and restore immudb database without stopping immudb engine. Database remains accessible during backup process. It is possibly to perform full or incremental/differential backup and restore. + +Both backup and restore functions can use streams or files as a source/destination. + +Backup file is not compressed, assuming user may use any suitable method (see examples for bzip2 compression). + + + + + +## Backup + +``` +immuadmin hot-backup [-o [--append]] [--start-tx] +``` + +### Full backup + +To run full database backup, execute `immuadmin hot-backup ` command, specifying the optional backup file name with `-o` options. If `-o` option is not specified, output is sent to `stdout`. + +If backup file is specified with `-o` option, the file is created. If file already exists, backup process fails. + +### Incremental backup + +When backup database up to the existing file, `immuadmin` tools finds the last backed up database transaction in file, verifies its checksum and appends only database changes, made after this transaction. `immuadmin` requires user to specify `--append` command line option to append to existing file. + +When backup up to the stream, `immuadmin` doesn't have information about last backup up transaction, however user can specify the ID of the transaction to start from with `--start-tx` command line option. It allows user to implement incremental/differential backup strategy, using streams. + + + + + +## Restore +``` +immuadmin hot-restore [-i ] [--append] [--force] [--force-replica] +``` + +### Full restore + +To run full restore, execute `immuadmin hot-restore ` command, specifying the optional backup file name with `-o` options. If `-o` option is not specified, input data is read from `stdin`. + +If database already exists, restore process fails. + +::: tip +To boost performance while restoring from a backup, immudb could be run without strong durability warranties i.e. `./immudb --synced=false`. If a reboot or crash occurs while restoring from a backup, you may have to restart the restore process. + +Synced mode prevents data loss from unexpected crashes and shutdowns, but affects performance (default true), which is why you may want to restart immudb server with default settings after restoring. +::: + +### Incremental restore + +If database already exists, it is possible to append new data from backup file to the database. In this case user has to specify `--append` flag. + +#### Transaction overlap/gap handling + +`immuadmin` tries to verify that backup file and database where data are being restored to have the same origin. To do this `immuadmin` finds the last transaction in source database and the same transaction in the backup file, and check transaction signatures. If transactions don't match, restore isn't possible. + +When there is no overlap between transactions in database and file, transaction verification is not possible. However, if there is no gap between transactions, `immuadmin` allows to bypass verification with `--force` command line option. If there is a gap between last transaction in database and first transaction in file, restore isn't possible. + +### Transaction verification + +During restore process `immuadmin` checks if checksum, reported by database after restoration of the transaction, matches the one stored in the file during backup process. It allows to detect backup file accidental or malicious corruption. + +### Replica flag handling + +It is possible to restore data only to the replica database. During full restore database automatically created as replica (replica flag is switched off after restore), but for incremental restore `immuadmin` assumes database is already in replica mode (user can use `immuadmin database update --replication-is-replica` command to switch on replica mode). + +However, it is possible to automatically switch on and off replica mode for incremental backup using `--force-replica` command line option. + + + + + +### Verifying backup file + +``` +immuadmin hot-restore --verify [-i ] +``` +It is possible to verify backup file/stream using `immuadmin hot-restore --verify` command. It only checks the correctness of database file, e.g. file format and correct sequence of transactions in file. The only way to detect data corruption is to restore data. + + + + + +## Examples + +Full backup to file: +``` +immuadmin hot-backup foo -o foo.backup +immuadmin hot-backup foo > foo.backup +``` + +Incremental backup to file: +``` +immuadmin hot-backup foo -o foo.backup --append +``` +Incremental backup with `bzip2` compression: +``` +immuadmin hot-backup foo --start-tx 123 | bzip2 > foo.bz2 +``` +Full restore +``` +immuadmin hot-restore bar -i foo.backup +immuadmin hot-restore bar < foo.backup +``` +Full restore from `bzip2`-compressed file +``` +bunzip2 foo.bz2 -c | immuadmin hot-restore bar +``` +Incremental restore with automatic switching of replica mode +``` +immuadmin hot-restore bar -i foo.backup --append --force-replica +``` + +Copy database: +``` +immuadmin hot-backup foo | immuadmin hot-restore bar +``` + + diff --git a/src/1.9.4/production/backwards-compatibility.md b/src/1.9.4/production/backwards-compatibility.md new file mode 100644 index 0000000000..316d413310 --- /dev/null +++ b/src/1.9.4/production/backwards-compatibility.md @@ -0,0 +1,67 @@ +# Backwards compatibility + + + +### immudb 1.1 proof compatibility + +immudb 1.2 introduced KV metadata to support new features such as logical deletion or data expiration. +This change required updates to the way a transaction hash is calculated. +The downside of such change is that immudb clients using immudb 1.2+ +needed an updated method of proof calculation in order to verify newly added data. + +In some cases it is very hard or impossible to update the verification code on the client side. +If this is the case, immudb offers a way to disable metadata to maintain compatibility with older clients. + + + + + +#### Enabling the 1.1 proof compatibility mode + +*Note: backwards compatibility mode is currently not available for the `detaultdb` database.* + +When creating new database, the mode can be specified with: + +```bash +$ ./immuadmin database create --write-tx-header-version 0 +``` + +Enabling compatibility mode for existing databases can be done by: + +```bash +$ ./immuadmin database update --write-tx-header-version 0 +``` + +*Note: immudb restart is needed to make this change effective.* + +In order to re-enable metadata-enhanced proofs, +update database settings with `--write-tx-header-version 1` option. + + + + + +#### Limitations of 1.1 compatibility mode + +Switching to 1.1-compatible proof mode will disable metadata support and thus will make the following operations unavailable: + +* For KV interface: + * Logical deletion + * Data expiration + * Non-indexable entries +* For SQL interface: + * Logical deletion + * Updates to indexed columns + +Make sure to test your application before enabling the 1.1 compatibility mode. + +#### Working with database that already contains metadata-enhanced entries + +Even though old clients can not validate proofs for metadata-enhanced records, +those can still read the data without proofs as long as those don't use metadata. +Operations such as `Get`, `Scan` or `History` will not cause errors in such workloads. + +If proofs are needed, KV entries that were previously added with metadata should +be re-added to the database after enabling immudb 1.1 compatibility mode. + + diff --git a/src/1.9.4/production/fips.md b/src/1.9.4/production/fips.md new file mode 100644 index 0000000000..a57145bd8e --- /dev/null +++ b/src/1.9.4/production/fips.md @@ -0,0 +1,43 @@ +# FIPS 140-2 + + + +The Federal Information Processing Standard (FIPS) 140-2 publication describes United States government-approved security requirements for cryptographic modules. [FIPS-140](https://csrc.nist.gov/publications/detail/fips/140/2/final) series is a collection of computer security standards set by the National Institute of Standards and Technology (NIST) for the United States government. FIPS 140–2 defines the critical security parameters vendors must use for encryption before selling their products to the U.S government. + +For a fully FIPS-compliant deployment of immudb a few things are required: + +- immudb must be compiled with a FIPS validated cryptographic module +- immudb must be configured to use FIPS-approved cryptographic algorithms +- immudb components (immuadmin and immuclient) must be compiled with a FIPS-validated cryptographic module + +For immudb, adherence to FIPS 140-2 is ensured by: + +- Using FIPS approved / NIST-recommended cryptographic algorithms through the use of `goboring/golang` container image. Since the native go crypto standard library is not FIPS compliant, we use the Google-provided Go implementation that has patches on top of standard Go to enable integrating BoringCrypto. immudb components are built with this image as a build base. +- Enabling [`fipsonly`](https://go.googlesource.com/go/+/dev.boringcrypto/src/crypto/tls/fipsonly/fipsonly.go) mode to restrict all TLS configuration in immudb binaries to FIPS-approved settings. + + + + + +### Limitations + +- Currently the builds with FIPS-compliance are only available on `linux-amd64` architecture. +- There is an overhead in calling into BoringCrypto via cgo for the crypto library functions, which incurs a performance penalty. The library performs slower than the built-in crypto library. Hence you could see a performance drop of ~15% when using a FIPS-compliant immudb server. + + + + + +### Using FIPS-compliant binaries + +You can download the immudb binary from the [latest releases](https://github.com/codenotary/immudb/releases) on Github. The FIPS-compliant binaries have a `-fips` suffix. (e.g. immudb-v1.4.x-Linux-amd64-fips) + + + + + +### Using FIPS-compliant docker images + +You can pull immudb FIPS-compliant docker images from [DockerHub](https://hub.docker.com/r/codenotary/immudb) and run it in a ready-to-use container. The FIPS-compliant docker images have a `-fips` suffix. (e.g. codenotary/immudb-fips:latest) + + diff --git a/src/1.9.4/production/index-maintenance.md b/src/1.9.4/production/index-maintenance.md new file mode 100644 index 0000000000..de90601e26 --- /dev/null +++ b/src/1.9.4/production/index-maintenance.md @@ -0,0 +1,140 @@ +# Index cleaning + + + +Maintaining a healthy disk usage is crucial. immudb has two operations operations aiming to remove unreferenced data from the index. +A full index clean-up is achieved by calling `CompactIndex`, which is a routine that creates a fresh index based on the current state, removing all intermediate data generated over time. +The index is generated asynchronously, so new transactions may take place while it is created. As a result, if the server is constantly overloaded, there will likely be blocking times when the newly compacted index replaces the current one. + +In the case of continuous load on the server, the `FlushIndex` operation may be used instead. It will dump the current index into disk while partly removing unreferenced data. The `cleanupPercentage` attribute indicates how much space will be scanned for unreferenced data. Even though this operation blocks transaction processing, choosing a small percentage e.g. 0.1 may not significantly hinder normal operations while reducing used storage space. + +Partial compaction may be triggered automatically by immudb. Database settings can be modified to set the `cleanupPercentage` attribute to non-zero in order to accomplish this. + + + +:::: tabs + +::: tab Go +<<< @/src/code-examples/go/maintenance-index/main.go +::: + +::: tab Java + +```java +package io.codenotary.immudb.helloworld; + +import io.codenotary.immudb4j.*; + +public class App { + + public static void main(String[] args) { + FileImmuStateHolder stateHolder = FileImmuStateHolder.newBuilder() + .withStatesFolder("./immudb_states") + .build(); + + ImmuClient client = ImmuClient.newBuilder() + .withServerUrl("127.0.0.1") + .withServerPort(3322) + .withStateHolder(stateHolder) + .build(); + + client.login("immudb", "immudb"); + + // partial index cleanup + client.flushIndex(0.1, false); + + // full async index cleanup + client.cleanIndex(); + } + +} +``` +::: + +::: tab .NET + +```csharp + +var client = new ImmuClient(); +await client.Open("immudb", "immudb", "defaultdb"); + +await client.FlushIndex(0.1f, false); + +await client.Close(); + +``` + +::: + +::: tab Python +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Python sdk github project](https://github.com/codenotary/immudb-py/issues/new) +::: + +::: tab Node.js +This feature is not yet supported or not documented. +Do you want to make a feature request or help out? Open an issue on [Node.js sdk github project](https://github.com/codenotary/immudb-node/issues/new) +::: + +::: tab Others +If you're using another development language, please refer to the [immugw](../connecting/immugw.md) option. +::: + +:::: + + + +## How indexing works + +immudb uses a btree to index key-value entries. While the key is the same submitted by the client, the value stored in the btree is an offset to the file where the actual value as stored, its size and hash value. + +The btree is keep in memory as new data is inserted, getting a key or even the historical values of a key can directly be made by using a mutex lock on the btree but scanning by prefix requires the tree to be stored into disk, this is referred as a snapshot. +The persistence is implemented in append-only mode, thus whenever a snapshot is created (btree flushed to disk), updated and new nodes are appended to the file, while new or updated nodes may be linked to unmodified nodes (already written into disk) and those unmodified nodes are not rewritten. + +The snapshot creation does not necessarily take place upon each scan by prefix, it's possible to reuse an already created one, client can provide his requirements on how new the snapshot should be by providing a transaction ID which at least must be indexed (sinceTx). + +After some time, several snapshots may be created (specified by flushAfter properties of the btree and the scan requests), the file backing the btree will hold several old snapshots. Thus the clean index process will dump to a different location only the latest snapshot but in this case also writing the unmodified nodes. Once that dump is done, the index folder is replaced by the new one. + +While the clean process is made, no data is indexed and there will be an extra disk space requirement due to the new dump. Once completed, a considerable disk space will be reduced by removing the previously indexed data (older snapshots). +The btree and clean up process is something specific to indexing. And will not lock transaction processing as indexing is asynchronously generated. + + + + + +## compactor tool + +To manage index compaction, you can use the [compactor](https://github.com/codenotary/immudb-tools/tree/main/compactor) tool, +part of the [immudb-tools](https://github.com/codenotary/immudb-tools) repository. + +This tool can be used to perform periodic maintenance on your database indexes, or to configure online compaction. + +The maintenance can be performed in three different ways: +- online compaction +- percentage compaction +- full flush + +In all three modes, new indexes are calculated and old one are discarded. Indexes are organized in chunk files; each time a file only contains discarded indexes, it is automatically deleted. + +### Online compaction + +This kind of compaction is performed by immudb during normal write operations: once the amount of new written data reaches the percentage threshold configured per one database, immudb cleans up specified percentage of the index data, discarding old unreferenced data. + +For every database, users can specify a percentage of total written data to be reindexed on every write. + +The compactor tool can be used to enable this mode, and to set the percentage threshold. Once this is done, there is no need to run compactor tool periodically: the compaction will happen automatically. + + +### Flush compaction + +In this mode, the tool calls for immudb to immediately perform a partial compaction, reindexing the oldest data up to the specified percentage. It is similar to the previous mode, but it is performed immediately and must be periodically issued. +The advantage is that you have control on the time when compaction is performed, so that you can leverage periods of less intense activity (e.g.: weekends or nights). + + +### Full compaction + +All indexes are rebuilt. Very resource intensive, but it gives you the most compact representation of indexes. + +You can get more information in the [README](https://github.com/codenotary/immudb-tools/tree/main/compactor) + + diff --git a/src/1.9.4/production/monitoring.md b/src/1.9.4/production/monitoring.md new file mode 100644 index 0000000000..f6978988b4 --- /dev/null +++ b/src/1.9.4/production/monitoring.md @@ -0,0 +1,517 @@ +# Health Monitoring + + + +## Prometheus metrics + +immudb exposes a Prometheus end-point, by default on port 9497 on `/metrics`.
+ +```bash +$ curl -s http://localhost:9497/metrics +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 1.3355e-05 +go_gc_duration_seconds{quantile="0.25"} 1.3615e-05 +go_gc_duration_seconds{quantile="0.5"} 1.9991e-05 +go_gc_duration_seconds{quantile="0.75"} 3.0348e-05 +go_gc_duration_seconds{quantile="1"} 3.3859e-05 +go_gc_duration_seconds_sum 0.000151623 +go_gc_duration_seconds_count 7 +# HELP go_goroutines Number of goroutines that currently exist. +... +``` + +Querying metrics with a simple curl command is not a very practical solution. immudb has predefined Grafana dashboard visualizing some of the key metrics. This dashboard can be downloaded from [immudb github repository][grafana-dashboard]. + +[grafana-dashboard]: https://github.com/codenotary/immudb/blob/master/tools/monitoring/grafana-dashboard.json + +
+ +![immudb grafana stats](/immudb/grafana-immudb.png) + +
+ +You can also use `immuadmin stats` to see these metrics without additional tools: + +```bash +./immuadmin stats +``` + +
+ +![immuadmin stats](/immudb/immuadmin-stats.png) + +
+ +immudb exports the standard Go metrics, so dashboards like [Go metrics](https://grafana.com/grafana/dashboards/10826) work out of the box. + +
+ +![immuadmin stats](/immudb/grafana-go.jpg) + +
+ +For very simple cases, you can use `immuadmin status` from monitoring scripts to ping the server: + +```bash +$ ./immuadmin status +OK - server is reachable and responding to queries +``` + + + + + +## Database size + +Following metric contains the information about the disk space usage in bytes for each individual database: + +```bash +$ curl -s localhost:9497/metrics | grep immudb_db_size_bytes +# HELP immudb_db_size_bytes Database size in bytes. +# TYPE immudb_db_size_bytes gauge +immudb_db_size_bytes{db="defaultdb"} 2929 +immudb_db_size_bytes{db="systemdb"} 3789 +``` + +The official [grafana dashboard][grafana-dashboard] is using this metric +to show the amount of disk space used by databases and the growth of disk usage over time. + +Those dashboards can be used to forecast disk usage and make sure it’s not getting out of control. + +
+ +![database size](/immudb/metrics-dbsize.jpg) + +
+ + + + + +## Database entries + +Following prometheus metric calculates the number of new KV entries added to the database since the start of the process: + +```bash +$ curl -s http://localhost:9497/metrics | grep immudb_number_of_stored_entries +# HELP immudb_number_of_stored_entries Number of key-value entries currently stored by the database. +# TYPE immudb_number_of_stored_entries gauge +immudb_number_of_stored_entries{db="defaultdb"} 1 +immudb_number_of_stored_entries{db="systemdb"} 2 +``` + +[Grafana dashboard][grafana-dashboard] show the amount of new DB entries and the insertion rate. +This chart does not display the total amount of KV entries and will be reset to zero upon immudb restart. + +
+ +![database entries](/immudb/metrics-entries.jpg) + +
+ + + + + +## Indexer metrics + +Immudb does expose metrics related to internal indexing process: + +```bash +$ curl -s http://localhost:9497/metrics | grep 'immudb_last_.*_trx_id' +# HELP immudb_last_committed_trx_id The highest id of committed transaction +# TYPE immudb_last_committed_trx_id gauge +immudb_last_committed_trx_id{db="defaultdb"} 1 +immudb_last_committed_trx_id{db="systemdb"} 2 +# HELP immudb_last_indexed_trx_id The highest id of indexed transaction +# TYPE immudb_last_indexed_trx_id gauge +immudb_last_indexed_trx_id{db="defaultdb"} 1 +immudb_last_indexed_trx_id{db="systemdb"} 2 +``` + +Those metrics are used on various graphs on the [Grafana dashboard][grafana-dashboard]. + + + + + +### Indexed % + +This graph shows the total percentage of all transactions in the database that has been indexed so far. +In a healthy situation, this chart should remain at or close to 100%. +If this value starts dropping down, that means that the data ingestion rate is higher than the indexing rate +indicating that additional rate limiting should be added to db writers. + +
+ +![database entries](/immudb/metrics-indexed-percent.jpg) + +
+ + + + + +### Indexing / Commit rate + +This chart shows the rate of new transactions added to the database and the rate of indexing those transactions. If the indexing rate is smaller than the commit rate, this means that the database isn’t keeping up with the indexing. In applications where only synchronous writes are performed or where data can be immediately indexed, the indexing rate line (Idx) and commit rate line (Cmt) will overlap. + +
+ +![database entries](/immudb/metrics-indexing-rate.jpg) + +
+ + + + + +### TRXs Left to Index + +This chart shows the number of transactions waiting for indexing. This value should be close to zero and should have a decreasing tendency. + +
+ +![database entries](/immudb/metrics-indexing-left.jpg) + +
+ + + + + +### TRX Count + +This chart shows the total number of transactions in the database. + +
+ +![database entries](/immudb/metrics-trx-count.jpg) + +
+ + + + + +## Btree metrics + +immudb exposes various metrics per btree instance. The `id` label is a relative path for the location of the btree on disk. +Currently there's a single btree instance per one database. + +```bash +$ curl -s http://localhost:9497/metrics | grep 'immudb_btree_' +# HELP immudb_btree_depth Btree depth +# TYPE immudb_btree_depth gauge +immudb_btree_depth{id="data/defaultdb/index"} 1 +immudb_btree_depth{id="data/systemdb/index"} 1 +# HELP immudb_btree_flushed_entries_last_cycle Numbers of btree entries written to disk during the last flush process +# TYPE immudb_btree_flushed_entries_last_cycle gauge +immudb_btree_flushed_entries_last_cycle{id="data/defaultdb/index"} 1 +immudb_btree_flushed_entries_last_cycle{id="data/systemdb/index"} 2 +# HELP immudb_btree_flushed_entries_total Number of btree entries written to disk during flush since the immudb process was started +# TYPE immudb_btree_flushed_entries_total counter +immudb_btree_flushed_entries_total{id="data/defaultdb/index"} 1 +immudb_btree_flushed_entries_total{id="data/systemdb/index"} 3 +# HELP immudb_btree_flushed_nodes_last_cycle Numbers of btree nodes written to disk during the last flush process +# TYPE immudb_btree_flushed_nodes_last_cycle gauge +immudb_btree_flushed_nodes_last_cycle{id="data/defaultdb/index",kind="inner"} 0 +immudb_btree_flushed_nodes_last_cycle{id="data/defaultdb/index",kind="leaf"} 1 +immudb_btree_flushed_nodes_last_cycle{id="data/systemdb/index",kind="inner"} 0 +immudb_btree_flushed_nodes_last_cycle{id="data/systemdb/index",kind="leaf"} 1 +# HELP immudb_btree_flushed_nodes_total Number of btree nodes written to disk during flush since the immudb process was started +# TYPE immudb_btree_flushed_nodes_total counter +immudb_btree_flushed_nodes_total{id="data/defaultdb/index",kind="inner"} 0 +immudb_btree_flushed_nodes_total{id="data/defaultdb/index",kind="leaf"} 2 +immudb_btree_flushed_nodes_total{id="data/systemdb/index",kind="inner"} 0 +immudb_btree_flushed_nodes_total{id="data/systemdb/index",kind="leaf"} 3 +# HELP immudb_btree_leaf_node_entries Histogram of number of entries in as single leaf btree node, calculated when visiting btree nodes +# TYPE immudb_btree_leaf_node_entries histogram +immudb_btree_leaf_node_entries_bucket{id="data/defaultdb/index",le="1"} 7 +immudb_btree_leaf_node_entries_bucket{id="data/defaultdb/index",le="2"} 7 +.... +immudb_btree_leaf_node_entries_sum{id="data/defaultdb/index"} 4 +immudb_btree_leaf_node_entries_count{id="data/defaultdb/index"} 7 +immudb_btree_leaf_node_entries_bucket{id="data/systemdb/index",le="1"} 8 +immudb_btree_leaf_node_entries_bucket{id="data/systemdb/index",le="2"} 11 +.... +immudb_btree_leaf_node_entries_sum{id="data/systemdb/index"} 11 +immudb_btree_leaf_node_entries_count{id="data/systemdb/index"} 11 +# HELP immudb_btree_nodes_data_begin Beginning offset for btree nodes data file +# TYPE immudb_btree_nodes_data_begin gauge +immudb_btree_nodes_data_begin{id="data/defaultdb/index"} 0 +immudb_btree_nodes_data_begin{id="data/systemdb/index"} 0 +# HELP immudb_btree_nodes_data_end End offset for btree nodes data appendable +# TYPE immudb_btree_nodes_data_end gauge +immudb_btree_nodes_data_end{id="data/defaultdb/index"} 100 +immudb_btree_nodes_data_end{id="data/systemdb/index"} 281 +``` + +[Grafana dashboard][grafana-dashboard] exposes both some basic and more advanced btree statistics. + + + + + +### Btree Cache Size / Btree Cache Hit % + +Those two charts show internal statistics about immudb btree cache. +In order to avoid reading large amounts of data on every btree operation, +immudb keeps an in-memory cache of recently used btree nodes. +The amount of nodes in the cache is shown in the first chart, +it is capped at the maximum amount of cache entries. + +The second chart shows how effective the cache is +presenting the percentage of btree node lookups that were optimized with the cache. +For small databases, it’s very likely that this hit ratio will be close to 100%, +but it will drop down once the amount of data increases. +There’s no single rule on what value we should expect here. +In our internal tests even 40% cache hit ratios with workloads using keys with random distribution were still yielding very good performance results. +To get higher cache utilization, applications should prefer working on keys close to themselves - +such as using sequentially increasing numbers where newly inserted data will end up +in the btree portion close to previously accessed entries. + +
+ +![database entries](/immudb/metrics-btree-hit.jpg) + +
+ + + + + +### Btree Depth + +This chart shows the depth of the tree. +Since btrees are auto-balancing data structures, +this depth will have a logarithmic tendency. +The depth of the tree indicates what is the amount of nodes traversed by each btree operation. + +
+ +![database entries](/immudb/metrics-btree-depth.jpg) + +
+ + + + + +### Btree Child Node Count Distributions + +These graphs show the distribution of the amount of child nodes. +In a healthy btree like the one below, +the amount of child nodes should be focused around a single value (40 in the example). +Also the amount of child nodes should be kept at sane levels - +values below 10 or above few hundred are a good indication that the btree isn’t performing well +and the application should consider using keys of different, +more uniform and shorter lengths for its data. + +_Note: These statistics are gathered when traversing the btree, if there’s no activity in the database the distribution will be flat._ + +
+ +![database entries](/immudb/metrics-btree-distribution.jpg) + +
+ + + + + +### Flush Statistics + +immudb keeps recent btree changes in memory to reduce the amount of data to be written to disk. +In order to persist those changes, there’s a btree flush process called once a threshold of new and modified entries is reached. + +
+ +![database entries](/immudb/metrics-btree-flush-entries.jpg) +![database entries](/immudb/metrics-btree-flush-nodes.jpg) + +
+ +These metrics are calculated for nodes (both inner and leaf ones) and separately for KV entries in the leaf nodes. + +The flush rate shows the rate of written nodes / entries per second. +It clearly shows where the flush process started and where it ended. + +Flush progress metrics for each flush cycle starts at zero and reach the total amount of entries processed during such single flush operation. +The next flush will repeat the process by starting from zero reaching the maximum value. +By looking at those maximum values, we can see how much data needs to be written to disk during flush operations. +During normal DB operation, it should be steady over time. +An unbound growth of those maximums could indicate that the flush operation is too aggressive and the threshold should be adjusted. + + + + + +### Compaction Statistics + +Similarly to flush statistics, immudb exposes the same set of values for full compaction. + +_Note: these values are gathered for overall compaction that fully rewrites the btree structure. immudb 1.2.3 introduced a new online compaction mode that gradually removes unused btree data during flush operation. This new compaction mode is not included in those charts._ + +
+ +![database entries](/immudb/metrics-btree-compaction-entries.jpg) +![database entries](/immudb/metrics-btree-compaction-nodes.jpg) + +
+ + + + + +### Data Size for Btree Nodes + +immudb internally uses append-only files to store data. +That is also used for btree nodes. +We don’t replace existing data on disk. +Instead we append a new modified version at the end of the data stream. +immudb 1.2.3 introduced new online compaction of btree data that deals with garbage +that naturally accumulates at the beginning of such data stream over time. +This chart can be used to ensure that the amount of data being used for btree nodes +is being reduced whenever a cleanup operation is performed. + +
+ +![database entries](/immudb/metrics-btree-data-size.jpg) + +
+ + + + + +## S3 storage metrics + +Various metrics are exposed when working with remote storage such as S3. +Those can be used to help analyzing s3 performance and resource consumption. + +```bash +$ curl -s http://localhost:9497/metrics | grep 'remote' +# HELP immudb_remoteapp_chunk_bytes Total number of bytes stored in chunks +# TYPE immudb_remoteapp_chunk_bytes gauge +.... +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Active"} 133631 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Cleaning"} 0 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="DownloadError"} 0 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Downloading"} 0 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Invalid"} 0 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Local"} 0 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Remote"} 1.048767e+06 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="UploadError"} 0 +immudb_remoteapp_chunk_bytes{path="defaultdb/aht/tree/",state="Uploading"} 0 +... +# HELP immudb_remoteapp_chunk_count Number of chunks used for immudb remote storage +# TYPE immudb_remoteapp_chunk_count gauge +.... +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="Active"} 1 +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="Cleaning"} 0 +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="DownloadError"} 0 +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="Downloading"} 0 +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="Invalid"} 0 +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="Local"} 0 +immudb_remoteapp_chunk_count{path="defaultdb/aht/tree/",state="Remote"} 1 +.... +# HELP immudb_remoteapp_corrupted_metadata Number of corrupted metadata detections in an immudb remote storage +# TYPE immudb_remoteapp_corrupted_metadata counter +immudb_remoteapp_corrupted_metadata 0 +# HELP immudb_remoteapp_download_events Immudb remote storage download event counters +# TYPE immudb_remoteapp_download_events counter +immudb_remoteapp_download_events{event="cancelled"} 0 +immudb_remoteapp_download_events{event="failed"} 0 +immudb_remoteapp_download_events{event="finished"} 0 +immudb_remoteapp_download_events{event="retried"} 0 +immudb_remoteapp_download_events{event="started"} 0 +immudb_remoteapp_download_events{event="succeeded"} 0 +# HELP immudb_remoteapp_open_time Histogram of the total time required to open a chunk of data stored on an immudb remote storage +# TYPE immudb_remoteapp_open_time histogram +immudb_remoteapp_open_time_bucket{le="0.1"} 0 +immudb_remoteapp_open_time_bucket{le="0.25"} 0 +.... +immudb_remoteapp_open_time_sum 0.747774699 +immudb_remoteapp_open_time_count 2 +# HELP immudb_remoteapp_read_bytes Total number of bytes read from immudb remote storage (including cached reads) +# TYPE immudb_remoteapp_read_bytes counter +immudb_remoteapp_read_bytes 0 +# HELP immudb_remoteapp_read_events Read event counters for immudb remote storage +# TYPE immudb_remoteapp_read_events counter +immudb_remoteapp_read_events{event="errors"} 0 +immudb_remoteapp_read_events{event="total_reads"} 0 +# HELP immudb_remoteapp_s3_download_bytes Number data bytes (excluding headers) downloaded from s3 +# TYPE immudb_remoteapp_s3_download_bytes counter +immudb_remoteapp_s3_download_bytes 2.097657e+06 +# HELP immudb_remoteapp_s3_upload_bytes Number data bytes (excluding headers) uploaded to s3 +# TYPE immudb_remoteapp_s3_upload_bytes counter +immudb_remoteapp_s3_upload_bytes 2.097657e+06 +# HELP immudb_remoteapp_uncached_read_bytes Direct (uncached) read byte counters for immudb remote storage +# TYPE immudb_remoteapp_uncached_read_bytes counter +immudb_remoteapp_uncached_read_bytes 2.097645e+06 +# HELP immudb_remoteapp_uncached_read_events Direct (uncached) read event counters for immudb remote storage +# TYPE immudb_remoteapp_uncached_read_events counter +immudb_remoteapp_uncached_read_events{event="errors"} 0 +immudb_remoteapp_uncached_read_events{event="total_reads"} 2 +# HELP immudb_remoteapp_upload_events Immudb remote storage upload event counters +# TYPE immudb_remoteapp_upload_events counter +immudb_remoteapp_upload_events{event="cancelled"} 0 +immudb_remoteapp_upload_events{event="failed"} 0 +immudb_remoteapp_upload_events{event="finished"} 2 +immudb_remoteapp_upload_events{event="retried"} 0 +immudb_remoteapp_upload_events{event="started"} 2 +immudb_remoteapp_upload_events{event="succeeded"} 2 +# HELP immudb_remoteapp_upload_time Histogram of the total time required to upload a chunk to the remote storage +# TYPE immudb_remoteapp_upload_time histogram +immudb_remoteapp_upload_time_bucket{le="0.1"} 0 +immudb_remoteapp_upload_time_bucket{le="0.25"} 1 +.... +immudb_remoteapp_upload_time_sum 0.983413478 +immudb_remoteapp_upload_time_count 2 +``` + +The section related to S3 in the [Grafana dashboard][grafana-dashboard] is in its early days +but already shows some basic insights into what the data performance is using immudb +with an AWS S3 backed data store. +The first chart shows the histogram of a single s3 object read duration, +the second one shows the inbound traffic needed for immudb operations. +If those values are high, +you should consider switching back to local disk and an alternative remote storage back-end such as EBS volumes. + +
+ +![database entries](/immudb/metrics-s3.jpg) + +
+ + + + + +## immuguard health check tool + +To help you check global immudb health, and reliably deploy immudb in production environment, you can use immuguard, +the health check tool you can find in the [immudb-tools](https://github.com/codenotary/immudb-tools) repository. + +This simple tool periodically polls immudb, checking the state of every loaded database, and exposes a simple REST endpoint +that can be used by docker or kubernetes probes. You simply have to configure it to talk to immudb, by providing its address +and some credentials. + +Then you can poll endpoint `/immustatus` on port 8085. If immudb is operating normally and all databases are responsive, you will get a 200 HTTP return code, and the string "OK". You will get a 500 HTTP code instead, and the string "FAIL" if immudb is not responding. + +This configuration snippet shows how to use the `/immustatus` endpoint on kubernetes: + +```yaml +livenessProbe: + httpGet: + path: /immustatus + port: 8085 + failureThreshold: 5 + periodSeconds: 60 +``` + +You can find more information about it on its [README](https://github.com/codenotary/immudb-tools/tree/main/immuguard) page. + + diff --git a/src/1.9.4/production/performance-guide.md b/src/1.9.4/production/performance-guide.md new file mode 100644 index 0000000000..da873a8e28 --- /dev/null +++ b/src/1.9.4/production/performance-guide.md @@ -0,0 +1,198 @@ +# Performance guide + + + +## Dependency on the B-Tree index + +Immudb is built in a layered approach. +The most low-level layer in immudb is an immutable log of changes. +An atomic entry in this log level corresponds to a single transaction. +Each transaction has an ID assigned - those IDs are increasing monotonically +thus it is easy to reference a specific transaction ID. + +Each transaction contains a list of Key-Metadata-Value entries which correspond to +changes to the database made within such transaction. + +Transactions ending up in immudb are protected from tampering attacks +by the parallel Merkle tree structure built from the transaction data. + +By default immudb builds an additional index based on a B-tree structure for fast key lookup. +This B-Tree is built in an asynchronous routine in parallel to the write operations. +This asynchronous nature of indexing can be used to gain significant performance gains +by avoiding strict dependency on the indexing. + +```text ++------------------+ +| SQL | ++---------+--------+ + | + v ++------------------+ +| KV (Indexed) | ++---------+--------+ + | + v ++------------------+ +| Log | ++---------+--------+ +``` + +### Using immudb as an immutable ledger + +To achieve the best performance immudb can be used as an immutable ledger only. +In such case the index can be completely ignored to avoid performance and time penalty due to indexing. +Below is the list of operations that do not rely on the index at all. + +* Data should be inserted into the database in asynchronous mode + that is enabled by setting the `noWait` GRPC argument to `true`. + To avoid dependency on the index one should also avoid using + conditional writes and [ExecAll][exec-all] operations. +* To read the data from the database, use one of the following operations: + * [GetAt][get-at-since] to get value of a key set at given tx ID, + * [GetAtTxID][get-at-TxID] to get all entries from given transaction, + * [TxScan][tx-scan] to perform an enhanced scan of entries within one transaction. + Standard retrieval of the most recent value for given key should not be used. + In most cases this means that the transaction ID must be stored in some external persistent storage. + +The SQL layer heavily depends on the B-tree index thus it can not be used when immudb is +treated as an immutable ledger only. + +[get-at-since]: ../develop/reading.md#get-at-and-since-a-transaction +[get-at-TxID]: ../develop/reading.md#get-at-txid +[tx-scan]: ../develop/transactions.md#txscan +[exec-all]: ../develop/transactions.md#execall +[cond-write]: ../develop/reading.md#conditional-writes + +### Indexed KV layer - asynchronous + +Working with Key-Value entries is hard without the ability to quickly get the value behind some key +which require the B-Tree index structure. + +The B-Tree index is built in an asynchronous mode - once data is inserted into the transaction log, +a separate routine periodically updates it. +By default all immudb operations wait for the index to be up to date +to ensure that the most recent writes are already fully processed. +In some use cases such waiting for the indexer is not necessary and can be skipped +or reduced that leads to a much greater performance of the application. +Below is the list of operations that adjust the index processing requirements. + +* Data can be inserted into the database in asynchronous mode + that is enabled by setting the `noWait` GRPC argument to `true`. + By doing so, the data is inserted as quickly as possible into the + transaction log and right after that, the response is sent to the caller + without waiting for the index at all. + To avoid dependency on the btree one must also avoid using + [conditional writes][cond-write] and [ExecAll][exec-all] operations that implicitly require + up-to-date index. +* Reading the data from the database should be done by using one of the following operations: + * [GetAt][get-at-since] to get value of a key set at given tx ID, + * [GetAtTxID][get-at-TxID] to get all entries from given transaction, + * [GetSince][get-at-since] where the indexer is only required to process up to given transaction ID, + * [TxScan][tx-scan] to perform an enhanced scan of entries within one transaction. + +The SQL layer heavily depends on the B-tree index thus it can not be used when relaxed indexing requirements are used. + +### Indexed KV layer - synchronous + +This mode of operation is very similar to the asynchronous one but with the requirement +that the B-tree index must be up-to-date. This is the default mode that immudb operates in. + +When using immudb in synchronous mode, all functionalities of immudb's KV interface can be used. +Certain operations such as [ExecAll][exec-all] or [conditional writes][cond-write] require up-to-date index +and should only be used when there's a guarantee that those will meet the performance +requirements of the application. + +### SQL + +When immudb is used as an SQL database, all operations require an up to date index. +This means that optimizations that relax B-tree indexing requirements can not be used. + +The SQL layer in immudb is built on top of the KV layer. +For each row in SQL table one entry is generated for the primary key. +That entry contains the whole row data. + +In addition to that, for every index on that table, one additional KV entry +is added that keeps the reference to the original entry created for the primary key. + + + + + +## Data modelling + +Applications can structure their data in many different ways. +The chosen data model will affect the performance of the immudb. +Below are some tips that can help optimizing and selecting the correct model for the data. + +### KV layer - key length + +Short keys should be preferred over long ones. Long keys naturally increase the disk space usage +increasing the overall IO pressure but also affect the shape of the B-Tree. The longer keys are used, +the deeper B-tree will become and thus a longer B-tree traversal needs to be performed to find specific +key entry. Larger B-tree will also mean that the internal B-tree cache will perform much worse reducing +hit/miss ratio. + +### KV layer - sequential keys + +Sequentially built keys (like those based on monotonically increasing numbers) +should be preferred over randomly generated ones. +This directly relates to hit/miss ration of the the B-tree cache. +Sequential keys tend to use similar parts of the B-tree thus there is a much higher +probability that B-tree traversal will use cached nodes. + +This is especially important when immudb is used with an S3 remote storage. +Any B-tree cache miss will result in a heavy read operation for the S3 server. + +### SQL layer - indexes + +All SQL operations in immudb are lowered to operations on the KV layer. +To optimize the performance of the SQL engine it is thus important to +understand how immudb generates keys for the KV layer from SQL data. + +Low-level keys generated by the SQL layer are directly related to +SQL indexes. Each index consists of serialized values of columns +that are part of the index. This means that the more columns are in +the index, the longer keys will be produced. Same happens with +column types. Small types such as INTEGER will result in short +low-level key where larger ones (such as VARCHAR with large limit) +will produce very long keys. + +Each table can have multiple indexes where each new index will +generate new entries inserted into the KV layer. It is thus important +to avoid creating unnecessary indexes. + + + + + +## immudb replicas + +immudb offers [replication](replication.md) mechanism where replicas follow the leader +node cloning its data. + +Such replica nodes can be used to handle read operations reducing the +load in the leader node. In such scenario it is important to ensure +that the replica is following the leader in asynchronous mode +resulting in an eventual consistency guarantees. + +Replication can be configured in various ways including tree-like topology +and multiple replication levels (like [replicas of replicas](replication.md#replica-of-a-replica)). +With such feature, the immudb cluster can be scaled to a very large topology with the ability +to handle huge read workloads. + + + + + +## embedded immudb vs standalone service + +immudb can be easily [embedded](../embedded/embedding.md) into other golang applications. +By doing so, the application does access immudb data directly without additional TCP connection. +That way the additional cost of handling TCP connectivity, GRPC serialization etc. are removed. + +It is important to note however that the embedded immudb interface is much simpler +than the GRPC API - e.g. it has no direct support for references or sorted sets. +In addition to that, any networking-related features such as replication or backups +must be handled by the application itself. + + diff --git a/src/1.9.4/production/planning.md b/src/1.9.4/production/planning.md new file mode 100644 index 0000000000..af95a41cb4 --- /dev/null +++ b/src/1.9.4/production/planning.md @@ -0,0 +1,35 @@ + +# Planning + + + +Before running a database in production, it is important to plan: + +- Computing resources +- Disk space +- Configuration +- Backups +- Health Monitoring + +### Computing Resources + +immudb was designed to have a stable memory/CPU footprint. + +Memory is pre-allocated based on specified maximum concurrency, maximum number of entries per transaction, cache sizes, etc. + +With the default settings, it's possible to stress immudb and memory usage should stay around 1.5GB (assuming low-sized values). Otherwise, memory will be needed to maintain the values within a transaction during commit time. + +### Disk space and data location + +immudb is an immutable database, this means all history is preserved and therefore disk usage is higher than a normal database. + +Data is stored in the directory specified by the `dir` option. + + + + + + + + + diff --git a/src/1.9.4/production/replication.md b/src/1.9.4/production/replication.md new file mode 100644 index 0000000000..a5a879afac --- /dev/null +++ b/src/1.9.4/production/replication.md @@ -0,0 +1,169 @@ + +# Replication + + + +### Replication strategy + +immudb includes support for replication by means of a follower approach. A database can be created or configured either to be a primary or a replica of another database. + +
+ +![replication using grpc clients](/immudb/replication-servers.jpg) + +
+ +During replication, primary databases have a passive role. The grpc endpoint `ExportTx` is used by replicas to fetch unseen committed transactions from the primary. + +Replicas are read only and any direct write operation will be rejected. Using replicas allow to distribute query loads. + +
+ +![replicator fetches committed txs via grpc calls and replicate them using in-process method invocations](/immudb/replication-comm.jpg) + +
+ + + + +### Replication and users + +As shown in the diagram above, the replicator fetches committed transaction from the primary via grpc calls. Internally, the replicator instantiates an immudb client (using the official golang SDK) and fetches unseen committed transactions from the primary. In order to do so, the replicator requires valid user credentials with admin permissions, otherwise the primary will reject any request. + + + + + +### Creating a replica + +Creating a replica of an existent database using immuadmin is super easy: + +```bash +$ ./immuadmin login immudb +Password: +logged in +$ ./immuadmin database create \ + --replication-is-replica \ + --replication-primary-username=immudb \ + --replication-primary-password=immudb \ + --replication-primary-database=defaultdb \ + replicadb +database 'replicadb' {replica: true} successfully created +``` + +::: tip + +Display all database creation flags with: + +```bash +$ ./immuadmin help database create +``` + +::: + +### Creating a second immudb instance to replicate systemdb and defaultdb behaves similarly + +Start immudb with enabled replication: + +```bash +$ ./immudb \ + --replication-is-replica \ + --replication-primary-username=immudb \ + --replication-primary-password=immudb \ + --replication-primary-host=127.0.0.1 +``` + +::: tip + +Display all replication flags: + +```bash +$ ./immudb --help +``` + +::: + + + + + +### Multiple replicas + +It's possible to create multiple replicas of a database. Each replica works independently of the others. + +
+ +![multiple replicas of the same primary database](/immudb/replication-multiple.jpg) + +
+ +Given the primary database acts in passive mode, there are no special steps needed in order to create more replicas. Thus, by repeating the same steps to create the first replica it's possible to configure new ones. + + + + + +### Replica of a replica + +In case many replicas are needed or the primary database is under heavy load, it's possible to delegate the creation of replicas to an existent replica. This way, the primary database is not affected by the total number of replicas being created. + +
+ +![a replica indirectly following the primary](/immudb/replication-chain.jpg) + +
+ + + + + +### External replicator + +By creating a database as a replica but with disabled replication, no replicator is created for the database and an external process could be used to replicate committed transactions from the primary. The grpc endpoint `ReplicateTx` may be used to externally replicate a transaction. + + + + + +### Heterogeneous settings + +Replication is configured per database. Thus, the same immudb server may hold several primary and replica databases at the same time. + +
+ +a single immudb server can hold multiple primary and replica databases + +
+ + + + + +### Replicator tool + +You may need to keep a copy of every database on one immudb instance on another, so that when a new database is created +on the main instance, a replicated database is created on the replica. + +In that case you can use the [replicator tool](https://github.com/codenotary/immudb-tools/tree/main/replicator), part of the +[immudb tools](https://github.com/codenotary/immudb-tools). + +This tool connects to two immudb instances, one main instance and a replica. Periodically, scans the list of databases +present on the main instance and it compares that with the list of databases present on the replica. If it finds any new +databases that are missing on the replicas, it will recreate it on the replica and it will configure it to start following +its counterpart on the main. + +If necessary (usually it is) it will also create the replication user on the main instance for the new database(s). + +Using this tool you won't need to manually configure replicated databases on replica instance(s). + +You can have more information about this tool on its [README page](https://github.com/codenotary/immudb-tools/tree/main/replicator). + + + + + + + + + + diff --git a/src/1.9.4/production/request-metadata.md b/src/1.9.4/production/request-metadata.md new file mode 100644 index 0000000000..22eb8d6235 --- /dev/null +++ b/src/1.9.4/production/request-metadata.md @@ -0,0 +1,26 @@ +# Request Metadata + +To enhance the auditing process, immudb can be configured to inject request information - such as user identifiers, IP addresses, and other relevant details - into transaction metadata. + + + +## Enabling request metadata logging + +Request metadata logging can be enabled by enabling the `--log-request-metdata` flag of the immudb command (or by setting +the corresponding `IMMUDB_LOG_REQUEST_METADATA` env var to `TRUE`). + +For example, when running the immudb docker image: + +```bash +$ docker run -e IMMUDB_LOG_REQUEST_METADATA=TRUE -d --net host --name immudb codenotary/immudb:latest +``` + +### Why should I enable request metadata? + +When this functionality is enabled, each transaction includes comprehensive metadata that provides context for the request executed by the immudb server. This metadata allows auditors and administrators to easily retrieve detailed information about the context of each transaction (see how to retrieve transaction metadata from an [SQL table](../develop/sql/querying.md#transaction-metadata)). + +Specifically, it enables the identification of who initiated the transaction, from which IP address, and any other pertinent request details. For example, if there is a need to investigate suspicious activity or trace the source of a particular change, this request metadata offers a clear and concise trail of the relevant data. + +Note that despite the extra information can increase the storage overhead, the benefits of enhanced transparency and accountability often outweigh the extra storage cost, ensuring that all actions within the database can be thoroughly examined and verified. + + diff --git a/src/1.9.4/production/retention.md b/src/1.9.4/production/retention.md new file mode 100644 index 0000000000..f8b37b1825 --- /dev/null +++ b/src/1.9.4/production/retention.md @@ -0,0 +1,118 @@ +# Retention + + + +## Data Retention + +Data retention refers to the practice of keeping data for a specific period of time before deleting it. This practice is commonly used in various industries and organizations to comply with legal and regulatory requirements, as well as to manage storage space and maintain data integrity. + +One of the primary benefits of data retention is its ability to help maintain disk space. By setting a retention period, organizations can automatically delete data that is no longer needed, freeing up disk space for new data. This can be particularly useful for organizations that deal with large amounts of data, such as those in the financial or healthcare industries, where storing vast amounts of data can be costly. + +In addition to helping maintain disk space, data retention also helps organizations manage their data more efficiently. By setting specific retention policies, organizations can ensure that data is only stored for as long as necessary and is deleted once it is no longer needed. This can help prevent data from being accidentally or maliciously retained beyond its usefulness, reducing the risk of data breaches and other security incidents. + + + + + + +## Data Retention in immudb + +In immudb, the data retention feature only deletes data that is stored in the value log, leaving the proofs and schema configuration data intact. This is an important aspect of the retention feature because it ensures that the immudb database remains functional and that the proofs and schema configuration data required for immudb to operate correctly are not deleted. + +The value log is where the actual values are stored in immudb. By only deleting data in the value log, the retention feature can remove old data from the immudb database while leaving the proofs and schema configuration data intact, hence freeing up disk space. + +For example, suppose an organization has set a retention period of six months for their immudb database. After six months, any data that is older than six months will be automatically deleted from the value log. + + + + + +## Settings + +Data retention is enabled per database. You can truncate data from the database in two ways: + +#### 1) While creating a database +```bash +Usage: + immuadmin database create {database_name} --retention-period={retention_period} --truncation-frequency={truncation_frequency} +Flags: + --retention-period duration duration of time to retain data in storage + --truncation-frequency duration set the truncation frequency for the database (default 24h0m0s) +``` +A background process is setup on creation of the database which runs every `truncation-frequency` seconds, and then truncates the data beyond the `retention-period` + +Please note that the default value of the `truncation-frequency` is set to 24 hours, and it does not need to be set explicitly when creating/updating a database. + +#### 2) Manually truncating data through immuadmin + +The following flags in the `immuadmin` tool will help in truncating data up to data retention period for your database. +```bash +Usage: + immuadmin database truncate [flags] + +Examples: +truncate --yes-i-know-what-i-am-doing {database_name} --retention-period {retention_period} + +Flags: + -h, --help help for truncate + --retention-period duration duration of time to retain data in storage + --yes-i-know-what-i-am-doing safety flag to confirm database truncation +``` + + + + + +## Setup + +This setup guides you through a simple demonstration of how data retention works in immudb. + +#### Before you begin + +Make sure you already have [immudb installed](../running/download.md). + +> Since you're running a local cluster, all nodes use the same hostname (`localhost`). + +#### Step 1. Start the cluster + +1. Run the immudb server: + + ```bash + $ immudb --dir test_data + ``` + +2. In a new terminal, use the [`immuadmin`](../connecting/clitools.md) command to create a database on the immudb server: + + Login to immudb + + ```shell + $ immuadmin login immudb + ``` + + Create a database `db` that sets up the retention period to 1 day. + + > Note that the default value of the `truncation-frequency` is set to 24 hours, and it does not need to be set explicitly when creating/updating a database. + + ```shell + $ immuadmin database create testdb \ + --retention-period=24h + ``` + + At this point, the `testdb` has been created on the server, and when every 24 hours, the data greater than the `retention-period` will be deleted from the value-log. + +3. Alternatively, you can use the [`immuadmin`](../connecting/clitools.md) command to truncate an existing database which has not been setup with retention period: + + Login to immudb + + ```shell + $ immuadmin login immudb -p 3324 + ``` + + ```shell + $ immuadmin database truncate --yes-i-know-what-i-am-doing=true testdb \ + --retention-period=24h + ``` + + At this point, the data beyond the retention period will be deleted in`testdb`. + + diff --git a/src/1.9.4/production/s3-storage.md b/src/1.9.4/production/s3-storage.md new file mode 100644 index 0000000000..46bb711542 --- /dev/null +++ b/src/1.9.4/production/s3-storage.md @@ -0,0 +1,18 @@ +# S3 Storage Backend + + + +immudb can store its data in the Amazon S3 service (or a compatible alternative). The following example shows how to run immudb with the S3 storage enabled: + +```bash +export IMMUDB_S3_STORAGE=true +export IMMUDB_S3_ACCESS_KEY_ID= +export IMMUDB_S3_SECRET_KEY= +export IMMUDB_S3_BUCKET_NAME= +export IMMUDB_S3_LOCATION= +export IMMUDB_S3_PATH_PREFIX=testing-001 +export IMMUDB_S3_ENDPOINT="https://${IMMUDB_S3_BUCKET_NAME}.s3.amazonaws.com" + +./immudb +``` + \ No newline at end of file diff --git a/src/1.9.4/production/sync-replication.md b/src/1.9.4/production/sync-replication.md new file mode 100644 index 0000000000..a8bfa55273 --- /dev/null +++ b/src/1.9.4/production/sync-replication.md @@ -0,0 +1,429 @@ +# Synchronous Replication + + + +## Synchronous Replication + +Replication is a common technique used in distributed databases to achieve scalable data distribution for better fault tolerance. Multiple replicas of a primary database server are created for higher durability. One of the replication methods is to update each replica as part of a single atomic transaction, also known as synchronous replication. Consensus algorithms apply this approach to achieve strong consistency on a replicated data set. immudb now supports the option for synchronous replication. + +### Architecture + +In synchronous replication, each commit of a write transaction will wait until there is a confirmation that the commit has been committed to both the primary and quorum of replica server(s). This method minimizes the possibility of data loss. + +immudb uses a quorum-based technique to enforce consistent operation in a distributed cluster. A quorum of replicas is used to ensure that synchronous replication is achieved even when replication is not completed across all replica servers. A quorum is a majority of the number of replicas in a cluster setup. The quorum can be set when creating or updating the database on the primary node. + +The primary server will wait for acknowledgment from a quorum of replica server(s) that each transaction is durably stored before proceeding. The drawback is that if enough replica server(s) go down or can’t commit a transaction, and the quorum is not reached, the primary server goes into a hung state. + +![synchronous replication](/immudb/replication-sync.png) + +Comparing this to the asynchronous replication mode, the primary server does not need to wait for transaction-completion acknowledgment from the replica server. The replication transactions queue up on the replica server, and the two servers can remain out-of-sync for a specified time until the processing completes. + +![asynchronous replication](/immudb/replication-async.png) + +immudb provides support for synchronous replication by means of a follower approach. There are two grpc endpoint used for replication: + +- `ExportTx`: Used by replicas to fetch precommitted transactions from the primary database server, and also to send the current database state to update the primary server. + +- `ReplicateTx`: Used by replicas to commit precommitted transactions (fetched from the primary) on the replica server. + +The primary server keeps a record of the current state of each replica. The current state of each replica is updated through the `ExportTx` grpc call from the replica server. So when a new transaction request comes to the primary server, it precommits the transaction, and checks if a quorum (on the transaction) has been reached by the replica server(s) by checking their state continuously. If the quorum was reached, the transaction is marked as successful. + +
+ +![how synchronous replication works](/immudb/replication-state.png) + +
+ + + + + +## Deciding on number of servers in a cluster + +Synchronous replication in a cluster can function only if the majority of servers are up and running. In systems with enabled data replication, it is important to consider the throughput of write operations. Every time data is written to the cluster, it needs to be copied to multiple replicas. Every additional server adds some overhead to complete this write. The latency of data write is directly proportional to the number of servers forming the quorum. + + + + + +## Settings + +Synchronous replication is enabled per database. The following flags in the `immuadmin` tool will help in setting up synchronous replication for your database. + +```bash +Flags: + --replication-allow-tx-discarding allow precommitted transactions to be discarded if the replica diverges from the primary + --replication-commit-concurrency uint32 number of concurrent replications (default 10) + --replication-is-replica set database as a replica + --replication-prefetch-tx-buffer-size uint32 maximum number of prefeched transactions (default 100) + --replication-primary-database string set primary database to be replicated + --replication-primary-host string set primary database host + --replication-primary-password string set password used for replication to connect to the primary database + --replication-primary-port uint32 set primary database port + --replication-primary-username string set username used for replication to connect to the primary database + --replication-sync-acks uint32 set a minimum number of replica acknowledgements required before transactions can be committed + --replication-sync-enabled enable synchronous replication + +``` + + + + + +## Setup + +This setup guides you through a simple demonstration of how synchronous replication works in immudb. Starting with a 2-node local cluster, you'll write some data and verify that it replicates in sync. + +#### Before you begin + +Make sure you already have [immudb installed](../running/download.md). + +> Since you're running a local cluster, all nodes use the same hostname (`localhost`). + +#### Step 1. Start the cluster + +1. Run the primary server: + + ```bash + $ immudb --dir data_primary + ``` + +2. In a new terminal, start replica server: + + ```bash + $ immudb --dir data_replica \ + --port=3324 \ + --pgsql-server=false \ + --metrics-server=false + ``` + +3. In a new terminal, use the [`immuadmin`](../connecting/clitools.md) command to create a database on the primary server: + + Login to immudb + + ```shell + $ immuadmin login immudb + ``` + + Create a database `db` that requires 1 confirmation from the synchronous replicas to do the commit. + + > Note that the number of confirmations needed (`--replication-sync-acks` option) should be set to `ceil(number of replicas/2)` + to achieve majority-based quorum. + + ```shell + $ immuadmin database create primarydb \ + --replication-sync-acks 1 \ + --replication-sync-enabled + ``` + + At this point, the `primarydb` has been created on the primary server. + +4. Use the [`immuadmin`](../connecting/clitools.md) command to create a database on the replica server: + + Login to immudb + + ```shell + $ immuadmin login immudb -p 3324 + ``` + + Create a database `replicadb` which will sync from the primary server's database `primarydb` + + ```shell + $ immuadmin database create replicadb -p 3324 \ + --replication-is-replica \ + --replication-primary-host 127.0.0.1 \ + --replication-primary-port 3322 \ + --replication-primary-database primarydb \ + --replication-primary-username immudb \ + --replication-primary-password immudb \ + --replication-sync-enabled \ + --replication-prefetch-tx-buffer-size 1000 \ + --replication-commit-concurrency 100 + ``` + + At this point, the `replicadb` has been created on the replica server to sync with the `primarydb` on primary server. + +#### Step 2. Send a request + +1. Use the [`immuclient`](../connecting/clitools.md) command to commit a transaction on the primary server: + + Login to immudb + + ```shell + $ immuclient login immudb + ``` + + Select database + + ```shell + $ immuclient use primarydb + ``` + + Set a value + + ```shell + $ immuclient safeset foo bar + ``` + +2. Verify the transaction on the replica server using the [`immuclient`](../connecting/clitools.md) command: + + Login to immudb + + ```shell + $ immuclient login immudb -p 3324 + ``` + + Select database + + ```shell + $ immuclient use primarydb -p 3324 + ``` + + Verify the key + + ```shell + $ immuclient safeget foo -p 3324 + ``` + +#### Step 3. Stop the replica server + +1. Stop the replica server running on port 3325 + +2. Send a transaction to the primary server: + + Login to immudb + + ```shell + $ immuclient login immudb + ``` + + Select database + + ```shell + $ immuclient use primarydb + ``` + + Set a value + + ```shell + $ immuclient safeset foo bar + ``` + + The client will block. This is because the primarydb requires 1 sync replica, and since the replica server is down, there is no ack from the replica server, hence synchronous transaction is blocked. + + + + + +## Recovering from a replica loss + +The primary node will continue read and write operations as long as the required quorum of replicas can send write confirmation to the primary node. +If there are not enough confirmations, write operations will be queued and will wait for enough replicas to synchronize with the cluster. +Read operations in such cases will continue to work. + +The simplest way to recover the replica is to simply add a new replica into the cluster and setup replication in the same way as during +the initial cluster setup, e.g.: + +```shell +$ immuadmin database create replicadb -p 3324 \ + --replication-is-replica \ + --replication-primary-host 127.0.0.1 \ + --replication-primary-port 3322 \ + --replication-primary-database primarydb \ + --replication-primary-username immudb \ + --replication-primary-password immudb \ + --replication-sync-enabled \ + --replication-prefetch-tx-buffer-size 1000 \ + --replication-commit-concurrency 100 +``` + +The new replica will start fetching transactions from the primary node and as soon as it synchronizes all transactions +it will become a valid member of the quorum for transaction confirmation. + +### Speeding up initial replica synchronization + +The synchronization process of a new replica may take a lot of time if the database is large or has to handle a lot of normal traffic. +Such replica will fetch all transactions performing additional checksum calculations and validations. +That way the security of the whole cluster is further hardened revealing tampering attempts in any transaction +in the database including those transactions that were not accessed for a very long time. + +There are situations however when the speed of recovery is crucial. +In such a situations the data of the database may be copied from another cluster node. +This should be done while the database is unloaded: + +#### Step 1. Create replica database + +```shell +$ immuadmin database create replicadb -p 3324 \ + --replication-is-replica \ + --replication-primary-host 127.0.0.1 \ + --replication-primary-port 3322 \ + --replication-primary-database primarydb \ + --replication-primary-username immudb \ + --replication-primary-password immudb \ + --replication-sync-enabled \ + --replication-prefetch-tx-buffer-size 1000 \ + --replication-commit-concurrency 100 +database 'replicadb' {replica: true} successfully created +``` + +#### Step 2. Unload replica from the database + +Once database is unloaded, we can safely work on the files of that database. + +```shell +$ immuadmin database unload replicadb +database 'replicadb' successfully unloaded +``` + +#### Step 3. Copy files from other node + +```shell +$ rsync -ave --delete \ + :/replicadb/ \ + :/replicadb/ +sending incremental file list +.... + +sent 590,357,187 bytes received 230 bytes 168,673,547.71 bytes/sec +total size is 590,212,158 speedup is 1.00 +``` + +> Note: if there are writes on the database happening during the sync, it is necessary to +> unload the source replica before copying files to avoid inconsistencies between database files. + +#### Step 4. Load database on new replica + +```shell +$ immuadmin database load replicadb +database 'replicadb' successfully unloaded +``` + + + + + +## Recovering from a primary loss + +Current immudb cluster setup requires the primary node to be always predefined. +This means that in case of a primary node loss, +it is necessary to manually promote a replica to become the primary node. +Generally, electing the new primary depends on the number of available instances, +their precommit state, and the replication-sync-acks setting on the primary. + +#### Step 1. Inspect states of all replicas in the cluster and choose the new primary node + +```shell +$ immuclient login immudb +Password: +Successfully logged in + +$ immuclient use replicadb +Now using replicadb + +$ immuclient status +database: replicadb +txID: 734931 +hash: 5e2f2feec159bc19c952a7a93832338a178936c5b258d0c906b7c145faf3a4b5 +precommittedTxID: 734931 +precommittedHash: 5e2f2feec159bc19c952a7a93832338a178936c5b258d0c906b7c145faf3a4b5 +``` + +It's important to carefully choose the new primary node in order to avoid losing committed transactions. +It is generally a good idea to promote some instance as a primary that has already precommitted the largest +transaction contained in at least `replication-sync-acks` instances. + +In the following scenario, we consider a three-node cluster with an unreachable primary: + +```shell +# state in replica1 +precommittedTxID: 734931 +precommittedHash: 5e2f2feec159bc19c952a7a93832338a178936c5b258d0c906b7c145faf3a4b5 + +# state in replica2 +precommittedTxID: 734920 +precommittedHash: 2a4f41c3d5b03ff014ca30b53d23ee3a098936c3b2a8a0d6e9b3b540cac166a1 +``` + +In the event that the primary node becomes unavailable, a replica with a higher precommittedTxID should be chosen as the primary. +If `replication-sync-acks` is 2, both replicas must acknowledge precommit before the primary can commit. +In the scenario above, this would mean 734920 was the most recent committed transaction. Therefore, replica2 could also be selected as the new primary. + +#### Step 2. Switch the selected replica to become new primary + +```shell +$ immuadmin database update replicadb -p 3324 \ + --replication-sync-enabled \ + --replication-sync-acks 1 \ + --replication-is-replica=false +database 'replicadb' {replica: false} successfully updated +``` + +> Note that the number of required sync replicas may be temporarily lowered due to the loss of the primary node. + +#### Step 3. Switch other replicas to follow new primary + +```shell +$ immuadmin database update replicadb -p 3325 \ + --replication-primary-host 127.0.0.1 \ + --replication-primary-port 3324 \ + --replication-primary-database replicadb +``` + +#### Step 4. Truncate precommitted transactions on other replicas if needed + +It may happen that the new replica will reject synchronizing with the new primary. +In such case immudb will report an error in logs: + +```text +immudb 2022/10/11 15:57:42 ERROR: replica precommit state at 'replicadb' diverged from primary's +``` + +To fix this issue the replica may need to discard precommited transactions. +This can be easily instructed with the flag `replication-allow-tx-discarding` as follows: + +```shell +$ immuadmin database update replicadb -p 3325 --replication-allow-tx-discarding +``` + +In the case immudb instance itself is run a replica, to fix that issue please restart immudb with the `--replication-allow-tx-discarding` flag that will discard any transaction on the replica that has not yet been fully committed. + +#### Step 5. Start a new replica to restore original cluster size + +Because the primary node was irrecoverably lost, a new replica should be spawned in its place. +Please refer to the previous section dealing with the loss of replica for more details +on how to add a replacement replica node. + +#### Step 6. Point immudb clients to the new primary node + +Clients performing write operations should now be switched to the new primary node. + + + + + +## Changing configuration of a locked primary database + +In most cases the primary database can be easily updated and the change will be applied without the need for a restart. +That way the primary node can change the number of required confirmations, +enable/disable synchronous replication and even be converted to a replica. + +There can be a situation though where the database is already blocked with writes waiting for confirmations from replicas. +This could happen if replicas became unavailable +or as a result of misconfiguration where the replicas quorum value was set to some large value. + +In this situation trying to change the configuration of the database will block as well and will be unblocked once +the database itself continues committing transactions. + +If the database can not be fixed to restore commits (e.g. if it is impossible to add enough synced replicas quickly enough), +the following workaround can be used (please note that it requires immudb restart): + +1. Update database settings, e.g. run `immuadmin database update` command - that operation will block indefinitely but will + already persist new database settings +2. Restart the immudb database instance - upon restart, the configuration of the database is read and applied from persistent settings + thus it will apply the configuration set in the previous step. + +With this approach, the number of required confirmations can be lowered down to the correct value +or disabled to switch to asynchronous replication. + + diff --git a/src/1.9.4/releasenotes.md b/src/1.9.4/releasenotes.md new file mode 100644 index 0000000000..c8e150e8ec --- /dev/null +++ b/src/1.9.4/releasenotes.md @@ -0,0 +1,8 @@ +# Release Notes + + + +- [immudb release notes](https://github.com/codenotary/immudb/releases) +- [immugw release notes](https://github.com/codenotary/immugw/releases) + + \ No newline at end of file diff --git a/src/1.9.4/running/build.md b/src/1.9.4/running/build.md new file mode 100644 index 0000000000..a1898036d6 --- /dev/null +++ b/src/1.9.4/running/build.md @@ -0,0 +1,46 @@ +# Building from source + + + +### Build the binaries + +Building binaries requires a Linux operating system. + +To build the binaries yourself, simply clone [immudb repository](https://github.com/codenotary/immudb) and run: + +```bash +make all +``` + +immudb can be cross compiled for different systems and architectures by setting `GOOS` and `GOARCH` variables, i.e.: + +```bash +GOOS=windows GOARCH=amd64 make all +``` + + + + + +### macOS specific + +The community already added immudb to [HomeBrew](https://formulae.brew.sh/formula/immudb), therefore you can simply run +```bash +brew install immudb +``` + + + + + +### Build the Docker images + +If you want to build the container images yourself, simply clone [immudb repository](https://github.com/codenotary/immudb) and run: + +```bash +docker build -t myown/immudb:latest -f Dockerfile . +docker build -t myown/immuadmin:latest -f Dockerfile.immuadmin . +docker build -t myown/immuclient:latest -f Dockerfile.immuclient . +``` + + diff --git a/src/1.9.4/running/configuration.md b/src/1.9.4/running/configuration.md new file mode 100644 index 0000000000..1f39db841c --- /dev/null +++ b/src/1.9.4/running/configuration.md @@ -0,0 +1,167 @@ +# Configuration + + + +This page describes how to set different settings in immudb. + +Settings can be specified as command line options to immudb (see `immudb -h`), in a configuration file, or as environment variables. + + + + + +### Settings + +| Parameter | Default | Description | +| ------------------------------------- | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `address` | `0.0.0.0` | bind address | +| `admin-password` | `immudb` | admin password as plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) | +| `auth` | `true` | enable auth | +| `certificate` | `` | server certificate file path | +| `clientcas` | `` | clients certificates list. Aka certificate authority | +| `config` | `` | config file (default path are configs or $HOME. Default filename is immudb. | +| `detached` | `false` | run immudb in background | +| `devmode` | `false` | enable dev mode: accept remote connections without auth | +| `dir` | `./data` | data folder | +| `force-admin-password` | `false` | if true, reset the admin password to the one passed through admin-password option upon startup | +| `log-request-metadata` | `false` | log request information in transaction metadata | +| `grpc-reflection` | `true` | GRPC reflection server enabled | +| `logfile` | `` | log path with filename. E.g. /tmp/immudb/immudb.log | +| `logformat` | `text` | log format e.g. text/json | +| `maintenance` | `false` | override the authentication flag | +| `max-recv-msg-size` | `33554432` | max message size in bytes the server can receive | +| `max-session-age-time` | infinity | max session age time is a duration after which session will be forcibly closed | +| `max-session-inactivity-time` | `3m0s` | max session inactivity time is a duration after which an active session is declared inactive by the server. A session is kept active if server is still receiving requests from client (keep-alive or other methods) | +| `max-sessions` | `100` | maximum number of simultaneously opened sessions | +| `metrics-server` | `true` | enable or disable Prometheus endpoint | +| `metrics-server-port` | `9477` | Prometheus endpoint port | +| `mtls` | `false` | enable mutual tls | +| `no-histograms` | `false` | disable collection of histogram metrics like query durations | +| `pgsql-server` | `true` | enable or disable pgsql server | +| `pgsql-server-port` | `5432` | pgsql server port | +| `pidfile` | `` | pid path with filename. E.g. /var/run/immudb.pid | +| `pkey` | `` | server private key path | +| `port` | `3322` | port number | +| `pprof` | `false` | add pprof profiling endpoint on the metrics server | +| `replication-allow-tx-discarding` | `false` | allow precommitted transactions to be discarded if the replica diverges from the primary | +| `replication-commit-concurrency` | `10` | number of concurrent replications | +| `replication-is-replica` | `false` | set systemdb and defaultdb as replica | +| `replication-prefetch-tx-buffer-size` | `100` | maximum number of prefeched transactions | +| `replication-primary-host` | `` | primary database host (if replica=true) | +| `replication-primary-password` | `` | password in the primary database used for replication of systemdb and defaultdb | +| `replication-primary-port` | `3322` | primary database port (if replica=true) (default 3322) | +| `replication-primary-username` | `` | username in the primary database used for replication of systemdb and defaultdb | +| `replication-skip-integrity-check` | `false` | disable integrity check when reading data during replication | +| `replication-sync-acks` | `0` | set a minimum number of replica acknowledgements required before transactions can be committed | +| `replication-sync-enabled` | `false` | enable synchronous replication | +| `replication-wait-for-indexing` | `false` | wait for indexing to be up to date during replication | +| `s3-access-key-id` | `` | s3 access key id | +| `s3-bucket-name` | `` | s3 bucket name | +| `s3-endpoint` | `` | s3 endpoint | +| `s3-external-identifier` | `` | use the remote identifier if there is no local identifier | +| `s3-instance-metadata-url` | `http://169.254.169.254` | s3 instance metadata url | +| `s3-location` | `` | s3 location (region) | +| `s3-path-prefix` | `` | s3 path prefix (multiple immudb instances can share the same bucket if they have different prefixes) | +| `s3-role` | `` | role name for role-based authentication attempt for s3 storage | +| `s3-role-enabled` | `false` | enable role-based authentication for s3 storage | +| `s3-secret-key` | `` | s3 secret access key | +| `s3-storage` | `false` | enable or disable s3 storage | +| `session-timeout` | `2m0s` | session timeout is a duration after which an inactive session is forcibly closed by the server | +| `signingKey` | `` | signature private key path. If a valid one is provided, it enables the cryptographic signature of the root. E.g. "./../test/signer/ec3.key" | +| `swaggerui` | `true` | Swagger UI enabled | +| `synced` | `true` | synced mode prevents data lost under unexpected crashes but affects performance | +| `token-expiry-time` | `1440` | client authentication token expiration time. Minutes | +| `web-server` | `true` | enable or disable web/console server | +| `web-server-port` | `8080` | web/console server port | + + + + + +### Configuration file + +Settings can be specified in a [immudb.toml configuration file](https://raw.githubusercontent.com/codenotary/immudb/master/configs/immudb.toml). + +Which configuration file to use is set with the `--config` option. By default, immudb looks into the `configs` subfolder in the current directory. + +When running immudb as a service, `immudb service install` allows to specify the configuration file to use with the `--config` option. + +### Environment variables + +Settings specified via environment variables take override the configuration file. They are specified in the form of `IMMUDB_`, for example `IMMUDB_DIR` specifies the `dir` variable. + + + + + +### Logging Levels + +The `LOG_LEVEL` environment variable sets the log level to be emitted from immudb logs. Valid logging level settings are `error`, `warn`, `info`(default), and `debug`. Logs that are equal to, or above, the specified level will be emitted. Log level `error` has the highest level, `debug` being the lowest. + +You can set the `LOG_LEVEL` when running immudb either by setting the environment variable, or by running the command as below: + +``` +LOG_LEVEL=error ./immudb +``` + +#### Levels + +##### info + +The `info` severity is used for informational messages that do not require action. + +##### warn + +The `warn` severity is used for messages that may require special handling, but does not affect normal operation. + +##### error + +The `error` severity is used for messages that require special handling, where a normal database operation could not proceed as expected. It does not block the database. + +##### debug + +The `debug` severity is used for messages that are used for debugging purpose for the database. + +### Logging formats + +Two logging format options are available: `text` and `json`. The default logging format setting is the `text`. The `json` format is available when specified. + +#### Examples of log output: + +##### Normal + +###### Command: + +``` +./immudb +``` + +###### Output: + +```bash +immudb 2022/11/17 14:30:02 INFO: Creating database 'systemdb' {replica = false}... +immudb 2022/11/17 14:30:02 INFO: Binary Linking up to date at 'data/systemdb' +immudb 2022/11/17 14:30:02 INFO: Index 'data/systemdb/index' {ts=0, discarded_snapshots=0} successfully loaded +immudb 2022/11/17 14:30:02 INFO: Indexing in progress at 'data/systemdb' +immudb 2022/11/17 14:30:02 INFO: Flushing index 'data/systemdb/index' {ts=0, cleanup_percentage=0.00/0.00, since_cleanup=0} requested via SnapshotSince... +immudb 2022/11/17 14:30:02 INFO: Index 'data/systemdb/index' {ts=0, cleanup_percentage=0.00/0.00} successfully flushed +``` + +##### JSON + +###### Command: + +``` +./immudb --logformat=json +``` + +###### Output: + +```bash +{"caller":"codenotary/immudb/pkg/database/database.go:179","component":"github.com/codenotary/immudb/pkg/database.OpenDB","level":"info","message":"Opening database 'systemdb' {replica = false}...","module":"immudb","timestamp":"2022-11-17T14:32:28.890774+05:30"} +{"caller":"codenotary/immudb/embedded/store/immustore.go:553","component":"github.com/codenotary/immudb/embedded/store.OpenWith","level":"info","message":"Binary Linking up to date at 'data/systemdb'","module":"immudb","timestamp":"2022-11-17T14:32:28.898035+05:30"} +{"caller":"codenotary/immudb/embedded/tbtree/tbtree.go:351","component":"github.com/codenotary/immudb/embedded/tbtree.Open","level":"info","message":"Reading snapshots at 'data/systemdb/index/commit'...","module":"immudb","timestamp":"2022-11-17T14:32:28.898296+05:30"} +{"caller":"codenotary/immudb/embedded/tbtree/tbtree.go:669","component":"github.com/codenotary/immudb/embedded/tbtree.OpenWith","level":"info","message":"Index 'data/systemdb/index' {ts=2, discarded_snapshots=0} successfully loaded","module":"immudb","timestamp":"2022-11-17T14:32:28.904722+05:30"} +``` + + diff --git a/src/1.9.4/running/download.md b/src/1.9.4/running/download.md new file mode 100644 index 0000000000..3dfff38c05 --- /dev/null +++ b/src/1.9.4/running/download.md @@ -0,0 +1,29 @@ +# Running + +::: tip +To learn interactively and to get started with immudb from the command line and with programming languages, visit the immudb Playground at +::: + + + +You may download the immudb binary from [the latest releases on Github](https://github.com/codenotary/immudb/releases/latest). Once you have downloaded immudb, rename it to `immudb`, make sure to mark it as executable, then run it. The following example shows how to obtain v1.9DOM.1 for linux amd64: + +```bash +$ wget https://github.com/vchain-us/immudb/releases/download/v1.9DOM.1/immudb-v1.9DOM.1-linux-amd64 +$ mv immudb-v1.9DOM.1-linux-amd64 immudb +$ chmod +x immudb + +# run immudb in the foreground to see all output +$ ./immudb + +# or run immudb in the background +$ ./immudb -d +``` + +Alternatively, you may [pull immudb docker image from DockerHub](https://hub.docker.com/r/codenotary/immudb) and run it in a ready-to-use container: + +```bash +$ docker run -d --net host -it --rm --name immudb codenotary/immudb:latest +``` + + diff --git a/src/1.9.4/running/service.md b/src/1.9.4/running/service.md new file mode 100644 index 0000000000..0ac7c259f3 --- /dev/null +++ b/src/1.9.4/running/service.md @@ -0,0 +1,34 @@ + +# Running as a service + + + +Every operating system has different ways of running services. immudb provides a facility called `immudb service` to hide this complexity. + +To install the service run as root: + +```bash +$ ./immudb service install +``` + +This will for example, on Linux, install `/etc/systemd/system/immudb.service` and create the appropriate user to run the service. On other operating systems, the native method would be used. + +The `immudb service` command also allows to control the lifecycle of the service: + +```bash +$ ./immudb service start +$ ./immudb service stop +$ ./immudb service status +``` + +On Linux, `immudb service status` is equivalent to `systemctl status immudb.service`, and is what it does under the hoods. + + + + + +### macOS specific + +In case you want to run immudb as a service, please check the following [guideline](https://medium.com/swlh/how-to-use-launchd-to-run-services-in-macos-b972ed1e352). + + \ No newline at end of file diff --git a/src/1.9.4/samples/go.md b/src/1.9.4/samples/go.md new file mode 100644 index 0000000000..b84fc94831 --- /dev/null +++ b/src/1.9.4/samples/go.md @@ -0,0 +1,129 @@ + +# App samples in Go + + + +This section includes sample applications using immudb in Go. + +Although the applications are simple, they will provide fully functional samples to demonstrate how to write an application using immudb. + + + + + +## Hello Immutable World + +The classical `Hello World` sample adapted to immudb. + +This simple application is using the [official immudb go sdk](https://github.com/codenotary/immudb/tree/master/pkg/client) to connect, store and retrieve key-value data from immudb server. + +The full source code of this sample can be found at [Hello Immutable World](https://github.com/codenotary/immudb-client-examples/tree/master/go/hello-immutable-world). + +### Prerequisites + +In order to run this sample, immudb server must be already running. This step is quite simple and it's described at [Running immudb](https://docs.immudb.io/master/running/download.html). + +### Building and running the sample app + +To build and run the sample application, simply clone the [Hello Immutable World](https://github.com/codenotary/immudb-client-examples/tree/master/go/hello-immutable-world) and run: + +``` +go mod tidy +go build +./hello-immutable-world +``` + +The sample application will run and display an output similar to + +``` +Sucessfully set a verified entry: ('hello', 'immutable world') @ tx 1 +Sucessfully got verified entry: ('hello', 'immutable world') @ tx 1 +``` + + + + + +## WebApp using SQL + +The purpose of this sample application is to demonstrate the use of immudb using [Go standard APIs for SQL](https://pkg.go.dev/database/sql). + +This sample was written taking as a basis the tutorial [Building a simple app with Go and PostgreSQL](https://blog.logrocket.com/building-simple-app-go-postgresql/). We followed the same application structure even though the source code is different to show how immudb and PostgreSQL can be used in analogy. + +The full source code of this sample can be found at [WebApp using SQL](https://github.com/codenotary/immudb-client-examples/tree/master/go/todos-sample-stdlib) + + + + + +### Prerequisites + +In order to run this sample, immudb server must be already running. This step is quite simple and it's described at [Running immudb](https://docs.immudb.io/master/running/download.html). + +### Building and running the sample app + +To build and run the sample application, simply clone the [sample repository](https://github.com/codenotary/immudb-client-examples/tree/master/go/todos-sample-stdlib) and run: + +``` +go mod tidy +go build +./immudb-todo-webapp +``` + +The sample application should be up and running now. The port 3000 is used by default unless a different one is specified using `PORT` environment variable e.g. `PORT=3001 ./immudb-todo-webapp` + +::: tip +Database initialization statements might be stored in an external file as in this sample [sql initialization script](https://github.com/codenotary/immudb-client-examples/tree/master/go/stdlib-init-script). +::: + + + + + +## Command line app using SQL + +A simple reminder console app that stores all data in immudb. + +As in the previous sample, the purpose of this sample application is to demonstrate the use of immudb using [Go standard APIs for SQL](https://pkg.go.dev/database/sql). + +The full source code of this sample can be found at [Console sample using SQL](https://github.com/codenotary/immudb-client-examples/tree/master/go/immudb-reminder-app). + +### Prerequisites + +In order to run this sample, immudb server must be already running. This step is quite simple and it's described at [Running immudb](https://docs.immudb.io/master/running/download.html). + +### Building and running the sample app + +To build and run the sample application, simply clone the [Console sample using SQL](https://github.com/codenotary/immudb-client-examples/tree/master/go/immudb-reminder-app) and run: + +``` +go mod tidy +go build +./immudb-reminder-app +``` + +The sample application should be up and running now. + +Additionally, this sample application provides a simple way to specify connection settings. run `./immudb-reminder-app -h` to display all the available flags. + +``` +Usage of ./immudb-reminder-app: + -addr string + IP address of immudb server (default "localhost") + -db string + Name of the database to use (default "defaultdb") + -pass string + Password for authenticating to immudb (default "immudb") + -port string + Port number of immudb server (default "3322") + -user string + Username for authenticating to immudb (default "immudb") +``` + + + + +::: tip +Additional samples can be found at [immudb client samples repository](https://github.com/codenotary/immudb-client-examples/tree/master/go). +::: + diff --git a/src/master/develop/sql/datatypes.md b/src/master/develop/sql/datatypes.md index 8da5406f78..fe55554aba 100644 --- a/src/master/develop/sql/datatypes.md +++ b/src/master/develop/sql/datatypes.md @@ -1,14 +1,15 @@ # Data types -| Name | Description | Length constraints | -|-----------|-------------|--------------------| -| INTEGER | Signed 64-bit integer value. Usually referred to as `BIGINT` in other databases. | - | -| BOOLEAN | A boolean value, either `TRUE` or `FALSE` | - | -| VARCHAR | UTF8-encoded text | Maximum number of bytes in the UTF-8 encoded representation of the string | -| BLOB | sequence of bytes | Maximum number of bytes in the sequence | -| TIMESTAMP | datetime value with microsecond precision | - | -| FLOAT | IEEE-754 64-bit floating-point number | - | -| UUID | Universally Unique Identifier (UUID), 128-bit value | - | +| Name | Description | Length constraints | +| --------- | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| INTEGER | Signed 64-bit integer value. Usually referred to as `BIGINT` in other databases. | - | +| BOOLEAN | A boolean value, either `TRUE` or `FALSE` | - | +| VARCHAR | UTF8-encoded text | Maximum number of bytes in the UTF-8 encoded representation of the string | +| BLOB | Sequence of bytes | Maximum number of bytes in the sequence | +| TIMESTAMP | Datetime value with microsecond precision | - | +| FLOAT | IEEE-754 64-bit floating-point number | - | +| UUID | Universally Unique Identifier (UUID), 128-bit value | - | +| JSON | RFC 8259 textual JSON data | Maximum number of bytes in the textual representation of the data |

@@ -36,4 +37,21 @@ e.g. `BLOB[16]` represents a sequence of up to 16 bytes. Timestamp values are internally stored as a 64-bit signed integer being a number of microseconds since the epoch time. Those values are not associated with any timezone, whenever a conversion is needed, it is considered to be in UTC. - \ No newline at end of file + + + + +### JSON primitive types and internal SQL types + +Except for `lists` and `objects`, each JSON type is backed by a specific immudb internal SQL type. The mapping between JSON primitive types and SQL internal types is reported in the following table: + +
+ +| JSON primitive type | Internal SQL type | Description | +| ------------------- | ----------------- | ------------------------------------------------------------------- | +| STRING | VARCHAR | control characters, quotation marks and backslashes must be escaped | +| NUMBER | FLOAT | `NaN`, `Infinity` and `-Infinity` are not supported | +| BOOLEAN | BOOLEAN | only lowercase `true` and `false` are valid | +| NULL | NULL | - | + + diff --git a/src/master/develop/sql/indexes.md b/src/master/develop/sql/indexes.md index e797dbf1e5..4b605a9524 100644 --- a/src/master/develop/sql/indexes.md +++ b/src/master/develop/sql/indexes.md @@ -21,9 +21,13 @@ CREATE INDEX IF NOT EXISTS ON customers(active); CREATE UNIQUE INDEX ON customers(email); ``` -Index can only be added to an empty table. +Currently, indexes are subject to the following limitations: -Index do not have explicit name and is referenced by the ordered list of indexed columns. +- Index can only be added to an empty table; + +- Indexes on JSON columns or subfields of them are not supported; + +- Each index doesn't have an explicit name and is referenced by the ordered list of indexed columns. ### Column value constraints @@ -53,7 +57,7 @@ With this clause the `CREATE INDEX` statement will not fail if an index with sam This includes a use case where the table is not empty which can be used to simplify database schema initialization. Note: If the index already exists, it is not compared against the provided index definition neither it is - updated to match it. +updated to match it. @@ -67,4 +71,4 @@ An index can be physically deleted. Table data is not deleted and can be queried DROP INDEX ON customers(surname); ``` - \ No newline at end of file + diff --git a/src/master/develop/sql/privileges.md b/src/master/develop/sql/privileges.md new file mode 100644 index 0000000000..7cfbfcffcb --- /dev/null +++ b/src/master/develop/sql/privileges.md @@ -0,0 +1,42 @@ +# Privileges + +In addition to database level permissions, each user is associated to a subset of the following SQL privileges: + +- `SELECT` +- `CREATE` +- `INSERT` +- `UPDATE` +- `DELETE` +- `DROP` +- `ALTER` + +Each privilege grants the possibility to execute the corresponding SQL statement. Any attempt to run a statement without owning the corresponding privilege will be promply blocked by the `ErrAccessDenied` error. +

+ + + +### Show privileges + +To show the SQL privileges associated to users, you can use the following statements: + +```sql +SHOW GRANTS; -- show all users privileges + +SHOW GRANTS FOR user; -- show the list of privileges connected to a specific user +``` + + + + + +### Alter privileges + +Users with `admin` permission can grant and revoke SQL privileges to other users: + +```sql +GRANT ALL PRIVILEGES TO myuser; -- grant all privileges to user + +REVOKE SELECT, UPDATE ON DATABASE mydb TO USER myuser; -- revoke specific privileges to user +``` + + diff --git a/src/master/develop/sql/querying.md b/src/master/develop/sql/querying.md index 685ab59081..6e33b49532 100644 --- a/src/master/develop/sql/querying.md +++ b/src/master/develop/sql/querying.md @@ -112,7 +112,7 @@ immudb has a basic supports for the `IN` operator. A `NOT` prefix negates the value of the `IN` operator. Note: Currently the list for the `IN` operator can not be - calculated using a sub-query. +calculated using a sub-query. ```sql SELECT product @@ -157,11 +157,11 @@ Available aggregation functions: -* COUNT -* SUM -* MAX -* MIN -* AVG +- COUNT +- SUM +- MAX +- MIN +- AVG @@ -237,8 +237,8 @@ INNER JOIN ( ``` Note: the context of a sub-query does not propagate outside, - e.g. it is not possible to reference a table from a sub-query - in the `WHERE` clause outside of the sub-query. +e.g. it is not possible to reference a table from a sub-query +in the `WHERE` clause outside of the sub-query. @@ -267,7 +267,7 @@ Duplicate rows are excluded by default. Using `UNION ALL` will leave duplicate r ```sql SELECT AVG(age) FROM customers -UNION ALL +UNION ALL SELECT AVG(age) FROM sellers ``` @@ -347,6 +347,20 @@ SELECT * FROM (HISTORY OF mytable) WHERE _rev = @rev +### Transaction metadata + +immudb can be configured to inject request information (**user**, **ip address**, etc...) to transaction metadata (see [Request Metadata](../../production/request-metadata.md)). When this functionality is enabled, +such information can be retrieved by querying the special `_tx_metadata` column, whose type is `JSON` (refer to [Querying JSON columns](#querying-json-columns) section). + +```sql +SELECT _tx_metadata->'ip' as ip_addr FROM customers +WHERE _tx_metadata->'usr' = 'username'; +``` + + + + + ### Skipping entries with OFFSET clause Using `OFFSET` clause in SQL queries can be used to skip an initial list of entries from the result set. @@ -366,4 +380,46 @@ ORDER BY customer_name LIMIT 50 OFFSET 100 ``` - \ No newline at end of file + + + + +## Querying JSON columns + +immudb supports a `JSON` data type designed to store data in the `JavaScript Object Notation` format. It allows for flexible and efficient storage and querying of JSON objects, which can be particularly useful in applications that require a combination of structured and semi-structured data. + +```sql +CREATE TABLE items ( + id INTEGER AUTO_INCREMENT, + details JSON, + + PRIMARY KEY id +); +``` + +Internally, JSON data are represented in their textual format. Columns can be then populated by any valid JSON string: + +```sql +INSERT INTO items(id, details) + VALUES + (1, '{"name": "Alice", "age": 25}'), + (2, '{"name": "Bob", "address": {"city": "Los Angeles", "state": "CA"}}'), + (3, '["apple", "banana", "cherry"]'), + (4, '[{"product": "book", "price": 12.99}, {"product": "pen", "price": 1.49}]'), + (5, '"N/A"') +``` + +The `ARROW (->)` operator, used to access nested fields, is crucial when working with the JSON type. The following shows valid examples of queries over a JSON field: + +```sql +SELECT * FROM items +WHERE details->'name' IS NOT NULL +ORDER BY details->'name'; + +SELECT * FROM items +WHERE json_typeof(details->'1') != 'OBJECT' AND details->'1' = 'banana'; + +SELECT details->'1'->'price' as price FROM items; +``` + + diff --git a/src/master/production/auditor.md b/src/master/production/auditor.md index af89bcba57..db1c869e36 100644 --- a/src/master/production/auditor.md +++ b/src/master/production/auditor.md @@ -82,6 +82,7 @@ Global Flags: ``` To get the (signed) state in combination with the immuclient with auditor capabilities: + ```bash immuclient audit-mode --audit-username {immudb-username} --audit-password {immudb-pw} --server-signing-pub-key {state-public-key} ``` @@ -91,6 +92,7 @@ immuclient audit-mode --audit-username {immudb-username} --audit-password {immud ## Running immuclient Auditor as a service + immuclient as Auditor can be installed in the system with the following command: Install service: @@ -100,25 +102,9 @@ immuclient audit-mode install ``` In this case, all parameters are written into the `immuclient` configuration file: -* Linux: `/etc/immudb/immuclient.toml` -* Windows: `C:\ProgramData\ImmuClient\config\immuclient.toml` - - - - - -## Running immuclient Auditor with docker -We also provide a docker image starting immuclient as Auditor: -```bash -docker pull codenotary/auditor -``` - -Then it's possible to run the command with: - -```bash -docker run -it -e IMMUCLIENT_IMMUDB_ADDRESS="ip" -e IMMUCLIENT_AUDIT_USERNAME="immudb" -e IMMUCLIENT_AUDIT_PASSWORD="immudb" codenotary/auditor -``` +- Linux: `/etc/immudb/immuclient.toml` +- Windows: `C:\ProgramData\ImmuClient\config\immuclient.toml` @@ -163,4 +149,4 @@ NOTE: it's not possible to know at which transaction the database was tampered. A proper setup of one immuclient instance can fit most of cases, but there are ways to increase the security on detecting tampering. A single instance can go offline for any reason: network problems, hardware failures or attacks. Therefore a good practice can be to have multiple Auditor instances running in different zones. - \ No newline at end of file + diff --git a/src/master/production/monitoring.md b/src/master/production/monitoring.md index 3a95c5ae69..f6978988b4 100644 --- a/src/master/production/monitoring.md +++ b/src/master/production/monitoring.md @@ -7,7 +7,7 @@ immudb exposes a Prometheus end-point, by default on port 9497 on `/metrics`.
```bash -$ curl -s http://localhost:9497/metrics +$ curl -s http://localhost:9497/metrics # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. # TYPE go_gc_duration_seconds summary go_gc_duration_seconds{quantile="0"} 1.3355e-05 @@ -309,7 +309,7 @@ values below 10 or above few hundred are a good indication that the btree isn’ and the application should consider using keys of different, more uniform and shorter lengths for its data. -*Note: These statistics are gathered when traversing the btree, if there’s no activity in the database the distribution will be flat.* +_Note: These statistics are gathered when traversing the btree, if there’s no activity in the database the distribution will be flat._
@@ -352,7 +352,7 @@ An unbound growth of those maximums could indicate that the flush operation is t Similarly to flush statistics, immudb exposes the same set of values for full compaction. -*Note: these values are gathered for overall compaction that fully rewrites the btree structure. immudb 1.2.3 introduced a new online compaction mode that gradually removes unused btree data during flush operation. This new compaction mode is not included in those charts.* +_Note: these values are gathered for overall compaction that fully rewrites the btree structure. immudb 1.2.3 introduced a new online compaction mode that gradually removes unused btree data during flush operation. This new compaction mode is not included in those charts._
@@ -497,19 +497,19 @@ the health check tool you can find in the [immudb-tools](https://github.com/code This simple tool periodically polls immudb, checking the state of every loaded database, and exposes a simple REST endpoint that can be used by docker or kubernetes probes. You simply have to configure it to talk to immudb, by providing its address -and some credentials. +and some credentials. Then you can poll endpoint `/immustatus` on port 8085. If immudb is operating normally and all databases are responsive, you will get a 200 HTTP return code, and the string "OK". You will get a 500 HTTP code instead, and the string "FAIL" if immudb is not responding. This configuration snippet shows how to use the `/immustatus` endpoint on kubernetes: ```yaml - livenessProbe: - httpGet: - path: /immustatus - port: 8085 - failureThreshold: 5 - periodSeconds: 60 +livenessProbe: + httpGet: + path: /immustatus + port: 8085 + failureThreshold: 5 + periodSeconds: 60 ``` You can find more information about it on its [README](https://github.com/codenotary/immudb-tools/tree/main/immuguard) page. diff --git a/src/master/production/request-metadata.md b/src/master/production/request-metadata.md new file mode 100644 index 0000000000..22eb8d6235 --- /dev/null +++ b/src/master/production/request-metadata.md @@ -0,0 +1,26 @@ +# Request Metadata + +To enhance the auditing process, immudb can be configured to inject request information - such as user identifiers, IP addresses, and other relevant details - into transaction metadata. + + + +## Enabling request metadata logging + +Request metadata logging can be enabled by enabling the `--log-request-metdata` flag of the immudb command (or by setting +the corresponding `IMMUDB_LOG_REQUEST_METADATA` env var to `TRUE`). + +For example, when running the immudb docker image: + +```bash +$ docker run -e IMMUDB_LOG_REQUEST_METADATA=TRUE -d --net host --name immudb codenotary/immudb:latest +``` + +### Why should I enable request metadata? + +When this functionality is enabled, each transaction includes comprehensive metadata that provides context for the request executed by the immudb server. This metadata allows auditors and administrators to easily retrieve detailed information about the context of each transaction (see how to retrieve transaction metadata from an [SQL table](../develop/sql/querying.md#transaction-metadata)). + +Specifically, it enables the identification of who initiated the transaction, from which IP address, and any other pertinent request details. For example, if there is a need to investigate suspicious activity or trace the source of a particular change, this request metadata offers a clear and concise trail of the relevant data. + +Note that despite the extra information can increase the storage overhead, the benefits of enhanced transparency and accountability often outweigh the extra storage cost, ensuring that all actions within the database can be thoroughly examined and verified. + + diff --git a/src/master/running/configuration.md b/src/master/running/configuration.md index a7feda8ab5..1f39db841c 100644 --- a/src/master/running/configuration.md +++ b/src/master/running/configuration.md @@ -1,4 +1,3 @@ - # Configuration @@ -13,66 +12,67 @@ Settings can be specified as command line options to immudb (see `immudb -h`), i ### Settings -| Parameter | Default | Description | -|---------------------------------|------------|------------------------------------------------------------------------------------------------------| -| `address` | `0.0.0.0` | bind address | -| `admin-password` | `immudb` | admin password as plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) | -| `auth` | `true` | enable auth | -| `certificate` | `` | server certificate file path | -| `clientcas` | `` | clients certificates list. Aka certificate authority | -| `config` | `` | config file (default path are configs or $HOME. Default filename is immudb. | -| `detached` | `false` | run immudb in background | -| `devmode` | `false` | enable dev mode: accept remote connections without auth | -| `dir` | `./data` | data folder | -| `force-admin-password` | `false` | if true, reset the admin password to the one passed through admin-password option upon startup | -| `grpc-reflection` | `true` | GRPC reflection server enabled | -| `logfile` | `` | log path with filename. E.g. /tmp/immudb/immudb.log | -| `logformat` | `text` | log format e.g. text/json | -| `maintenance` | `false` | override the authentication flag | -| `max-recv-msg-size` | `33554432` | max message size in bytes the server can receive | -| `max-session-age-time` | infinity | max session age time is a duration after which session will be forcibly closed | -| `max-session-inactivity-time` | `3m0s` | max session inactivity time is a duration after which an active session is declared inactive by the server. A session is kept active if server is still receiving requests from client (keep-alive or other methods) | -| `max-sessions` | `100` | maximum number of simultaneously opened sessions | -| `metrics-server` | `true` | enable or disable Prometheus endpoint | -| `metrics-server-port` | `9477` | Prometheus endpoint port | -| `mtls` | `false` | enable mutual tls | -| `no-histograms` | `false` | disable collection of histogram metrics like query durations | -| `pgsql-server` | `true` | enable or disable pgsql server | -| `pgsql-server-port` | `5432` | pgsql server port | -| `pidfile` | `` | pid path with filename. E.g. /var/run/immudb.pid | -| `pkey` | `` | server private key path | -| `port` | `3322` | port number | -| `pprof` | `false` | add pprof profiling endpoint on the metrics server | -| `replication-allow-tx-discarding` | `false` | allow precommitted transactions to be discarded if the replica diverges from the primary | -| `replication-commit-concurrency` | `10` | number of concurrent replications | -| `replication-is-replica` | `false` | set systemdb and defaultdb as replica | -| `replication-prefetch-tx-buffer-size` | `100`| maximum number of prefeched transactions | -| `replication-primary-host` | `` | primary database host (if replica=true) | -| `replication-primary-password` | `` | password in the primary database used for replication of systemdb and defaultdb | -| `replication-primary-port` | `3322` | primary database port (if replica=true) (default 3322) | -| `replication-primary-username` | `` | username in the primary database used for replication of systemdb and defaultdb | -| `replication-skip-integrity-check` | `false` | disable integrity check when reading data during replication | -| `replication-sync-acks` | `0` | set a minimum number of replica acknowledgements required before transactions can be committed | -| `replication-sync-enabled` | `false` | enable synchronous replication | -| `replication-wait-for-indexing` | `false` | wait for indexing to be up to date during replication | -| `s3-access-key-id` | `` | s3 access key id | -| `s3-bucket-name` | `` | s3 bucket name | -| `s3-endpoint` | `` | s3 endpoint | -| `s3-external-identifier` | `` | use the remote identifier if there is no local identifier | -| `s3-instance-metadata-url` | `http://169.254.169.254` | s3 instance metadata url | -| `s3-location` | `` | s3 location (region) | -| `s3-path-prefix` | `` | s3 path prefix (multiple immudb instances can share the same bucket if they have different prefixes) | -| `s3-role` | `` | role name for role-based authentication attempt for s3 storage | -| `s3-role-enabled` | `false` | enable role-based authentication for s3 storage | -| `s3-secret-key` | `` | s3 secret access key | -| `s3-storage` | `false` | enable or disable s3 storage | -| `session-timeout` | `2m0s` | session timeout is a duration after which an inactive session is forcibly closed by the server | -| `signingKey` | `` | signature private key path. If a valid one is provided, it enables the cryptographic signature of the root. E.g. "./../test/signer/ec3.key" | -| `swaggerui` | `true` | Swagger UI enabled | -| `synced` | `true` | synced mode prevents data lost under unexpected crashes but affects performance | -| `token-expiry-time` | `1440` | client authentication token expiration time. Minutes | -| `web-server` | `true` | enable or disable web/console server | -| `web-server-port` | `8080` | web/console server port | +| Parameter | Default | Description | +| ------------------------------------- | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `address` | `0.0.0.0` | bind address | +| `admin-password` | `immudb` | admin password as plain-text or base64 encoded (must be prefixed with 'enc:' if it is encoded) | +| `auth` | `true` | enable auth | +| `certificate` | `` | server certificate file path | +| `clientcas` | `` | clients certificates list. Aka certificate authority | +| `config` | `` | config file (default path are configs or $HOME. Default filename is immudb. | +| `detached` | `false` | run immudb in background | +| `devmode` | `false` | enable dev mode: accept remote connections without auth | +| `dir` | `./data` | data folder | +| `force-admin-password` | `false` | if true, reset the admin password to the one passed through admin-password option upon startup | +| `log-request-metadata` | `false` | log request information in transaction metadata | +| `grpc-reflection` | `true` | GRPC reflection server enabled | +| `logfile` | `` | log path with filename. E.g. /tmp/immudb/immudb.log | +| `logformat` | `text` | log format e.g. text/json | +| `maintenance` | `false` | override the authentication flag | +| `max-recv-msg-size` | `33554432` | max message size in bytes the server can receive | +| `max-session-age-time` | infinity | max session age time is a duration after which session will be forcibly closed | +| `max-session-inactivity-time` | `3m0s` | max session inactivity time is a duration after which an active session is declared inactive by the server. A session is kept active if server is still receiving requests from client (keep-alive or other methods) | +| `max-sessions` | `100` | maximum number of simultaneously opened sessions | +| `metrics-server` | `true` | enable or disable Prometheus endpoint | +| `metrics-server-port` | `9477` | Prometheus endpoint port | +| `mtls` | `false` | enable mutual tls | +| `no-histograms` | `false` | disable collection of histogram metrics like query durations | +| `pgsql-server` | `true` | enable or disable pgsql server | +| `pgsql-server-port` | `5432` | pgsql server port | +| `pidfile` | `` | pid path with filename. E.g. /var/run/immudb.pid | +| `pkey` | `` | server private key path | +| `port` | `3322` | port number | +| `pprof` | `false` | add pprof profiling endpoint on the metrics server | +| `replication-allow-tx-discarding` | `false` | allow precommitted transactions to be discarded if the replica diverges from the primary | +| `replication-commit-concurrency` | `10` | number of concurrent replications | +| `replication-is-replica` | `false` | set systemdb and defaultdb as replica | +| `replication-prefetch-tx-buffer-size` | `100` | maximum number of prefeched transactions | +| `replication-primary-host` | `` | primary database host (if replica=true) | +| `replication-primary-password` | `` | password in the primary database used for replication of systemdb and defaultdb | +| `replication-primary-port` | `3322` | primary database port (if replica=true) (default 3322) | +| `replication-primary-username` | `` | username in the primary database used for replication of systemdb and defaultdb | +| `replication-skip-integrity-check` | `false` | disable integrity check when reading data during replication | +| `replication-sync-acks` | `0` | set a minimum number of replica acknowledgements required before transactions can be committed | +| `replication-sync-enabled` | `false` | enable synchronous replication | +| `replication-wait-for-indexing` | `false` | wait for indexing to be up to date during replication | +| `s3-access-key-id` | `` | s3 access key id | +| `s3-bucket-name` | `` | s3 bucket name | +| `s3-endpoint` | `` | s3 endpoint | +| `s3-external-identifier` | `` | use the remote identifier if there is no local identifier | +| `s3-instance-metadata-url` | `http://169.254.169.254` | s3 instance metadata url | +| `s3-location` | `` | s3 location (region) | +| `s3-path-prefix` | `` | s3 path prefix (multiple immudb instances can share the same bucket if they have different prefixes) | +| `s3-role` | `` | role name for role-based authentication attempt for s3 storage | +| `s3-role-enabled` | `false` | enable role-based authentication for s3 storage | +| `s3-secret-key` | `` | s3 secret access key | +| `s3-storage` | `false` | enable or disable s3 storage | +| `session-timeout` | `2m0s` | session timeout is a duration after which an inactive session is forcibly closed by the server | +| `signingKey` | `` | signature private key path. If a valid one is provided, it enables the cryptographic signature of the root. E.g. "./../test/signer/ec3.key" | +| `swaggerui` | `true` | Swagger UI enabled | +| `synced` | `true` | synced mode prevents data lost under unexpected crashes but affects performance | +| `token-expiry-time` | `1440` | client authentication token expiration time. Minutes | +| `web-server` | `true` | enable or disable web/console server | +| `web-server-port` | `8080` | web/console server port | @@ -164,5 +164,4 @@ immudb 2022/11/17 14:30:02 INFO: Index 'data/systemdb/index' {ts=0, cleanup_perc {"caller":"codenotary/immudb/embedded/tbtree/tbtree.go:669","component":"github.com/codenotary/immudb/embedded/tbtree.OpenWith","level":"info","message":"Index 'data/systemdb/index' {ts=2, discarded_snapshots=0} successfully loaded","module":"immudb","timestamp":"2022-11-17T14:32:28.904722+05:30"} ``` -