diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 63a3ca4eeb531..3bdd7c5f42812 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,6 @@ # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners # Order is important; the last matching pattern takes the most precedence. +* @dutow @dAdAbird /contrib/pg_tde/documentation/ @nastena1606 @Andriciuc /.github/ @artemgavrilov diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f22e97afc0b8a..70d227e18bea3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -10,9 +10,7 @@ on: - contrib/pg_tde/documentation/** env: - # Avoid failures on slow recovery - PGCTLTIMEOUT: 120 - PG_TEST_TIMEOUT_DEFAULT: 300 + PGCTLTIMEOUT: 120 # Avoid failures on slow recovery jobs: collect: diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 462de6fa0fb67..b1196d41f4fbe 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -17,9 +17,7 @@ env: LSAN_OPTIONS: log_path=${{ github.workspace }}/sanitize.log print_suppressions=0 suppressions=${{ github.workspace }}/ci_scripts/suppressions/lsan.supp ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-14 EXTRA_REGRESS_OPTS: "--temp-config=${{ github.workspace }}/test_postgresql.conf" - # Avoid failures on slow recovery PGCTLTIMEOUT: 120 - PG_TEST_TIMEOUT_DEFAULT: 300 jobs: run: diff --git a/ci_scripts/suppressions/lsan.supp b/ci_scripts/suppressions/lsan.supp index 3b7d93af62dda..087f3164fc72c 100644 --- a/ci_scripts/suppressions/lsan.supp +++ b/ci_scripts/suppressions/lsan.supp @@ -51,11 +51,3 @@ leak:bin/pg_waldump/pg_waldump.c #pg_dump leak:getSchemaData leak:dumpDumpableObject - -#pg_rewind -leak:decide_file_actions -leak:GenerateRecoveryConfig - -# Returns strdup'd string which never gets freed in frontend tools. Trying to -# free it leads to comiler complains that it discards 'const' qualifier. -leak:get_progname diff --git a/contrib/pg_tde/.gitignore b/contrib/pg_tde/.gitignore index 47f98d48e68f3..044af8f3c5cae 100644 --- a/contrib/pg_tde/.gitignore +++ b/contrib/pg_tde/.gitignore @@ -11,9 +11,7 @@ __pycache__ /configure~ /log /results -/src/bin/pg_tde_archive_decrypt -/src/bin/pg_tde_change_key_provider -/src/bin/pg_tde_restore_encrypt +/src/pg_tde_change_key_provider /t/results /tmp_check diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index 3eba916acc935..dd9f5541c6d19 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -51,16 +51,10 @@ src/libkmip/libkmip/src/kmip_bio.o \ src/libkmip/libkmip/src/kmip_locate.o \ src/libkmip/libkmip/src/kmip_memset.o -SCRIPTS_built = src/bin/pg_tde_archive_decrypt \ -src/bin/pg_tde_change_key_provider \ -src/bin/pg_tde_restore_encrypt +SCRIPTS_built = src/pg_tde_change_key_provider EXTRA_INSTALL += contrib/pg_buffercache contrib/test_decoding -EXTRA_CLEAN += src/bin/pg_tde_archive_decrypt.o \ -src/bin/pg_tde_change_key_provider.o \ -src/bin/pg_tde_restore_encrypt.o \ -xlogreader.c \ -xlogreader.o +EXTRA_CLEAN += src/pg_tde_change_key_provider.o ifdef USE_PGXS PG_CONFIG = pg_config @@ -77,21 +71,9 @@ endif override SHLIB_LINK += -lcurl -lcrypto -lssl -src/bin/pg_tde_change_key_provider: src/bin/pg_tde_change_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a +src/pg_tde_change_key_provider: src/pg_tde_change_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a $(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -src/bin/pg_tde_archive_decrypt: src/bin/pg_tde_archive_decrypt.o xlogreader.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtdexlog.a $(top_builddir)/src/libtde/libtde.a - $(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) - -src/bin/pg_tde_restore_encrypt: src/bin/pg_tde_restore_encrypt.o xlogreader.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtdexlog.a $(top_builddir)/src/libtde/libtde.a - $(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) - -xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% - rm -f $@ && $(LN_S) $< . - -xlogreader.o: xlogreader.c - $(CC) $(CPPFLAGS) -DFRONTEND -c $< -o $@ - # Fetches typedefs list for PostgreSQL core and merges it with typedefs defined in this project. # https://wiki.postgresql.org/wiki/Running_pgindent_on_non-core_code_or_development_code update-typedefs: diff --git a/contrib/pg_tde/documentation/docs/advanced-topics/tech-reference.md b/contrib/pg_tde/documentation/docs/advanced-topics/index.md similarity index 56% rename from contrib/pg_tde/documentation/docs/advanced-topics/tech-reference.md rename to contrib/pg_tde/documentation/docs/advanced-topics/index.md index 309a0b1bf73aa..9df169821bcd3 100644 --- a/contrib/pg_tde/documentation/docs/advanced-topics/tech-reference.md +++ b/contrib/pg_tde/documentation/docs/advanced-topics/index.md @@ -1,7 +1,7 @@ -# Overview of technical capabilities +# Technical Reference This section covers the internal components and tools that power `pg_tde`. Use it to understand how encryption is implemented, fine-tune a configuration, leverage advanced CLI tools and functions for diagnostics and customization. -[Architecture](../architecture/architecture.md){.md-button} [GUC Variables](../variables.md){.md-button} [Functions](../functions.md){.md-button} +[Architecture](../architecture/index.md){.md-button} [GUC Variables](../variables.md){.md-button} [Functions](../functions.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/architecture/architecture.md b/contrib/pg_tde/documentation/docs/architecture/index.md similarity index 94% rename from contrib/pg_tde/documentation/docs/architecture/architecture.md rename to contrib/pg_tde/documentation/docs/architecture/index.md index bc1df2f25df6e..ae42285564fbf 100644 --- a/contrib/pg_tde/documentation/docs/architecture/architecture.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -22,7 +22,7 @@ The following sections break down the key architectural components of this desig * Temporary tables * Write Ahead Log (WAL), still in beta. **Do not enable this feature in production environments**. -**Extension** means that `pg_tde` should be implemented only as an extension, possibly compatible with any PostgreSQL distribution, including the open source community version. This requires changes in the PostgreSQL core to make it more extensible. Therefore, `pg_tde` currently works only with the [Percona Server for PostgreSQL](https://docs.percona.com/postgresql/17/index.html) - a binary replacement of community PostgreSQL and included in Percona Distribution for PostgreSQL. +**Extension** means that `pg_tde` should be implemented only as an extension, possibly compatible with any PostgreSQL distribution, including the open source community version. This requires changes in the PostgreSQL core to make it more extensible. Therefore, `pg_tde` currently works only with the [Percona Server for PostgreSQL :octicons-link-external-16:](https://docs.percona.com/postgresql/17/index.html) - a binary replacement of community PostgreSQL and included in Percona Distribution for PostgreSQL. ## Main components @@ -30,7 +30,7 @@ The main components of `pg_tde` are the following: * **Core server changes** focus on making the server more extensible, allowing the main logic of `pg_tde` to remain separate, as an extension. Core changes also add encryption-awareness to some command line tools that have to work directly with encrypted tables or encrypted WAL files. - [Percona Server for PostgreSQL location](https://github.com/percona/postgres/tree/{{tdebranch}}) + [Percona Server for PostgreSQL location :octicons-link-external-16:](https://github.com/percona/postgres/tree/{{tdebranch}}) * The **`pg_tde` extension itself** implements the encryption code by hooking into the extension points introduced in the core changes, and the already existing extension points in the PostgreSQL server. @@ -50,7 +50,7 @@ In the future these could be extracted into separate shared libraries with an op `pg_tde` uses one principal key per database. Every internal key for the given database is encrypted using this principal key. -Internal keys are used for specific database files: each file with a different [Object Identifier (OID)](https://www.postgresql.org/docs/current/datatype-oid.html) has a different internal key. +Internal keys are used for specific database files: each file with a different [Object Identifier (OID) :octicons-link-external-16:](https://www.postgresql.org/docs/current/datatype-oid.html) has a different internal key. This means that, for example, a table with 4 indexes will have at least 5 internal keys - one for the table, and one for each index. @@ -82,12 +82,9 @@ Later decisions are made using a slightly modified Storage Manager (SMGR) API: w ### WAL encryption -!!! note - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - WAL encryption is controlled globally via a global GUC variable, `pg_tde.wal_encrypt`, that requires a server restart. -WAL keys also contain the [LSN](https://www.postgresql.org/docs/17/wal-internals.html) of the first WAL write after key creation. This allows `pg_tde` to know which WAL ranges are encrypted or not and with which key. +WAL keys also contain the [LSN :octicons-link-external-16:](https://www.postgresql.org/docs/17/wal-internals.html) of the first WAL write after key creation. This allows `pg_tde` to know which WAL ranges are encrypted or not and with which key. The setting only controls writes so that only WAL writes are encrypted when WAL encryption is enabled. This means that WAL files can contain both encrypted and unencrypted data, depending on what the status of this variable was when writing the data. @@ -147,8 +144,8 @@ Principal keys are stored externally in a Key Management Services (KMS). In `pg_ The following key providers are supported: -* [HashiCorp Vault](https://developer.hashicorp.com/vault/docs/what-is-vault) KV2 secrets engine -* [OpenBao](https://openbao.org/) implementation of Vault +* [HashiCorp Vault :octicons-link-external-16:](https://developer.hashicorp.com/vault/docs/what-is-vault) KV2 secrets engine +* [OpenBao :octicons-link-external-16:](https://openbao.org/) implementation of Vault * KMIP compatible servers * A local file storage. This storage is intended only for development and testing and is not recommended for production use. @@ -167,7 +164,7 @@ Key provider configuration or location may change. For example, a service is mov In certain cases you can't use SQL functions to manage key providers. For example, if the key provider changed while the server wasn't running and is therefore unaware of these changes. The startup can fail if it needs to access the encryption keys. -For such situations, `pg_tde` also provides [command line tools](../command-line-tools/cli-tools.md) to recover the database. +For such situations, `pg_tde` also provides [command line tools](../command-line-tools/index.md) to recover the database. ### Sensitive key provider information diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md b/contrib/pg_tde/documentation/docs/command-line-tools/index.md similarity index 93% rename from contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md rename to contrib/pg_tde/documentation/docs/command-line-tools/index.md index e0edeb4b5de32..b854eab1b304a 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools/cli-tools.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools/index.md @@ -1,4 +1,4 @@ -# Overview of pg_tde CLI tools +# pg_tde CLI Tools The `pg_tde` extension introduces new command-line utilities and extends some existing PostgreSQL tools to support encrypted WAL and tables. These include: diff --git a/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md b/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md index 0362eadcef7f0..a3ac50df4df02 100644 --- a/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md +++ b/contrib/pg_tde/documentation/docs/command-line-tools/pg-waldump.md @@ -2,9 +2,6 @@ [`pg_waldump` :octicons-link-external-16:](https://www.postgresql.org/docs/current/pgwaldump.html) is a tool to display a human-readable rendering of the Write-Ahead Log (WAL) of a PostgreSQL database cluster. -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - To read encrypted WAL records, `pg_waldump` supports the following additional arguments: * `keyring_path` is the directory where the keyring configuration files for WAL are stored. The following files are included: diff --git a/contrib/pg_tde/documentation/docs/faq.md b/contrib/pg_tde/documentation/docs/faq.md index b244818371144..7b8843c0ce2a0 100644 --- a/contrib/pg_tde/documentation/docs/faq.md +++ b/contrib/pg_tde/documentation/docs/faq.md @@ -94,9 +94,6 @@ The principal key is used to encrypt the internal keys. The principal key is sto ### WAL encryption -!!! note - WAL encryption is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - WAL encryption is done globally for the entire database cluster. All modifications to any database within a PostgreSQL cluster are written to the same WAL to maintain data consistency and integrity and ensure that PostgreSQL cluster can be restored to a consistent state. Therefore, WAL is encrypted globally. When you turn on WAL encryption, `pg_tde` encrypts entire WAL files starting from the first WAL write after the server was started with the encryption turned on. @@ -140,9 +137,9 @@ Since the `SET ACCESS METHOD` command drops hint bits and this may affect the pe You must restart the database in the following cases to apply the changes: * after you enabled the `pg_tde` extension -* when enabling WAL encryption, which is currently in beta. **Do not enable this feature in production environments**. +* when enabling WAL encryption -After that, no database restart is required. When you create or alter the table using the `tde_heap` access method, the files are marked as those that require encryption. The encryption happens at the storage manager level, before a transaction is written to disk. Read more about [how tde_heap works](index/table-access-method.md#how-tde_heap-works-with-pg_tde). +After that, no database restart is required. When you create or alter the table using the `tde_heap` access method, the files are marked as those that require encryption. The encryption happens at the storage manager level, before a transaction is written to disk. Read more about [how tde_heap works](index/table-access-method.md#how-tde_heap-works). ## What happens to my data if I lose a principal key? diff --git a/contrib/pg_tde/documentation/docs/features.md b/contrib/pg_tde/documentation/docs/features.md index aa8703fc42bca..ce9cd10826a81 100644 --- a/contrib/pg_tde/documentation/docs/features.md +++ b/contrib/pg_tde/documentation/docs/features.md @@ -5,7 +5,7 @@ The Percona Server for PostgreSQL provides an extended Storage Manager API that The following features are available for the extension: -* [Table encryption](test.md#encrypt-data-in-a-new-table), including: +* Table encryption, including: * Data tables * Index data for encrypted tables * TOAST tables @@ -14,9 +14,9 @@ The following features are available for the extension: !!! note Metadata of those tables is not encrypted. -* Single-tenancy support via a [global keyring provider](global-key-provider-configuration/set-principal-key.md) -* [Multi-tenancy support](how-to/multi-tenant-setup.md) +* Single-tenancy support via a global keyring provider +* Multi-tenancy support * Table-level granularity for encryption and access control -* Multiple [Key management options](global-key-provider-configuration/index.md) +* Multiple Key management options -[Learn more about TDE and pg_tde :material-arrow-right:](index/about-tde.md){.md-button} [Get started with installation :material-arrow-right:](install.md){.md-button} +[Overview](index/index.md){.md-button} [Get Started](install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 73f2da208faa0..c18a570a9eea7 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -2,9 +2,6 @@ The `pg_tde` extension provides functions for managing different aspects of its operation: -!!! note - If no error is reported when running the commands below, the operation completed successfully. - ## Key provider management A key provider is a system or service responsible for managing encryption keys. `pg_tde` supports the following key providers: @@ -283,8 +280,6 @@ SELECT pg_tde_set_server_key_using_global_key_provider( ); ``` -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. ======= The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/index.md similarity index 79% rename from contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md rename to contrib/pg_tde/documentation/docs/global-key-provider-configuration/index.md index 800f505925715..f8fbf49d5ab29 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/overview.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/index.md @@ -1,4 +1,4 @@ -# Key management overview +# Configure Key Management (KMS) In production environments, storing encryption keys locally on the PostgreSQL server can introduce security risks. To enhance security, `pg_tde` supports integration with external Key Management Systems (KMS) through a Global Key Provider interface. @@ -13,6 +13,4 @@ To use an external KMS with `pg_tde`, follow these two steps: Select your prefered configuration from the links below: -[KMIP Configuration :material-arrow-right:](kmip-server.md){.md-button} -[Vault Configuration :material-arrow-right:](vault.md){.md-button} -[Keyring File Configuration (not recommended) :material-arrow-right:](keyring.md){.md-button} +[KMIP Configuration :material-arrow-right:](kmip-server.md){.md-button} [Vault Configuration :material-arrow-right:](vault.md){.md-button} [Keyring File Configuration (not recommended) :material-arrow-right:](keyring.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/keyring.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/keyring.md index fd60e4fcf220c..82829aefd1d12 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/keyring.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/keyring.md @@ -1,4 +1,4 @@ -# Keyring file configuration +# Keyring File Configuration This setup is intended for development and stores the keys unencrypted in the specified data file. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-fortanix.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-fortanix.md index db08752133248..edc09daf51af8 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-fortanix.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-fortanix.md @@ -1,4 +1,4 @@ -# Fortanix KMIP server configuration +# Fortanix KMIP Server Configuration `pg_tde` is compatible with Fortanix Data Security Manager (DSM) via the KMIP protocol. For a full setup guide, see [the Fortanix KMIP documentation here](https://support.fortanix.com/docs/users-guide-account-client-configurations?highlight=KMIP#23-kmip-clients). diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md index 91bc1cefd3b7d..7b11a694a3d26 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md @@ -1,4 +1,4 @@ -# Using OpenBao as a key provider +# Using OpenBao as a Key Provider You can configure `pg_tde` to use OpenBao as a global key provider for managing encryption keys securely. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md index 488ca0007749b..46d93e7f31a63 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md @@ -1,4 +1,4 @@ -# KMIP configuration +# KMIP Configuration To use a Key Management Interoperability Protocol (KMIP) server with `pg_tde`, you must configure it as a global key provider. This setup enables `pg_tde` to securely fetch and manage encryption keys from a centralized key management appliance. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-thales.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-thales.md index d31ba22e9fdd2..daf6bd5e16039 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-thales.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-thales.md @@ -1,4 +1,4 @@ -# Thales KMIP server configuration +# Thales KMIP Server Configuration `pg_tde` is compatible with the Thales CipherTrust Manager via the KMIP protocol. For a full setup guide, see [the Thales documentation](https://thalesdocs.com/ctp/cm/2.19/reference/kmip-ref/index.html?). diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md index 1d4e55788d5df..c069f44e98209 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md @@ -1,12 +1,9 @@ -# Global Principal Key configuration +# Global Principal Key Configuration You can configure a default principal key using a global key provider. This key will be used by all databases that do not have their own encryption keys configured. The function **both** sets the principal key and rotates internal keys as needed. ## Create a default principal key -!!! note - The sample output below is for demonstration purposes only. Be sure to replace the key name and provider with your actual values. - To create a global principal key, run: ```sql @@ -16,18 +13,6 @@ SELECT pg_tde_create_key_using_global_key_provider( ); ``` -??? example "Sample output" - ```sql - postgres=# SELECT pg_tde_create_key_using_global_key_provider( - 'keytest1', - 'file-keyring' - ); - pg_tde_create_key_using_global_key_provider - --------------------------------------------- - - (1 row) - ``` - ## Configure a default principal key To configure a global principal key, run: @@ -39,26 +24,11 @@ SELECT pg_tde_set_default_key_using_global_key_provider( ); ``` -??? example "Sample output" - ```sql - postgres=# SELECT pg_tde_set_default_key_using_global_key_provider( - 'keytest1', - 'file-keyring' - ); - pg_tde_set_default_key_using_global_key_provider - -------------------------------------------------- - - (1 row) - ``` - ## Parameter description * `key-name` is the name under which the principal key is stored in the provider. * `global_vault_provider` is the name of the global key provider you previously configured. -!!! note - If no error is reported, the action completed successfully. - ## How key generation works The key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. @@ -66,6 +36,22 @@ The key material (actual cryptographic key) is auto-generated by `pg_tde` and st !!! note This process sets the **default principal key for the entire server**. Any database without a key explicitly configured will fall back to this key. +## Example + +This example is for testing purposes only. Replace the key name and provider name with your values: + +```sql +SELECT pg_tde_create_key_using_global_key_provider( + 'test-db-master-key', + 'file-vault' +); + +SELECT pg_tde_set_key_using_global_key_provider( + 'test-db-master-key', + 'file-vault' +); +``` + ## Next steps [Validate Encryption with pg_tde :material-arrow-right:](../test.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md index 00bac8074d457..d60024a62e7c3 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md @@ -1,4 +1,4 @@ -# Vault configuration +# Vault Configuration You can configure `pg_tde` to use HashiCorp Vault as a global key provider for managing encryption keys securely. diff --git a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md index 92bc892285a39..703461c74b3d5 100644 --- a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md +++ b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md @@ -2,15 +2,12 @@ The steps below describe how to set up multi-tenancy with `pg_tde`. Multi-tenancy allows you to encrypt different databases with different keys. This provides granular control over data and enables you to introduce different security policies and access controls for each database so that only authorized users of specific databases have access to the data. -If you don't need multi-tenancy, use the global key provider. See the configuration steps from the [Configure pg_tde :octicons-link-external-16:](../setup.md) section. +If you don't need multi-tenancy, use the global key provider. See the configuration steps from the [Configure pg_tde](../setup.md) section. -For how to enable WAL encryption, refer to the [Configure WAL Encryption :octicons-link-external-16:](../wal-encryption.md) section. +For how to enable WAL encryption, refer to the [Configure WAL Encryption](../wal-encryption.md) section. --8<-- "kms-considerations.md" -!!! note - If no error is reported when running the commands below, the operation completed successfully. - ## Enable extension Load the `pg_tde` at startup time. The extension requires additional shared memory; therefore, add the `pg_tde` value for the `shared_preload_libraries` parameter and restart the `postgresql` cluster. @@ -53,7 +50,7 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo ## Key provider configuration -You must do these steps for every database where you have created the extension. For more information on configurations, please see the [Configure Key Management (KMS) :octicons-link-external-16:](../global-key-provider-configuration/overview.md) topic. +You must do these steps for every database where you have created the extension. For more information on configurations, please see the [Configure Key Management (KMS)](../global-key-provider-configuration/index.md) topic. 1. Set up a key provider. diff --git a/contrib/pg_tde/documentation/docs/index.md b/contrib/pg_tde/documentation/docs/index.md index e8b3ed6a5e6a7..5f2a091a250ff 100644 --- a/contrib/pg_tde/documentation/docs/index.md +++ b/contrib/pg_tde/documentation/docs/index.md @@ -6,36 +6,6 @@ Percona Transparent Data Encryption for PostgreSQL (`pg_tde`) is an open source, There is no safe upgrade path from the previous versions, such as Release Candidate 2, to the General Availability (GA) version of `pg_tde`. We recommend starting with a **clean installation** for GA deployments. Avoid using RC environments in production. -
- -### :material-progress-download: Installation guide { .title } - -Get started quickly with the step-by-step installation instructions. - -[How to install `pg_tde` :material-arrow-right:](install.md){ .md-button } - -
- -### :rocket: Features { .title } - -Explore what features Percona's `pg_tde` extension brings to PostgreSQL. - -[Check what you can do with `pg_tde` :material-arrow-right:](features.md){ .md-button } - -
- -### :material-cog-refresh-outline: Architecture { .title } - -Understand how `pg_tde` integrates into PostgreSQL with Percona's architecture. Learn how keys are managed, how encryption is applied, and how our design ensures performance and security. - -[Check what’s under the hood for `pg_tde` :material-arrow-right:](architecture/architecture.md){.md-button} - -
- -### :loudspeaker: What's new? { .title } - -Learn about the releases and changes in `pg_tde`. - -[Check what’s new in the latest version :material-arrow-right:](release-notes/{{latestreleasenotes}}.md){.md-button} -
-
+[Overview](index/index.md){.md-button} +[Get Started](install.md){.md-button} +[What's new in pg_tde {{release}}](release-notes/release-notes.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/about-tde.md b/contrib/pg_tde/documentation/docs/index/about-tde.md deleted file mode 100644 index e2952ece5b8d2..0000000000000 --- a/contrib/pg_tde/documentation/docs/index/about-tde.md +++ /dev/null @@ -1,12 +0,0 @@ -# About Transparent Data Encryption - -Transparent Data Encryption (TDE) protects your data at rest by ensuring that even if the underlying storage is compromised, the data remains unreadable without the correct decryption keys. This is especially critical for environments handling sensitive, regulated, or high-value information. - -This process runs transparently in the background, with minimal impact on database operations. - -Use the links below to explore how `pg_tde` helps secure your PostgreSQL deployments: - -[Discover the benefits of pg_tde :material-arrow-right:](how-tde-helps.md){.md-button} -[Understand pg_tde limitations :material-arrow-right:](tde-limitations.md){.md-button} -[View the full feature list :material-arrow-right:](../features.md){.md-button} -[Get started with installation :material-arrow-right:](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md b/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md index 8bce6c474c017..e21bdffbccfe2 100644 --- a/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md +++ b/contrib/pg_tde/documentation/docs/index/how-does-tde-work.md @@ -1,4 +1,4 @@ -# How pg_tde works +# How TDE Works To encrypt the data, two types of keys are used: @@ -7,7 +7,7 @@ To encrypt the data, two types of keys are used: !!! note - For more information on managing and storing principal keys externally, see [Configure Global Key Provider](../global-key-provider-configuration/overview.md). + For more information on managing and storing principal keys externally, see [Configure Global Key Provider](../global-key-provider-configuration/index.md). You have the following options to store and manage principal keys externally: @@ -24,4 +24,4 @@ The internal key itself is encrypted using the principal key. The principal key Similarly when the user queries the encrypted table, the principal key is retrieved from the key store to decrypt the internal key. Then the same unique internal key for that table is used to decrypt the data, and unencrypted data gets returned to the user. So, effectively, every TDE table has a unique key, and each table key is encrypted using the principal key. -[Understand the encrypted data scope :material-arrow-right:](tde-encrypts.md){.md-button} +[Encrypted Data Scope](tde-encrypts.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/how-tde-helps.md b/contrib/pg_tde/documentation/docs/index/how-tde-helps.md index 1f213c20b2e0b..d4397dc330043 100644 --- a/contrib/pg_tde/documentation/docs/index/how-tde-helps.md +++ b/contrib/pg_tde/documentation/docs/index/how-tde-helps.md @@ -1,6 +1,4 @@ -# Benefits of pg_tde - -The benefits of using `pg_tde` are outlined below for different users and organizations. +# TDE Benefits ## Benefits for organizations @@ -13,10 +11,10 @@ The benefits of using `pg_tde` are outlined below for different users and organi * **Operational simplicity:** Works transparently without requiring major application changes. * **Defense in depth:** Adds another layer of protection to existing controls like TLS (encryption in transit), access control, and role-based permissions. -When combined with the external Key Management Systems (KMS), `pg_tde` enables centralized control, auditing, and rotation of encryption keys—critical for secure production environments. +When combined with the external Key Management Systems (KMS), TDE enables centralized control, auditing, and rotation of encryption keys—critical for secure production environments. !!! admonition "See also" - You can find more information on Transparent Data Encryption (TDE) in [this article](https://www.percona.com/blog/transparent-data-encryption-tde/). + Percona Blog: [Transparent Data Encryption (TDE)](https://www.percona.com/blog/transparent-data-encryption-tde/) -[Learn how pg_tde works :material-arrow-right:](how-does-tde-work.md){.md-button} +[How TDE works](how-does-tde-work.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/index.md b/contrib/pg_tde/documentation/docs/index/index.md new file mode 100644 index 0000000000000..8c988a00128d4 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/index/index.md @@ -0,0 +1,7 @@ +# What is Transparent Data Encryption (TDE)? + +Transparent Data Encryption (TDE) protects your data at rest by ensuring that even if the underlying storage is compromised, the data remains unreadable without the correct encryption keys. This is especially critical for environments handling sensitive, regulated, or high-value information. + +Encryption happens transparently in the background, with minimal impact on database operations. + +[TDE Benefits](how-tde-helps.md){.md-button} [Check the full feature list](../features.md){.md-button} [Get Started](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/supported-versions.md b/contrib/pg_tde/documentation/docs/index/supported-versions.md index 7d31c7d45664f..d943451450d58 100644 --- a/contrib/pg_tde/documentation/docs/index/supported-versions.md +++ b/contrib/pg_tde/documentation/docs/index/supported-versions.md @@ -1,4 +1,4 @@ -# Versions and Supported PostgreSQL deployments +# Versions and Supported PostgreSQL Deployments The `pg_tde` extension is available for [Percona Server for PostgreSQL 17.x](https://docs.percona.com/postgresql/17/postgresql-server.html), an open source, drop-in replacement for PostgreSQL Community. This version provides the `tde_heap` access method and offers [full encryption capabilities](../features.md), including encryption of tables, indexes and WAL data. @@ -18,4 +18,4 @@ By using our PostgreSQL distribution, you get: Still unsure which deployment fits your needs? [Contact our experts](https://www.percona.com/about/contact) to find the best solution for your environment. -[Get started with installation :material-arrow-right:](../install.md){.md-button} +[Get Started: Install pg_tde :material-arrow-right:](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/table-access-method.md b/contrib/pg_tde/documentation/docs/index/table-access-method.md index 90957facded97..b308cc1aedbcb 100644 --- a/contrib/pg_tde/documentation/docs/index/table-access-method.md +++ b/contrib/pg_tde/documentation/docs/index/table-access-method.md @@ -1,4 +1,4 @@ -# Table Access Methods and pg_tde +# Table Access Methods and TDE A table access method is the way how PostgreSQL stores the data in a table. The default table access method is `heap`. PostgreSQL organizes data in a heap structure, meaning there is no particular order to the rows in the table. Each row is stored independently and identified by its unique row identifier (TID). @@ -28,7 +28,7 @@ CREATE TABLE table_name ( ) USING tde_heap; ``` -### How tde_heap works with pg_tde +### How tde_heap works The `tde_heap` access method works on top of the default `heap` access method and is a marker to point which tables require encryption. It uses the custom storage manager TDE SMGR, which becomes active only after you installed the `pg_tde` extension. @@ -88,4 +88,4 @@ Here is how you can set the new default table access method: SELECT pg_reload_conf(); ``` -[Understand pg_tde's limitations :material-arrow-right:](tde-limitations.md){.md-button} +[Limitations of TDE](tde-limitations.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/tde-encrypts.md b/contrib/pg_tde/documentation/docs/index/tde-encrypts.md index 19e9357057c50..1bc2c70cef3e4 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-encrypts.md +++ b/contrib/pg_tde/documentation/docs/index/tde-encrypts.md @@ -1,4 +1,4 @@ -# Encrypted data scope +# Encrypted Data Scope `pg_tde` encrypts the following components: @@ -7,4 +7,6 @@ * **Write-Ahead Log (WAL) data** for the entire database cluster. This includes WAL data in encrypted and non-encrypted tables. * **Indexes** on encrypted tables. -[Check out the table access methods :material-arrow-right:](table-access-method.md){.md-button} +!!! + +[Table Access Methods and TDE](table-access-method.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/index/tde-limitations.md b/contrib/pg_tde/documentation/docs/index/tde-limitations.md index 8e3376671aea7..10c1627b789ce 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-limitations.md +++ b/contrib/pg_tde/documentation/docs/index/tde-limitations.md @@ -3,6 +3,5 @@ The following are current limitations of `pg_tde`: * System tables, which include statistics data and database statistics, are currently **not encrypted**. -* The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. [View the versions and supported deployments :material-arrow-right:](supported-versions.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/install.md b/contrib/pg_tde/documentation/docs/install.md index 542c876c2726b..273bb899cd29c 100644 --- a/contrib/pg_tde/documentation/docs/install.md +++ b/contrib/pg_tde/documentation/docs/install.md @@ -39,4 +39,4 @@ Follow the configuration steps below to continue: If you’ve already completed these steps, feel free to skip ahead to a later section: - [Configure Key Management (KMS)](global-key-provider-configuration/overview.md){.md-button} [Validate Encryption with pg_tde](test.md){.md-button} [Configure WAL encryption](wal-encryption.md){.md-button} + [Configure Key Management (KMS)](global-key-provider-configuration/index.md){.md-button} [Validate Encryption with pg_tde](test.md){.md-button} [Configure WAL encryption](wal-encryption.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/release-notes/alpha1.md b/contrib/pg_tde/documentation/docs/release-notes/alpha1.md index 401819f529ba1..766c9e28b136e 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/alpha1.md +++ b/contrib/pg_tde/documentation/docs/release-notes/alpha1.md @@ -1,6 +1,6 @@ # pg_tde Alpha 1 (2024-03-28) -`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get started](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/release-notes/beta.md b/contrib/pg_tde/documentation/docs/release-notes/beta.md index f01faac5fc92e..11985962c0b20 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/beta.md +++ b/contrib/pg_tde/documentation/docs/release-notes/beta.md @@ -1,6 +1,6 @@ # pg_tde Beta (2024-06-30) -`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get started](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/release-notes/beta2.md b/contrib/pg_tde/documentation/docs/release-notes/beta2.md index 1cd220d7d4785..1680c9f58a4ad 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/beta2.md +++ b/contrib/pg_tde/documentation/docs/release-notes/beta2.md @@ -1,13 +1,13 @@ # pg_tde Beta 2 (2024-12-16) -`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get started](../install.md){.md-button} !!! important This version of Percona Transparent Data Encryption extension **is - not recommended for production environments yet**. We encourage you to test it and [give your feedback](https://forums.percona.com/c/postgresql/pg-tde-transparent-data-encryption-tde/82). + not recommended for production environments yet**. We encourage you to test it and [give your feedback :octicons-link-external-16:](https://forums.percona.com/c/postgresql/pg-tde-transparent-data-encryption-tde/82). This will help us improve the product and make it production-ready faster. diff --git a/contrib/pg_tde/documentation/docs/release-notes/rc.md b/contrib/pg_tde/documentation/docs/release-notes/rc.md index 416801e1eaa07..d3ff780ff9187 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/rc.md +++ b/contrib/pg_tde/documentation/docs/release-notes/rc.md @@ -1,6 +1,6 @@ # pg_tde Release Candidate 1 ({{date.RC}}) -`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get started](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/release-notes/rc2.md b/contrib/pg_tde/documentation/docs/release-notes/rc2.md index 0d25d1d49b591..c2d174f7e1f55 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/rc2.md +++ b/contrib/pg_tde/documentation/docs/release-notes/rc2.md @@ -1,6 +1,6 @@ # pg_tde Release Candidate 2 ({{date.RC2}}) -`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/about-tde.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +`pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get Started](../install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md index e5035a8a1d66e..96a7ce3b1ea2a 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md @@ -1,6 +1,6 @@ # pg_tde 1.0 ({{date.GA10}}) -The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. [Get Started](../install.md){.md-button} @@ -8,7 +8,7 @@ The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE) : * **`pg_tde` 1.0 is now GA (Generally Available)** -And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE) :octicons-link-external-16:](../index/index.md). This milestone brings production-level data protection to PostgreSQL workloads. +And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE)](../index/index.md). This milestone brings production-level data protection to PostgreSQL workloads. * **WAL encryption is still in Beta** @@ -16,7 +16,7 @@ The WAL encryption feature is currently still in beta and is not effective unles ## Upgrade considerations -`pg_tde` ({{tdeversion}}) is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`. +`pg_tde` {{tdeversion}} is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`. ## Known issues @@ -33,29 +33,29 @@ Adjust the limits with caution since it affects other processes running in your ### New Features -- [PG-1257 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1257) – Added SQL function to remove the current principal key +- [PG-1257](https://perconadev.atlassian.net/browse/PG-1257) – Added SQL function to remove the current principal key ### Improvements -- [PG-1617 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1617) – Removed relation key cache -- [PG-1635 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1635) – User-facing TDE functions now return void -- [PG-1605 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1605) – Removed undeclared dependencies for `pg_tde_grant_database_key_management_to_role()` +- [PG-1617](https://perconadev.atlassian.net/browse/PG-1617) – Removed relation key cache +- [PG-1635](https://perconadev.atlassian.net/browse/PG-1635) – User-facing TDE functions now return void +- [PG-1605](https://perconadev.atlassian.net/browse/PG-1605) – Removed undeclared dependencies for `pg_tde_grant_database_key_management_to_role()` ### Bugs Fixed -- [PG-1581 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1581) – Fixed PostgreSQL crashes on table access when KMIP key is unavailable after restart -- [PG-1583 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1583) – Fixed a crash when dropping the `pg_tde` extension with CASCADE after changing the key provider file -- [PG-1585 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1585) – Fixed the vault provider re-addition that failed after server restart with a new token -- [PG-1592 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1592) – Improve error logs when Server Key Info is requested without being created -- [PG-1593 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1593) – Fixed runtime failures when invalid Vault tokens are allowed during key provider creation -- [PG-1600 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1600) – Fixed Postmaster error when dropping a table with an unavailable key provider -- [PG-1606 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1606) – Fixed missing superuser check in role grant function leads to misleading errors -- [PG-1607 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1607) – Improved CA parameter order and surrounding documentation for clearer interpretation -- [PG-1608 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1608) – Updated and fixed global key configuration parameters in documentation -- [PG-1613 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1613) – Tested and improved the `pg_tde_change_key_provider` CLI utility -- [PG-1637 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1637) – Fixed unused keys in key files which caused issues after OID wraparound -- [PG-1651 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1651) – Fixed the CLI tool when working with Vault key export/import -- [PG-1652 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1652) – Fixed when the server fails to find encryption keys after CLI-based provider change -- [PG-1662 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1662) – Fixed the creation of inconsistent encryption status when altering partitioned tables -- [PG-1663 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1663) – Fixed the indexes on partitioned tables which were not encrypted -- [PG-1700 :octicons-link-external-16:](https://perconadev.atlassian.net/browse/PG-1700) – Fixed the error hint when the principal key is missing +- [PG-1581](https://perconadev.atlassian.net/browse/PG-1581) – Fixed PostgreSQL crashes on table access when KMIP key is unavailable after restart +- [PG-1583](https://perconadev.atlassian.net/browse/PG-1583) – Fixed a crash when dropping the `pg_tde` extension with CASCADE after changing the key provider file +- [PG-1585](https://perconadev.atlassian.net/browse/PG-1585) – Fixed the vault provider re-addition that failed after server restart with a new token +- [PG-1592](https://perconadev.atlassian.net/browse/PG-1592) – Improve error logs when Server Key Info is requested without being created +- [PG-1593](https://perconadev.atlassian.net/browse/PG-1593) – Fixed runtime failures when invalid Vault tokens are allowed during key provider creation +- [PG-1600](https://perconadev.atlassian.net/browse/PG-1600) – Fixed Postmaster error when dropping a table with an unavailable key provider +- [PG-1606](https://perconadev.atlassian.net/browse/PG-1606) – Fixed missing superuser check in role grant function leads to misleading errors +- [PG-1607](https://perconadev.atlassian.net/browse/PG-1607) – Improved CA parameter order and surrounding documentation for clearer interpretation +- [PG-1608](https://perconadev.atlassian.net/browse/PG-1608) – Updated and fixed global key configuration parameters in documentation +- [PG-1613](https://perconadev.atlassian.net/browse/PG-1613) – Tested and improved the `pg_tde_change_key_provider` CLI utility +- [PG-1637](https://perconadev.atlassian.net/browse/PG-1637) – Fixed unused keys in key files which caused issues after OID wraparound +- [PG-1651](https://perconadev.atlassian.net/browse/PG-1651) – Fixed the CLI tool when working with Vault key export/import +- [PG-1652](https://perconadev.atlassian.net/browse/PG-1652) – Fixed when the server fails to find encryption keys after CLI-based provider change +- [PG-1662](https://perconadev.atlassian.net/browse/PG-1662) – Fixed the creation of inconsistent encryption status when altering partitioned tables +- [PG-1663](https://perconadev.atlassian.net/browse/PG-1663) – Fixed the indexes on partitioned tables which were not encrypted +- [PG-1700](https://perconadev.atlassian.net/browse/PG-1700) – Fixed the error hint when the principal key is missing diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md index 38d8536e2ba4f..d1e7b8e1424d8 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md @@ -1,19 +1,10 @@ -# Percona Transparent Data Encryption for PostgreSQL release notes index - -This page lists all release notes for `pg_tde`, organized by year and version. Use it to track new features, fixes, and updates across major and minor versions. - -## 2025 - -* [1.0](release-notes-v1.0.md) ({{date.GA10}}) -* [Release Candidate 2 (RC2)](rc2.md) ({{date.RC2}}) -* [Release Candidate 1 (RC1)](rc.md) ({{date.RC}}) - -## 2024 - -* [Beta 2](beta2.md) (2024-12-16) -* [Beta 1](beta.md) (2024-06-30) -* [Alpha 1](alpha1.md) (2024-03-28) - -## 2023 - -* [MVP](mvp.md) (2023-12-12) +# Release notes index + +* [Percona Transparent Database Encryption for PostgreSQL 1.0](release-notes-v1.0.md) ({{date.GA10}}) +* [pg_tde Release Candidate 2 (RC2)](rc2.md) ({{date.RC2}}) +* [pg_tde Release Candidate 1 (RC1)](rc2.md) ({{date.RC}}) +* [pg_tde Release Candidate](rc.md) ({{date.RC}}) +* [pg_tde Beta2](beta2.md) (2024-12-16) +* [pg_tde Beta](beta.md) (2024-06-30) +* [pg_tde Alpha1](alpha1.md) (2024-03-28) +* [pg_tde MVP](mvp.md) (2023-12-12) diff --git a/contrib/pg_tde/documentation/docs/setup.md b/contrib/pg_tde/documentation/docs/setup.md index 8b0d1f24136c3..73cafba216c50 100644 --- a/contrib/pg_tde/documentation/docs/setup.md +++ b/contrib/pg_tde/documentation/docs/setup.md @@ -65,4 +65,4 @@ psql -d template1 -c 'CREATE EXTENSION pg_tde;' ## Next steps -[Configure Key Management (KMS) :material-arrow-right:](global-key-provider-configuration/overview.md){.md-button} +[Configure Key Management (KMS) :material-arrow-right:](global-key-provider-configuration/index.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/test.md b/contrib/pg_tde/documentation/docs/test.md index be73e73863e76..2da3b4b33f17a 100644 --- a/contrib/pg_tde/documentation/docs/test.md +++ b/contrib/pg_tde/documentation/docs/test.md @@ -59,4 +59,4 @@ ALTER TABLE table_name SET ACCESS METHOD tde_heap; ## Next steps -[Configure WAL Encryption (tech preview) :material-arrow-right:](wal-encryption.md){.md-button} +[Configure WAL Encryption :material-arrow-right:](wal-encryption.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/variables.md b/contrib/pg_tde/documentation/docs/variables.md index 74bdd62138193..e0acbf03131f0 100644 --- a/contrib/pg_tde/documentation/docs/variables.md +++ b/contrib/pg_tde/documentation/docs/variables.md @@ -19,9 +19,6 @@ This variable only controls new writes to the WAL, it doesn't affect existing WA Enabling WAL encryption requires a configured global principal key. Refer to the [WAL encryption configuration](wal-encryption.md) topic for more information. -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - ## pg_tde.enforce_encryption **Type** - boolean
diff --git a/contrib/pg_tde/documentation/docs/wal-encryption.md b/contrib/pg_tde/documentation/docs/wal-encryption.md index 33547d0e359c4..5064085c4bd18 100644 --- a/contrib/pg_tde/documentation/docs/wal-encryption.md +++ b/contrib/pg_tde/documentation/docs/wal-encryption.md @@ -1,9 +1,6 @@ -# Configure WAL Encryption (tech preview) +# Configure WAL Encryption -!!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. - -Before enabling WAL encryption, follow the steps below to create a principal key and configure it for WAL: +Before enabling WAL encryption, follow the steps below to create a principal key and configure it for WAL: 1. Create the `pg_tde` extension if it does not exist: @@ -15,7 +12,7 @@ Before enabling WAL encryption, follow the steps below to create a principal key === "With KMIP server" - Make sure you have obtained the root certificate for the KMIP server and the keypair for the client. The client key needs permissions to create / read keys on the server. Find the [configuration guidelines for the HashiCorp Vault Enterprise KMIP Secrets Engine](https://developer.hashicorp.com/vault/tutorials/enterprise/kmip-engine). + Make sure you have obtained the root certificate for the KMIP server and the keypair for the client. The client key needs permissions to create / read keys on the server. Find the [configuration guidelines for the HashiCorp Vault Enterprise KMIP Secrets Engine :octicons-link-external-16:](https://developer.hashicorp.com/vault/tutorials/enterprise/kmip-engine). For testing purposes, you can use the PyKMIP server which enables you to set up required certificates. To use a real KMIP server, make sure to obtain the valid certificates issued by the key management appliance. @@ -117,4 +114,6 @@ Before enabling WAL encryption, follow the steps below to create a principal key Now WAL files start to be encrypted for both encrypted and unencrypted tables. For more technical references related to architecture, variables or functions, see: -[Technical Reference](advanced-topics/tech-reference.md){.md-button} +[Technical Reference](advanced-topics/index.md){.md-button} + +💬 Need help customizing this for your infrastructure? [Contact Percona support :octicons-link-external-16:](get-help.md) diff --git a/contrib/pg_tde/documentation/snippets/kms-considerations.md b/contrib/pg_tde/documentation/snippets/kms-considerations.md index 5ef51732fc76a..c0e6bfdadfb91 100644 --- a/contrib/pg_tde/documentation/snippets/kms-considerations.md +++ b/contrib/pg_tde/documentation/snippets/kms-considerations.md @@ -1,3 +1,3 @@ ## Considerations -You can use external key providers to manage encryption keys. The recommended approach is to use the Key Management Store (KMS). For more information, see [Configure Key Management (KMS) :octicons-link-external-16:](../global-key-provider-configuration/overview.md). +You can use external key providers to manage encryption keys. The recommended approach is to use the Key Management Store (KMS). For more information, see [Configure Key Management (KMS)](../global-key-provider-configuration/index.md). diff --git a/contrib/pg_tde/documentation/variables.yml b/contrib/pg_tde/documentation/variables.yml index bf698a07c2272..2938e44a53a96 100644 --- a/contrib/pg_tde/documentation/variables.yml +++ b/contrib/pg_tde/documentation/variables.yml @@ -1,6 +1,5 @@ #Variables used throughout the docs -latestreleasenotes: 'release-notes-v1.0' tdeversion: '1.0' release: '1.0' pgversion17: '17.5' diff --git a/contrib/pg_tde/expected/default_principal_key.out b/contrib/pg_tde/expected/default_principal_key.out index 21319d568ea60..72ec579b6de3b 100644 --- a/contrib/pg_tde/expected/default_principal_key.out +++ b/contrib/pg_tde/expected/default_principal_key.out @@ -45,7 +45,7 @@ SELECT provider_id, provider_name, key_name -- fails SELECT pg_tde_delete_global_key_provider('file-provider'); -ERROR: cannot delete provider which is currently in use +ERROR: Can't delete a provider which is currently in use SELECT id, name FROM pg_tde_list_all_global_key_providers(); id | name ----+--------------- diff --git a/contrib/pg_tde/expected/delete_principal_key.out b/contrib/pg_tde/expected/delete_principal_key.out index 92b8299c2b725..550bcc217b2dd 100644 --- a/contrib/pg_tde/expected/delete_principal_key.out +++ b/contrib/pg_tde/expected/delete_principal_key.out @@ -134,46 +134,6 @@ SELECT pg_tde_delete_key(); (1 row) --- Delete default key even if it's configured for a database or server key, as --- long as it's unused. Regardless how the key was set, we unset it if it's the --- same key as is used as a default key. This is probably a bug. -SELECT pg_tde_set_default_key_using_global_key_provider('test-db-key','file-provider'); - pg_tde_set_default_key_using_global_key_provider --------------------------------------------------- - -(1 row) - -SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); - pg_tde_set_key_using_global_key_provider ------------------------------------------- - -(1 row) - -SELECT pg_tde_set_server_key_using_global_key_provider('test-db-key','file-provider'); -WARNING: The WAL encryption feature is currently in beta and may be unstable. Do not use it in production environments! - pg_tde_set_server_key_using_global_key_provider -------------------------------------------------- - -(1 row) - -SELECT pg_tde_delete_default_key(); - pg_tde_delete_default_key ---------------------------- - -(1 row) - -SELECT pg_tde_key_info(); -- No key configured - pg_tde_key_info ------------------ - (,,,) -(1 row) - -SELECT pg_tde_server_key_info(); -- No key configured - pg_tde_server_key_info ------------------------- - (,,,) -(1 row) - SELECT pg_tde_delete_global_key_provider('file-provider'); pg_tde_delete_global_key_provider ----------------------------------- diff --git a/contrib/pg_tde/expected/key_provider.out b/contrib/pg_tde/expected/key_provider.out index 7be4f59044fd2..5563df5aa4d9c 100644 --- a/contrib/pg_tde/expected/key_provider.out +++ b/contrib/pg_tde/expected/key_provider.out @@ -91,7 +91,7 @@ SELECT id, name FROM pg_tde_list_all_global_key_providers(); -- fails SELECT pg_tde_delete_database_key_provider('file-provider'); -ERROR: cannot delete provider which is currently in use +ERROR: Can't delete a provider which is currently in use SELECT id, name FROM pg_tde_list_all_database_key_providers(); id | name ----+---------------- @@ -127,7 +127,7 @@ SELECT pg_tde_set_key_using_global_key_provider('test-db-key', 'file-keyring'); -- fails SELECT pg_tde_delete_global_key_provider('file-keyring'); -ERROR: cannot delete provider which is currently in use +ERROR: Can't delete a provider which is currently in use SELECT id, name FROM pg_tde_list_all_global_key_providers(); id | name ----+--------------- diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 5785f3148a25c..42a6bbe6afff3 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -122,7 +122,6 @@ tap_tests = [ 't/rotate_key.pl', 't/tde_heap.pl', 't/unlogged_tables.pl', - 't/wal_archiving.pl', 't/wal_encrypt.pl', ] @@ -152,7 +151,7 @@ pg_tde_frontend = static_library('pg_tde_frontend', ) pg_tde_change_key_provider_sources = files( - 'src/bin/pg_tde_change_key_provider.c', + 'src/pg_tde_change_key_provider.c', ) pg_tde_change_key_provider = executable('pg_tde_change_key_provider', @@ -164,31 +163,3 @@ pg_tde_change_key_provider = executable('pg_tde_change_key_provider', link_with: [pg_tde_frontend] ) contrib_targets += pg_tde_change_key_provider - -pg_tde_archive_decrypt_sources = files( - 'src/bin/pg_tde_archive_decrypt.c', -) + xlogreader_sources - -pg_tde_archive_decrypt = executable('pg_tde_archive_decrypt', - pg_tde_archive_decrypt_sources, - dependencies: [frontend_code], - c_args: ['-DFRONTEND'], # needed for xlogreader et al - kwargs: default_bin_args, - include_directories: [postgres_inc, pg_tde_inc], - link_with: [pg_tde_frontend] -) -contrib_targets += pg_tde_archive_decrypt - -pg_tde_restore_encrypt_sources = files( - 'src/bin/pg_tde_restore_encrypt.c', -) + xlogreader_sources - -pg_tde_restore_encrypt = executable('pg_tde_restore_encrypt', - pg_tde_restore_encrypt_sources, - dependencies: [frontend_code], - c_args: ['-DFRONTEND'], # needed for xlogreader et al - kwargs: default_bin_args, - include_directories: [postgres_inc, pg_tde_inc], - link_with: [pg_tde_frontend] -) -contrib_targets += pg_tde_restore_encrypt diff --git a/contrib/pg_tde/sql/delete_principal_key.sql b/contrib/pg_tde/sql/delete_principal_key.sql index 142eedde0a56d..abc4c574b5616 100644 --- a/contrib/pg_tde/sql/delete_principal_key.sql +++ b/contrib/pg_tde/sql/delete_principal_key.sql @@ -53,16 +53,5 @@ SELECT pg_tde_delete_default_key(); DROP TABLE test_table; SELECT pg_tde_delete_key(); - --- Delete default key even if it's configured for a database or server key, as --- long as it's unused. Regardless how the key was set, we unset it if it's the --- same key as is used as a default key. This is probably a bug. -SELECT pg_tde_set_default_key_using_global_key_provider('test-db-key','file-provider'); -SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); -SELECT pg_tde_set_server_key_using_global_key_provider('test-db-key','file-provider'); -SELECT pg_tde_delete_default_key(); -SELECT pg_tde_key_info(); -- No key configured -SELECT pg_tde_server_key_info(); -- No key configured - SELECT pg_tde_delete_global_key_provider('file-provider'); DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 379e56bc54cf3..f285f2e24a0a8 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -54,24 +54,23 @@ typedef struct TDEFileHeader static WALKeyCacheRec *tde_wal_key_cache = NULL; static WALKeyCacheRec *tde_wal_key_last_rec = NULL; -static void pg_tde_initialize_map_entry(TDEMapEntry *map_entry, const TDEPrincipalKey *principal_key, const RelFileLocator *rlocator, const InternalKey *rel_key_data); -static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, const InternalKey *rel_key_data, TDEPrincipalKey *principal_key); static bool pg_tde_find_map_entry(const RelFileLocator *rlocator, TDEMapEntryType key_type, char *db_map_path, TDEMapEntry *map_entry); static InternalKey *tde_decrypt_rel_key(TDEPrincipalKey *principal_key, TDEMapEntry *map_entry); static int pg_tde_open_file_basic(const char *tde_filename, int fileFlags, bool ignore_missing); -static int pg_tde_open_file_read(const char *tde_filename, bool ignore_missing, off_t *curr_pos); -static int pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos); static void pg_tde_file_header_read(const char *tde_filename, int fd, TDEFileHeader *fheader, off_t *bytes_read); -static int pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written); static bool pg_tde_read_one_map_entry(int fd, TDEMapEntry *map_entry, off_t *offset); static void pg_tde_read_one_map_entry2(int keydata_fd, int32 key_index, TDEMapEntry *map_entry, Oid databaseId); +static int pg_tde_open_file_read(const char *tde_filename, bool ignore_missing, off_t *curr_pos); static WALKeyCacheRec *pg_tde_add_wal_key_to_cache(InternalKey *cached_key, XLogRecPtr start_lsn); #ifndef FRONTEND +static int pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written); static void pg_tde_sign_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key); static void pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path); +static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, const InternalKey *rel_key_data, TDEPrincipalKey *principal_key); static int keyrotation_init_file(const TDESignedPrincipalKeyInfo *signed_key_info, char *rotated_filename, const char *filename, off_t *curr_pos); static void finalize_key_rotation(const char *path_old, const char *path_new); +static int pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos); void pg_tde_save_smgr_key(RelFileLocator rel, const InternalKey *rel_key_data) @@ -104,6 +103,37 @@ tde_sprint_key(InternalKey *k) return buf; } +/* + * Generates a new internal key for WAL and adds it to the key file. + * + * We have a special function for WAL as it is being called during recovery + * start so there should be no XLog records and aquired locks. The key is + * always created with start_lsn = InvalidXLogRecPtr. Which will be updated + * with the actual lsn by the first WAL write. + */ +void +pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocator, TDEMapEntryType entry_type) +{ + TDEPrincipalKey *principal_key; + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + + principal_key = GetPrincipalKey(newrlocator->dbOid, LW_EXCLUSIVE); + if (principal_key == NULL) + { + ereport(ERROR, + errmsg("principal key not configured"), + errhint("Use pg_tde_set_server_key_using_global_key_provider() to configure one.")); + } + + /* TODO: no need in generating key if TDE_KEY_TYPE_WAL_UNENCRYPTED */ + pg_tde_generate_internal_key(rel_key_data, entry_type); + + pg_tde_write_key_map_entry(newrlocator, rel_key_data, principal_key); + + LWLockRelease(tde_lwlock_enc_keys()); +} + /* * Deletes the key file for a given database. */ @@ -172,6 +202,155 @@ pg_tde_save_principal_key(const TDEPrincipalKey *principal_key, bool write_xlog) CloseTransientFile(map_fd); } +/* + * Write TDE file header to a TDE file. + */ +static int +pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written) +{ + TDEFileHeader fheader; + + Assert(signed_key_info); + + fheader.file_version = PG_TDE_FILEMAGIC; + fheader.signed_key_info = *signed_key_info; + *bytes_written = pg_pwrite(fd, &fheader, TDE_FILE_HEADER_SIZE, 0); + + if (*bytes_written != TDE_FILE_HEADER_SIZE) + { + ereport(ERROR, + errcode_for_file_access(), + errmsg("could not write tde file \"%s\": %m", tde_filename)); + } + + if (pg_fsync(fd) != 0) + { + ereport(data_sync_elevel(ERROR), + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", tde_filename)); + } + + ereport(DEBUG2, errmsg("Wrote the header to %s", tde_filename)); + + return fd; +} + +static void +pg_tde_sign_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key) +{ + signed_key_info->data = principal_key->keyInfo; + + if (!RAND_bytes(signed_key_info->sign_iv, MAP_ENTRY_IV_SIZE)) + ereport(ERROR, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL))); + + AesGcmEncrypt(principal_key->keyData, + signed_key_info->sign_iv, MAP_ENTRY_IV_SIZE, + (unsigned char *) &signed_key_info->data, sizeof(signed_key_info->data), + NULL, 0, + NULL, + signed_key_info->aead_tag, MAP_ENTRY_AEAD_TAG_SIZE); +} + +static void +pg_tde_initialize_map_entry(TDEMapEntry *map_entry, const TDEPrincipalKey *principal_key, const RelFileLocator *rlocator, const InternalKey *rel_key_data) +{ + map_entry->spcOid = rlocator->spcOid; + map_entry->relNumber = rlocator->relNumber; + map_entry->type = rel_key_data->type; + map_entry->enc_key = *rel_key_data; + + if (!RAND_bytes(map_entry->entry_iv, MAP_ENTRY_IV_SIZE)) + ereport(ERROR, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL))); + + AesGcmEncrypt(principal_key->keyData, + map_entry->entry_iv, MAP_ENTRY_IV_SIZE, + (unsigned char *) map_entry, offsetof(TDEMapEntry, enc_key), + rel_key_data->key, INTERNAL_KEY_LEN, + map_entry->enc_key.key, + map_entry->aead_tag, MAP_ENTRY_AEAD_TAG_SIZE); +} + +static void +pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path) +{ + int bytes_written = 0; + + bytes_written = pg_pwrite(fd, map_entry, MAP_ENTRY_SIZE, *offset); + + if (bytes_written != MAP_ENTRY_SIZE) + { + ereport(ERROR, + errcode_for_file_access(), + errmsg("could not write tde map file \"%s\": %m", db_map_path)); + } + if (pg_fsync(fd) != 0) + { + ereport(data_sync_elevel(ERROR), + errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", db_map_path)); + } + + *offset += bytes_written; +} + +/* + * The caller must hold an exclusive lock on the key file to avoid + * concurrent in place updates leading to data conflicts. + */ +void +pg_tde_write_key_map_entry(const RelFileLocator *rlocator, const InternalKey *rel_key_data, TDEPrincipalKey *principal_key) +{ + char db_map_path[MAXPGPATH]; + int map_fd; + off_t curr_pos = 0; + TDEMapEntry write_map_entry; + TDESignedPrincipalKeyInfo signed_key_Info; + + Assert(rlocator); + + pg_tde_set_db_file_path(rlocator->dbOid, db_map_path); + + pg_tde_sign_principal_key_info(&signed_key_Info, principal_key); + + /* Open and validate file for basic correctness. */ + map_fd = pg_tde_open_file_write(db_map_path, &signed_key_Info, false, &curr_pos); + + /* + * Read until we find an empty slot. Otherwise, read until end. This seems + * to be less frequent than vacuum. So let's keep this function here + * rather than overloading the vacuum process. + */ + while (1) + { + TDEMapEntry read_map_entry; + off_t prev_pos = curr_pos; + + if (!pg_tde_read_one_map_entry(map_fd, &read_map_entry, &curr_pos)) + { + curr_pos = prev_pos; + break; + } + + if (read_map_entry.type == MAP_ENTRY_EMPTY) + { + curr_pos = prev_pos; + break; + } + } + + /* Initialize map entry and encrypt key */ + pg_tde_initialize_map_entry(&write_map_entry, principal_key, rlocator, rel_key_data); + + /* Write the given entry at curr_pos; i.e. the free entry. */ + pg_tde_write_one_map_entry(map_fd, &write_map_entry, &curr_pos, db_map_path); + + CloseTransientFile(map_fd); +} + /* * Mark relation map entry as free and overwrite the key * @@ -350,7 +529,7 @@ pg_tde_delete_principal_key(Oid dbOid) char path[MAXPGPATH]; Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_EXCLUSIVE)); - Assert(pg_tde_count_encryption_keys(dbOid) == 0); + Assert(pg_tde_count_relations(dbOid) == 0); pg_tde_set_db_file_path(dbOid, path); @@ -362,8 +541,6 @@ pg_tde_delete_principal_key(Oid dbOid) durable_unlink(path, ERROR); } -#endif /* !FRONTEND */ - /* * It's called by seg_write inside crit section so no pallocs, hence * needs keyfile_path @@ -432,155 +609,37 @@ pg_tde_wal_last_key_set_lsn(XLogRecPtr lsn, const char *keyfile_path) CloseTransientFile(fd); } -static void -pg_tde_sign_principal_key_info(TDESignedPrincipalKeyInfo *signed_key_info, const TDEPrincipalKey *principal_key) -{ - signed_key_info->data = principal_key->keyInfo; - - if (!RAND_bytes(signed_key_info->sign_iv, MAP_ENTRY_IV_SIZE)) - ereport(ERROR, - errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL))); - - AesGcmEncrypt(principal_key->keyData, - signed_key_info->sign_iv, MAP_ENTRY_IV_SIZE, - (unsigned char *) &signed_key_info->data, sizeof(signed_key_info->data), - NULL, 0, - NULL, - signed_key_info->aead_tag, MAP_ENTRY_AEAD_TAG_SIZE); -} - -static void -pg_tde_initialize_map_entry(TDEMapEntry *map_entry, const TDEPrincipalKey *principal_key, const RelFileLocator *rlocator, const InternalKey *rel_key_data) -{ - map_entry->spcOid = rlocator->spcOid; - map_entry->relNumber = rlocator->relNumber; - map_entry->type = rel_key_data->type; - map_entry->enc_key = *rel_key_data; - - if (!RAND_bytes(map_entry->entry_iv, MAP_ENTRY_IV_SIZE)) - ereport(ERROR, - errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate iv for key map: %s", ERR_error_string(ERR_get_error(), NULL))); - - AesGcmEncrypt(principal_key->keyData, - map_entry->entry_iv, MAP_ENTRY_IV_SIZE, - (unsigned char *) map_entry, offsetof(TDEMapEntry, enc_key), - rel_key_data->key, INTERNAL_KEY_LEN, - map_entry->enc_key.key, - map_entry->aead_tag, MAP_ENTRY_AEAD_TAG_SIZE); -} - -static void -pg_tde_write_one_map_entry(int fd, const TDEMapEntry *map_entry, off_t *offset, const char *db_map_path) -{ - int bytes_written = 0; - - bytes_written = pg_pwrite(fd, map_entry, MAP_ENTRY_SIZE, *offset); - - if (bytes_written != MAP_ENTRY_SIZE) - { - ereport(ERROR, - errcode_for_file_access(), - errmsg("could not write tde map file \"%s\": %m", db_map_path)); - } - if (pg_fsync(fd) != 0) - { - ereport(data_sync_elevel(ERROR), - errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", db_map_path)); - } - - *offset += bytes_written; -} - /* - * Generates a new internal key for WAL and adds it to the key file. + * Open for write and Validate File Header: + * header: {Format Version, Principal Key Name} * - * We have a special function for WAL as it is being called during recovery - * start so there should be no XLog records and aquired locks. The key is - * always created with start_lsn = InvalidXLogRecPtr. Which will be updated - * with the actual lsn by the first WAL write. + * Returns the file descriptor in case of a success. Otherwise, error + * is raised. */ -void -pg_tde_create_wal_key(InternalKey *rel_key_data, const RelFileLocator *newrlocator, TDEMapEntryType entry_type) +static int +pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos) { - TDEPrincipalKey *principal_key; + int fd; + TDEFileHeader fheader; + off_t bytes_read = 0; + off_t bytes_written = 0; + int file_flags = O_RDWR | O_CREAT | PG_BINARY | (truncate ? O_TRUNC : 0); - LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_EXCLUSIVE)); - principal_key = GetPrincipalKey(newrlocator->dbOid, LW_EXCLUSIVE); - if (principal_key == NULL) - { - ereport(ERROR, - errmsg("principal key not configured"), - errhint("Use pg_tde_set_server_key_using_global_key_provider() to configure one.")); - } + fd = pg_tde_open_file_basic(tde_filename, file_flags, false); - /* TODO: no need in generating key if TDE_KEY_TYPE_WAL_UNENCRYPTED */ - pg_tde_generate_internal_key(rel_key_data, entry_type); + pg_tde_file_header_read(tde_filename, fd, &fheader, &bytes_read); - pg_tde_write_key_map_entry(newrlocator, rel_key_data, principal_key); + /* In case it's a new file, let's add the header now. */ + if (bytes_read == 0 && signed_key_info) + pg_tde_file_header_write(tde_filename, fd, signed_key_info, &bytes_written); -#ifdef FRONTEND - free(principal_key); -#endif - LWLockRelease(tde_lwlock_enc_keys()); + *curr_pos = bytes_read + bytes_written; + return fd; } -/* - * The caller must hold an exclusive lock on the key file to avoid - * concurrent in place updates leading to data conflicts. - */ -void -pg_tde_write_key_map_entry(const RelFileLocator *rlocator, const InternalKey *rel_key_data, TDEPrincipalKey *principal_key) -{ - char db_map_path[MAXPGPATH]; - int map_fd; - off_t curr_pos = 0; - TDEMapEntry write_map_entry; - TDESignedPrincipalKeyInfo signed_key_Info; - - Assert(rlocator); - - pg_tde_set_db_file_path(rlocator->dbOid, db_map_path); - - pg_tde_sign_principal_key_info(&signed_key_Info, principal_key); - - /* Open and validate file for basic correctness. */ - map_fd = pg_tde_open_file_write(db_map_path, &signed_key_Info, false, &curr_pos); - - /* - * Read until we find an empty slot. Otherwise, read until end. This seems - * to be less frequent than vacuum. So let's keep this function here - * rather than overloading the vacuum process. - */ - while (1) - { - TDEMapEntry read_map_entry; - off_t prev_pos = curr_pos; - - if (!pg_tde_read_one_map_entry(map_fd, &read_map_entry, &curr_pos)) - { - curr_pos = prev_pos; - break; - } - - if (read_map_entry.type == MAP_ENTRY_EMPTY) - { - curr_pos = prev_pos; - break; - } - } - - /* Initialize map entry and encrypt key */ - pg_tde_initialize_map_entry(&write_map_entry, principal_key, rlocator, rel_key_data); - - /* Write the given entry at curr_pos; i.e. the free entry. */ - pg_tde_write_one_map_entry(map_fd, &write_map_entry, &curr_pos, db_map_path); - - CloseTransientFile(map_fd); -} +#endif /* !FRONTEND */ /* * Returns true if we find a valid match; e.g. type is not set to @@ -613,15 +672,17 @@ pg_tde_find_map_entry(const RelFileLocator *rlocator, TDEMapEntryType key_type, } /* - * Counts number of encryption keys in a key file. + * Counts number of encrypted objects in a database. * * Does not check if objects actually exist but just that they have keys in - * the key file. + * the map file. For the only current caller, checking if we can use + * FILE_COPY, this is good enough but for other workloads where a false + * positive is more harmful this might not be. * - * Works even if the database has no key file. + * Works even if the database has no map file. */ int -pg_tde_count_encryption_keys(Oid dbOid) +pg_tde_count_relations(Oid dbOid) { char db_map_path[MAXPGPATH]; File map_fd; @@ -680,28 +741,6 @@ tde_decrypt_rel_key(TDEPrincipalKey *principal_key, TDEMapEntry *map_entry) return rel_key_data; } -/* - * Open a TDE file: - * - * Returns the file descriptor in case of a success. Otherwise, error - * is raised except when ignore_missing is true and the file does not exit. - */ -static int -pg_tde_open_file_basic(const char *tde_filename, int fileFlags, bool ignore_missing) -{ - int fd; - - fd = OpenTransientFile(tde_filename, fileFlags); - if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) - { - ereport(ERROR, - errcode_for_file_access(), - errmsg("could not open tde file \"%s\": %m", tde_filename)); - } - - return fd; -} - /* * Open for read and Validate File Header: * header: {Format Version, Principal Key Name} @@ -729,32 +768,24 @@ pg_tde_open_file_read(const char *tde_filename, bool ignore_missing, off_t *curr } /* - * Open for write and Validate File Header: - * header: {Format Version, Principal Key Name} + * Open a TDE file: * * Returns the file descriptor in case of a success. Otherwise, error - * is raised. + * is raised except when ignore_missing is true and the file does not exit. */ static int -pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos) +pg_tde_open_file_basic(const char *tde_filename, int fileFlags, bool ignore_missing) { int fd; - TDEFileHeader fheader; - off_t bytes_read = 0; - off_t bytes_written = 0; - int file_flags = O_RDWR | O_CREAT | PG_BINARY | (truncate ? O_TRUNC : 0); - - Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_EXCLUSIVE)); - fd = pg_tde_open_file_basic(tde_filename, file_flags, false); - - pg_tde_file_header_read(tde_filename, fd, &fheader, &bytes_read); - - /* In case it's a new file, let's add the header now. */ - if (bytes_read == 0 && signed_key_info) - pg_tde_file_header_write(tde_filename, fd, signed_key_info, &bytes_written); + fd = OpenTransientFile(tde_filename, fileFlags); + if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) + { + ereport(ERROR, + errcode_for_file_access(), + errmsg("could not open tde file \"%s\": %m", tde_filename)); + } - *curr_pos = bytes_read + bytes_written; return fd; } @@ -781,39 +812,6 @@ pg_tde_file_header_read(const char *tde_filename, int fd, TDEFileHeader *fheader } } -/* - * Write TDE file header to a TDE file. - */ -static int -pg_tde_file_header_write(const char *tde_filename, int fd, const TDESignedPrincipalKeyInfo *signed_key_info, off_t *bytes_written) -{ - TDEFileHeader fheader; - - Assert(signed_key_info); - - fheader.file_version = PG_TDE_FILEMAGIC; - fheader.signed_key_info = *signed_key_info; - *bytes_written = pg_pwrite(fd, &fheader, TDE_FILE_HEADER_SIZE, 0); - - if (*bytes_written != TDE_FILE_HEADER_SIZE) - { - ereport(ERROR, - errcode_for_file_access(), - errmsg("could not write tde file \"%s\": %m", tde_filename)); - } - - if (pg_fsync(fd) != 0) - { - ereport(data_sync_elevel(ERROR), - errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", tde_filename)); - } - - ereport(DEBUG2, errmsg("Wrote the header to %s", tde_filename)); - - return fd; -} - /* * Returns true if a map entry if found or false if we have reached the end of * the file. @@ -1021,9 +1019,6 @@ pg_tde_read_last_wal_key(void) /* No keys */ if (fsize == TDE_FILE_HEADER_SIZE) { -#ifdef FRONTEND - pfree(principal_key); -#endif LWLockRelease(lock_pk); CloseTransientFile(fd); return NULL; @@ -1033,9 +1028,6 @@ pg_tde_read_last_wal_key(void) pg_tde_read_one_map_entry2(fd, file_idx, &map_entry, rlocator.dbOid); rel_key_data = tde_decrypt_rel_key(principal_key, &map_entry); -#ifdef FRONTEND - pfree(principal_key); -#endif LWLockRelease(lock_pk); CloseTransientFile(fd); diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_smgr.c b/contrib/pg_tde/src/access/pg_tde_xlog_smgr.c index 5578167a9b471..bd08072aefdcc 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog_smgr.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog_smgr.c @@ -19,11 +19,11 @@ #include "encryption/enc_tde.h" #include "pg_tde.h" #include "pg_tde_defines.h" +#include "pg_tde_guc.h" #ifdef FRONTEND #include "pg_tde_fe.h" #else -#include "pg_tde_guc.h" #include "port/atomics.h" #endif @@ -32,21 +32,15 @@ static ssize_t tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offs TimeLineID tli, XLogSegNo segno, int segSize); static ssize_t tdeheap_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset, TimeLineID tli, - XLogSegNo segno, int segSize); + XLogSegNo segno); static const XLogSmgr tde_xlog_smgr = { .seg_read = tdeheap_xlog_seg_read, .seg_write = tdeheap_xlog_seg_write, }; -static void *EncryptionCryptCtx = NULL; - -/* TODO: can be swapped out to the disk */ -static InternalKey EncryptionKey = -{ - .type = MAP_ENTRY_EMPTY, - .start_lsn = InvalidXLogRecPtr, -}; +#ifndef FRONTEND +static Size TDEXLogEncryptBuffSize(void); /* * Must be the same as in replication/walsender.c @@ -55,37 +49,26 @@ static InternalKey EncryptionKey = */ #define MAX_SEND_SIZE (XLOG_BLCKSZ * 16) -/* - * Since the backend code needs to use atomics and shared memory while the - * frotnend code cannot do that we provide two separate implementations of some - * data structures and the functions which operate one them. - */ - -#ifndef FRONTEND +static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, + off_t offset, TimeLineID tli, + XLogSegNo segno); typedef struct EncryptionStateData { + char *segBuf; char db_map_path[MAXPGPATH]; pg_atomic_uint64 enc_key_lsn; /* to sync with readers */ } EncryptionStateData; static EncryptionStateData *EncryptionState = NULL; -static char *EncryptionBuf; - -static XLogRecPtr -TDEXLogGetEncKeyLsn() -{ - return (XLogRecPtr) pg_atomic_read_u64(&EncryptionState->enc_key_lsn); -} - -static void -TDEXLogSetEncKeyLsn(XLogRecPtr start_lsn) +/* TODO: can be swapped out to the disk */ +static InternalKey EncryptionKey = { - pg_atomic_write_u64(&EncryptionState->enc_key_lsn, start_lsn); -} - -static Size TDEXLogEncryptBuffSize(void); + .type = MAP_ENTRY_EMPTY, + .start_lsn = InvalidXLogRecPtr, +}; +static void *EncryptionCryptCtx = NULL; static int XLOGChooseNumBuffers(void); @@ -143,6 +126,7 @@ void TDEXLogShmemInit(void) { bool foundBuf; + char *allocptr; EncryptionState = (EncryptionStateData *) ShmemInitStruct("TDE XLog Encryption State", @@ -153,9 +137,11 @@ TDEXLogShmemInit(void) if (EncryptXLog) { - EncryptionBuf = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, ((char *) EncryptionState) + sizeof(EncryptionStateData)); + allocptr = ((char *) EncryptionState) + sizeof(EncryptionStateData); + allocptr = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, allocptr); + EncryptionState->segBuf = allocptr; - Assert((char *) EncryptionState + TDEXLogEncryptStateSize() >= (char *) EncryptionBuf + TDEXLogEncryptBuffSize()); + Assert((char *) EncryptionState + TDEXLogEncryptStateSize() >= (char *) EncryptionState->segBuf + TDEXLogEncryptBuffSize()); } pg_atomic_init_u64(&EncryptionState->enc_key_lsn, 0); @@ -163,43 +149,39 @@ TDEXLogShmemInit(void) elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", TDEXLogEncryptStateSize()); } -#else /* !FRONTEND */ - -typedef struct EncryptionStateData +/* + * Encrypt XLog page(s) from the buf and write to the segment file. + */ +static ssize_t +TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset, + TimeLineID tli, XLogSegNo segno) { - char db_map_path[MAXPGPATH]; - XLogRecPtr enc_key_lsn; /* to sync with reader */ -} EncryptionStateData; - -static EncryptionStateData EncryptionStateD = {0}; + char iv_prefix[16]; + InternalKey *key = &EncryptionKey; + char *enc_buff = EncryptionState->segBuf; -static EncryptionStateData *EncryptionState = &EncryptionStateD; + Assert(count <= TDEXLogEncryptBuffSize()); -static char EncryptionBuf[MAX_SEND_SIZE]; +#ifdef TDE_XLOG_DEBUG + elog(DEBUG1, "write encrypted WAL, size: %lu, offset: %ld [%lX], seg: %X/%X, key_start_lsn: %X/%X", + count, offset, offset, LSN_FORMAT_ARGS(segno), LSN_FORMAT_ARGS(key->start_lsn)); +#endif -static XLogRecPtr -TDEXLogGetEncKeyLsn() -{ - return (XLogRecPtr) EncryptionState->enc_key_lsn; -} + CalcXLogPageIVPrefix(tli, segno, key->base_iv, iv_prefix); + pg_tde_stream_crypt(iv_prefix, offset, + (char *) buf, count, + enc_buff, key, &EncryptionCryptCtx); -static void -TDEXLogSetEncKeyLsn(XLogRecPtr start_lsn) -{ - EncryptionState->enc_key_lsn = EncryptionKey.start_lsn; + return pg_pwrite(fd, enc_buff, count, offset); } -#endif /* FRONTEND */ +#endif /* !FRONTEND */ void -TDEXLogSmgrInit() -{ - SetXLogSmgr(&tde_xlog_smgr); -} - -void -TDEXLogSmgrInitWrite(bool encrypt_xlog) +TDEXLogSmgrInit(void) { +#ifndef FRONTEND + /* TODO: move to the separate func, it's not an SMGR init */ InternalKey *key = pg_tde_read_last_wal_key(); /* @@ -207,7 +189,7 @@ TDEXLogSmgrInitWrite(bool encrypt_xlog) * attacks on CTR ciphers based on comparing the WAL generated by two * divergent copies of the same cluster. */ - if (encrypt_xlog) + if (EncryptXLog) { pg_tde_create_wal_key(&EncryptionKey, &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_WAL_ENCRYPTED); @@ -220,81 +202,45 @@ TDEXLogSmgrInitWrite(bool encrypt_xlog) else if (key) { EncryptionKey = *key; - TDEXLogSetEncKeyLsn(EncryptionKey.start_lsn); + pg_atomic_write_u64(&EncryptionState->enc_key_lsn, EncryptionKey.start_lsn); } if (key) pfree(key); pg_tde_set_db_file_path(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID).dbOid, EncryptionState->db_map_path); -} - -void -TDEXLogSmgrInitWriteReuseKey() -{ - InternalKey *key = pg_tde_read_last_wal_key(); - - if (key) - { - EncryptionKey = *key; - TDEXLogSetEncKeyLsn(EncryptionKey.start_lsn); - pfree(key); - } - - pg_tde_set_db_file_path(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID).dbOid, EncryptionState->db_map_path); -} - -/* - * Encrypt XLog page(s) from the buf and write to the segment file. - */ -static ssize_t -TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset, - TimeLineID tli, XLogSegNo segno) -{ - char iv_prefix[16]; - InternalKey *key = &EncryptionKey; - char *enc_buff = EncryptionBuf; - -#ifndef FRONTEND - Assert(count <= TDEXLogEncryptBuffSize()); -#endif -#ifdef TDE_XLOG_DEBUG - elog(DEBUG1, "write encrypted WAL, size: %lu, offset: %ld [%lX], seg: %X/%X, key_start_lsn: %X/%X", - count, offset, offset, LSN_FORMAT_ARGS(segno), LSN_FORMAT_ARGS(key->start_lsn)); #endif - - CalcXLogPageIVPrefix(tli, segno, key->base_iv, iv_prefix); - pg_tde_stream_crypt(iv_prefix, offset, - (char *) buf, count, - enc_buff, key, &EncryptionCryptCtx); - - return pg_pwrite(fd, enc_buff, count, offset); + SetXLogSmgr(&tde_xlog_smgr); } static ssize_t tdeheap_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset, - TimeLineID tli, XLogSegNo segno, int segSize) + TimeLineID tli, XLogSegNo segno) { +#ifndef FRONTEND + /* * Set the last (most recent) key's start LSN if not set. * * This func called with WALWriteLock held, so no need in any extra sync. */ - if (EncryptionKey.type != MAP_ENTRY_EMPTY && TDEXLogGetEncKeyLsn() == 0) + if (EncryptionKey.type != MAP_ENTRY_EMPTY && + pg_atomic_read_u64(&EncryptionState->enc_key_lsn) == 0) { XLogRecPtr lsn; - XLogSegNoOffsetToRecPtr(segno, offset, segSize, lsn); + XLogSegNoOffsetToRecPtr(segno, offset, wal_segment_size, lsn); pg_tde_wal_last_key_set_lsn(lsn, EncryptionState->db_map_path); EncryptionKey.start_lsn = lsn; - TDEXLogSetEncKeyLsn(lsn); + pg_atomic_write_u64(&EncryptionState->enc_key_lsn, lsn); } - if (EncryptionKey.type == TDE_KEY_TYPE_WAL_ENCRYPTED) + if (EncryptXLog) return TDEXLogWriteEncryptedPages(fd, buf, count, offset, tli, segno); else +#endif return pg_pwrite(fd, buf, count, offset); } @@ -327,7 +273,8 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, keys = pg_tde_fetch_wal_keys(InvalidXLogRecPtr); } - write_key_lsn = TDEXLogGetEncKeyLsn(); +#ifndef FRONTEND + write_key_lsn = pg_atomic_read_u64(&EncryptionState->enc_key_lsn); if (!XLogRecPtrIsInvalid(write_key_lsn)) { @@ -344,6 +291,7 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, keys = pg_tde_get_wal_cache_keys(); } } +#endif XLogSegNoOffsetToRecPtr(segno, offset, segSize, data_start); XLogSegNoOffsetToRecPtr(segno, offset + readsz, segSize, data_end); @@ -394,6 +342,11 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset, #endif pg_tde_stream_crypt(iv_prefix, dec_off, dec_buf, dec_sz, dec_buf, &curr_key->key, &curr_key->crypt_ctx); + + if (dec_off + dec_sz == offset) + { + break; + } } } } diff --git a/contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c b/contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c deleted file mode 100644 index f3d52d81cf0a2..0000000000000 --- a/contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c +++ /dev/null @@ -1,228 +0,0 @@ -#include "postgres_fe.h" - -#include -#include -#include - -#include "access/xlog_internal.h" -#include "access/xlog_smgr.h" -#include "common/logging.h" - -#include "access/pg_tde_fe_init.h" -#include "access/pg_tde_xlog_smgr.h" - -#define TMPFS_DIRECTORY "/dev/shm" - -static bool -is_segment(const char *filename) -{ - return strspn(filename, "0123456789ABCDEF") == XLOG_FNAME_LEN && - (filename[XLOG_FNAME_LEN] == '\0' || strcmp(filename + XLOG_FNAME_LEN, ".partial") == 0); -} - -static void -write_decrypted_segment(const char *segpath, const char *segname, const char *tmppath) -{ - int segfd; - int tmpfd; - off_t fsize; - int r; - int w; - TimeLineID tli; - XLogSegNo segno; - PGAlignedXLogBlock buf; - off_t pos = 0; - - segfd = open(segpath, O_RDONLY | PG_BINARY, 0); - if (segfd < 0) - pg_fatal("could not open file \"%s\": %m", segpath); - - tmpfd = open(tmppath, O_CREAT | O_WRONLY | PG_BINARY, 0666); - if (tmpfd < 0) - pg_fatal("could not open file \"%s\": %m", tmppath); - - /* - * WalSegSz extracted from the first page header but it might be - * encrypted. But we need to know the segment seize to decrypt it (it's - * required for encryption offset calculations). So we get the segment - * size from the file's actual size. XLogLongPageHeaderData->xlp_seg_size - * there is "just as a cross-check" anyway. - */ - fsize = lseek(segfd, 0, SEEK_END); - XLogFromFileName(segname, &tli, &segno, fsize); - - r = xlog_smgr->seg_read(segfd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize); - - if (r == XLOG_BLCKSZ) - { - XLogLongPageHeader longhdr = (XLogLongPageHeader) buf.data; - int walsegsz = longhdr->xlp_seg_size; - - if (walsegsz != fsize) - pg_fatal("mismatch of segment size in WAL file \"%s\" (header: %d bytes, file size: %ld bytes)", - segname, walsegsz, fsize); - - if (!IsValidWalSegSize(walsegsz)) - { - pg_log_error(ngettext("invalid WAL segment size in WAL file \"%s\" (%d byte)", - "invalid WAL segment size in WAL file \"%s\" (%d bytes)", - walsegsz), - segname, walsegsz); - pg_log_error_detail("The WAL segment size must be a power of two between 1 MB and 1 GB."); - exit(1); - } - } - else if (r < 0) - pg_fatal("could not read file \"%s\": %m", - segpath); - else - pg_fatal("could not read file \"%s\": read %d of %d", - segpath, r, XLOG_BLCKSZ); - - pos += r; - - w = write(tmpfd, buf.data, XLOG_BLCKSZ); - - if (w < 0) - pg_fatal("could not write file \"%s\": %m", tmppath); - else if (w != r) - pg_fatal("could not write file \"%s\": wrote %d of %d", - tmppath, w, r); - - while (1) - { - r = xlog_smgr->seg_read(segfd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize); - - if (r == 0) - break; - else if (r < 0) - pg_fatal("could not read file \"%s\": %m", segpath); - - pos += r; - - w = write(tmpfd, buf.data, r); - - if (w < 0) - pg_fatal("could not write file \"%s\": %m", tmppath); - else if (w != r) - pg_fatal("could not write file \"%s\": wrote %d of %d", - tmppath, w, r); - } - - close(tmpfd); - close(segfd); -} - -static void -usage(const char *progname) -{ - printf(_("%s wraps an archive command to make it archive unencrypted WAL.\n\n"), progname); - printf(_("Usage:\n %s %%p \n\n"), progname); - printf(_("Options:\n")); - printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -?, --help show this help, then exit\n")); -} - -int -main(int argc, char *argv[]) -{ - const char *progname; - char *sourcepath; - char *sep; - char *sourcename; - char tmpdir[MAXPGPATH] = TMPFS_DIRECTORY "/pg_tde_archiveXXXXXX"; - char tmppath[MAXPGPATH]; - bool issegment; - pid_t child; - int status; - int r; - - pg_logging_init(argv[0]); - progname = get_progname(argv[0]); - - if (argc > 1) - { - if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) - { - usage(progname); - exit(0); - } - if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) - { - puts("pg_tde_archive_deceypt (PostgreSQL) " PG_VERSION); - exit(0); - } - } - - if (argc < 3) - { - pg_log_error("too few arguments"); - pg_log_error_detail("Try \"%s --help\" for more information.", progname); - exit(1); - } - - sourcepath = argv[1]; - - pg_tde_fe_init("pg_tde"); - TDEXLogSmgrInit(); - - sep = strrchr(sourcepath, '/'); - - if (sep != NULL) - sourcename = sep + 1; - else - sourcename = sourcepath; - - issegment = is_segment(sourcename); - - if (issegment) - { - char *s; - - if (mkdtemp(tmpdir) == NULL) - pg_fatal("could not create temporary directory \"%s\": %m", tmpdir); - - s = stpcpy(tmppath, tmpdir); - s = stpcpy(s, "/"); - stpcpy(s, sourcename); - - for (int i = 2; i < argc; i++) - if (strcmp(sourcepath, argv[i]) == 0) - argv[i] = tmppath; - - write_decrypted_segment(sourcepath, sourcename, tmppath); - } - - child = fork(); - if (child == 0) - { - if (execvp(argv[2], argv + 2) < 0) - pg_fatal("exec failed: %m"); - } - else if (child < 0) - pg_fatal("could not create background process: %m"); - - r = waitpid(child, &status, 0); - if (r == (pid_t) -1) - pg_fatal("could not wait for child process: %m"); - if (r != child) - pg_fatal("child %d died, expected %d", (int) r, (int) child); - if (status != 0) - { - char *reason = wait_result_to_str(status); - - pg_fatal("%s", reason); - /* keep lsan happy */ - free(reason); - } - - if (issegment) - { - if (unlink(tmppath) < 0) - pg_log_warning("could not remove file \"%s\": %m", tmppath); - if (rmdir(tmpdir) < 0) - pg_log_warning("could not remove directory \"%s\": %m", tmpdir); - } - - return 0; -} diff --git a/contrib/pg_tde/src/bin/pg_tde_restore_encrypt.c b/contrib/pg_tde/src/bin/pg_tde_restore_encrypt.c deleted file mode 100644 index de808b4ef735e..0000000000000 --- a/contrib/pg_tde/src/bin/pg_tde_restore_encrypt.c +++ /dev/null @@ -1,224 +0,0 @@ -#include "postgres_fe.h" - -#include -#include -#include - -#include "access/xlog_internal.h" -#include "access/xlog_smgr.h" -#include "common/logging.h" - -#include "access/pg_tde_fe_init.h" -#include "access/pg_tde_xlog_smgr.h" - -#define TMPFS_DIRECTORY "/dev/shm" - -/* - * Partial WAL segments are archived but never automatically fetched from the - * archive by the restore_command. We support them here for symmetry though - * since if someone would want to fetch a partial segment from the archive and - * write it to pg_wal then they would want it encrypted. - */ -static bool -is_segment(const char *filename) -{ - return strspn(filename, "0123456789ABCDEF") == XLOG_FNAME_LEN && - (filename[XLOG_FNAME_LEN] == '\0' || strcmp(filename + XLOG_FNAME_LEN, ".partial") == 0); -} - -static void -write_encrypted_segment(const char *segpath, const char *segname, const char *tmppath) -{ - int tmpfd; - int segfd; - PGAlignedXLogBlock buf; - int r; - int w; - int pos = 0; - XLogLongPageHeader longhdr; - int walsegsz; - TimeLineID tli; - XLogSegNo segno; - - tmpfd = open(tmppath, O_RDONLY | PG_BINARY, 0); - if (tmpfd < 0) - pg_fatal("could not open file \"%s\": %m", tmppath); - - segfd = open(segpath, O_CREAT | O_WRONLY | PG_BINARY, 0666); - if (segfd < 0) - pg_fatal("could not open file \"%s\": %m", segpath); - - r = read(tmpfd, buf.data, XLOG_BLCKSZ); - - if (r < 0) - pg_fatal("could not read file \"%s\": %m", tmppath); - else if (r != XLOG_BLCKSZ) - pg_fatal("could not read file \"%s\": read %d of %d", - tmppath, r, XLOG_BLCKSZ); - - longhdr = (XLogLongPageHeader) buf.data; - walsegsz = longhdr->xlp_seg_size; - - if (!IsValidWalSegSize(walsegsz)) - { - pg_log_error(ngettext("invalid WAL segment size in WAL file \"%s\" (%d byte)", - "invalid WAL segment size in WAL file \"%s\" (%d bytes)", - walsegsz), - segname, walsegsz); - pg_log_error_detail("The WAL segment size must be a power of two between 1 MB and 1 GB."); - exit(1); - } - - XLogFromFileName(segname, &tli, &segno, walsegsz); - - TDEXLogSmgrInitWriteReuseKey(); - - w = xlog_smgr->seg_write(segfd, buf.data, r, pos, tli, segno, walsegsz); - - if (w < 0) - pg_fatal("could not write file \"%s\": %m", segpath); - else if (w != r) - pg_fatal("could not write file \"%s\": wrote %d of %d", - segpath, w, r); - - pos += w; - - while (1) - { - r = read(tmpfd, buf.data, XLOG_BLCKSZ); - - if (r == 0) - break; - else if (r < 0) - pg_fatal("could not read file \"%s\": %m", tmppath); - - w = xlog_smgr->seg_write(segfd, buf.data, r, pos, tli, segno, walsegsz); - - if (w < 0) - pg_fatal("could not write file \"%s\": %m", segpath); - else if (w != r) - pg_fatal("could not write file \"%s\": wrote %d of %d", - segpath, w, r); - - pos += w; - } - - close(segfd); - close(tmpfd); -} - -static void -usage(const char *progname) -{ - printf(_("%s wraps a restore command to make it write encrypted WAL to pg_wal.\n\n"), progname); - printf(_("Usage:\n %s %%f %%p \n\n"), progname); - printf(_("Options:\n")); - printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -?, --help show this help, then exit\n")); -} - -int -main(int argc, char *argv[]) -{ - const char *progname; - char *sourcename; - char *targetpath; - char *sep; - char *targetname; - char tmpdir[MAXPGPATH] = TMPFS_DIRECTORY "/pg_tde_restoreXXXXXX"; - char tmppath[MAXPGPATH]; - bool issegment; - pid_t child; - int status; - int r; - - pg_logging_init(argv[0]); - progname = get_progname(argv[0]); - - if (argc > 1) - { - if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) - { - usage(progname); - exit(0); - } - if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) - { - puts("pg_tde_restore_encrypt (PostgreSQL) " PG_VERSION); - exit(0); - } - } - - if (argc < 4) - { - pg_log_error("too few arguments"); - pg_log_error_detail("Try \"%s --help\" for more information.", progname); - exit(1); - } - - sourcename = argv[1]; - targetpath = argv[2]; - - pg_tde_fe_init("pg_tde"); - TDEXLogSmgrInit(); - - sep = strrchr(targetpath, '/'); - - if (sep != NULL) - targetname = sep + 1; - else - targetname = targetpath; - - issegment = is_segment(sourcename); - - if (issegment) - { - char *s; - - if (mkdtemp(tmpdir) == NULL) - pg_fatal("could not create temporary directory \"%s\": %m", tmpdir); - - s = stpcpy(tmppath, tmpdir); - s = stpcpy(s, "/"); - stpcpy(s, targetname); - - for (int i = 2; i < argc; i++) - if (strcmp(targetpath, argv[i]) == 0) - argv[i] = tmppath; - } - - child = fork(); - if (child == 0) - { - if (execvp(argv[3], argv + 3) < 0) - pg_fatal("exec failed: %m"); - } - else if (child < 0) - pg_fatal("could not create background process: %m"); - - r = waitpid(child, &status, 0); - if (r == (pid_t) -1) - pg_fatal("could not wait for child process: %m"); - if (r != child) - pg_fatal("child %d died, expected %d", (int) r, (int) child); - if (status != 0) - { - char *reason = wait_result_to_str(status); - - pg_fatal("%s", reason); - /* keep lsan happy */ - free(reason); - } - - if (issegment) - { - write_encrypted_segment(targetpath, sourcename, tmppath); - - if (unlink(tmppath) < 0) - pg_log_warning("could not remove file \"%s\": %m", tmppath); - if (rmdir(tmpdir) < 0) - pg_log_warning("could not remove directory \"%s\": %m", tmpdir); - } - - return 0; -} diff --git a/contrib/pg_tde/src/catalog/tde_keyring.c b/contrib/pg_tde/src/catalog/tde_keyring.c index 4ca489885058c..0f7b53164c1e2 100644 --- a/contrib/pg_tde/src/catalog/tde_keyring.c +++ b/contrib/pg_tde/src/catalog/tde_keyring.c @@ -309,8 +309,7 @@ pg_tde_delete_key_provider_internal(PG_FUNCTION_ARGS, Oid db_oid) if (provider_used) { ereport(ERROR, - errcode(ERRCODE_OBJECT_IN_USE), - errmsg("cannot delete provider which is currently in use")); + errmsg("Can't delete a provider which is currently in use")); } delete_key_provider_info(provider_name, db_oid); diff --git a/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c b/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c index 1297e16e9e924..e8714cb02692b 100644 --- a/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c +++ b/contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c @@ -264,7 +264,6 @@ json_kring_object_field_start(void *state, char *fname, bool isnull) break; } - pfree(fname); return JSON_SUCCESS; } diff --git a/contrib/pg_tde/src/catalog/tde_principal_key.c b/contrib/pg_tde/src/catalog/tde_principal_key.c index 178461905a93c..38f962b4794f2 100644 --- a/contrib/pg_tde/src/catalog/tde_principal_key.c +++ b/contrib/pg_tde/src/catalog/tde_principal_key.c @@ -700,13 +700,12 @@ pg_tde_delete_key(PG_FUNCTION_ARGS) * If database has something encryted, we can try to fallback to the * default principal key */ - if (pg_tde_count_encryption_keys(MyDatabaseId) != 0) + if (pg_tde_count_relations(MyDatabaseId) != 0) { default_principal_key = GetPrincipalKeyNoDefault(DEFAULT_DATA_TDE_OID, LW_EXCLUSIVE); if (default_principal_key == NULL) { ereport(ERROR, - errcode(ERRCODE_OBJECT_IN_USE), errmsg("cannot delete principal key"), errdetail("There are encrypted tables in the database."), errhint("Set default principal key as fallback option or decrypt all tables before deleting principal key.")); @@ -719,7 +718,6 @@ pg_tde_delete_key(PG_FUNCTION_ARGS) if (pg_tde_is_same_principal_key(principal_key, default_principal_key)) { ereport(ERROR, - errcode(ERRCODE_OBJECT_IN_USE), errmsg("cannot delete principal key"), errdetail("There are encrypted tables in the database.")); } @@ -787,10 +785,9 @@ pg_tde_delete_default_key(PG_FUNCTION_ARGS) * delete default principal key if there are encrypted tables in * the database. */ - if (pg_tde_count_encryption_keys(dbOid) != 0) + if (pg_tde_count_relations(dbOid) != 0) { ereport(ERROR, - errcode(ERRCODE_OBJECT_IN_USE), errmsg("cannot delete default principal key"), errhint("There are encrypted tables in the database with id: %u.", dbOid)); } @@ -801,23 +798,8 @@ pg_tde_delete_default_key(PG_FUNCTION_ARGS) } /* - * The default key may have been used as server key, check if there are - * any WAL encryption keys that uses it. - */ - principal_key = GetPrincipalKeyNoDefault(GLOBAL_DATA_TDE_OID, LW_EXCLUSIVE); - if (pg_tde_is_same_principal_key(default_principal_key, principal_key)) - { - if (pg_tde_count_encryption_keys(GLOBAL_DATA_TDE_OID) != 0) - ereport(ERROR, - errcode(ERRCODE_OBJECT_IN_USE), - errmsg("cannot delete default principal key"), - errhint("There are WAL encryption keys.")); - dbs = lappend_oid(dbs, GLOBAL_DATA_TDE_OID); - } - - /* - * Remove empty key files for OIDs that have no encryption keys as we - * cannot leave references to the default principal key. + * Remove empty key map files for databases that has no encrypted tables + * as we cannot leave reference to the default principal key. */ foreach_oid(dbOid, dbs) { diff --git a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h index e6d6d982404ec..f3177544cf551 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h +++ b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h @@ -92,7 +92,7 @@ extern bool pg_tde_has_smgr_key(RelFileLocator rel); extern InternalKey *pg_tde_get_smgr_key(RelFileLocator rel); extern void pg_tde_free_key_map_entry(RelFileLocator rel); -extern int pg_tde_count_encryption_keys(Oid dbOid); +extern int pg_tde_count_relations(Oid dbOid); extern void pg_tde_delete_tde_files(Oid dbOid); diff --git a/contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h b/contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h index 6b434de3d3c93..cb714ed34438a 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h +++ b/contrib/pg_tde/src/include/access/pg_tde_xlog_smgr.h @@ -10,7 +10,5 @@ extern Size TDEXLogEncryptStateSize(void); extern void TDEXLogShmemInit(void); extern void TDEXLogSmgrInit(void); -extern void TDEXLogSmgrInitWrite(bool encrypt_xlog); -extern void TDEXLogSmgrInitWriteReuseKey(void); #endif /* PG_TDE_XLOGSMGR_H */ diff --git a/contrib/pg_tde/src/include/pg_tde_fe.h b/contrib/pg_tde/src/include/pg_tde_fe.h index 86d4f2377998d..6ef811b2b98ec 100644 --- a/contrib/pg_tde/src/include/pg_tde_fe.h +++ b/contrib/pg_tde/src/include/pg_tde_fe.h @@ -67,8 +67,6 @@ static int tde_fe_error_level = 0; -#define data_sync_elevel(elevel) (elevel) - /* * ------------- */ diff --git a/contrib/pg_tde/src/pg_tde.c b/contrib/pg_tde/src/pg_tde.c index 977ba0ccd35c8..626d5493b8fb3 100644 --- a/contrib/pg_tde/src/pg_tde.c +++ b/contrib/pg_tde/src/pg_tde.c @@ -68,7 +68,6 @@ tde_shmem_startup(void) PrincipalKeyShmemInit(); TDEXLogShmemInit(); TDEXLogSmgrInit(); - TDEXLogSmgrInitWrite(EncryptXLog); } void diff --git a/contrib/pg_tde/src/bin/pg_tde_change_key_provider.c b/contrib/pg_tde/src/pg_tde_change_key_provider.c similarity index 100% rename from contrib/pg_tde/src/bin/pg_tde_change_key_provider.c rename to contrib/pg_tde/src/pg_tde_change_key_provider.c diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 3dcb787c7dd87..9f089f5c36eee 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -643,7 +643,7 @@ pg_tde_proccess_utility(PlannedStmt *pstmt, int count; LWLockAcquire(tde_lwlock_enc_keys(), LW_SHARED); - count = pg_tde_count_encryption_keys(dbOid); + count = pg_tde_count_relations(dbOid); LWLockRelease(tde_lwlock_enc_keys()); if (count > 0) diff --git a/contrib/pg_tde/t/pg_resetwal_basic.pl b/contrib/pg_tde/t/pg_resetwal_basic.pl deleted file mode 100644 index 78890f633cf83..0000000000000 --- a/contrib/pg_tde/t/pg_resetwal_basic.pl +++ /dev/null @@ -1,150 +0,0 @@ - -# Copyright (c) 2021-2024, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -unlink('/tmp/pg_resetwal_basic.per'); - -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; -$node->append_conf( - 'postgresql.conf', q{ -track_commit_timestamp = on - -# WAL Encryption -shared_preload_libraries = 'pg_tde' -}); - -$node->start; -$node->safe_psql('postgres', "CREATE EXTENSION pg_tde;"); -$node->safe_psql('postgres', - "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal', '/tmp/pg_resetwal_basic.per');" -); -$node->safe_psql('postgres', - "SELECT pg_tde_create_key_using_global_key_provider('server-key', 'file-keyring-wal');" -); -$node->safe_psql('postgres', - "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-wal');" -); - -$node->append_conf( - 'postgresql.conf', q{ -pg_tde.wal_encrypt = on -}); -$node->stop; - - -command_like([ 'pg_resetwal', '-n', $node->data_dir ], - qr/checkpoint/, 'pg_resetwal -n produces output'); - - -# Permissions on PGDATA should be default -SKIP: -{ - skip "unix-style permissions not supported on Windows", 1 - if ($windows_os); - - ok(check_mode_recursive($node->data_dir, 0700, 0600), - 'check PGDATA permissions'); -} - -command_ok([ 'pg_resetwal', '-D', $node->data_dir ], 'pg_resetwal runs'); -$node->start; -is($node->safe_psql("postgres", "SELECT 1;"), - 1, 'server running and working after reset'); - -command_fails_like( - [ 'pg_resetwal', $node->data_dir ], - qr/lock file .* exists/, - 'fails if server running'); - -$node->stop('immediate'); -command_fails_like( - [ 'pg_resetwal', $node->data_dir ], - qr/database server was not shut down cleanly/, - 'does not run after immediate shutdown'); -command_ok( - [ 'pg_resetwal', '-f', $node->data_dir ], - 'runs after immediate shutdown with force'); -$node->start; -is($node->safe_psql("postgres", "SELECT 1;"), - 1, 'server running and working after forced reset'); - -$node->stop; - -# check various command-line handling - -# Note: This test intends to check that a nonexistent data directory -# gives a reasonable error message. Because of the way the code is -# currently structured, you get an error about readings permissions, -# which is perhaps suboptimal, so feel free to update this test if -# this gets improved. - - -# run with control override options - -my $out = (run_command([ 'pg_resetwal', '-n', $node->data_dir ]))[0]; -$out =~ /^Database block size: *(\d+)$/m or die; -my $blcksz = $1; - -my @cmd = ('pg_resetwal', '-D', $node->data_dir); - -# some not-so-critical hardcoded values -push @cmd, '-e', 1; -push @cmd, '-l', '00000001000000320000004B'; -push @cmd, '-o', 100_000; -push @cmd, '--wal-segsize', 1; - -# these use the guidance from the documentation - -sub get_slru_files -{ - opendir(my $dh, $node->data_dir . '/' . $_[0]) or die $!; - my @files = sort grep { /[0-9A-F]+/ } readdir $dh; - closedir $dh; - return @files; -} - -my (@files, $mult); - -@files = get_slru_files('pg_commit_ts'); -# XXX: Should there be a multiplier, similar to the other options? -# -c argument is "old,new" -push @cmd, - '-c', - sprintf("%d,%d", hex($files[0]) == 0 ? 3 : hex($files[0]), hex($files[-1])); - -@files = get_slru_files('pg_multixact/offsets'); -$mult = 32 * $blcksz / 4; -# -m argument is "new,old" -push @cmd, '-m', - sprintf("%d,%d", - (hex($files[-1]) + 1) * $mult, - hex($files[0]) == 0 ? 1 : hex($files[0] * $mult)); - -@files = get_slru_files('pg_multixact/members'); -$mult = 32 * int($blcksz / 20) * 4; -push @cmd, '-O', (hex($files[-1]) + 1) * $mult; - -@files = get_slru_files('pg_xact'); -$mult = 32 * $blcksz * 4; -push @cmd, - '-u', (hex($files[0]) == 0 ? 3 : hex($files[0]) * $mult), - '-x', ((hex($files[-1]) + 1) * $mult); - -command_ok([ @cmd, '-n' ], 'runs with control override options, dry run'); -command_ok(\@cmd, 'runs with control override options'); -command_like( - [ 'pg_resetwal', '-n', $node->data_dir ], - qr/^Latest checkpoint's NextOID: *100000$/m, - 'spot check that control changes were applied'); - -$node->start; -ok(1, 'server started after reset'); - -done_testing(); diff --git a/contrib/pg_tde/t/pg_resetwal_corrupted.pl b/contrib/pg_tde/t/pg_resetwal_corrupted.pl deleted file mode 100644 index 74cc66a2c8030..0000000000000 --- a/contrib/pg_tde/t/pg_resetwal_corrupted.pl +++ /dev/null @@ -1,92 +0,0 @@ - -# Copyright (c) 2021-2024, PostgreSQL Global Development Group - -# Tests for handling a corrupted pg_control - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -unlink('/tmp/pg_resetwal_corrupted.per'); - -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; -$node->append_conf( - 'postgresql.conf', q{ - -# WAL Encryption -shared_preload_libraries = 'pg_tde' -}); - -$node->start; -$node->safe_psql('postgres', "CREATE EXTENSION pg_tde;"); -$node->safe_psql('postgres', - "SELECT pg_tde_add_global_key_provider_file('file-keyring-wal', '/tmp/pg_waldump_corrupted.per');" -); -$node->safe_psql('postgres', - "SELECT pg_tde_create_key_using_global_key_provider('server-key', 'file-keyring-wal');" -); -$node->safe_psql('postgres', - "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'file-keyring-wal');" -); - -$node->append_conf( - 'postgresql.conf', q{ -pg_tde.wal_encrypt = on -}); -$node->stop; - -my $pg_control = $node->data_dir . '/global/pg_control'; -my $size = -s $pg_control; - -# Read out the head of the file to get PG_CONTROL_VERSION in -# particular. -my $data; -open my $fh, '<', $pg_control or BAIL_OUT($!); -binmode $fh; -read $fh, $data, 16 or die $!; -close $fh; - -# Fill pg_control with zeros -open $fh, '>', $pg_control or BAIL_OUT($!); -binmode $fh; -print $fh pack("x[$size]"); -close $fh; - -command_checks_all( - [ 'pg_resetwal', '-n', $node->data_dir ], - 0, - [qr/pg_control version number/], - [ - qr/pg_resetwal: warning: pg_control exists but is broken or wrong version; ignoring it/ - ], - 'processes corrupted pg_control all zeroes'); - -# Put in the previously saved header data. This uses a different code -# path internally, allowing us to process a zero WAL segment size. -open $fh, '>', $pg_control or BAIL_OUT($!); -binmode $fh; -print $fh $data, pack("x[" . ($size - 16) . "]"); -close $fh; - -command_checks_all( - [ 'pg_resetwal', '-n', $node->data_dir ], - 0, - [qr/pg_control version number/], - [ - qr/\Qpg_resetwal: warning: pg_control specifies invalid WAL segment size (0 bytes); proceed with caution\E/ - ], - 'processes zero WAL segment size'); - -# now try to run it -command_fails_like( - [ 'pg_resetwal', $node->data_dir ], - qr/not proceeding because control file values were guessed/, - 'does not run when control file values were guessed'); -command_ok([ 'pg_resetwal', '-f', $node->data_dir ], - 'runs with force when control file values were guessed'); - -done_testing(); diff --git a/contrib/pg_tde/t/pgtde.pm b/contrib/pg_tde/t/pgtde.pm index 8f02dce151555..eb5c02b24ec17 100644 --- a/contrib/pg_tde/t/pgtde.pm +++ b/contrib/pg_tde/t/pgtde.pm @@ -6,7 +6,6 @@ use PostgreSQL::Test::Utils; use File::Basename; use File::Compare; use Test::More; -use Time::HiRes qw(usleep); # Expected .out filename of TAP testcase being executed. These are already part of repo under t/expected/*. our $expected_filename_with_path; diff --git a/contrib/pg_tde/t/wal_archiving.pl b/contrib/pg_tde/t/wal_archiving.pl deleted file mode 100644 index 9bd37904a0792..0000000000000 --- a/contrib/pg_tde/t/wal_archiving.pl +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use File::Basename; -use Test::More; -use lib 't'; -use pgtde; - -unlink('/tmp/wal_archiving.per'); - -# Test archive_command - -my $primary = PostgreSQL::Test::Cluster->new('primary'); -my $archive_dir = $primary->archive_dir; -$primary->init(allows_streaming => 1); -$primary->append_conf('postgresql.conf', - "shared_preload_libraries = 'pg_tde'"); -$primary->append_conf('postgresql.conf', "wal_level = 'replica'"); -$primary->append_conf('postgresql.conf', "autovacuum = off"); -$primary->append_conf('postgresql.conf', "checkpoint_timeout = 1h"); -$primary->append_conf('postgresql.conf', "archive_mode = on"); -$primary->append_conf('postgresql.conf', - "archive_command = 'pg_tde_archive_decrypt %p cp %p $archive_dir/%f'"); -$primary->start; - -$primary->safe_psql('postgres', "CREATE EXTENSION pg_tde;"); - -$primary->safe_psql('postgres', - "SELECT pg_tde_add_global_key_provider_file('keyring', '/tmp/wal_archiving.per');" -); -$primary->safe_psql('postgres', - "SELECT pg_tde_create_key_using_global_key_provider('server-key', 'keyring');" -); -$primary->safe_psql('postgres', - "SELECT pg_tde_set_server_key_using_global_key_provider('server-key', 'keyring');" -); - -$primary->append_conf('postgresql.conf', "pg_tde.wal_encrypt = on"); - -$primary->backup('backup', backup_options => [ '-X', 'none' ]); - -$primary->safe_psql('postgres', - "CREATE TABLE t1 AS SELECT 'foobar_plain' AS x"); - -$primary->restart; - -$primary->safe_psql('postgres', - "CREATE TABLE t2 AS SELECT 'foobar_enc' AS x"); - -my $data_dir = $primary->data_dir; - -like( - `strings $data_dir/pg_wal/0000000100000000000000??`, - qr/foobar_plain/, - 'should find foobar_plain in WAL'); -unlike( - `strings $data_dir/pg_wal/0000000100000000000000??`, - qr/foobar_enc/, - 'should not find foobar_enc in WAL'); - -$primary->stop; - -like( - `strings $archive_dir/0000000100000000000000??`, - qr/foobar_plain/, - 'should find foobar_plain in archive'); -like( - `strings $archive_dir/0000000100000000000000??`, - qr/foobar_enc/, - 'should find foobar_enc in archive'); - -# Test restore_command - -my $replica = PostgreSQL::Test::Cluster->new('replica'); -$replica->init_from_backup($primary, 'backup'); -$replica->append_conf('postgresql.conf', - "restore_command = 'pg_tde_restore_encrypt %f %p cp $archive_dir/%f %p'"); -$replica->append_conf('postgresql.conf', "recovery_target_action = promote"); -$replica->set_recovery_mode; -$replica->start; - -$data_dir = $replica->data_dir; - -unlike( - `strings $data_dir/pg_wal/0000000100000000000000??`, - qr/foobar_plain/, - 'should not find foobar_plain in WAL since it is encrypted'); -unlike( - `strings $data_dir/pg_wal/0000000100000000000000??`, - qr/foobar_enc/, - 'should not find foobar_enc in WAL since it is encrypted'); - -my $result = $replica->safe_psql('postgres', - 'SELECT * FROM t1 UNION ALL SELECT * FROM t2'); - -is($result, "foobar_plain\nfoobar_enc", 'b'); - -$replica->stop; - -done_testing(); diff --git a/contrib/pg_tde/t/wal_encrypt.pl b/contrib/pg_tde/t/wal_encrypt.pl index 2c23d75808a52..7db5cd28dbf2a 100644 --- a/contrib/pg_tde/t/wal_encrypt.pl +++ b/contrib/pg_tde/t/wal_encrypt.pl @@ -15,8 +15,8 @@ $node->init; $node->append_conf('postgresql.conf', "shared_preload_libraries = 'pg_tde'"); $node->append_conf('postgresql.conf', "wal_level = 'logical'"); -# We don't test that it can't start: the test framework doesn't have an easy way to do this -#$node->append_conf('postgresql.conf', "pg_tde.wal_encrypt = 1"); +# NOT testing that it can't start: the test framework doesn't have an easy way to do this +#$node->append_conf('postgresql.conf', "pg_tde.wal_encrypt = 1"}); $node->start; PGTDE::psql($node, 'postgres', "CREATE EXTENSION pg_tde;"); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e0e8d2daf0071..c1d7988066ff6 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -2446,7 +2446,7 @@ XLogWrite(XLogwrtRqst WriteRqst, TimeLineID tli, bool flexible) INSTR_TIME_SET_ZERO(start); pgstat_report_wait_start(WAIT_EVENT_WAL_WRITE); - written = xlog_smgr->seg_write(openLogFile, from, nleft, startoffset, tli, openLogSegNo, wal_segment_size); + written = xlog_smgr->seg_write(openLogFile, from, nleft, startoffset, tli, openLogSegNo); pgstat_report_wait_end(); /* @@ -3491,7 +3491,7 @@ XLogFileCopy(TimeLineID destTLI, XLogSegNo destsegno, } errno = 0; pgstat_report_wait_start(WAIT_EVENT_WAL_COPY_WRITE); - if ((int) xlog_smgr->seg_write(fd, buffer.data, sizeof(buffer), offset, destTLI, destsegno, wal_segment_size) != (int) sizeof(buffer)) + if ((int) xlog_smgr->seg_write(fd, buffer.data, sizeof(buffer), offset, destTLI, destsegno) != (int) sizeof(buffer)) { int save_errno = errno; diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 7cdfe1a648731..973ef75f138d1 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -944,7 +944,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr, TimeLineID tli) byteswritten = xlog_smgr->seg_write(recvFile, buf, segbytes, (off_t) startoff, recvFileTLI, - recvSegNo, wal_segment_size); + recvSegNo); if (byteswritten <= 0) { char xlogfname[MAXFNAMELEN]; diff --git a/src/bin/pg_resetwal/.gitignore b/src/bin/pg_resetwal/.gitignore index ddb4f6fb35745..56bade5ea4401 100644 --- a/src/bin/pg_resetwal/.gitignore +++ b/src/bin/pg_resetwal/.gitignore @@ -1,5 +1,2 @@ /pg_resetwal /tmp_check/ - -# Source files copied from src/backend/access/transam/ -/xlogreader.c diff --git a/src/bin/pg_resetwal/Makefile b/src/bin/pg_resetwal/Makefile index f39e8378417ce..4228a5a772a9f 100644 --- a/src/bin/pg_resetwal/Makefile +++ b/src/bin/pg_resetwal/Makefile @@ -21,23 +21,8 @@ OBJS = \ $(WIN32RES) \ pg_resetwal.o -ifeq ($(enable_percona_ext),yes) -override CPPFLAGS := -DFRONTEND $(CPPFLAGS) - -OBJS += \ - $(top_srcdir)/src/fe_utils/simple_list.o \ - $(top_builddir)/src/libtde/libtdexlog.a \ - $(top_builddir)/src/libtde/libtde.a \ - xlogreader.o - -override CPPFLAGS := -I$(top_srcdir)/contrib/pg_tde/src/include -I$(top_srcdir)/contrib/pg_tde/src/libkmip/libkmip/include $(CPPFLAGS) -endif - all: pg_resetwal -xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% - rm -f $@ && $(LN_S) $< . - pg_resetwal: $(OBJS) | submake-libpgport $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) @@ -51,7 +36,7 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/pg_resetwal$(X)' clean distclean: - rm -f pg_resetwal$(X) $(OBJS) xlogreader.c + rm -f pg_resetwal$(X) $(OBJS) rm -rf tmp_check check: diff --git a/src/bin/pg_resetwal/meson.build b/src/bin/pg_resetwal/meson.build index ea7a3094452d5..c1239528db27f 100644 --- a/src/bin/pg_resetwal/meson.build +++ b/src/bin/pg_resetwal/meson.build @@ -10,22 +10,10 @@ if host_system == 'windows' '--FILEDESC', 'pg_resetwal - reset PostgreSQL WAL log']) endif -link_w = [] -include_dirs = [postgres_inc] - -if percona_ext == true - link_w = [pg_tde_frontend] - include_dirs = [postgres_inc, pg_tde_inc] - pg_resetwal_sources += xlogreader_sources -endif - pg_resetwal = executable('pg_resetwal', pg_resetwal_sources, dependencies: [frontend_code], - c_args: ['-DFRONTEND'], # needed for xlogreader et al kwargs: default_bin_args, - include_directories: include_dirs, - link_with: link_w ) bin_targets += pg_resetwal diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 3cebcb6adabe2..e9dcb5a6d89d1 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -59,13 +59,6 @@ #include "pg_getopt.h" #include "storage/large_object.h" -#ifdef PERCONA_EXT -#include "pg_tde.h" -#include "access/pg_tde_fe_init.h" -#include "access/pg_tde_xlog_smgr.h" -#include "access/xlog_smgr.h" -#endif - static ControlFileData ControlFile; /* pg_control values */ static XLogSegNo newXlogSegNo; /* new XLOG segment # */ static bool guessed = false; /* T if we had to guess at any values */ @@ -351,15 +344,6 @@ main(int argc, char *argv[]) } #endif -#ifdef PERCONA_EXT - { - char tde_path[MAXPGPATH]; - snprintf(tde_path, sizeof(tde_path), "%s/%s", DataDir, PG_TDE_DATA_DIR); - pg_tde_fe_init(tde_path); - TDEXLogSmgrInit(); - } -#endif - get_restricted_token(); /* Set mask based on PGDATA permissions */ @@ -408,11 +392,7 @@ main(int argc, char *argv[]) WalSegSz = ControlFile.xlog_seg_size; if (log_fname != NULL) - { XLogFromFileName(log_fname, &minXlogTli, &minXlogSegNo, WalSegSz); - free(log_fname); - } - /* * Also look at existing segment files to set up newXlogSegNo @@ -508,22 +488,6 @@ main(int argc, char *argv[]) exit(1); } -#ifdef PERCONA_EXT - /* - * The only thing that WAL will contain after reset is a checkpoint, so we can - * write it in unencrypted form. There is no sensitive data in it. But we still - * need to use TDE smgr because WAL key change may be needed. If the last WAL key - * had type "encrypted" new key will be created with type "unencrypted" to mark - * the beginning of a new unencrypted WAL record. If the last WAL key had type - * "unencrypted" it will be reused. On the startup server may create a new key - * with appropriate type according to encryption settings. - * - * We are doing a write initialization only here and not at the startup because we - * want to be sure that everything is checked and ready for writing at this point. - */ - TDEXLogSmgrInitWrite(false); -#endif - /* * Else, do the dirty deed. */ @@ -643,8 +607,6 @@ read_controlfile(void) memcpy(&ControlFile, buffer, sizeof(ControlFile)); - free(buffer); - /* return false if WAL segment size is not valid */ if (!IsValidWalSegSize(ControlFile.xlog_seg_size)) { @@ -658,8 +620,6 @@ read_controlfile(void) return true; } - free(buffer); - /* Looks like it's a mess. */ pg_log_warning("pg_control exists but is broken or wrong version; ignoring it"); return false; @@ -1174,13 +1134,7 @@ WriteEmptyXLOG(void) pg_fatal("could not open file \"%s\": %m", path); errno = 0; -#ifdef PERCONA_EXT - if (xlog_smgr->seg_write(fd, buffer.data, XLOG_BLCKSZ, 0, - ControlFile.checkPointCopy.ThisTimeLineID, - newXlogSegNo, WalSegSz) != XLOG_BLCKSZ) -#else if (write(fd, buffer.data, XLOG_BLCKSZ) != XLOG_BLCKSZ) -#endif { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) @@ -1188,11 +1142,6 @@ WriteEmptyXLOG(void) pg_fatal("could not write file \"%s\": %m", path); } -#ifdef PERCONA_EXT - /* If we used xlog smgr, we need to update the file offset */ - lseek(fd, XLOG_BLCKSZ, SEEK_CUR); -#endif - /* Fill the rest of the file with zeroes */ memset(buffer.data, 0, XLOG_BLCKSZ); for (nbytes = XLOG_BLCKSZ; nbytes < WalSegSz; nbytes += XLOG_BLCKSZ) diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 931dda9250150..f6726c1df485c 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -376,7 +376,7 @@ main(int argc, char **argv) char tde_path[MAXPGPATH]; snprintf(tde_path, sizeof(tde_path), "%s/%s", datadir_target, PG_TDE_DATA_DIR); pg_tde_fe_init(tde_path); - TDEXLogSmgrInit(); + TDEXLogSmgrInit(); } #endif /* diff --git a/src/include/access/xlog_smgr.h b/src/include/access/xlog_smgr.h index b1f7c4c425f48..808a07f502fc1 100644 --- a/src/include/access/xlog_smgr.h +++ b/src/include/access/xlog_smgr.h @@ -12,12 +12,12 @@ typedef struct XLogSmgr TimeLineID tli, XLogSegNo segno, int segSize); ssize_t (*seg_write) (int fd, const void *buf, size_t count, off_t offset, - TimeLineID tli, XLogSegNo segno, int segSize); + TimeLineID tli, XLogSegNo segno); } XLogSmgr; static inline ssize_t default_seg_write(int fd, const void *buf, size_t count, off_t offset, - TimeLineID tli, XLogSegNo segno, int segSize) + TimeLineID tli, XLogSegNo segno) { return pg_pwrite(fd, buf, count, offset); } diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index 588d6e987cd30..78353b03d70cb 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -384,4 +384,4 @@ */ /* #define TRACE_SYNCSCAN */ -#define PERCONA_API_VERSION 2 +#define PERCONA_API_VERSION 1