From bd0535f3b998c751702375a13acc193630ee869d Mon Sep 17 00:00:00 2001 From: Jay Kothari Date: Fri, 12 Apr 2024 19:40:07 -0400 Subject: [PATCH 1/5] basic setup to automate --- sql/prometheus_fdw--0.1.4--0.1.5.sql | 3 +++ src/init.rs | 32 ++++++++++++++++++++++++++++ src/lib.rs | 2 ++ 3 files changed, 37 insertions(+) create mode 100644 sql/prometheus_fdw--0.1.4--0.1.5.sql create mode 100644 src/init.rs diff --git a/sql/prometheus_fdw--0.1.4--0.1.5.sql b/sql/prometheus_fdw--0.1.4--0.1.5.sql new file mode 100644 index 0000000..ec7846b --- /dev/null +++ b/sql/prometheus_fdw--0.1.4--0.1.5.sql @@ -0,0 +1,3 @@ +CREATE FUNCTION create_table() + RETURNS VOID AS 'MODULE_PATHNAME', 'create_table' + LANGUAGE C STRICT; \ No newline at end of file diff --git a/src/init.rs b/src/init.rs new file mode 100644 index 0000000..58e766d --- /dev/null +++ b/src/init.rs @@ -0,0 +1,32 @@ +use pgrx::prelude::*; +use pgrx::spi::Spi; +use std::error::Error; + +#[pg_extern] +fn create_table() -> Result<(), Box> { + let queries = r#" + CREATE TABLE IF NOT EXISTS metric_labels ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + labels jsonb NOT NULL + ); + + CREATE TABLE IF NOT EXISTS metric_values ( + label_id INTEGER REFERENCES metric_labels (id), + time TIMESTAMP NOT NULL, + value DOUBLE PRECISION NOT NULL + ) PARTITION BY RANGE (time); + "#; + + Spi::run(queries); + + Ok(()) +} + +fn index_tables() { + // Implement indexing logic +} + +fn create_partitions(){ + // Implement partitioning logic +} diff --git a/src/lib.rs b/src/lib.rs index dd82ca9..25d92c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +mod init; + use pgrx::warning; use pgrx::{pg_sys, prelude::*, JsonB}; use reqwest::{self, Client}; From fe4dc6dffb681e6db9d588603cdb704c3fb9cc29 Mon Sep 17 00:00:00 2001 From: Jay Kothari Date: Fri, 12 Apr 2024 20:00:53 -0400 Subject: [PATCH 2/5] add indexing and partitioning function --- sql/prometheus_fdw--0.1.4--0.1.5.sql | 17 ++++++++-- src/init.rs | 46 +++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/sql/prometheus_fdw--0.1.4--0.1.5.sql b/sql/prometheus_fdw--0.1.4--0.1.5.sql index ec7846b..4c7e1ca 100644 --- a/sql/prometheus_fdw--0.1.4--0.1.5.sql +++ b/sql/prometheus_fdw--0.1.4--0.1.5.sql @@ -1,3 +1,14 @@ -CREATE FUNCTION create_table() - RETURNS VOID AS 'MODULE_PATHNAME', 'create_table' - LANGUAGE C STRICT; \ No newline at end of file +CREATE FUNCTION create_tables() + RETURNS void +AS 'MODULE_PATHNAME', 'create_tables' + LANGUAGE C STRICT; + +CREATE FUNCTION create_indexes() + RETURNS void +AS 'MODULE_PATHNAME', 'create_indexes' + LANGUAGE C STRICT; + +CREATE FUNCTION create_partitions(retention_period text) + RETURNS void +AS 'MODULE_PATHNAME', 'create_partitions' + LANGUAGE C STRICT; diff --git a/src/init.rs b/src/init.rs index 58e766d..c89a989 100644 --- a/src/init.rs +++ b/src/init.rs @@ -1,16 +1,15 @@ use pgrx::prelude::*; -use pgrx::spi::Spi; use std::error::Error; +/// Creates the necessary tables for metric tracking. #[pg_extern] -fn create_table() -> Result<(), Box> { +fn create_tables() -> Result<(), Box> { let queries = r#" CREATE TABLE IF NOT EXISTS metric_labels ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, labels jsonb NOT NULL ); - CREATE TABLE IF NOT EXISTS metric_values ( label_id INTEGER REFERENCES metric_labels (id), time TIMESTAMP NOT NULL, @@ -23,10 +22,43 @@ fn create_table() -> Result<(), Box> { Ok(()) } -fn index_tables() { - // Implement indexing logic +/// Creates indexes to optimize query performance. +#[pg_extern] +fn create_indexes() -> Result<(), Box> { + let queries = r#" + CREATE INDEX idx_metric_labels_name ON metric_labels (name); + CREATE INDEX idx_metric_labels_labels ON metric_labels USING GIN (labels); + CREATE INDEX idx_metric_values_time ON metric_values (time); + CREATE INDEX idx_metric_values_label_id ON metric_values (label_id); + "#; + + Spi::run(queries); + Ok(()) } -fn create_partitions(){ - // Implement partitioning logic +/// Sets up partitioning for the metric_values table and configures retention policy. +#[pg_extern] +fn create_partitions(retention_period: &str) -> Result<(), Box> { + let setup_partitioning = r#" + SELECT create_parent('public.metric_values', 'time', 'native', '1 day'); + "#; + + // Execute the partition setup query + Spi::run(setup_partitioning); + + let setup_retention = format!( + r#" + UPDATE part_config + SET retention = '{}', + retention_keep_table = false, + retention_keep_index = false, + infinite_time_partitions = true + WHERE parent_table = 'public.metric_values'; + "#, + retention_period + ); + + // Execute the retention setup query + Spi::run(&setup_retention); + Ok(()) } From d79670b27289a95e1d16df7194ff17442b80ef1f Mon Sep 17 00:00:00 2001 From: Jay Kothari Date: Tue, 23 Apr 2024 08:00:55 -0400 Subject: [PATCH 3/5] add basic setup steps as postgresql functions --- src/init.rs | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/init.rs b/src/init.rs index c89a989..0cbe4c7 100644 --- a/src/init.rs +++ b/src/init.rs @@ -1,6 +1,48 @@ use pgrx::prelude::*; use std::error::Error; +#[pg_extern] +fn basic_setup(base_url: &str, step: Option<&str>) -> Result<(), Box> { + let queries = format!( + r#" + -- Enable the extensions + CREATE EXTENSION IF NOT EXISTS prometheus_fdw CASCADE; + CREATE EXTENSION IF NOT EXISTS pg_partman CASCADE; + CREATE EXTENSION IF NOT EXISTS pg_cron CASCADE; + + -- Create the FDW + CREATE FOREIGN DATA WRAPPER prometheus_wrapper + HANDLER prometheus_fdw_handler + VALIDATOR prometheus_fdw_validator; + + -- Configure connection to server + CREATE SERVER my_prometheus_server + FOREIGN DATA WRAPPER prometheus_wrapper + OPTIONS ( + base_url '{}'); + + -- Create FDW table we can query to get metrics + CREATE FOREIGN TABLE metrics ( + metric_name TEXT, + metric_labels JSONB, + metric_time BIGINT, + metric_value FLOAT8 + ) + SERVER my_prometheus_server + OPTIONS ( + object 'metrics', + step '{}' + ); + "#, + base_url, + step.unwrap_or("10m") + ); + + Spi::run(&queries); + + Ok(()) +} + /// Creates the necessary tables for metric tracking. #[pg_extern] fn create_tables() -> Result<(), Box> { @@ -17,7 +59,7 @@ fn create_tables() -> Result<(), Box> { ) PARTITION BY RANGE (time); "#; - Spi::run(queries); + Spi::run(&queries); Ok(()) } From 4a1a0e679f4116286cb20f00a26c81f0c73d4bd0 Mon Sep 17 00:00:00 2001 From: Jay Kothari Date: Tue, 23 Apr 2024 08:08:40 -0400 Subject: [PATCH 4/5] update function signature --- sql/prometheus_fdw--0.1.4--0.1.5.sql | 5 +++++ src/init.rs | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sql/prometheus_fdw--0.1.4--0.1.5.sql b/sql/prometheus_fdw--0.1.4--0.1.5.sql index 4c7e1ca..48a5ed3 100644 --- a/sql/prometheus_fdw--0.1.4--0.1.5.sql +++ b/sql/prometheus_fdw--0.1.4--0.1.5.sql @@ -1,3 +1,8 @@ +CREATE FUNCTION basic_setup(base_url text) + RETURNS void +AS 'MODULE_PATHNAME', 'basic_setup' + LANGUAGE C STRICT; + CREATE FUNCTION create_tables() RETURNS void AS 'MODULE_PATHNAME', 'create_tables' diff --git a/src/init.rs b/src/init.rs index 0cbe4c7..0060807 100644 --- a/src/init.rs +++ b/src/init.rs @@ -2,7 +2,7 @@ use pgrx::prelude::*; use std::error::Error; #[pg_extern] -fn basic_setup(base_url: &str, step: Option<&str>) -> Result<(), Box> { +fn basic_setup(base_url: &str) -> Result<(), Box> { let queries = format!( r#" -- Enable the extensions @@ -34,8 +34,7 @@ fn basic_setup(base_url: &str, step: Option<&str>) -> Result<(), Box> step '{}' ); "#, - base_url, - step.unwrap_or("10m") + base_url, "10m" ); Spi::run(&queries); From 6be09bb5a607ac67af974c3ddb2225e05346e7db Mon Sep 17 00:00:00 2001 From: Jay Kothari Date: Tue, 23 Apr 2024 08:39:44 -0400 Subject: [PATCH 5/5] basic setup for mkdocs --- .github/workflows/pages-deployment.yml | 16 +++++++ .gitignore | 3 +- docs/hello_world/basic_setup.md | 6 +++ docs/index.md | 64 ++++++++++++++++++++++++++ docs/practical_example/setup.md | 43 +++++++++++++++++ mkdocs.yml | 39 ++++++++++++++++ 6 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pages-deployment.yml create mode 100644 docs/hello_world/basic_setup.md create mode 100644 docs/index.md create mode 100644 docs/practical_example/setup.md create mode 100644 mkdocs.yml diff --git a/.github/workflows/pages-deployment.yml b/.github/workflows/pages-deployment.yml new file mode 100644 index 0000000..a959f32 --- /dev/null +++ b/.github/workflows/pages-deployment.yml @@ -0,0 +1,16 @@ +name: github pages deployment +on: + push: + branches: + - main + - master +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.x + - run: pip install mkdocs-material + - run: mkdocs gh-deploy --force diff --git a/.gitignore b/.gitignore index 020c65b..ec91acd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ **/*.rs.bk .vscode/ META.json -/prometheus_fdw-* \ No newline at end of file +/prometheus_fdw-*site/ +/site \ No newline at end of file diff --git a/docs/hello_world/basic_setup.md b/docs/hello_world/basic_setup.md new file mode 100644 index 0000000..b0d4144 --- /dev/null +++ b/docs/hello_world/basic_setup.md @@ -0,0 +1,6 @@ +# Prometheus FDW basic demo + +- **Dockerfile**: A Postgres database with prometheus_fdw installed +- **setup-prometheus-fdw.sql**: A script to show how to configure +- **sample-query.sql**: A sample query against Prometheus +- **run-demo.sh**: A script showing the whole process of running the demo diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..b45b4db --- /dev/null +++ b/docs/index.md @@ -0,0 +1,64 @@ +## Prometheus_fdw + +Prometheus_fdw is an integration of Prometheus monitoring data into Postgres. It enables querying for Prometheus metrics directly within Postgres, bridging the gap between Prometheus monitoring and Postgres's robust database capabilities. +[![Static Badge](https://img.shields.io/badge/%40tembo-community?logo=slack&label=slack)](https://join.slack.com/t/tembocommunity/shared_invite/zt-20dtnhcmo-pLNV7_Aobi50TdTLpfQ~EQ) +[![PGXN version](https://badge.fury.io/pg/prometheus_fdw.svg)](https://pgxn.org/dist/prometheus_fdw/) + +### Pre-requisistes + +- Install `prometheus_fdw` +- (Optional) install `pg_partman` and `pg_cron` + +### Quick start + +`create extension prometheus_fdw;` + +Create the foreign data wrapper: + +```sql +create foreign data wrapper prometheus_wrapper + handler prometheus_fdw_handler + validator prometheus_fdw_validator; +``` + +Create the server: + +```sql +create server my_prometheus_server + foreign data wrapper prometheus_wrapper + options ( + base_url ''); +``` + +Create Foreign Table: + +```sql +CREATE FOREIGN TABLE IF NOT EXISTS metrics ( + metric_name TEXT, + metric_labels JSONB, + metric_time BIGINT, + metric_value FLOAT8 + ) +server my_prometheus_server +options ( + object 'metrics', + step '10m' +); +``` + +## Queries + +To simply run the fdw and look at values + +```sql +SELECT + * +FROM metrics +WHERE + metric_name='container_cpu_usage_seconds_total' + AND metric_time > 1696046800 AND metric_time < 1696133000; +``` + +## Examples + +Please see the `examples/` directory to find a basic example and a practical example. In the practical example, metrics are automatically synced into the database using `pg_cron`, and automatically expired using `pg_partman`. Performance is optimized using indexes and partitioning. diff --git a/docs/practical_example/setup.md b/docs/practical_example/setup.md new file mode 100644 index 0000000..93b441f --- /dev/null +++ b/docs/practical_example/setup.md @@ -0,0 +1,43 @@ +# Practical example + +- **Dockerfile**: A Postgres database with prometheus_fdw installed +- **setup-prometheus-fdw.sql**: A script to show how to configure +- **setup-cache.sql**: Setup tables for local storage of metrics data +- **setup-metrics-sync.sql**: Automatically sync data from Prometheus to Postgres +- **run-demo.sh**: A script showing the whole process of running the demo + +## Data model + +This data model was inspired by the [Crunchy Data postgresql-prometheus-adapter](https://github.com/CrunchyData/postgresql-prometheus-adapter). + +**metric_labels**: Stores the metric name labels. + +``` + id | name | labels +----+------------------------------------+------------------------- + 1 | container_cpu_usage_seconds_total | {"pod": "my-pod-1", ...} + 2 | container_cpu_usage_seconds_total | {"pod": "my-pod-2", ...} + 3 | container_memory_working_set_bytes | {"pod": "my-pod-1", ...} + 4 | container_memory_working_set_bytes | {"pod": "my-pod-2", ...} +``` + +**metric_values**: A partitioned table that stores metric values, when they happened, and the corresponding labels. + +``` + label_id | time | value +----------+------------+---------- + 4320 | 1702678142 | 12214272 + 4320 | 1702678742 | 11923456 + 4320 | 1702679342 | 12230656 + 4320 | 1702679942 | 11804672 + 4320 | 1702677542 | 11870208 + 4331 | 1702679942 | 53743616 + 4331 | 1702678142 | 54022144 + 4331 | 1702678742 | 53903360 + 4331 | 1702679342 | 53288960 + 4331 | 1702677542 | 53514240 +``` + +## Example query + +The query in **sample-query.sql** is an example of showing the current memory utilization of each container in kube-system. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..b3f59c1 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,39 @@ +site_name: Prometheus FDW +theme: + name: material + palette: + - scheme: default + primary: teal + toggle: + icon: material/weather-night + name: dark mode + - scheme: slate + primary: teal + toggle: + icon: material/weather-sunny + name: light mode +nav: + - Prometheus FDW: 'index.md' + - hello world: + - 'hello_world/basic_setup.md' + - practical example: + - 'practical_example/setup.md' +markdown_extensions: +- toc: + permalink: true +- markdown.extensions.codehilite: + guess_lang: false +- codehilite: +- admonition +- extra +- pymdownx.snippets: + check_paths: true +- pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true +- pymdownx.inlinehilite +- pymdownx.superfences +plugins: + - search + - mkdocstrings