diff --git a/.github/workflows/pages-deployment.yml b/.github/workflows/pages-deployment.yml new file mode 100644 index 0000000..a959f32 --- /dev/null +++ b/.github/workflows/pages-deployment.yml @@ -0,0 +1,16 @@ +name: github pages deployment +on: + push: + branches: + - main + - master +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.x + - run: pip install mkdocs-material + - run: mkdocs gh-deploy --force diff --git a/.gitignore b/.gitignore index 020c65b..ec91acd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ **/*.rs.bk .vscode/ META.json -/prometheus_fdw-* \ No newline at end of file +/prometheus_fdw-*site/ +/site \ No newline at end of file diff --git a/docs/hello_world/basic_setup.md b/docs/hello_world/basic_setup.md new file mode 100644 index 0000000..b0d4144 --- /dev/null +++ b/docs/hello_world/basic_setup.md @@ -0,0 +1,6 @@ +# Prometheus FDW basic demo + +- **Dockerfile**: A Postgres database with prometheus_fdw installed +- **setup-prometheus-fdw.sql**: A script to show how to configure +- **sample-query.sql**: A sample query against Prometheus +- **run-demo.sh**: A script showing the whole process of running the demo diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..b45b4db --- /dev/null +++ b/docs/index.md @@ -0,0 +1,64 @@ +## Prometheus_fdw + +Prometheus_fdw is an integration of Prometheus monitoring data into Postgres. It enables querying for Prometheus metrics directly within Postgres, bridging the gap between Prometheus monitoring and Postgres's robust database capabilities. +[![Static Badge](https://img.shields.io/badge/%40tembo-community?logo=slack&label=slack)](https://join.slack.com/t/tembocommunity/shared_invite/zt-20dtnhcmo-pLNV7_Aobi50TdTLpfQ~EQ) +[![PGXN version](https://badge.fury.io/pg/prometheus_fdw.svg)](https://pgxn.org/dist/prometheus_fdw/) + +### Pre-requisistes + +- Install `prometheus_fdw` +- (Optional) install `pg_partman` and `pg_cron` + +### Quick start + +`create extension prometheus_fdw;` + +Create the foreign data wrapper: + +```sql +create foreign data wrapper prometheus_wrapper + handler prometheus_fdw_handler + validator prometheus_fdw_validator; +``` + +Create the server: + +```sql +create server my_prometheus_server + foreign data wrapper prometheus_wrapper + options ( + base_url ''); +``` + +Create Foreign Table: + +```sql +CREATE FOREIGN TABLE IF NOT EXISTS metrics ( + metric_name TEXT, + metric_labels JSONB, + metric_time BIGINT, + metric_value FLOAT8 + ) +server my_prometheus_server +options ( + object 'metrics', + step '10m' +); +``` + +## Queries + +To simply run the fdw and look at values + +```sql +SELECT + * +FROM metrics +WHERE + metric_name='container_cpu_usage_seconds_total' + AND metric_time > 1696046800 AND metric_time < 1696133000; +``` + +## Examples + +Please see the `examples/` directory to find a basic example and a practical example. In the practical example, metrics are automatically synced into the database using `pg_cron`, and automatically expired using `pg_partman`. Performance is optimized using indexes and partitioning. diff --git a/docs/practical_example/setup.md b/docs/practical_example/setup.md new file mode 100644 index 0000000..93b441f --- /dev/null +++ b/docs/practical_example/setup.md @@ -0,0 +1,43 @@ +# Practical example + +- **Dockerfile**: A Postgres database with prometheus_fdw installed +- **setup-prometheus-fdw.sql**: A script to show how to configure +- **setup-cache.sql**: Setup tables for local storage of metrics data +- **setup-metrics-sync.sql**: Automatically sync data from Prometheus to Postgres +- **run-demo.sh**: A script showing the whole process of running the demo + +## Data model + +This data model was inspired by the [Crunchy Data postgresql-prometheus-adapter](https://github.com/CrunchyData/postgresql-prometheus-adapter). + +**metric_labels**: Stores the metric name labels. + +``` + id | name | labels +----+------------------------------------+------------------------- + 1 | container_cpu_usage_seconds_total | {"pod": "my-pod-1", ...} + 2 | container_cpu_usage_seconds_total | {"pod": "my-pod-2", ...} + 3 | container_memory_working_set_bytes | {"pod": "my-pod-1", ...} + 4 | container_memory_working_set_bytes | {"pod": "my-pod-2", ...} +``` + +**metric_values**: A partitioned table that stores metric values, when they happened, and the corresponding labels. + +``` + label_id | time | value +----------+------------+---------- + 4320 | 1702678142 | 12214272 + 4320 | 1702678742 | 11923456 + 4320 | 1702679342 | 12230656 + 4320 | 1702679942 | 11804672 + 4320 | 1702677542 | 11870208 + 4331 | 1702679942 | 53743616 + 4331 | 1702678142 | 54022144 + 4331 | 1702678742 | 53903360 + 4331 | 1702679342 | 53288960 + 4331 | 1702677542 | 53514240 +``` + +## Example query + +The query in **sample-query.sql** is an example of showing the current memory utilization of each container in kube-system. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..b3f59c1 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,39 @@ +site_name: Prometheus FDW +theme: + name: material + palette: + - scheme: default + primary: teal + toggle: + icon: material/weather-night + name: dark mode + - scheme: slate + primary: teal + toggle: + icon: material/weather-sunny + name: light mode +nav: + - Prometheus FDW: 'index.md' + - hello world: + - 'hello_world/basic_setup.md' + - practical example: + - 'practical_example/setup.md' +markdown_extensions: +- toc: + permalink: true +- markdown.extensions.codehilite: + guess_lang: false +- codehilite: +- admonition +- extra +- pymdownx.snippets: + check_paths: true +- pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true +- pymdownx.inlinehilite +- pymdownx.superfences +plugins: + - search + - mkdocstrings diff --git a/sql/prometheus_fdw--0.1.4--0.1.5.sql b/sql/prometheus_fdw--0.1.4--0.1.5.sql new file mode 100644 index 0000000..48a5ed3 --- /dev/null +++ b/sql/prometheus_fdw--0.1.4--0.1.5.sql @@ -0,0 +1,19 @@ +CREATE FUNCTION basic_setup(base_url text) + RETURNS void +AS 'MODULE_PATHNAME', 'basic_setup' + LANGUAGE C STRICT; + +CREATE FUNCTION create_tables() + RETURNS void +AS 'MODULE_PATHNAME', 'create_tables' + LANGUAGE C STRICT; + +CREATE FUNCTION create_indexes() + RETURNS void +AS 'MODULE_PATHNAME', 'create_indexes' + LANGUAGE C STRICT; + +CREATE FUNCTION create_partitions(retention_period text) + RETURNS void +AS 'MODULE_PATHNAME', 'create_partitions' + LANGUAGE C STRICT; diff --git a/src/init.rs b/src/init.rs new file mode 100644 index 0000000..0060807 --- /dev/null +++ b/src/init.rs @@ -0,0 +1,105 @@ +use pgrx::prelude::*; +use std::error::Error; + +#[pg_extern] +fn basic_setup(base_url: &str) -> Result<(), Box> { + let queries = format!( + r#" + -- Enable the extensions + CREATE EXTENSION IF NOT EXISTS prometheus_fdw CASCADE; + CREATE EXTENSION IF NOT EXISTS pg_partman CASCADE; + CREATE EXTENSION IF NOT EXISTS pg_cron CASCADE; + + -- Create the FDW + CREATE FOREIGN DATA WRAPPER prometheus_wrapper + HANDLER prometheus_fdw_handler + VALIDATOR prometheus_fdw_validator; + + -- Configure connection to server + CREATE SERVER my_prometheus_server + FOREIGN DATA WRAPPER prometheus_wrapper + OPTIONS ( + base_url '{}'); + + -- Create FDW table we can query to get metrics + CREATE FOREIGN TABLE metrics ( + metric_name TEXT, + metric_labels JSONB, + metric_time BIGINT, + metric_value FLOAT8 + ) + SERVER my_prometheus_server + OPTIONS ( + object 'metrics', + step '{}' + ); + "#, + base_url, "10m" + ); + + Spi::run(&queries); + + Ok(()) +} + +/// Creates the necessary tables for metric tracking. +#[pg_extern] +fn create_tables() -> Result<(), Box> { + let queries = r#" + CREATE TABLE IF NOT EXISTS metric_labels ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + labels jsonb NOT NULL + ); + CREATE TABLE IF NOT EXISTS metric_values ( + label_id INTEGER REFERENCES metric_labels (id), + time TIMESTAMP NOT NULL, + value DOUBLE PRECISION NOT NULL + ) PARTITION BY RANGE (time); + "#; + + Spi::run(&queries); + + Ok(()) +} + +/// Creates indexes to optimize query performance. +#[pg_extern] +fn create_indexes() -> Result<(), Box> { + let queries = r#" + CREATE INDEX idx_metric_labels_name ON metric_labels (name); + CREATE INDEX idx_metric_labels_labels ON metric_labels USING GIN (labels); + CREATE INDEX idx_metric_values_time ON metric_values (time); + CREATE INDEX idx_metric_values_label_id ON metric_values (label_id); + "#; + + Spi::run(queries); + Ok(()) +} + +/// Sets up partitioning for the metric_values table and configures retention policy. +#[pg_extern] +fn create_partitions(retention_period: &str) -> Result<(), Box> { + let setup_partitioning = r#" + SELECT create_parent('public.metric_values', 'time', 'native', '1 day'); + "#; + + // Execute the partition setup query + Spi::run(setup_partitioning); + + let setup_retention = format!( + r#" + UPDATE part_config + SET retention = '{}', + retention_keep_table = false, + retention_keep_index = false, + infinite_time_partitions = true + WHERE parent_table = 'public.metric_values'; + "#, + retention_period + ); + + // Execute the retention setup query + Spi::run(&setup_retention); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index dd82ca9..25d92c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +mod init; + use pgrx::warning; use pgrx::{pg_sys, prelude::*, JsonB}; use reqwest::{self, Client};