From 4fe89b0a4021c01c1358b9bb1032309e850f2782 Mon Sep 17 00:00:00 2001 From: Takahiro Ebato Date: Mon, 12 Feb 2024 19:49:57 +0900 Subject: [PATCH] Enhance documentation (#2) --- README.md | 100 +++++++++++++++++ sql-insight-cli/Cargo.toml | 3 +- sql-insight-cli/README.md | 101 ++++++++++++++++++ sql-insight-cli/src/main.rs | 2 + sql-insight/Cargo.toml | 2 +- sql-insight/README.md | 1 + .../src/extractor/crud_table_extractor.rs | 20 ++++ sql-insight/src/extractor/table_extractor.rs | 26 +++++ sql-insight/src/formatter.rs | 18 ++++ sql-insight/src/lib.rs | 25 +++++ sql-insight/src/normalizer.rs | 34 ++++++ 11 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 README.md create mode 100644 sql-insight-cli/README.md create mode 120000 sql-insight/README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..869196d --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# sql-insight + +A toolkit for SQL query analysis, formatting, and transformation. +Leveraging the comprehensive parsing capabilities of [sqlparser-rs](https://github.com/sqlparser-rs/sqlparser-rs), it can handle various SQL dialects. + +[![Crates.io](https://img.shields.io/crates/v/sql-insight.svg)](https://crates.io/crates/sql-insight) +[![Docs.rs](https://docs.rs/sql-insight/badge.svg)](https://docs.rs/sql-insight) +[![Rust](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml/badge.svg?branch=master)](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml) +[![codecov](https://codecov.io/gh/takaebato/sql-insight/graph/badge.svg?token=Z1KYAWA3HY)](https://codecov.io/gh/takaebato/sql-insight) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) + +## Features + +- **SQL Formatting**: Format SQL queries to standardized form, improving readability and maintainability. +- **SQL Normalization**: Convert SQL queries into a normalized form, making them easier to analyze and process. +- **Table Extraction**: Extract tables referenced in SQL queries, clarifying the data sources involved. +- **CRUD Table Extraction**: Identify the create, read, update, and delete operations, along with the tables involved in each operation within SQL queries. + +## Installation + +Add `sql_insight` to your `Cargo.toml` file: + +```toml +[dependencies] +sql-insight = { version = "0.1.0" } +``` + +## Usage + +### SQL Formatting + +Format SQL queries according to different dialects: + +```rust +use sqlparser::dialect::GenericDialect; + +let dialect = GenericDialect {}; +let formatted_sql = sql_insight::format(&dialect, "SELECT * \n from users WHERE id = 1").unwrap(); +assert_eq!(formatted_sql, ["SELECT * FROM users WHERE id = 1"]); +``` + +### SQL Normalization + +Normalize SQL queries to abstract away literals: + +```rust +use sqlparser::dialect::GenericDialect; + +let dialect = GenericDialect {}; +let normalized_sql = sql_insight::normalize(&dialect, "SELECT * \n from users WHERE id = 1").unwrap(); +assert_eq!(normalized_sql, ["SELECT * FROM users WHERE id = ?"]); +``` + +### Table Extraction + +Extract table references from SQL queries: + +```rust +use sqlparser::dialect::GenericDialect; + +let dialect = GenericDialect {}; +let tables = sql_insight::extract_tables(&dialect, "SELECT * FROM catalog.schema.`users` as users_alias").unwrap(); +println!("{:?}", tables); +``` + +This outputs: + +``` +[Ok(Tables([TableReference { catalog: Some(Ident { value: "catalog", quote_style: None }), schema: Some(Ident { value: "schema", quote_style: None }), name: Ident { value: "users", quote_style: Some('`') }, alias: Some(Ident { value: "users_alias", quote_style: None }) }]))] +``` + +### CRUD Table Extraction + +Identify CRUD operations and the tables involved in each operation within SQL queries: + +```rust +use sqlparser::dialect::GenericDialect; + +let dialect = GenericDialect {}; +let crud_tables = sql_insight::extract_crud_tables(&dialect, "INSERT INTO users (name) SELECT name FROM employees").unwrap(); +println!("{:?}", crud_tables); +``` + +This outputs: + +``` +[Ok(CrudTables { create_tables: [TableReference { catalog: None, schema: None, name: Ident { value: "users", quote_style: None }, alias: None }], read_tables: [TableReference { catalog: None, schema: None, name: Ident { value: "employees", quote_style: None }, alias: None }], update_tables: [], delete_tables: [] })] +``` + +## Supported SQL Dialects + +`sql-insight` supports a comprehensive range of SQL dialects through [sqlparser-rs](https://github.com/sqlparser-rs/sqlparser-rs). For details on supported dialects, please refer to the documentation. + +## Contributing + +Contributions to `sql-insight` are welcome! Whether it's adding new features, fixing bugs, or improving documentation, feel free to fork the repository and submit a pull request. + +## License + +`sql-insight` is distributed under the [MIT license](https://github.com/takaebato/sql-insight/blob/master/LICENSE.txt). diff --git a/sql-insight-cli/Cargo.toml b/sql-insight-cli/Cargo.toml index ad435c4..c60c425 100644 --- a/sql-insight-cli/Cargo.toml +++ b/sql-insight-cli/Cargo.toml @@ -2,7 +2,7 @@ name = "sql-insight-cli" description = "SQL Insight is a tool to parse SQL queries and provide insight into the queries." documentation = "https://docs.rs/sql-insight-cli/" -keywords = ["sql", "query", "cli", "insight", "parser"] +keywords = ["sql", "query", "cli","toolkit", "insight"] include = [ "src/**/*.rs", "Cargo.toml", @@ -20,6 +20,7 @@ authors = { workspace = true } [[bin]] name = "sql-insight" path = "src/main.rs" +doc = false [dependencies] sql-insight = { path = "../sql-insight", version = "0.1.0" } diff --git a/sql-insight-cli/README.md b/sql-insight-cli/README.md new file mode 100644 index 0000000..467dc56 --- /dev/null +++ b/sql-insight-cli/README.md @@ -0,0 +1,101 @@ +# sql-insight-cli + +A command-line interface built on top of the [sql-insight](https://github.com/takaebato/sql-insight/tree/master/sql-insight). It provides a set of commands that `sql-insight` supports. + +[![Crates.io](https://img.shields.io/crates/v/sql-insight-cli.svg)](https://crates.io/crates/sql-insight-cli) +[![Rust](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml/badge.svg?branch=master)](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml) +[![codecov](https://codecov.io/gh/takaebato/sql-insight/graph/badge.svg?token=Z1KYAWA3HY)](https://codecov.io/gh/takaebato/sql-insight) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) + +## Features + +- **SQL Formatting**: Format SQL queries to standardized form, improving readability and maintainability. +- **SQL Normalization**: Convert SQL queries into a normalized form, making them easier to analyze and process. +- **Table Extraction**: Extract tables referenced in SQL queries, clarifying the data sources involved. +- **CRUD Table Extraction**: Identify the create, read, update, and delete operations, along with the tables involved in each operation within SQL queries. + +Additional Features: + +- **File and Interactive Mode Support**: Process SQL queries directly from files or via an interactive CLI session. + +## Installation + +Install `sql-insight-cli` using Cargo: + +```bash +cargo install sql-insight-cli +``` + +## Usage + +`sql-insight-cli` supports the following commands. Commands can process input directly from the command line, from a file using the --file option, or interactively. + +### General Options + +- `--file `: Read SQL queries from the specified file instead of command line arguments. +- interactive mode: Launch an interactive CLI session to input SQL queries. Enter this mode by running the command without a SQL argument nor --file option. To exit, type `exit`, `quit` or press `Ctrl + C`. + +### Formatting SQL + +Format SQL queries to a standardized style: + +```bash +sql-insight format "SELECT * \n FROM users WHERE id = 1;" +``` + +This outputs: + +```sql +SELECT * FROM users WHERE id = 1 +``` + +### Normalizing SQL + +Normalize SQL queries, abstracting values to placeholders: + +```bash +sql-insight normalize "SELECT * \n FROM users WHERE id = 1;" +``` + +This outputs: + +```sql +SELECT * FROM users WHERE id = ? +``` + +### Table Extraction + +Identify tables involved in SQL queries: + +```bash +sql-insight extract-tables "SELECT * FROM catalog.schema.users as users_alias" +``` + +This outputs: + +``` +catalog.schema.users AS users_alias +``` + +### CRUD Table Extraction + +Extract and identify CRUD operations and involved tables: + +```bash +sql-insight extract-crud "INSERT INTO users (name) SELECT name FROM employees" +``` + +This outputs: + +``` +Create: [users], Read: [employees], Update: [], Delete: [] +``` + +## Supported SQL Dialects +`sql-insight-cli` leverages [sqlparser-rs](https://github.com/sqlparser-rs/sqlparser-rs) for parsing, supporting a wide range of SQL dialects. For a detailed list, please refer to the sqlparser-rs documentation. + +## Contributing +Contributions to `sql-insight-cli` are welcome! Whether it's adding new features, fixing bugs, or improving documentation, feel free to fork the repository and submit a pull request. + +## License +`sql-insight-cli` is licensed under the [MIT License](https://github.com/takaebato/sql-insight/blob/master/sql-insight-cli/LICENSE.txt). diff --git a/sql-insight-cli/src/main.rs b/sql-insight-cli/src/main.rs index 6778282..a14fca8 100644 --- a/sql-insight-cli/src/main.rs +++ b/sql-insight-cli/src/main.rs @@ -28,6 +28,8 @@ struct CommonOptions { #[clap(value_parser, group = "source")] sql: Option, /// The dialect of the input SQL. Might be required for parsing dialect-specific syntax. + /// Available dialects: ansi, bigquery, clickhouse, duckdb, generic, hive, mssql, mysql, postgres, redshift, snowflake, sqlite. + /// Default: generic. #[clap(short, long)] dialect: Option, /// The file containing the SQL to operate on diff --git a/sql-insight/Cargo.toml b/sql-insight/Cargo.toml index c814179..5fc39f5 100644 --- a/sql-insight/Cargo.toml +++ b/sql-insight/Cargo.toml @@ -2,7 +2,7 @@ name = "sql-insight" description = "SQL Insight is a tool to parse SQL queries and provide insight into the queries." documentation = "https://docs.rs/sql-insight/" -keywords = ["sql", "query", "tool", "insight", "parser"] +keywords = ["sql", "query", "toolkit", "insight"] include = [ "src/**/*.rs", "Cargo.toml", diff --git a/sql-insight/README.md b/sql-insight/README.md new file mode 120000 index 0000000..32d46ee --- /dev/null +++ b/sql-insight/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/sql-insight/src/extractor/crud_table_extractor.rs b/sql-insight/src/extractor/crud_table_extractor.rs index 4b7715d..9314bf4 100644 --- a/sql-insight/src/extractor/crud_table_extractor.rs +++ b/sql-insight/src/extractor/crud_table_extractor.rs @@ -1,3 +1,7 @@ +//! A Extractor that extracts CRUD tables from SQL queries. +//! +//! See [`extract_crud_tables`](crate::extract_crud_tables()) as the entry point for extracting CRUD tables from SQL. + use std::fmt; use std::ops::ControlFlow; @@ -8,6 +12,19 @@ use sqlparser::ast::{MergeClause, Statement, Visit, Visitor}; use sqlparser::dialect::Dialect; use sqlparser::parser::Parser; +/// Convenience function to extract CRUD tables from SQL. +/// +/// ## Example +/// +/// ```rust +/// use sqlparser::dialect::GenericDialect; +/// +/// let dialect = GenericDialect {}; +/// let sql = "INSERT INTO t1 (a) SELECT a FROM t2"; +/// let result = sql_insight::extract_crud_tables(&dialect, sql).unwrap(); +/// println!("{:#?}", result); +/// assert_eq!(result[0].as_ref().unwrap().to_string(), "Create: [t1], Read: [t2], Update: [], Delete: []"); +/// ``` pub fn extract_crud_tables( dialect: &dyn Dialect, sql: &str, @@ -15,6 +32,7 @@ pub fn extract_crud_tables( CrudTableExtractor::extract(dialect, sql) } +/// [`CrudTables`] represents the tables involved in CRUD operations. #[derive(Default, Debug, PartialEq)] pub struct CrudTables { pub create_tables: Vec, @@ -47,6 +65,7 @@ impl CrudTables { } } +/// A visitor to extract CRUD tables from SQL. #[derive(Default, Debug)] pub struct CrudTableExtractor { create_tables: Vec, @@ -146,6 +165,7 @@ impl Visitor for CrudTableExtractor { } impl CrudTableExtractor { + /// Extract CRUD tables from SQL. pub fn extract( dialect: &dyn Dialect, sql: &str, diff --git a/sql-insight/src/extractor/table_extractor.rs b/sql-insight/src/extractor/table_extractor.rs index feea996..76feaba 100644 --- a/sql-insight/src/extractor/table_extractor.rs +++ b/sql-insight/src/extractor/table_extractor.rs @@ -1,3 +1,7 @@ +//! A Extractor that extracts tables from SQL queries. +//! +//! See [`extract_tables`](crate::extract_tables()) as the entry point for extracting tables from SQL. + use core::fmt; use std::ops::ControlFlow; @@ -7,6 +11,19 @@ use sqlparser::ast::{Ident, ObjectName, Statement, TableFactor, TableWithJoins, use sqlparser::dialect::Dialect; use sqlparser::parser::Parser; +/// Convenience function to extract tables from SQL. +/// +/// ## Example +/// +/// ```rust +/// use sqlparser::dialect::GenericDialect; +/// +/// let dialect = GenericDialect {}; +/// let sql = "SELECT a FROM t1 INNER JOIN t2 ON t1.id = t2.id"; +/// let result = sql_insight::extract_tables(&dialect, sql).unwrap(); +/// println!("{:#?}", result); +/// assert_eq!(result[0].as_ref().unwrap().to_string(), "t1, t2"); +/// ``` pub fn extract_tables( dialect: &dyn Dialect, sql: &str, @@ -14,6 +31,9 @@ pub fn extract_tables( TableExtractor::extract(dialect, sql) } +/// [`TableReference`] represents a qualified table with alias. +/// In this crate, this is the canonical representation of a table. +/// Tables found during analyzing an AST are stored as `TableReference`. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct TableReference { pub catalog: Option, @@ -115,6 +135,7 @@ impl TryFrom<&ObjectName> for TableReference { } } +/// [`Tables`] represents a list of [`TableReference`] that found in SQL. #[derive(Debug, PartialEq)] pub struct Tables(pub Vec); @@ -130,10 +151,14 @@ impl fmt::Display for Tables { } } +/// A visitor to extract tables from SQL. #[derive(Default, Debug)] pub struct TableExtractor { + // All tables found in the SQL including aliases, must be resolved to original tables. all_tables: Vec, + // Original tables found in the SQL, used to resolve aliases. original_tables: Vec, + // Flag to indicate if the current relation is part of a `TableFactor::Table` relation_of_table: bool, } @@ -185,6 +210,7 @@ impl Visitor for TableExtractor { } impl TableExtractor { + /// Extract tables from SQL. pub fn extract(dialect: &dyn Dialect, sql: &str) -> Result>, Error> { let statements = Parser::parse_sql(dialect, sql)?; let results = statements diff --git a/sql-insight/src/formatter.rs b/sql-insight/src/formatter.rs index 1e45cbe..a89e1fe 100644 --- a/sql-insight/src/formatter.rs +++ b/sql-insight/src/formatter.rs @@ -1,15 +1,33 @@ +//! A Formatter that formats SQL into a standardized format. +//! +//! See [`format`](crate::format()) as the entry point for formatting SQL. + use crate::error::Error; use sqlparser::dialect::Dialect; use sqlparser::parser::Parser; +/// Convenience function to format SQL. +/// +/// ## Example +/// +/// ```rust +/// use sqlparser::dialect::GenericDialect; +/// +/// let dialect = GenericDialect {}; +/// let sql = "SELECT a FROM t1 \n WHERE b = 1"; +/// let result = sql_insight::format(&dialect, sql).unwrap(); +/// assert_eq!(result, ["SELECT a FROM t1 WHERE b = 1"]); +/// ``` pub fn format(dialect: &dyn Dialect, sql: &str) -> Result, Error> { Formatter::format(dialect, sql) } +/// Formatter for SQL. #[derive(Debug, Default)] pub struct Formatter; impl Formatter { + /// Format SQL. pub fn format(dialect: &dyn Dialect, sql: &str) -> Result, Error> { let statements = Parser::parse_sql(dialect, sql)?; Ok(statements diff --git a/sql-insight/src/lib.rs b/sql-insight/src/lib.rs index 1a493d9..63b693d 100644 --- a/sql-insight/src/lib.rs +++ b/sql-insight/src/lib.rs @@ -1,3 +1,28 @@ +//! # sql-insight +//! +//! `sql-insight` is a toolkit designed for SQL query analysis, formatting, and transformation. +//! +//! ## Main Functionalities +//! +//! - **SQL Formatting**: Format SQL queries into a standardized format. See the [`formatter`] module for more information. +//! - **SQL Normalization**: Normalize SQL queries by abstracting literals. See the [`normalizer`] module for more information. +//! - **Table Extraction**: Extract tables within SQL queries. See the [`table_extractor`] module for more information. +//! - **CRUD Table Extraction**: Extract CRUD tables from SQL queries. See the [`crud_table_extractor`] module for more information. +//! +//! ## Quick Start +//! +//! Here's a quick example to get you started with SQL formatting: +//! +//! ```rust +//! use sqlparser::dialect::GenericDialect; +//! +//! let dialect = GenericDialect {}; +//! let normalized_sql = sql_insight::format(&dialect, "SELECT * \n from users WHERE id = 1").unwrap(); +//! assert_eq!(normalized_sql, ["SELECT * FROM users WHERE id = 1"]); +//! ``` +//! +//! For more comprehensive examples and usage, refer to [crates.io](https://crates.io/crates/sql-insight) or the documentation of each module. + pub mod error; pub mod extractor; pub mod formatter; diff --git a/sql-insight/src/normalizer.rs b/sql-insight/src/normalizer.rs index 00c2614..fceac3a 100644 --- a/sql-insight/src/normalizer.rs +++ b/sql-insight/src/normalizer.rs @@ -1,3 +1,7 @@ +//! A Normalizer that converts SQL queries to a canonical form. +//! +//! See [`normalize`](crate::normalize()) as the entry point for normalizing SQL. + use std::ops::ControlFlow; use crate::error::Error; @@ -6,10 +10,35 @@ use sqlparser::ast::{Expr, VisitMut, VisitorMut}; use sqlparser::dialect::Dialect; use sqlparser::parser::Parser; +/// Convenience function to normalize SQL with default options. +/// +/// ## Example +/// +/// ```rust +/// use sqlparser::dialect::GenericDialect; +/// +/// let dialect = GenericDialect {}; +/// let sql = "SELECT a FROM t1 WHERE b = 1 AND c in (2, 3) AND d LIKE '%foo'"; +/// let result = sql_insight::normalize(&dialect, sql).unwrap(); +/// assert_eq!(result, ["SELECT a FROM t1 WHERE b = ? AND c IN (?, ?) AND d LIKE ?"]); +/// ``` pub fn normalize(dialect: &dyn Dialect, sql: &str) -> Result, Error> { Normalizer::normalize(dialect, sql, NormalizerOptions::new()) } +/// Convenience function to normalize SQL with options. +/// +/// ## Example +/// +/// ```rust +/// use sqlparser::dialect::GenericDialect; +/// use sql_insight::NormalizerOptions; +/// +/// let dialect = GenericDialect {}; +/// let sql = "SELECT a FROM t1 WHERE b = 1 AND c in (2, 3, 4)"; +/// let result = sql_insight::normalize_with_options(&dialect, sql, NormalizerOptions::new().with_unify_in_list(true)).unwrap(); +/// assert_eq!(result, ["SELECT a FROM t1 WHERE b = ? AND c IN (...)"]); +/// ``` pub fn normalize_with_options( dialect: &dyn Dialect, sql: &str, @@ -18,8 +47,11 @@ pub fn normalize_with_options( Normalizer::normalize(dialect, sql, options) } +/// Options for normalizing SQL. #[derive(Default, Clone)] pub struct NormalizerOptions { + /// Unify IN lists to a single form when all elements are literal values. + /// For example, `IN (1, 2, 3)` becomes `IN (...)`. pub unify_in_list: bool, } @@ -34,6 +66,7 @@ impl NormalizerOptions { } } +/// A visitor for SQL AST nodes that normalizes SQL queries. #[derive(Default)] pub struct Normalizer { pub options: NormalizerOptions, @@ -72,6 +105,7 @@ impl Normalizer { self } + /// Normalize SQL. pub fn normalize( dialect: &dyn Dialect, sql: &str,