Enhance documentation (#2)

README.md

@@ -0,0 +1,100 @@
+# sql-insight
+
+A toolkit for SQL query analysis, formatting, and transformation.
+Leveraging the comprehensive parsing capabilities of [sqlparser-rs](https://github.com/sqlparser-rs/sqlparser-rs), it can handle various SQL dialects.
+
+[![Crates.io](https://img.shields.io/crates/v/sql-insight.svg)](https://crates.io/crates/sql-insight)
+[![Docs.rs](https://docs.rs/sql-insight/badge.svg)](https://docs.rs/sql-insight)
+[![Rust](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml/badge.svg?branch=master)](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml)
+[![codecov](https://codecov.io/gh/takaebato/sql-insight/graph/badge.svg?token=Z1KYAWA3HY)](https://codecov.io/gh/takaebato/sql-insight)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+## Features
+
+- **SQL Formatting**: Format SQL queries to standardized form, improving readability and maintainability.
+- **SQL Normalization**: Convert SQL queries into a normalized form, making them easier to analyze and process.
+- **Table Extraction**: Extract tables referenced in SQL queries, clarifying the data sources involved.
+- **CRUD Table Extraction**: Identify the create, read, update, and delete operations, along with the tables involved in each operation within SQL queries.
+
+## Installation
+
+Add `sql_insight` to your `Cargo.toml` file:
+
+```toml
+[dependencies]
+sql-insight = { version = "0.1.0" }
+```
+
+## Usage
+
+### SQL Formatting
+
+Format SQL queries according to different dialects:
+
+```rust
+use sqlparser::dialect::GenericDialect;
+
+let dialect = GenericDialect {};
+let formatted_sql = sql_insight::format(&dialect, "SELECT * \n from users   WHERE id = 1").unwrap();
+assert_eq!(formatted_sql, ["SELECT * FROM users WHERE id = 1"]);
+```
+
+### SQL Normalization
+
+Normalize SQL queries to abstract away literals:
+
+```rust
+use sqlparser::dialect::GenericDialect;
+
+let dialect = GenericDialect {};
+let normalized_sql = sql_insight::normalize(&dialect, "SELECT * \n from users   WHERE id = 1").unwrap();
+assert_eq!(normalized_sql, ["SELECT * FROM users WHERE id = ?"]);
+```
+
+### Table Extraction
+
+Extract table references from SQL queries:
+
+```rust
+use sqlparser::dialect::GenericDialect;
+
+let dialect = GenericDialect {};
+let tables = sql_insight::extract_tables(&dialect, "SELECT * FROM catalog.schema.`users` as users_alias").unwrap();
+println!("{:?}", tables);
+```
+
+This outputs:
+
+```
+[Ok(Tables([TableReference { catalog: Some(Ident { value: "catalog", quote_style: None }), schema: Some(Ident { value: "schema", quote_style: None }), name: Ident { value: "users", quote_style: Some('`') }, alias: Some(Ident { value: "users_alias", quote_style: None }) }]))]
+```
+
+### CRUD Table Extraction
+
+Identify CRUD operations and the tables involved in each operation within SQL queries:
+
+```rust
+use sqlparser::dialect::GenericDialect;
+
+let dialect = GenericDialect {};
+let crud_tables = sql_insight::extract_crud_tables(&dialect, "INSERT INTO users (name) SELECT name FROM employees").unwrap();
+println!("{:?}", crud_tables);
+```
+
+This outputs:
+
+```
+[Ok(CrudTables { create_tables: [TableReference { catalog: None, schema: None, name: Ident { value: "users", quote_style: None }, alias: None }], read_tables: [TableReference { catalog: None, schema: None, name: Ident { value: "employees", quote_style: None }, alias: None }], update_tables: [], delete_tables: [] })]
+```
+
+## Supported SQL Dialects
+
+`sql-insight` supports a comprehensive range of SQL dialects through [sqlparser-rs](https://github.com/sqlparser-rs/sqlparser-rs). For details on supported dialects, please refer to the documentation.
+
+## Contributing
+
+Contributions to `sql-insight` are welcome! Whether it's adding new features, fixing bugs, or improving documentation, feel free to fork the repository and submit a pull request.
+
+## License
+
+`sql-insight` is distributed under the [MIT license](https://github.com/takaebato/sql-insight/blob/master/LICENSE.txt).

sql-insight-cli/Cargo.toml

@@ -2,7 +2,7 @@
 name = "sql-insight-cli"
 description = "SQL Insight is a tool to parse SQL queries and provide insight into the queries."
 documentation = "https://docs.rs/sql-insight-cli/"
-keywords = ["sql", "query", "cli", "insight", "parser"]
+keywords = ["sql", "query", "cli","toolkit", "insight"]
 include = [
     "src/**/*.rs",
     "Cargo.toml",
@@ -20,6 +20,7 @@ authors = { workspace = true }
 [[bin]]
 name = "sql-insight"
 path = "src/main.rs"
+doc = false
 
 [dependencies]
 sql-insight = { path = "../sql-insight", version = "0.1.0" }

sql-insight-cli/README.md

@@ -0,0 +1,101 @@
+# sql-insight-cli
+
+A command-line interface built on top of the [sql-insight](https://github.com/takaebato/sql-insight/tree/master/sql-insight). It provides a set of commands that `sql-insight` supports.
+
+[![Crates.io](https://img.shields.io/crates/v/sql-insight-cli.svg)](https://crates.io/crates/sql-insight-cli)
+[![Rust](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml/badge.svg?branch=master)](https://github.com/takaebato/sql-insight/actions/workflows/rust.yaml)
+[![codecov](https://codecov.io/gh/takaebato/sql-insight/graph/badge.svg?token=Z1KYAWA3HY)](https://codecov.io/gh/takaebato/sql-insight)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+## Features
+
+- **SQL Formatting**: Format SQL queries to standardized form, improving readability and maintainability.
+- **SQL Normalization**: Convert SQL queries into a normalized form, making them easier to analyze and process.
+- **Table Extraction**: Extract tables referenced in SQL queries, clarifying the data sources involved.
+- **CRUD Table Extraction**: Identify the create, read, update, and delete operations, along with the tables involved in each operation within SQL queries.
+
+Additional Features:
+
+- **File and Interactive Mode Support**: Process SQL queries directly from files or via an interactive CLI session.
+
+## Installation
+
+Install `sql-insight-cli` using Cargo:
+
+```bash
+cargo install sql-insight-cli
+```
+
+## Usage
+
+`sql-insight-cli` supports the following commands. Commands can process input directly from the command line, from a file using the --file option, or interactively.
+
+### General Options
+
+- `--file <path>`: Read SQL queries from the specified file instead of command line arguments.
+- interactive mode: Launch an interactive CLI session to input SQL queries. Enter this mode by running the command without a SQL argument nor --file option. To exit, type `exit`, `quit` or press `Ctrl + C`.
+
+### Formatting SQL
+
+Format SQL queries to a standardized style:
+
+```bash
+sql-insight format "SELECT *  \n FROM users         WHERE id = 1;"
+```
+
+This outputs:
+
+```sql
+SELECT * FROM users WHERE id = 1
+```
+
+### Normalizing SQL
+
+Normalize SQL queries, abstracting values to placeholders:
+
+```bash
+sql-insight normalize "SELECT *  \n FROM users         WHERE id = 1;"
+```
+
+This outputs:
+
+```sql
+SELECT * FROM users WHERE id = ?
+```
+
+### Table Extraction
+
+Identify tables involved in SQL queries:
+
+```bash
+sql-insight extract-tables "SELECT * FROM catalog.schema.users as users_alias"
+```
+
+This outputs:
+
+```
+catalog.schema.users AS users_alias
+```
+
+### CRUD Table Extraction
+
+Extract and identify CRUD operations and involved tables:
+
+```bash
+sql-insight extract-crud "INSERT INTO users (name) SELECT name FROM employees"
+```
+
+This outputs:
+
+```
+Create: [users], Read: [employees], Update: [], Delete: []
+```
+
+## Supported SQL Dialects
+`sql-insight-cli` leverages [sqlparser-rs](https://github.com/sqlparser-rs/sqlparser-rs) for parsing, supporting a wide range of SQL dialects. For a detailed list, please refer to the sqlparser-rs documentation.
+
+## Contributing
+Contributions to `sql-insight-cli` are welcome! Whether it's adding new features, fixing bugs, or improving documentation, feel free to fork the repository and submit a pull request.
+
+## License
+`sql-insight-cli` is licensed under the [MIT License](https://github.com/takaebato/sql-insight/blob/master/sql-insight-cli/LICENSE.txt).

sql-insight-cli/src/main.rs

@@ -28,6 +28,8 @@ struct CommonOptions {
     #[clap(value_parser, group = "source")]
     sql: Option<String>,
     /// The dialect of the input SQL. Might be required for parsing dialect-specific syntax.
+    /// Available dialects: ansi, bigquery, clickhouse, duckdb, generic, hive, mssql, mysql, postgres, redshift, snowflake, sqlite.
+    /// Default: generic.
     #[clap(short, long)]
     dialect: Option<String>,
     /// The file containing the SQL to operate on

sql-insight/Cargo.toml

@@ -2,7 +2,7 @@
 name = "sql-insight"
 description = "SQL Insight is a tool to parse SQL queries and provide insight into the queries."
 documentation = "https://docs.rs/sql-insight/"
-keywords = ["sql", "query", "tool", "insight", "parser"]
+keywords = ["sql", "query", "toolkit", "insight"]
 include = [
     "src/**/*.rs",
     "Cargo.toml",

sql-insight/README.md

@@ -0,0 +1 @@
+../README.md

sql-insight/src/extractor/crud_table_extractor.rs

@@ -1,3 +1,7 @@
+//! A Extractor that extracts CRUD tables from SQL queries.
+//!
+//! See [`extract_crud_tables`](crate::extract_crud_tables()) as the entry point for extracting CRUD tables from SQL.
+
 use std::fmt;
 use std::ops::ControlFlow;
 
@@ -8,13 +12,27 @@ use sqlparser::ast::{MergeClause, Statement, Visit, Visitor};
 use sqlparser::dialect::Dialect;
 use sqlparser::parser::Parser;
 
+/// Convenience function to extract CRUD tables from SQL.
+///
+/// ## Example
+///
+/// ```rust
+/// use sqlparser::dialect::GenericDialect;
+///
+/// let dialect = GenericDialect {};
+/// let sql = "INSERT INTO t1 (a) SELECT a FROM t2";
+/// let result = sql_insight::extract_crud_tables(&dialect, sql).unwrap();
+/// println!("{:#?}", result);
+/// assert_eq!(result[0].as_ref().unwrap().to_string(), "Create: [t1], Read: [t2], Update: [], Delete: []");
+/// ```
 pub fn extract_crud_tables(
     dialect: &dyn Dialect,
     sql: &str,
 ) -> Result<Vec<Result<CrudTables, Error>>, Error> {
     CrudTableExtractor::extract(dialect, sql)
 }
 
+/// [`CrudTables`] represents the tables involved in CRUD operations.
 #[derive(Default, Debug, PartialEq)]
 pub struct CrudTables {
     pub create_tables: Vec<TableReference>,
@@ -47,6 +65,7 @@ impl CrudTables {
     }
 }
 
+/// A visitor to extract CRUD tables from SQL.
 #[derive(Default, Debug)]
 pub struct CrudTableExtractor {
     create_tables: Vec<TableReference>,
@@ -146,6 +165,7 @@ impl Visitor for CrudTableExtractor {
 }
 
 impl CrudTableExtractor {
+    /// Extract CRUD tables from SQL.
     pub fn extract(
         dialect: &dyn Dialect,
         sql: &str,

sql-insight/src/extractor/table_extractor.rs

@@ -1,3 +1,7 @@
+//! A Extractor that extracts tables from SQL queries.
+//!
+//! See [`extract_tables`](crate::extract_tables()) as the entry point for extracting tables from SQL.
+
 use core::fmt;
 use std::ops::ControlFlow;
 
@@ -7,13 +11,29 @@ use sqlparser::ast::{Ident, ObjectName, Statement, TableFactor, TableWithJoins,
 use sqlparser::dialect::Dialect;
 use sqlparser::parser::Parser;
 
+/// Convenience function to extract tables from SQL.
+///
+/// ## Example
+///
+/// ```rust
+/// use sqlparser::dialect::GenericDialect;
+///
+/// let dialect = GenericDialect {};
+/// let sql = "SELECT a FROM t1 INNER JOIN t2 ON t1.id = t2.id";
+/// let result = sql_insight::extract_tables(&dialect, sql).unwrap();
+/// println!("{:#?}", result);
+/// assert_eq!(result[0].as_ref().unwrap().to_string(), "t1, t2");
+/// ```
 pub fn extract_tables(
     dialect: &dyn Dialect,
     sql: &str,
 ) -> Result<Vec<Result<Tables, Error>>, Error> {
     TableExtractor::extract(dialect, sql)
 }
 
+/// [`TableReference`] represents a qualified table with alias.
+/// In this crate, this is the canonical representation of a table.
+/// Tables found during analyzing an AST are stored as `TableReference`.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TableReference {
     pub catalog: Option<Ident>,
@@ -115,6 +135,7 @@ impl TryFrom<&ObjectName> for TableReference {
     }
 }
 
+/// [`Tables`] represents a list of [`TableReference`] that found in SQL.
 #[derive(Debug, PartialEq)]
 pub struct Tables(pub Vec<TableReference>);
 
@@ -130,10 +151,14 @@ impl fmt::Display for Tables {
     }
 }
 
+/// A visitor to extract tables from SQL.
 #[derive(Default, Debug)]
 pub struct TableExtractor {
+    // All tables found in the SQL including aliases, must be resolved to original tables.
     all_tables: Vec<TableReference>,
+    // Original tables found in the SQL, used to resolve aliases.
     original_tables: Vec<TableReference>,
+    // Flag to indicate if the current relation is part of a `TableFactor::Table`
     relation_of_table: bool,
 }
 
@@ -185,6 +210,7 @@ impl Visitor for TableExtractor {
 }
 
 impl TableExtractor {
+    /// Extract tables from SQL.
     pub fn extract(dialect: &dyn Dialect, sql: &str) -> Result<Vec<Result<Tables, Error>>, Error> {
         let statements = Parser::parse_sql(dialect, sql)?;
         let results = statements

sql-insight/src/formatter.rs

@@ -1,15 +1,33 @@
+//! A Formatter that formats SQL into a standardized format.
+//!
+//! See [`format`](crate::format()) as the entry point for formatting SQL.
+
 use crate::error::Error;
 use sqlparser::dialect::Dialect;
 use sqlparser::parser::Parser;
 
+/// Convenience function to format SQL.
+///
+/// ## Example
+///
+/// ```rust
+/// use sqlparser::dialect::GenericDialect;
+///
+/// let dialect = GenericDialect {};
+/// let sql = "SELECT a FROM t1 \n WHERE b =   1";
+/// let result = sql_insight::format(&dialect, sql).unwrap();
+/// assert_eq!(result, ["SELECT a FROM t1 WHERE b = 1"]);
+/// ```
 pub fn format(dialect: &dyn Dialect, sql: &str) -> Result<Vec<String>, Error> {
     Formatter::format(dialect, sql)
 }
 
+/// Formatter for SQL.
 #[derive(Debug, Default)]
 pub struct Formatter;
 
 impl Formatter {
+    /// Format SQL.
     pub fn format(dialect: &dyn Dialect, sql: &str) -> Result<Vec<String>, Error> {
         let statements = Parser::parse_sql(dialect, sql)?;
         Ok(statements