From 74b4db73dbfb6668400ca1e2264e1b43e2394fca Mon Sep 17 00:00:00 2001 From: Tiark Rompf Date: Fri, 27 Dec 2024 19:04:14 -0500 Subject: [PATCH] pass over docs --- docs/.vitepress/config.mjs | 18 ++++- docs/development.md | 49 +++++++++++++ docs/examples.md | 13 ++++ docs/getting-started.md | 99 +++++++++++++++++++++++++ docs/index.md | 22 +++--- docs/overview.md | 147 ++++++++++++------------------------- docs/reference.md | 124 +++++++++++++++++++++++++++++++ 7 files changed, 357 insertions(+), 115 deletions(-) create mode 100644 docs/development.md create mode 100644 docs/examples.md create mode 100644 docs/getting-started.md create mode 100644 docs/reference.md diff --git a/docs/.vitepress/config.mjs b/docs/.vitepress/config.mjs index 8e2c5f5..00e51eb 100644 --- a/docs/.vitepress/config.mjs +++ b/docs/.vitepress/config.mjs @@ -17,16 +17,26 @@ export default defineConfig({ text: 'Introduction', items: [ { text: 'Overview', link: '/overview' }, - ], + { text: 'Getting Started', link: '/getting-started' }, + { text: 'Development Setup', link: '/development' }, + ] + }, { + text: 'Documentation', + items: [ + { text: 'Reference', link: '/reference' }, + { text: 'Examples', link: '/examples' }, + ] + },/*{ text: 'Examples', items: [ - { text: 'Markdown Examples', link: '/markdown-examples' }, - { text: 'Runtime API Examples', link: '/api-examples' } + { text: 'Markdown Examples', link: '/xx-markdown-examples' }, + { text: 'Runtime API Examples', link: '/xx-api-examples' } ] - } + }*/ ], socialLinks: [ + { icon: 'npm', link: 'https://www.npmjs.com/package/rhyme-lang' }, { icon: 'github', link: 'https://github.com/rhyme-lang' } ] } diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..c283b2d --- /dev/null +++ b/docs/development.md @@ -0,0 +1,49 @@ +--- +outline: deep +--- + +# Development Setup + + + +Clone the repo and run `npm install` to install all the dependencies. + +If you want to use the development version of the library you cloned in a different +project, you can run `npm link` in the root directory of the repo and then run +`npm link rhyme-lang` in your project directory. + +## Setup for Running in the Browser + +If you want to use the development version of the library in the browser, you can use +webpack to build the browser version of the library. +Use the following commands. + +```bash +npm install webpack webpack-cli --save-dev +./node_modules/.bin/webpack +``` + +This will generate a file `umd/rhyme-lang.min.js` that you can include in your HTML file. + +## Code Structure + +The code is broadly structured according to compiler phases. +The following four javascript files provide useful entry points into the codebase: +- `src/rhyme.js`: Contains the main APIs that are exposed to the user. +- `src/parser.js`: Contains the logic for the parser that provides a simple +textual interface for writing Rhyme expressions. +- `src/ir.js`: Contains the logic for creating the Rhyme intermediate representation (IR) from input query ASTs. +- `src/codegen.js`: Contains the logic for generating optimized javascript code from the Rhyme IR. + + +## Running tests + +Typing `npm test` will run all the tests that are in the `test` directory. + +Running tests that involve running generated C++ code will also need Niels Lohmann's +[C++ JSON library](https://github.com/nlohmann/json) installed. This can be done +either system-wide (e.g. using `brew install nlohmann-json`) or by cloning the +git repo in a Rhyme subdirectory called 'thirdparty'. + +If you're using VSCode, you can install [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) extension and run/debug individual tests. + diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..43d8dc7 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,13 @@ +--- +outline: deep +--- + +# Examples + +The Rhyme test suite contains many helpful usage examples: + +- [Miscellaneous Tests and Examples](https://github.com/rhyme-lang/rhyme/tree/main/test/semantics) + +- [Advent of Code](https://github.com/rhyme-lang/rhyme/tree/main/test/aoc) + +- [HTML and React demos](https://github.com/rhyme-lang/rhyme/tree/main/demos) \ No newline at end of file diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..95233ae --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,99 @@ +--- +outline: deep +--- + +# Getting Started + +Rhyme can be used in multiple different ways: as dependency in a node.js project, +directly in the browser, or as a command-line tool. + + +## Using Rhyme in Node Projects + +To get started with the latest release of Rhyme in your node project, +run the following command: + +```bash +npm install rhyme-lang +``` + +You can then import the library (as you would any other node module) and start using it: + +```javascript +const { api } = require('rhyme-lang') + +let data = [ + { key: "A", value: 10 }, + { key: "B", value: 20 }, + { key: "A", value: 30 } +] + +let query = { + total: api.sum("data.*.value"), + "data.*.key": api.sum("data.*.value"), +} +let res = api.compile(query)({ data }) +console.log("Result: " + JSON.stringify(res)) +``` + +Visit [documentation](https://rhyme-lang.github.io/docs/) to get a glimpse of what Rhyme can do. + + +## Using Rhyme in the Browser/Frontend + +The npm package `rhyme-lang` installed using the above commands is intended for use in node.js projects. +However, if you want to use Rhyme in the browser (especially the visualization features), +you can use `unpkg` CDN to get the browser version of the library. +Specifically, you can include the script from the following URL in your HTML file: +``` +https://unpkg.com/rhyme-lang/umd/rhyme-lang.min.js +``` + + +Shown below is a simple complete example HTML file: +```html + + + + Rhyme Example + + + +

Rhyme Example

+
+ + + + + + +``` + +## Using Rhyme as a Command-Line Tool + +You can also use Rhyme as a command-line tool to process JSON files. For this, install rhyme globally: + +```bash +npm install -g rhyme-lang +``` + +Then you can use it as follows: + +```bash +echo '[1,2,3,4]' | rhyme 'sum stdin.*' +10 +``` + +When given an argument ending in `.rh`, as in `rhyme query.rh`, Rhyme will treat it as a file name to load the query from. + + diff --git a/docs/index.md b/docs/index.md index 23fa268..c9db546 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,7 +5,7 @@ layout: home hero: name: "Rhyme" text: "An Expressive \nData-Centric \nQuery Language" - tagline: My great project tagline + tagline: Query nested data, produce nested data as result image: src: "https://avatars.githubusercontent.com/u/150201258?s=400&u=c165a8a5fc98d1ddc149652fcdb818e4222f3094&v=4" alt: Rhyme @@ -14,18 +14,18 @@ hero: text: Overview link: /overview - theme: alt - text: Markdown Examples - link: /markdown-examples + text: Getting Started + link: /getting-started - theme: alt - text: API Examples - link: /api-examples + text: Examples + link: /examples features: - - title: Feature A - details: Lorem ipsum dolor sit amet, consectetur adipiscing elit - - title: Feature B - details: Lorem ipsum dolor sit amet, consectetur adipiscing elit - - title: Feature C - details: Lorem ipsum dolor sit amet, consectetur adipiscing elit + - title: Tree-to-Tree Queries + details: Process nested structures (JSON, Tensors) as input, produce nested structures as result. + - title: Easy to Metaprogram + details: Compose query fragments in JS, following the structure of the input or output. + - title: Compile to JS (or C) + details: Queries are optimized and translated to low-level code for maximum performance. --- diff --git a/docs/overview.md b/docs/overview.md index 1b5cc80..f8ff78b 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -4,128 +4,75 @@ outline: deep # Rhyme - +Rhyme is a declarative query language designed for querying and transforming nested data +structures like JSON or tensors (nested arrays). It is designed to be easy to use, and to +express complex queries in a simple way, while also achieving high performance by constructing +an IR which gets optimized and translated to efficient JS or C code. -Rhyme is an expressive declarative query language designed for high-level data manipulation, with a primary focus on querying nested structures (e.g., JSON, Tensors, etc.) and producing nested structures as a result. +Rhyme takes inspiration from existing approaches like GraphQL, Datalog, JQ, XQuery, and +similar query languages, as well as from functional logic and array languages. -Rhyme is still at very early stages of development and therefore expect rough edges -and breaking changes. -However, we are actively working on it and would love to hear your feedback. +Rhyme can represent all the usual query operators like aggregations, joins, group-by, etc., +but also supports tensor computations in the style of einops. Rhyme also profits from integration +with the JS ecosystem, e.g., for producing visualizations as DOM trees, React components, etc. - +## Quick Examples -## Getting Started -To get started with the latest release of Rhyme in your node project, -run the following command: - -```bash -npm install rhyme-lang -``` - -You can then import the library (as you would any other node module) and start using it: - -```javascript -const { api } = require('rhyme-lang') - +Below is an example that computes a group-by aggregation based on keys: +```js let data = [ - { key: "A", value: 10 }, - { key: "B", value: 20 }, - { key: "A", value: 30 } + {"key": "A", "value": 30}, + {"key": "B", "value": 20}, + {"key": "A", "value": 45}, ] - +// rhyme-query let query = { - total: api.sum("data.*.value"), - "data.*.key": api.sum("data.*.value"), + "data.*.key": api.sum("data.*.value") } -let res = api.compile(query)({ data }) -console.log("Result: " + JSON.stringify(res)) -``` - -Visit [documentation](https://rhyme-lang.github.io/docs/) to get a glimpse of what Rhyme can do. - -## Using in the browser/frontend -Npm package `rhyme-lang` installed using above command is intended for use in nodejs projects. -However, if you want to use Rhyme in the browser (especially the visualization features), -you can use `unpkg` CDN to get the browser version of the library. -Specifically, you can include the script from the following URL in your HTML file: -``` -https://unpkg.com/rhyme-lang/umd/rhyme-lang.min.js +// compile +let fun = api.compile(query) +// run the compiled query +let result = fun({data}) ``` - -Shown below is a simple complete example HTML file: -```html - - - - Rhyme Example - - - -

Rhyme Example

-
- - - - - - +Likewise, we can also express other types of workloads like tensor computations: +```js +let A = [[1, 2], [3, 4]] +let B = [[5, 6], [7, 8]] + +// rhyme-query +let matmul = {"*i": {"*j": api.sum("A.*i.*k * B.*k.*j")}} +// compile +let fun = api.compile(matmul) +// run the compiled query +let result = fun({A, B}) ``` +To learn more about the different ways of using Rhyme, including different APIs, check out the [documentation](/docs/frontends). -## Development -### Setup -Clone the repo and run `npm install` to install all the dependencies. -If you want to use the development version of the library you cloned in a different -project, you can run `npm link` in the root directory of the repo and then run -`npm link rhyme-lang` in your project directory. +## Useful Links -### Setup for running on browser -If you want to use the development version of the library in the browser, you can use -webpack to build the browser version of the library. -Use the following commands. +- Slides presented at the 24th IFIP WG 2.11 Meeting (Dec 2024): [Rhyme: A Data-Centric Multi-Paradigm Query Language](https://docs.google.com/presentation/d/1yljJLrcbHdGiKhMhAoQkgLJnMqCgEc5Uy6Se0Dz0V0Y/view) -```bash -npm install webpack webpack-cli --save-dev -./node_modules/.bin/webpack -``` +- Poster presented at the Midwest PL Summit (Nov 2024): [Rhyme: A Data-Centric Expressive Query Language for Nested Data Structures](https://docs.google.com/drawings/d/16PsXYFohtb8WhrVIC3y53FeYj3xHDZvQUcSp5fyd0Bg/view) -This will generate a file `umd/rhyme-lang.min.js` that you can include in your HTML file. +- Paper published at FLOPS (Jun 2024): + [Rhyme: A Data-Centric Multi-Paradigm Query Language based on Functional Logic Metaprogramming](https://www.cs.purdue.edu/homes/rompf/papers/abeysinghe-preprint2401.pdf) -### Code Structure -Currently the code is structured into four main javascript files: -- `src/rhyme.js`: Contains the main APIs that are exposed to the user. -- `src/ir.js`: Contains the logic for creating the Rhyme intermediate representation (IR) from input query ASTs. -- `src/codegen.js`: Contains the logic for generating optimized javascript code from the Rhyme IR. -- `src/parser.js`: Contains the logic for a preliminary parser that provides a simple -textual interface for writing certain Rhyme expressions. +- Paper published at PADL (Jan 2024): + [Rhyme: A Data-Centric Expressive Query Language for Nested Data Structures](https://www.cs.purdue.edu/homes/rompf/papers/abeysinghe-padl24.pdf) +- An interactive blog post introducing (an early version of) Rhyme: + [Let's build a Query Language!](https://tiarkrompf.github.io/notes/?/js-queries/) -### Running tests -`npm test` will run all the tests that are in the `test` directory. -If you're using VSCode, you can install [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) extension and run/debug individual tests. +## Development +Rhyme is developed by members of [Prof. Tiark Rompf](https://tiarkrompf.github.io)'s'research group +at Purdue University and is available for use under a permissive MIT license. +Before using Rhyme for anything critical, please consider the usual caveats around research-oriented +software developed by a small team. -### Useful Links -- A recent publication on Rhyme (PADL 2024): - [Rhyme: A Data-Centric Expressive Query Language for Nested Data Structures](https://www.cs.purdue.edu/homes/rompf/papers/abeysinghe-padl24.pdf) -- A second publication (FLOPS 2024): - [Rhyme: A Data-Centric Multi-Paradigm Query Language based on Functional Logic Metaprogramming](https://www.cs.purdue.edu/homes/rompf/papers/abeysinghe-preprint2401.pdf) -- An interactive blog post introducing Rhyme: - [Let's build a Query Language!](https://tiarkrompf.github.io/notes/?/js-queries/) diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..8a99283 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,124 @@ +--- +outline: deep +--- + +# Reference + +Before diving into different kinds of operators, we will first look the steps involved in running a Rhyme query. +We assume that you have already installed the `rhyme-lang` package in your project. +If not, please head out to the [installation](/getting-started#installation) page. + +Rhyme works by compiling the given query into a function that can be run on data. +The compilation step is done by the `api.compile` function. +This returns a JS function which can be run on any input data to get the result of the query. + +```js +// import the library +let api = require("rhyme-lang") + +// define the query +let query = { + "data.*.key": api.sum("data.*.value") +} + +// use the api.compile function to compile the query +let func = api.compile(query) + +// run the compiled function on some sample data +let sampleData = [ + { key: "A", value: 10 }, + { key: "B", value: 20 }, + { key: "A", value: 30 } + ] +let result = func({"data": sampleData}) +``` + +Now that we have seen the basic steps involved in running a Rhyme query, let us look at the different kinds of operators available in Rhyme. + +### Simple Indexing +Like JQ, we can use the `.` operator to index into objects and arrays. For instance the query `data.0` will return the first element of the array `data`. + + +### Iterating +We can use the `*` operator to iterate over arrays and compute aggregations. For instance, the query `data.*.value` will +iterate through `data` and collect all the `value` attributes. + +```js +["data.*.value"] // collect all the values +``` + +Note that if no top-level aggregation is specified explicitly, the first value is returned. +For instance, the query `data.*.value` will return `10` for the sample data above. + +We will see later that these iterators can be named like `*A`, `*B`, `*item`, etc. instead of using the default `*` symbol +to allow for more complex queries. + +### Aggregations + +We can use aggregate operators to compute aggregations over iterated values. +For instance, the query `api.sum(data.*.value)` will compute the sum of all the values. +Below is a list of all the aggregate operators currently available in Rhyme. + +| Aggregation | Description | +| ----------- | ----------- | +| `api.sum` | Computes the sum of the values | +| `api.min` | Computes the minimum of the values | +| `api.max` | Computes the maximum of the values | +| `api.mean` | Computes the mean of the values | +| `api.count` | Computes the number of values | +| `api.array` or `[ ]` | Collects all the values into an array | + +### Arithmetic (Binary Operators) +Rhyme supports the usual basic arithmetic operations like, addition, +subtraction, multiplication, division, etc. + +| Opeartor | Description | +| -------- | ----------- | +| `api.plus` | Addition | +| `api.minus` | Subtraction | +| `api.mul` | Multiplication | +| `api.div` | Division | +| `api.fdiv` | Floor division | +| `api.mod` | Modulo | + +### Group-bys +Group-bys in Rhyme are implicitly expressed using keys. For instance, having `data.*.key` as the key would do +a group-by on the `key` attribute of the data and whenever `data.*` is iterated within this key, it will only +iterate over the values that have the same key. +We can use nested JSON objects to express group-bys on multiple keys. +For instance, the query below will do a group-by on the `key1` and `key2` attributes of the data. + +```js +{ + "data.*.key1": { + "data.*.key2": api.sum("data.*.value") + } +} +``` + +### Joins +Rhyme currently only support equi-joins. Joins between two objects are essentially expressed as doing a +lookup on the other object based on the key. +The following query will do a join between `data1` and `data2` based on the `key` attribute and compute +the sum of products of the `value` attributes. +`api.keyval` is used to specify expression as keys since JSON only supports string keys. + +```js +{ + "-": api.keyval(api.get("data2", "data1.*.key")) ( + api.sum(api.times("data2.*.value", "data1.*.value")) + ) +} +``` + +### UDFs +Rhyme supports user-defined functions (UDFs) which can be used to express custom computations. +UDFs are expressed as using normal JS functions and called using `api.apply`. +For instance, the following query will apply the `multiply10` function to all the values in `data.*.value`. + +```js +let udf = { + multiply10: x => x * 10 +} +let query = api.apply("multiply10", "data.*.value") +```