diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 57a9b55..0e56246 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -3,53 +3,213 @@ on: push: branches: ["main"] pull_request: + workflow_dispatch: env: CI: true + DEBUG: napi:* + APP_NAME: flight-sql-client + MACOSX_DEPLOYMENT_TARGET: '10.13' + CARGO_INCREMENTAL: '1' + FLIGHT_SQL_PATH: ./packages/flight-sql-client/ +permissions: + contents: write + id-token: write +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: build: - name: Build, lint, and test on Node ${{ matrix.node }} and ${{ matrix.os }} - - runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - node: [20, 22] - os: [ubuntu-latest, macOS-latest] - + settings: + - host: macos-13 + target: x86_64-apple-darwin + build: pnpm build:rust --target x86_64-apple-darwin && pnpm build:js + - host: ubuntu-latest + target: x86_64-unknown-linux-gnu + # using vm instead of docker, but leaving in case we need this for other targets + # docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian + build: pnpm build:rust --target x86_64-unknown-linux-gnu && pnpm build:js + test: cd tests/c1-integration && pnpm run test + - host: macos-latest + target: aarch64-apple-darwin + build: pnpm build:rust --target aarch64-apple-darwin && pnpm build:js + name: stable - ${{ matrix.settings.target }} - node@20 + runs-on: ${{ matrix.settings.host }} steps: - name: Checkout repository uses: actions/checkout@v4 - - - name: Use Node ${{ matrix.node }} + - name: Setup node uses: actions/setup-node@v4 + if: ${{ !matrix.settings.docker }} with: - node-version: ${{ matrix.node }} - + node-version: 20 - name: Install pnpm id: pnpm-install uses: pnpm/action-setup@v3 with: version: 9.8.0 run_install: false - - name: Get pnpm store directory id: pnpm-cache shell: bash run: | - echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 - name: Setup pnpm cache + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + - name: Setup pnpm cache + uses: actions/cache@v4 with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + key: ${{ runner.os }}_${{ matrix.settings.target }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install dependencies and build - run: pnpm install --frozen-lockfile && pnpm build - + ${{ runner.os }}_${{ matrix.settings.target }}-pnpm-store- + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + target/ + key: ${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }} + - name: Setup toolchain + run: ${{ matrix.settings.setup }} + if: ${{ matrix.settings.setup }} + shell: bash + - name: Install dependencies and buld + run: pnpm install --frozen-lockfile - name: Lint run: pnpm run lint:ci - - - name: Test - run: pnpm run test:ci + - name: Build in docker + uses: addnab/docker-run-action@v3 + if: ${{ matrix.settings.docker }} + with: + image: ${{ matrix.settings.docker }} + options: '--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build' + run: ${{ matrix.settings.build }} + - name: Build + run: ${{ matrix.settings.build }} + if: ${{ !matrix.settings.docker }} + shell: bash + - name: Run unit tests + run: pnpm test:ci + if: ${{ !matrix.settings.docker }} + shell: bash + # we only run integration tests on linux vm where we have access to docker we can talk to + - name: Integration tests + run: ${{ matrix.settings.test }} + if: ${{ !matrix.settings.test }} + shell: bash + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-${{ matrix.settings.target }} + path: ${{ env.FLIGHT_SQL_PATH }}${{ env.APP_NAME }}.*.node + if-no-files-found: error + test-macOS-binding: + name: Test bindings on ${{ matrix.settings.target }} - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + settings: + - host: macos-latest + target: aarch64-apple-darwin + architecture: arm64 + - host: macos-13 + target: x86_64-apple-darwin + architecture: x64 + node: + - '20' + - '22' + runs-on: ${{ matrix.settings.host }} + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + architecture: ${{ matrix.settings.architecture }} + - name: Install pnpm + id: pnpm-install + uses: pnpm/action-setup@v3 + with: + version: 9.8.0 + run_install: false + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + - name: Setup pnpm cache + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}_${{ matrix.settings.target }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}_${{ matrix.settings.target }}-pnpm-store- + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-${{ matrix.settings.target }} + path: ${{ env.FLIGHT_SQL_PATH }} + - name: List packages + working-directory: ${{ env.FLIGHT_SQL_PATH }} + run: ls -R . + shell: bash + - name: Test bindings + working-directory: ${{ env.FLIGHT_SQL_PATH }} + run: pnpm test + test-linux-x64-gnu-binding: + name: Test bindings on Linux-x64-gnu - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + node: + - '20' + - '22' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Install pnpm + id: pnpm-install + uses: pnpm/action-setup@v3 + with: + version: 9.8.0 + run_install: false + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + - name: Setup pnpm cache + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}_x86_64-unknown-linux-gnu-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}_x86_64-unknown-linux-gnu-pnpm-store- + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-x86_64-unknown-linux-gnu + path: ${{ env.FLIGHT_SQL_PATH }} + - name: List packages + working-directory: ${{ env.FLIGHT_SQL_PATH }} + run: ls -R . + shell: bash + - name: Test bindings + working-directory: ${{ env.FLIGHT_SQL_PATH }} + run: pnpm test \ No newline at end of file diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml deleted file mode 100644 index 8639466..0000000 --- a/.github/workflows/integration-test.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Integration tests -on: - push: - branches: ["main"] - pull_request: -env: - CI: true -jobs: - test: - name: Run integration tests - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Install pnpm - id: pnpm-install - uses: pnpm/action-setup@v3 - with: - version: 9.8.0 - run_install: false - - - name: Get pnpm store directory - id: pnpm-cache - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: pnpm-store - - - name: Install dependencies and build - run: pnpm install --frozen-lockfile && pnpm build - - - name: Test - run: cd tests/c1-integration && pnpm run test diff --git a/.gitignore b/.gitignore index 529d783..c4ed4d5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ coverage .vscode .turbo .ceramic-one + +# rust +target +Cargo.lock +# napi binaries +*.node \ No newline at end of file diff --git a/README.md b/README.md index 05d3472..70d5ba9 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ TypeScript client and utilities for [Ceramic One](https://github.com/ceramicnetw | Name | Description | Version | | ------------------------------------------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------- | | [events](./packages/events) | Events encoding, signing and other utilities | ![npm version](https://img.shields.io/npm/v/@ceramic-sdk/events.svg) | +| [flight-sql-client](./packages/flight-sql-client) | Flight SQL client for ceramic one using WASM | ![npm version](https://img.shields.io/npm/v/@ceramic-sdk/flight-sql-client.svg) | | [http-client](./packages/http-client) | HTTP client for Ceramic One | ![npm version](https://img.shields.io/npm/v/@ceramic-sdk/http-client.svg) | | [identifiers](./packages/identifiers) | Ceramic streams and commits identifiers | ![npm version](https://img.shields.io/npm/v/@ceramic-sdk/identifiers.svg) | | [model-protocol](./packages/model-protocol) | Model streams protocol | ![npm version](https://img.shields.io/npm/v/@ceramic-sdk/model-protocol.svg) | @@ -27,6 +28,10 @@ pnpm test # run all tests (unit and integration, requires docker to be running) pnpm test:ci # run only unit tests ``` +## CI + +In order to specify targets for WASM builds on CI, the build script is split into `pnpm build:rust` which allows passing `--target TARGET_TRIPLE` and `pnpm build:js`. + ## License -Dual licensed under MIT and Apache 2 +Dual licensed under MIT and Apache 2 \ No newline at end of file diff --git a/biome.json b/biome.json index fbac5cb..e8d42ce 100644 --- a/biome.json +++ b/biome.json @@ -7,7 +7,10 @@ "formatter": { "enabled": true, "formatWithErrors": false, - "ignore": [], + "ignore": [ + "./packages/flight-sql-client/index.d.ts", + "./packages/flight-sql-client/index.js" + ], "indentStyle": "space", "lineWidth": 80 }, @@ -23,7 +26,11 @@ }, "linter": { "enabled": true, - "ignore": ["*.gen.ts"], + "ignore": [ + "*.gen.ts", + "./packages/flight-sql-client/index.d.ts", + "./packages/flight-sql-client/index.js" + ], "rules": { "recommended": true } @@ -33,4 +40,4 @@ "clientKind": "git", "useIgnoreFile": true } -} +} \ No newline at end of file diff --git a/package.json b/package.json index 1db9c72..7cf9609 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "packageManager": "pnpm@9.8.0", "scripts": { "build": "pnpm --filter \"@ceramic-sdk/*\" build && turbo run build:js", + "build:rust": "pnpm --filter \"@ceramic-sdk/flight-sql-client\" build", + "build:js": "pnpm --filter \"@ceramic-sdk/*\" --filter !@ceramic-sdk/flight-sql-client build && turbo run build:js", "docs": "typedoc", "lint": "biome check --write apps/* packages/* tests/*", "lint:ci": "biome ci packages/* tests/*", diff --git a/packages/flight-sql-client/.cargo/config.toml b/packages/flight-sql-client/.cargo/config.toml new file mode 100644 index 0000000..1261fea --- /dev/null +++ b/packages/flight-sql-client/.cargo/config.toml @@ -0,0 +1,3 @@ +[target.aarch64-unknown-linux-musl] +linker = "aarch64-linux-musl-gcc" +rustflags = ["-C", "target-feature=-crt-static"] diff --git a/packages/flight-sql-client/Cargo.toml b/packages/flight-sql-client/Cargo.toml new file mode 100644 index 0000000..ebd85c3 --- /dev/null +++ b/packages/flight-sql-client/Cargo.toml @@ -0,0 +1,38 @@ +[package] +authors = [ + "Robert Pack ", + "David Estes ", +] +edition = "2021" +name = "flight-sql-client" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +crate-type = ["cdylib"] + +[dependencies] +arrow-array = "53" +arrow-cast = "53" +arrow-flight = { version = "53", features = ["flight-sql-experimental"] } +arrow-ipc = "53" +arrow-schema = "53" +futures = "0.3" +napi = { version = "2.12.2", default-features = false, features = [ + "napi8", + "tokio_rt", + "async", +] } +napi-derive = "2" +snafu = "0.8" +tokio = "1" +tonic = { version = "0.12", features = ["tls"] } +tracing-log = "0.2" +tracing-subscriber = "0.3" + +[build-dependencies] +napi-build = "2" + +[profile.release] +lto = true diff --git a/packages/flight-sql-client/LICENSE b/packages/flight-sql-client/LICENSE new file mode 100644 index 0000000..6c7f562 --- /dev/null +++ b/packages/flight-sql-client/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 N-API for Rust + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/flight-sql-client/README.md b/packages/flight-sql-client/README.md new file mode 100644 index 0000000..1005eb9 --- /dev/null +++ b/packages/flight-sql-client/README.md @@ -0,0 +1,84 @@ +# `@ceramic-sdk/flight-sql-client` + +A client library for interacting with [Arrow Flight SQL] enabled databases from Node.js. + +This library provides a thin wrapper around the flight-sql client implementation in +the [arrow-flight] crate. Node bindings are created with the help of [napi-rs]. Originally forked from lakehouse-rs [npm](https://www.npmjs.com/package/@lakehouse-rs/flight-sql-client) and [github](https://github.com/roeap/flight-sql-client-node). + +## Usage + +Install library + +```sh +yarn add @ceramic-sdk/flight-sql-client +# or +npm install @ceramic-sdk/flight-sql-client +# or +pnpm add @ceramic-sdk/flight-sql-client +``` + +Create a new client instance + +```ts +import { ClientOptions, createFlightSqlClient } from '@ceramic-sdk/flight-sql-client'; +import { tableFromIPC } from 'apache-arrow'; + +const options: ClientOptions = { + username: undefined, + password: undefined, + tls: false, + host: '127.0.0.1', + port: 5102, + headers: [], +}; + +const client = await createFlightSqlClient(options); +``` + +Execute a query against the service + +```ts +const buffer = await client.query('SELECT * FROM my_tyble'); +const table = tableFromIPC(buffer); +``` + +Or inspect some server metadata + +```ts +const buffer = await client.getTables({ includeSchema: true }); +const table = tableFromIPC(buffer); +``` + +## Development + +Requirements: + +- Rust +- node.js >= 18 +- Pnpm + +Install dependencies via + +```sh +pnpm i +``` + +Build native module + +```sh +pnpm build +``` + +Run tests + +```sh +pnpm test +``` + +## Release + +TODO + +[Arrow Flight SQL]: https://arrow.apache.org/docs/format/FlightSql.html +[arrow-flight]: https://crates.io/crates/arrow-flight +[napi-rs]: https://napi.rs/ diff --git a/packages/flight-sql-client/build.rs b/packages/flight-sql-client/build.rs new file mode 100644 index 0000000..9fc2367 --- /dev/null +++ b/packages/flight-sql-client/build.rs @@ -0,0 +1,5 @@ +extern crate napi_build; + +fn main() { + napi_build::setup(); +} diff --git a/packages/flight-sql-client/index.d.ts b/packages/flight-sql-client/index.d.ts new file mode 100644 index 0000000..1fcf58b --- /dev/null +++ b/packages/flight-sql-client/index.d.ts @@ -0,0 +1,88 @@ +/* tslint:disable */ +/* eslint-disable */ + +/* auto-generated by NAPI-RS */ + +/** A ':' separated key value pair */ +export interface KeyValue { + key: string + value: string +} +export interface ClientOptions { + /** + * Additional headers. + * + * Values should be key value pairs separated by ':' + */ + headers: Array + /** Username */ + username?: string + /** Password */ + password?: string + /** Auth token. */ + token?: string + /** Use TLS. */ + tls: boolean + /** Server host. */ + host: string + /** Server port. */ + port?: number +} +export declare function createFlightSqlClient(options: ClientOptions): Promise +export declare function rustCrateVersion(): string +export interface GetDbSchemasOptions { + /** + * Specifies the Catalog to search for the tables. + * An empty string retrieves those without a catalog. + * If omitted the catalog name should not be used to narrow the search. + */ + catalog?: string + /** + * Specifies a filter pattern for schemas to search for. + * When no db_schema_filter_pattern is provided, the pattern will not be used to narrow the search. + * In the pattern string, two special characters can be used to denote matching rules: + * - "%" means to match any substring with 0 or more characters. + * - "_" means to match any one character. + */ + dbSchemaFilterPattern?: string +} +export interface GetTablesOptions { + /** + * Specifies the Catalog to search for the tables. + * An empty string retrieves those without a catalog. + * If omitted the catalog name should not be used to narrow the search. + */ + catalog?: string + /** + * Specifies a filter pattern for schemas to search for. + * When no db_schema_filter_pattern is provided, the pattern will not be used to narrow the search. + * In the pattern string, two special characters can be used to denote matching rules: + * - "%" means to match any substring with 0 or more characters. + * - "_" means to match any one character. + */ + dbSchemaFilterPattern?: string + /** + * Specifies a filter pattern for tables to search for. + * When no table_name_filter_pattern is provided, all tables matching other filters are searched. + * In the pattern string, two special characters can be used to denote matching rules: + * - "%" means to match any substring with 0 or more characters. + * - "_" means to match any one character. + */ + tableNameFilterPattern?: string + /** + * Specifies a filter of table types which must match. + * The table types depend on vendor/implementation. + * It is usually used to separate tables from views or system tables. + * TABLE, VIEW, and SYSTEM TABLE are commonly supported. + */ + tableTypes?: Array + /** Specifies if the Arrow schema should be returned for found tables. */ + includeSchema?: boolean +} +export declare class FlightSqlClient { + query(query: string): Promise + preparedStatement(query: string, params: Array<[string, string]>): Promise + getCatalogs(): Promise + getDbSchemas(options: GetDbSchemasOptions): Promise + getTables(options: GetTablesOptions): Promise +} diff --git a/packages/flight-sql-client/index.js b/packages/flight-sql-client/index.js new file mode 100644 index 0000000..1063719 --- /dev/null +++ b/packages/flight-sql-client/index.js @@ -0,0 +1,317 @@ +/* tslint:disable */ +/* eslint-disable */ +/* prettier-ignore */ + +/* auto-generated by NAPI-RS */ + +const { existsSync, readFileSync } = require('fs') +const { join } = require('path') + +const { platform, arch } = process + +let nativeBinding = null +let localFileExisted = false +let loadError = null + +function isMusl() { + // For Node 10 + if (!process.report || typeof process.report.getReport !== 'function') { + try { + const lddPath = require('child_process').execSync('which ldd').toString().trim() + return readFileSync(lddPath, 'utf8').includes('musl') + } catch (e) { + return true + } + } else { + const { glibcVersionRuntime } = process.report.getReport().header + return !glibcVersionRuntime + } +} + +switch (platform) { + case 'android': + switch (arch) { + case 'arm64': + localFileExisted = existsSync(join(__dirname, 'flight-sql-client.android-arm64.node')) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.android-arm64.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-android-arm64') + } + } catch (e) { + loadError = e + } + break + case 'arm': + localFileExisted = existsSync(join(__dirname, 'flight-sql-client.android-arm-eabi.node')) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.android-arm-eabi.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-android-arm-eabi') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on Android ${arch}`) + } + break + case 'win32': + switch (arch) { + case 'x64': + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.win32-x64-msvc.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.win32-x64-msvc.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-win32-x64-msvc') + } + } catch (e) { + loadError = e + } + break + case 'ia32': + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.win32-ia32-msvc.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.win32-ia32-msvc.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-win32-ia32-msvc') + } + } catch (e) { + loadError = e + } + break + case 'arm64': + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.win32-arm64-msvc.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.win32-arm64-msvc.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-win32-arm64-msvc') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on Windows: ${arch}`) + } + break + case 'darwin': + localFileExisted = existsSync(join(__dirname, 'flight-sql-client.darwin-universal.node')) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.darwin-universal.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-darwin-universal') + } + break + } catch {} + switch (arch) { + case 'x64': + localFileExisted = existsSync(join(__dirname, 'flight-sql-client.darwin-x64.node')) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.darwin-x64.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-darwin-x64') + } + } catch (e) { + loadError = e + } + break + case 'arm64': + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.darwin-arm64.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.darwin-arm64.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-darwin-arm64') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on macOS: ${arch}`) + } + break + case 'freebsd': + if (arch !== 'x64') { + throw new Error(`Unsupported architecture on FreeBSD: ${arch}`) + } + localFileExisted = existsSync(join(__dirname, 'flight-sql-client.freebsd-x64.node')) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.freebsd-x64.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-freebsd-x64') + } + } catch (e) { + loadError = e + } + break + case 'linux': + switch (arch) { + case 'x64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-x64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-x64-musl.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-x64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-x64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-x64-gnu.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-x64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 'arm64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-arm64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-arm64-musl.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-arm64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-arm64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-arm64-gnu.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-arm64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 'arm': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-arm-musleabihf.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-arm-musleabihf.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-arm-musleabihf') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-arm-gnueabihf.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-arm-gnueabihf.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-arm-gnueabihf') + } + } catch (e) { + loadError = e + } + } + break + case 'riscv64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-riscv64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-riscv64-musl.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-riscv64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-riscv64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-riscv64-gnu.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-riscv64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 's390x': + localFileExisted = existsSync( + join(__dirname, 'flight-sql-client.linux-s390x-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./flight-sql-client.linux-s390x-gnu.node') + } else { + nativeBinding = require('@ceramic-sdk/flight-sql-client-linux-s390x-gnu') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on Linux: ${arch}`) + } + break + default: + throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`) +} + +if (!nativeBinding) { + if (loadError) { + throw loadError + } + throw new Error(`Failed to load native binding`) +} + +const { FlightSqlClient, createFlightSqlClient, rustCrateVersion } = nativeBinding + +module.exports.FlightSqlClient = FlightSqlClient +module.exports.createFlightSqlClient = createFlightSqlClient +module.exports.rustCrateVersion = rustCrateVersion diff --git a/packages/flight-sql-client/npm/darwin-arm64/README.md b/packages/flight-sql-client/npm/darwin-arm64/README.md new file mode 100644 index 0000000..550be66 --- /dev/null +++ b/packages/flight-sql-client/npm/darwin-arm64/README.md @@ -0,0 +1,3 @@ +# `@ceramic-sdk/flight-sql-client-darwin-arm64` + +This is the **aarch64-apple-darwin** binary for `@ceramic-sdk/flight-sql-client` diff --git a/packages/flight-sql-client/npm/darwin-arm64/package.json b/packages/flight-sql-client/npm/darwin-arm64/package.json new file mode 100644 index 0000000..d94be83 --- /dev/null +++ b/packages/flight-sql-client/npm/darwin-arm64/package.json @@ -0,0 +1,23 @@ +{ + "name": "@ceramic-sdk/flight-sql-client-darwin-arm64", + "version": "0.0.1", + "os": ["darwin"], + "cpu": ["arm64"], + "main": "flight-sql-client.darwin-arm64.node", + "files": ["flight-sql-client.darwin-arm64.node"], + "description": "A FlightSQL client for Node.js", + "keywords": ["ceramic", "FlightSQL"], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/ceramicnetwork/ceramic-sdk", + "directory": "packages/flight-sql-client/npm/darwin-arm64" + } +} diff --git a/packages/flight-sql-client/npm/darwin-x64/README.md b/packages/flight-sql-client/npm/darwin-x64/README.md new file mode 100644 index 0000000..f7d0d38 --- /dev/null +++ b/packages/flight-sql-client/npm/darwin-x64/README.md @@ -0,0 +1,3 @@ +# `@ceramic-sdk/flight-sql-client-darwin-x64` + +This is the **x86_64-apple-darwin** binary for `@ceramic-sdk/flight-sql-client` diff --git a/packages/flight-sql-client/npm/darwin-x64/package.json b/packages/flight-sql-client/npm/darwin-x64/package.json new file mode 100644 index 0000000..9b4a381 --- /dev/null +++ b/packages/flight-sql-client/npm/darwin-x64/package.json @@ -0,0 +1,23 @@ +{ + "name": "@ceramic-sdk/flight-sql-client-darwin-x64", + "version": "0.0.1", + "os": ["darwin"], + "cpu": ["x64"], + "main": "flight-sql-client.darwin-x64.node", + "files": ["flight-sql-client.darwin-x64.node"], + "description": "A FlightSQL client for Node.js", + "keywords": ["ceramic", "FlightSQL"], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/ceramicnetwork/ceramic-sdk", + "directory": "packages/flight-sql-client/npm/darwin-x64" + } +} diff --git a/packages/flight-sql-client/npm/linux-x64-gnu/README.md b/packages/flight-sql-client/npm/linux-x64-gnu/README.md new file mode 100644 index 0000000..92598c0 --- /dev/null +++ b/packages/flight-sql-client/npm/linux-x64-gnu/README.md @@ -0,0 +1,3 @@ +# `@ceramic-sdk/flight-sql-client-linux-x64-gnu` + +This is the **x86_64-unknown-linux-gnu** binary for `@ceramic-sdk/flight-sql-client` diff --git a/packages/flight-sql-client/npm/linux-x64-gnu/package.json b/packages/flight-sql-client/npm/linux-x64-gnu/package.json new file mode 100644 index 0000000..5108ae3 --- /dev/null +++ b/packages/flight-sql-client/npm/linux-x64-gnu/package.json @@ -0,0 +1,24 @@ +{ + "name": "@ceramic-sdk/flight-sql-client-linux-x64-gnu", + "version": "0.0.1", + "os": ["linux"], + "cpu": ["x64"], + "main": "flight-sql-client.linux-x64-gnu.node", + "files": ["flight-sql-client.linux-x64-gnu.node"], + "description": "A FlightSQL client for Node.js", + "keywords": ["ceramic", "FlightSQL"], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/ceramicnetwork/ceramic-sdk", + "directory": "packages/flight-sql-client/npm/linux-x64-gnu" + }, + "libc": ["glibc"] +} diff --git a/packages/flight-sql-client/package.json b/packages/flight-sql-client/package.json new file mode 100644 index 0000000..29e4a5d --- /dev/null +++ b/packages/flight-sql-client/package.json @@ -0,0 +1,56 @@ +{ + "name": "@ceramic-sdk/flight-sql-client", + "version": "0.0.1", + "description": "A FlightSQL client. Currently only supports Node.js", + "main": "index.js", + "types": "index.d.ts", + "files": ["index.js", "index.d.ts"], + "keywords": ["ceramic", "FlightSQL"], + "napi": { + "name": "flight-sql-client", + "triples": { + "defaults": false, + "additional": [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu" + ] + } + }, + "license": "MIT", + "devDependencies": { + "@napi-rs/cli": "^2.18.4", + "@swc-node/register": "^1.10.6", + "@taplo/cli": "^0.7.0" + }, + "engines": { + "node": ">= 10" + }, + "scripts": { + "artifacts": "napi artifacts", + "build:debug": "napi build --platform", + "build": "napi build --platform --release", + "test": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js", + "test:ci": "pnpm run test --ci --coverage", + "prepublishOnly": "napi prepublish -t npm", + "format": "pnpm format:rs && pnpm format:toml", + "format:toml": "taplo format", + "format:rs": "cargo fmt", + "universal": "napi universal", + "version": "napi version" + }, + "jest": { + "extensionsToTreatAsEsm": [".ts"], + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.js$": "$1" + }, + "transform": { + "^.+\\.(t|j)s$": [ + "@swc/jest", + { + "root": "../.." + } + ] + } + } +} diff --git a/packages/flight-sql-client/rustfmt.toml b/packages/flight-sql-client/rustfmt.toml new file mode 100644 index 0000000..3a26366 --- /dev/null +++ b/packages/flight-sql-client/rustfmt.toml @@ -0,0 +1 @@ +edition = "2021" diff --git a/packages/flight-sql-client/src/conversion.rs b/packages/flight-sql-client/src/conversion.rs new file mode 100644 index 0000000..7a4d7e0 --- /dev/null +++ b/packages/flight-sql-client/src/conversion.rs @@ -0,0 +1,48 @@ +use std::io::Cursor; +use std::ops::Deref; + +use arrow_array::RecordBatch; +use arrow_ipc::reader::FileReader; +use arrow_ipc::writer::FileWriter; +use arrow_schema::SchemaRef; +use snafu::prelude::*; + +use crate::error::{ArrowSnafu, Result}; + +#[allow(unused)] +pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<(Vec, SchemaRef)> { + let mut batches: Vec = Vec::new(); + let file_reader = FileReader::try_new(Cursor::new(slice), None).context(ArrowSnafu { + message: "failed to convert to record batch", + })?; + let schema = file_reader.schema().clone(); + for b in file_reader { + let record_batch = b.context(ArrowSnafu { + message: "failed to convert to record batch", + })?; + batches.push(record_batch); + } + Ok((batches, schema)) +} + +pub(crate) fn record_batch_to_buffer(batches: Vec) -> Result> { + if batches.is_empty() { + return Ok(Vec::new()); + } + + let schema = batches.first().unwrap().schema(); + let mut fr = FileWriter::try_new(Vec::new(), schema.deref()).context(ArrowSnafu { + message: "failed to convert to buffer", + })?; + for batch in batches.iter() { + fr.write(batch).context(ArrowSnafu { + message: "failed to convert to buffer", + })? + } + fr.finish().context(ArrowSnafu { + message: "failed to convert to buffer", + })?; + fr.into_inner().context(ArrowSnafu { + message: "failed to convert to buffer", + }) +} diff --git a/packages/flight-sql-client/src/error.rs b/packages/flight-sql-client/src/error.rs new file mode 100644 index 0000000..fe40905 --- /dev/null +++ b/packages/flight-sql-client/src/error.rs @@ -0,0 +1,32 @@ +use arrow_flight::error::FlightError; +use arrow_schema::ArrowError; +use snafu::Snafu; +use tonic::Status; + +pub type Result = std::result::Result; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + #[snafu(display("column '{name}' is missing"))] + MissingColumn { name: String }, + #[snafu(display("{message}"))] + Arrow { + source: ArrowError, + message: &'static str, + }, + Flight { + source: FlightError, + message: &'static str, + }, + Status { + source: Status, + message: &'static str, + }, +} + +impl From for napi::Error { + fn from(value: Error) -> Self { + napi::Error::from_reason(value.to_string()) + } +} diff --git a/packages/flight-sql-client/src/flight_client.rs b/packages/flight-sql-client/src/flight_client.rs new file mode 100644 index 0000000..ef1b378 --- /dev/null +++ b/packages/flight-sql-client/src/flight_client.rs @@ -0,0 +1,170 @@ +use std::{sync::Arc, time::Duration}; + +use arrow_array::{ArrayRef, Datum, RecordBatch, StringArray}; +use arrow_cast::{cast_with_options, CastOptions}; +use arrow_flight::{sql::client::FlightSqlServiceClient, FlightInfo}; +use arrow_schema::{ArrowError, Schema}; +use futures::TryStreamExt; +use napi_derive::napi; +use snafu::ResultExt; +use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; +use tracing_log::log::{debug, info}; + +use crate::error::{ArrowSnafu, FlightSnafu, Result}; + +/// A ':' separated key value pair +#[derive(Debug, Clone)] +#[napi(object)] +pub struct KeyValue { + pub key: String, + pub value: String, +} + +#[derive(Debug)] +#[napi(object)] +pub struct ClientOptions { + /// Additional headers. + /// + /// Values should be key value pairs separated by ':' + pub headers: Vec, + + /// Username + pub username: Option, + + /// Password + pub password: Option, + + /// Auth token. + pub token: Option, + + /// Use TLS. + pub tls: bool, + + /// Server host. + pub host: String, + + /// Server port. + pub port: Option, +} + +pub(crate) async fn execute_flight( + client: &mut FlightSqlServiceClient, + info: FlightInfo, +) -> Result> { + let schema = Arc::new(Schema::try_from(info.clone()).context(ArrowSnafu { + message: "creating schema from flight info", + })?); + let mut batches = Vec::with_capacity(info.endpoint.len() + 1); + batches.push(RecordBatch::new_empty(schema)); + + debug!("decoded schema"); + + for endpoint in info.endpoint { + let Some(ticket) = &endpoint.ticket else { + panic!("did not get ticket"); + }; + let flight_data = client.do_get(ticket.clone()).await.context(ArrowSnafu { + message: "do_get_request", + })?; + let mut flight_data: Vec<_> = flight_data + .try_collect() + .await + .context(FlightSnafu { + message: "collect data stream", + }) + .expect("collect data stream"); + batches.append(&mut flight_data); + } + + debug!("received data"); + + Ok(batches) +} + +#[allow(unused)] +fn construct_record_batch_from_params( + params: &[(String, String)], + parameter_schema: &Schema, +) -> Result { + let mut items = Vec::<(&String, ArrayRef)>::new(); + + for (name, value) in params { + let field = parameter_schema.field_with_name(name)?; + let value_as_array = StringArray::new_scalar(value); + let casted = cast_with_options( + value_as_array.get().0, + field.data_type(), + &CastOptions::default(), + )?; + items.push((name, casted)) + } + + RecordBatch::try_from_iter(items) +} + +#[allow(unused)] +fn setup_logging() { + tracing_log::LogTracer::init().expect("tracing log init"); + tracing_subscriber::fmt::init(); +} + +pub(crate) async fn setup_client( + args: ClientOptions, +) -> Result, ArrowError> { + let port = args.port.unwrap_or(if args.tls { 443 } else { 80 }); + + let protocol = if args.tls { "https" } else { "http" }; + + let mut endpoint = Endpoint::new(format!("{}://{}:{}", protocol, args.host, port)) + .map_err(|err| ArrowError::ExternalError(Box::new(err)))? + .connect_timeout(Duration::from_secs(20)) + .timeout(Duration::from_secs(20)) + .tcp_nodelay(true) // Disable Nagle's Algorithm since we don't want packets to wait + .tcp_keepalive(Option::Some(Duration::from_secs(3600))) + .http2_keep_alive_interval(Duration::from_secs(300)) + .keep_alive_timeout(Duration::from_secs(20)) + .keep_alive_while_idle(true); + + if args.tls { + let tls_config = ClientTlsConfig::new(); + endpoint = endpoint + .tls_config(tls_config) + .map_err(|err| ArrowError::ExternalError(Box::new(err)))?; + } + + let channel = endpoint + .connect() + .await + .map_err(|err| ArrowError::ExternalError(Box::new(err)))?; + + let mut client = FlightSqlServiceClient::new(channel); + info!("connected"); + + for kv in args.headers { + client.set_header(kv.key, kv.value); + } + + if let Some(token) = args.token { + client.set_token(token); + info!("token set"); + } + + match (args.username, args.password) { + (None, None) => {} + (Some(username), Some(password)) => { + client + .handshake(&username, &password) + .await + .expect("handshake"); + info!("performed handshake"); + } + (Some(_), None) => { + panic!("when username is set, you also need to set a password") + } + (None, Some(_)) => { + panic!("when password is set, you also need to set a username") + } + } + + Ok(client) +} diff --git a/packages/flight-sql-client/src/lib.rs b/packages/flight-sql-client/src/lib.rs new file mode 100644 index 0000000..bef1780 --- /dev/null +++ b/packages/flight-sql-client/src/lib.rs @@ -0,0 +1,194 @@ +#![deny(clippy::all)] + +mod conversion; +mod error; +mod flight_client; + +use arrow_array::{ArrayRef, Datum as _, RecordBatch, StringArray}; +use arrow_cast::CastOptions; +use arrow_flight::sql::{client::FlightSqlServiceClient, CommandGetDbSchemas, CommandGetTables}; +use arrow_schema::Schema; +use napi::bindgen_prelude::*; +use napi_derive::napi; +use snafu::prelude::*; +use tokio::sync::Mutex; +use tonic::transport::Channel; + +use crate::conversion::record_batch_to_buffer; +use crate::error::{ArrowSnafu, Result}; +use crate::flight_client::{execute_flight, setup_client, ClientOptions}; + +#[napi] +pub struct FlightSqlClient { + client: Mutex>, +} + +#[napi] +impl FlightSqlClient { + #[napi] + pub async fn query(&self, query: String) -> napi::Result { + let mut client = self.client.lock().await; + + let flight_info = client.execute(query, None).await.context(ArrowSnafu { + message: "failed to execute query", + })?; + + let batches = execute_flight(&mut client, flight_info).await?; + Ok(record_batch_to_buffer(batches)?.into()) + } + + #[napi] + pub async fn prepared_statement( + &self, + query: String, + params: Vec<(String, String)>, + ) -> napi::Result { + let mut client = self.client.lock().await; + let mut prepared_stmt = client.prepare(query, None).await.context(ArrowSnafu { + message: "failed to prepare statement", + })?; + let schema = prepared_stmt.parameter_schema().context(ArrowSnafu { + message: "failed to retrieve parameter schema from prepare statement", + })?; + prepared_stmt + .set_parameters(construct_record_batch_from_params(¶ms, schema)?) + .context(ArrowSnafu { + message: "failed to bind parameters", + })?; + let flight_info = prepared_stmt.execute().await.context(ArrowSnafu { + message: "failed to execute prepared statement", + })?; + let batches = execute_flight(&mut client, flight_info).await?; + Ok(record_batch_to_buffer(batches)?.into()) + } + + #[napi] + pub async fn get_catalogs(&self) -> napi::Result { + let mut client = self.client.lock().await; + let flight_info = client.get_catalogs().await.context(ArrowSnafu { + message: "failed to execute get catalogs", + })?; + let batches = execute_flight(&mut client, flight_info).await?; + Ok(record_batch_to_buffer(batches)?.into()) + } + + #[napi] + pub async fn get_db_schemas(&self, options: GetDbSchemasOptions) -> napi::Result { + let command = CommandGetDbSchemas { + catalog: options.catalog, + db_schema_filter_pattern: options.db_schema_filter_pattern, + }; + let mut client = self.client.lock().await; + let flight_info = client.get_db_schemas(command).await.context(ArrowSnafu { + message: "failed to execute get schemas", + })?; + let batches = execute_flight(&mut client, flight_info).await?; + Ok(record_batch_to_buffer(batches)?.into()) + } + + #[napi] + pub async fn get_tables(&self, options: GetTablesOptions) -> napi::Result { + let command = CommandGetTables { + catalog: options.catalog, + db_schema_filter_pattern: options.db_schema_filter_pattern, + table_name_filter_pattern: options.table_name_filter_pattern, + table_types: options.table_types.unwrap_or_default(), + include_schema: options.include_schema.unwrap_or_default(), + }; + let mut client = self.client.lock().await; + let flight_info = client.get_tables(command).await.context(ArrowSnafu { + message: "failed to execute get tables", + })?; + let batches = execute_flight(&mut client, flight_info).await?; + Ok(record_batch_to_buffer(batches)?.into()) + } +} + +#[napi] +pub async fn create_flight_sql_client( + options: ClientOptions, +) -> Result { + Ok(FlightSqlClient { + client: Mutex::new(setup_client(options).await.context(ArrowSnafu { + message: "failed setting up flight sql client", + })?), + }) +} + +#[napi] +pub fn rust_crate_version() -> &'static str { + env!("CARGO_PKG_VERSION") +} + +#[napi(object)] +pub struct GetDbSchemasOptions { + /// Specifies the Catalog to search for the tables. + /// An empty string retrieves those without a catalog. + /// If omitted the catalog name should not be used to narrow the search. + pub catalog: Option, + + /// Specifies a filter pattern for schemas to search for. + /// When no db_schema_filter_pattern is provided, the pattern will not be used to narrow the search. + /// In the pattern string, two special characters can be used to denote matching rules: + /// - "%" means to match any substring with 0 or more characters. + /// - "_" means to match any one character. + pub db_schema_filter_pattern: Option, +} + +#[napi(object)] +pub struct GetTablesOptions { + /// Specifies the Catalog to search for the tables. + /// An empty string retrieves those without a catalog. + /// If omitted the catalog name should not be used to narrow the search. + pub catalog: Option, + + /// Specifies a filter pattern for schemas to search for. + /// When no db_schema_filter_pattern is provided, the pattern will not be used to narrow the search. + /// In the pattern string, two special characters can be used to denote matching rules: + /// - "%" means to match any substring with 0 or more characters. + /// - "_" means to match any one character. + pub db_schema_filter_pattern: Option, + + /// Specifies a filter pattern for tables to search for. + /// When no table_name_filter_pattern is provided, all tables matching other filters are searched. + /// In the pattern string, two special characters can be used to denote matching rules: + /// - "%" means to match any substring with 0 or more characters. + /// - "_" means to match any one character. + pub table_name_filter_pattern: Option, + + /// Specifies a filter of table types which must match. + /// The table types depend on vendor/implementation. + /// It is usually used to separate tables from views or system tables. + /// TABLE, VIEW, and SYSTEM TABLE are commonly supported. + pub table_types: Option>, + + /// Specifies if the Arrow schema should be returned for found tables. + pub include_schema: Option, +} + +fn construct_record_batch_from_params( + params: &[(String, String)], + parameter_schema: &Schema, +) -> Result { + let mut items = Vec::<(&String, ArrayRef)>::new(); + + for (name, value) in params { + let field = parameter_schema.field_with_name(name).context(ArrowSnafu { + message: "failed to find field name in parameter schemas", + })?; + let value_as_array = StringArray::new_scalar(value); + let casted = arrow_cast::cast_with_options( + value_as_array.get().0, + field.data_type(), + &CastOptions::default(), + ) + .context(ArrowSnafu { + message: "failed to cast parameter", + })?; + items.push((name, casted)) + } + + RecordBatch::try_from_iter(items).context(ArrowSnafu { + message: "failed to build record batch", + }) +} diff --git a/packages/flight-sql-client/test/index.spec.ts b/packages/flight-sql-client/test/index.spec.ts new file mode 100644 index 0000000..589d280 --- /dev/null +++ b/packages/flight-sql-client/test/index.spec.ts @@ -0,0 +1,6 @@ +import { rustCrateVersion } from '../index' + +test('returns native code version', () => { + const r = rustCrateVersion() + expect(r).toBeTruthy() +}) diff --git a/packages/flight-sql-client/tsconfig.json b/packages/flight-sql-client/tsconfig.json new file mode 100644 index 0000000..bc3ef51 --- /dev/null +++ b/packages/flight-sql-client/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "target": "ES2018", + "strict": true, + "moduleResolution": "node", + "module": "CommonJS", + "noUnusedLocals": true, + "noUnusedParameters": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true + }, + "include": ["."], + "exclude": ["node_modules", "bench", "test"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 657301f..62498dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -185,6 +185,18 @@ importers: specifier: ^5.0.2 version: 5.0.2(typescript@5.6.2)(zod@3.23.8) + packages/flight-sql-client: + devDependencies: + '@napi-rs/cli': + specifier: ^2.18.4 + version: 2.18.4 + '@swc-node/register': + specifier: ^1.10.6 + version: 1.10.9(@swc/core@1.7.26(@swc/helpers@0.5.15))(@swc/types@0.1.12)(typescript@5.6.2) + '@taplo/cli': + specifier: ^0.7.0 + version: 0.7.0 + packages/http-client: dependencies: '@ceramic-sdk/events': @@ -533,6 +545,9 @@ importers: '@ceramic-sdk/events': specifier: workspace:^ version: link:../../packages/events + '@ceramic-sdk/flight-sql-client': + specifier: workspace:^ + version: link:../../packages/flight-sql-client '@ceramic-sdk/http-client': specifier: workspace:^ version: link:../../packages/http-client @@ -566,6 +581,9 @@ importers: '@types/cross-spawn': specifier: ^6.0.0 version: 6.0.6 + apache-arrow: + specifier: 18.0.0 + version: 18.0.0 cross-spawn: specifier: ^7.0.5 version: 7.0.5 @@ -845,6 +863,15 @@ packages: resolution: {integrity: sha512-eqBtI5dZrptXTCyadnhvU0di/KvumoByT7F8KB/8BLU7M1lltfEmvf/c5AnsyrWO9338ygCs2u5mKz1p1Zdj5A==} engines: {node: '>=14.14'} + '@emnapi/core@1.3.1': + resolution: {integrity: sha512-pVGjBIt1Y6gg3EJN8jTcfpP/+uuRksIo055oE/OBkDNcjZqVbfkWCksG1Jp4yZnj3iKWyWX8fdG/j6UDYPbFog==} + + '@emnapi/runtime@1.3.1': + resolution: {integrity: sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==} + + '@emnapi/wasi-threads@1.0.1': + resolution: {integrity: sha512-iIBu7mwkq4UQGeMEM8bLwNK962nXdhodeScX4slfQnRhEMMzvYivHhutCIk8uojvmASXXPC2WNEjwxFWk72Oqw==} + '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} engines: {node: '>=12'} @@ -1271,6 +1298,14 @@ packages: '@multiformats/base-x@4.0.1': resolution: {integrity: sha512-eMk0b9ReBbV23xXU693TAIrLyeO5iTgBZGSJfpqriG8UkYvr/hC9u9pyMlAakDNHWmbhMZCDs6KQO0jzKD8OTw==} + '@napi-rs/cli@2.18.4': + resolution: {integrity: sha512-SgJeA4df9DE2iAEpr3M2H0OKl/yjtg1BnRI5/JyowS71tUWhrfSu2LT0V3vlHET+g1hBVlrO60PmEXwUEKp8Mg==} + engines: {node: '>= 10'} + hasBin: true + + '@napi-rs/wasm-runtime@0.2.5': + resolution: {integrity: sha512-kwUxR7J9WLutBbulqg1dfOrMTwhMdXLdcGUhcbCcGwnPLt3gz19uHVdwH1syKVDbE022ZS2vZxOWflFLS0YTjw==} + '@noble/ciphers@0.4.1': resolution: {integrity: sha512-QCOA9cgf3Rc33owG0AYBB9wszz+Ul2kramWN8tXG44Gyciud/tbkEqvxRF/IpqQaBpRBNi9f4jdNxqB2CQCIXg==} @@ -1311,6 +1346,61 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@oxc-resolver/binding-darwin-arm64@1.12.0': + resolution: {integrity: sha512-wYe+dlF8npM7cwopOOxbdNjtmJp17e/xF5c0K2WooQXy5VOh74icydM33+Uh/SZDgwyum09/U1FVCX5GdeQk+A==} + cpu: [arm64] + os: [darwin] + + '@oxc-resolver/binding-darwin-x64@1.12.0': + resolution: {integrity: sha512-FZxxp99om+SlvBr1cjzF8A3TjYcS0BInCqjUlM+2f9m9bPTR2Bng9Zq5Q09ZQyrKJjfGKqlOEHs3akuVOnrx3Q==} + cpu: [x64] + os: [darwin] + + '@oxc-resolver/binding-freebsd-x64@1.12.0': + resolution: {integrity: sha512-BZi0iU6IEOnXGSkqt1OjTTkN9wfyaK6kTpQwL/axl8eCcNDc7wbv1vloHgILf7ozAY1TP75nsLYlASYI4B5kGA==} + cpu: [x64] + os: [freebsd] + + '@oxc-resolver/binding-linux-arm-gnueabihf@1.12.0': + resolution: {integrity: sha512-L2qnMEnZAqxbG9b1J3di/w/THIm+1fMVfbbTMWIQNMMXdMeqqDN6ojnOLDtuP564rAh4TBFPdLyEfGhMz6ipNA==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-gnu@1.12.0': + resolution: {integrity: sha512-otVbS4zeo3n71zgGLBYRTriDzc0zpruC0WI3ICwjpIk454cLwGV0yzh4jlGYWQJYJk0BRAmXFd3ooKIF+bKBHw==} + cpu: [arm64] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-musl@1.12.0': + resolution: {integrity: sha512-IStQDjIT7Lzmqg1i9wXvPL/NsYsxF24WqaQFS8b8rxra+z0VG7saBOsEnOaa4jcEY8MVpLYabFhTV+fSsA2vnA==} + cpu: [arm64] + os: [linux] + + '@oxc-resolver/binding-linux-x64-gnu@1.12.0': + resolution: {integrity: sha512-SipT7EVORz8pOQSFwemOm91TpSiBAGmOjG830/o+aLEsvQ4pEy223+SAnCfITh7+AahldYsJnVoIs519jmIlKQ==} + cpu: [x64] + os: [linux] + + '@oxc-resolver/binding-linux-x64-musl@1.12.0': + resolution: {integrity: sha512-mGh0XfUzKdn+WFaqPacziNraCWL5znkHRfQVxG9avGS9zb2KC/N1EBbPzFqutDwixGDP54r2gx4q54YCJEZ4iQ==} + cpu: [x64] + os: [linux] + + '@oxc-resolver/binding-wasm32-wasi@1.12.0': + resolution: {integrity: sha512-SZN6v7apKmQf/Vwiqb6e/s3Y2Oacw8uW8V2i1AlxtyaEFvnFE0UBn89zq6swEwE3OCajNWs0yPvgAXUMddYc7Q==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@oxc-resolver/binding-win32-arm64-msvc@1.12.0': + resolution: {integrity: sha512-GRe4bqCfFsyghruEn5bv47s9w3EWBdO2q72xCz5kpQ0LWbw+enPHtTjw3qX5PUcFYpKykM55FaO0hFDs1yzatw==} + cpu: [arm64] + os: [win32] + + '@oxc-resolver/binding-win32-x64-msvc@1.12.0': + resolution: {integrity: sha512-Z3llHH0jfJP4mlWq3DT7bK6qV+/vYe0+xzCgfc67+Tc/U3eYndujl880bexeGdGNPh87JeYznpZAOJ44N7QVVQ==} + cpu: [x64] + os: [win32] + '@redocly/ajv@8.11.2': resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} @@ -1535,6 +1625,22 @@ packages: '@stablelib/wipe@1.0.1': resolution: {integrity: sha512-WfqfX/eXGiAd3RJe4VU2snh/ZPwtSjLG4ynQ/vYzvghTh7dHFcI1wl+nrkWG6lGhukOxOsUHfv8dUXr58D0ayg==} + '@swc-node/core@1.13.3': + resolution: {integrity: sha512-OGsvXIid2Go21kiNqeTIn79jcaX4l0G93X2rAnas4LFoDyA9wAwVK7xZdm+QsKoMn5Mus2yFLCc4OtX2dD/PWA==} + engines: {node: '>= 10'} + peerDependencies: + '@swc/core': '>= 1.4.13' + '@swc/types': '>= 0.1' + + '@swc-node/register@1.10.9': + resolution: {integrity: sha512-iXy2sjP0phPEpK2yivjRC3PAgoLaT4sjSk0LDWCTdcTBJmR4waEog0E6eJbvoOkLkOtWw37SB8vCkl/bbh4+8A==} + peerDependencies: + '@swc/core': '>= 1.4.13' + typescript: '>= 4.3' + + '@swc-node/sourcemap-support@0.5.1': + resolution: {integrity: sha512-JxIvIo/Hrpv0JCHSyRpetAdQ6lB27oFYhv0PKCNf1g2gUXOjpeR1exrXccRxLMuAV5WAmGFBwRnNOJqN38+qtg==} + '@swc/cli@0.4.0': resolution: {integrity: sha512-4JdVrPtF/4rCMXp6Q1h5I6YkYZrCCcqod7Wk97ZQq7K8vNGzJUryBv4eHCvqx5sJOJBrbYm9fcswe1B0TygNoA==} engines: {node: '>= 16.14.0'} @@ -1705,9 +1811,16 @@ packages: resolution: {integrity: sha512-fBUj+lbSaw+VxoBN4J/WFE7dTx8x4XCTRAQvbiIyPJ8MY1KRVkdZV6cbLvg7MeDP6CxUcj6XNvWU6h0ic1Ipyg==} engines: {node: '>=12'} + '@taplo/cli@0.7.0': + resolution: {integrity: sha512-Ck3zFhQhIhi02Hl6T4ZmJsXdnJE+wXcJz5f8klxd4keRYgenMnip3JDPMGDRLbnC/2iGd8P0sBIQqI3KxfVjBg==} + hasBin: true + '@tokenizer/token@0.3.0': resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + '@tybys/wasm-util@0.9.0': + resolution: {integrity: sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==} + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -1723,6 +1836,12 @@ packages: '@types/cacheable-request@6.0.3': resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} + '@types/command-line-args@5.2.3': + resolution: {integrity: sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==} + + '@types/command-line-usage@5.0.4': + resolution: {integrity: sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==} + '@types/cross-spawn@6.0.6': resolution: {integrity: sha512-fXRhhUkG4H3TQk5dBhQ7m/JDdSNHKwR2BBia62lhwEIq9xGiQKLxd6LymNhn47SjXhsUEPmxi+PKw2OkW4LLjA==} @@ -1765,6 +1884,9 @@ packages: '@types/minimist@1.2.5': resolution: {integrity: sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==} + '@types/node@20.17.6': + resolution: {integrity: sha512-VEI7OdvK2wP7XHnsuXbAJnEpEkF6NjSN45QJlL4VGqZSXsnicpesdTWsg9RISeSdYd3yeRj/y3k5KGjUXYnFwQ==} + '@types/node@22.5.5': resolution: {integrity: sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==} @@ -1880,6 +2002,10 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + apache-arrow@18.0.0: + resolution: {integrity: sha512-gFlPaqN9osetbB83zC29AbbZqGiCuFH1vyyPseJ+B7SIbfBtESV62mMT/CkiIt77W6ykC/nTWFzTXFs0Uldg4g==} + hasBin: true + arch@2.2.0: resolution: {integrity: sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==} @@ -1889,6 +2015,14 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@6.2.2: + resolution: {integrity: sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==} + engines: {node: '>=12.17'} + arrify@1.0.1: resolution: {integrity: sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==} engines: {node: '>=0.10.0'} @@ -2014,6 +2148,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chalk-template@0.4.0: + resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} + engines: {node: '>=12'} + chalk@2.4.2: resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} engines: {node: '>=4'} @@ -2091,9 +2229,20 @@ packages: colorette@1.4.0: resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@7.0.3: + resolution: {integrity: sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==} + engines: {node: '>=12.20.0'} + commander@12.1.0: resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} engines: {node: '>=18'} @@ -2357,6 +2506,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -2369,6 +2522,9 @@ packages: resolution: {integrity: sha512-+iwzCJ7C5v5KgcBuueqVoNiHVoQpwiUK5XFLjf0affFTep+Wcw93tPvmb8tqujDNmzhBDPddnWV/qgWSXgq+Hg==} engines: {node: '>=12'} + flatbuffers@24.3.25: + resolution: {integrity: sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ==} + fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -2772,6 +2928,10 @@ packages: engines: {node: '>=4'} hasBin: true + json-bignum@0.0.3: + resolution: {integrity: sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==} + engines: {node: '>=0.8'} + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -2836,6 +2996,9 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.isequal@4.5.0: resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} @@ -3066,6 +3229,9 @@ packages: resolution: {integrity: sha512-uksVLsqG3pVdzzPvmAHpBK0wKxYItuzZr7SziusRPoz67tGV8rL1szZ6IdeUrbqLjGDwApBtN29eEE3IqGHOjg==} engines: {node: '>=4'} + oxc-resolver@1.12.0: + resolution: {integrity: sha512-YlaCIArvWNKCWZFRrMjhh2l5jK80eXnpYP+bhRc1J/7cW3TiyEY0ngJo73o/5n8hA3+4yLdTmXLNTQ3Ncz50LQ==} + p-cancelable@2.1.1: resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==} engines: {node: '>=8'} @@ -3444,6 +3610,9 @@ packages: source-map-support@0.5.13: resolution: {integrity: sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==} + source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + source-map@0.6.1: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} @@ -3549,6 +3718,10 @@ packages: tabbable@6.2.0: resolution: {integrity: sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==} + table-layout@4.1.1: + resolution: {integrity: sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==} + engines: {node: '>=12.17'} + test-exclude@6.0.0: resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} engines: {node: '>=8'} @@ -3680,6 +3853,14 @@ packages: engines: {node: '>=14.17'} hasBin: true + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@7.3.0: + resolution: {integrity: sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==} + engines: {node: '>=12.17'} + uc.micro@2.1.0: resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} @@ -3870,6 +4051,10 @@ packages: engines: {node: '>= 8'} hasBin: true + wordwrapjs@5.1.0: + resolution: {integrity: sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==} + engines: {node: '>=12.17'} + wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -4302,6 +4487,22 @@ snapshots: dependencies: codeco: 1.4.3 + '@emnapi/core@1.3.1': + dependencies: + '@emnapi/wasi-threads': 1.0.1 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.3.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.0.1': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.21.5': optional: true @@ -4702,6 +4903,15 @@ snapshots: '@multiformats/base-x@4.0.1': {} + '@napi-rs/cli@2.18.4': {} + + '@napi-rs/wasm-runtime@0.2.5': + dependencies: + '@emnapi/core': 1.3.1 + '@emnapi/runtime': 1.3.1 + '@tybys/wasm-util': 0.9.0 + optional: true + '@noble/ciphers@0.4.1': {} '@noble/curves@1.2.0': @@ -4738,6 +4948,41 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 + '@oxc-resolver/binding-darwin-arm64@1.12.0': + optional: true + + '@oxc-resolver/binding-darwin-x64@1.12.0': + optional: true + + '@oxc-resolver/binding-freebsd-x64@1.12.0': + optional: true + + '@oxc-resolver/binding-linux-arm-gnueabihf@1.12.0': + optional: true + + '@oxc-resolver/binding-linux-arm64-gnu@1.12.0': + optional: true + + '@oxc-resolver/binding-linux-arm64-musl@1.12.0': + optional: true + + '@oxc-resolver/binding-linux-x64-gnu@1.12.0': + optional: true + + '@oxc-resolver/binding-linux-x64-musl@1.12.0': + optional: true + + '@oxc-resolver/binding-wasm32-wasi@1.12.0': + dependencies: + '@napi-rs/wasm-runtime': 0.2.5 + optional: true + + '@oxc-resolver/binding-win32-arm64-msvc@1.12.0': + optional: true + + '@oxc-resolver/binding-win32-x64-msvc@1.12.0': + optional: true + '@redocly/ajv@8.11.2': dependencies: fast-deep-equal: 3.1.3 @@ -5016,6 +5261,31 @@ snapshots: '@stablelib/wipe@1.0.1': {} + '@swc-node/core@1.13.3(@swc/core@1.7.26(@swc/helpers@0.5.15))(@swc/types@0.1.12)': + dependencies: + '@swc/core': 1.7.26(@swc/helpers@0.5.15) + '@swc/types': 0.1.12 + + '@swc-node/register@1.10.9(@swc/core@1.7.26(@swc/helpers@0.5.15))(@swc/types@0.1.12)(typescript@5.6.2)': + dependencies: + '@swc-node/core': 1.13.3(@swc/core@1.7.26(@swc/helpers@0.5.15))(@swc/types@0.1.12) + '@swc-node/sourcemap-support': 0.5.1 + '@swc/core': 1.7.26(@swc/helpers@0.5.15) + colorette: 2.0.20 + debug: 4.3.7(supports-color@9.4.0) + oxc-resolver: 1.12.0 + pirates: 4.0.6 + tslib: 2.8.1 + typescript: 5.6.2 + transitivePeerDependencies: + - '@swc/types' + - supports-color + + '@swc-node/sourcemap-support@0.5.1': + dependencies: + source-map-support: 0.5.21 + tslib: 2.8.1 + '@swc/cli@0.4.0(@swc/core@1.7.26(@swc/helpers@0.5.15))(chokidar@3.6.0)': dependencies: '@mole-inc/bin-wrapper': 8.0.1 @@ -5083,7 +5353,6 @@ snapshots: '@swc/helpers@0.5.15': dependencies: tslib: 2.8.1 - optional: true '@swc/jest@0.2.36(@swc/core@1.7.26(@swc/helpers@0.5.15))': dependencies: @@ -5181,8 +5450,15 @@ snapshots: '@tanstack/virtual-file-routes@1.56.0': {} + '@taplo/cli@0.7.0': {} + '@tokenizer/token@0.3.0': {} + '@tybys/wasm-util@0.9.0': + dependencies: + tslib: 2.8.1 + optional: true + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.25.6 @@ -5211,6 +5487,10 @@ snapshots: '@types/node': 22.5.5 '@types/responselike': 1.0.3 + '@types/command-line-args@5.2.3': {} + + '@types/command-line-usage@5.0.4': {} + '@types/cross-spawn@6.0.6': dependencies: '@types/node': 22.5.5 @@ -5258,6 +5538,10 @@ snapshots: '@types/minimist@1.2.5': {} + '@types/node@20.17.6': + dependencies: + undici-types: 6.19.8 + '@types/node@22.5.5': dependencies: undici-types: 6.19.8 @@ -5359,6 +5643,18 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + apache-arrow@18.0.0: + dependencies: + '@swc/helpers': 0.5.15 + '@types/command-line-args': 5.2.3 + '@types/command-line-usage': 5.0.4 + '@types/node': 20.17.6 + command-line-args: 5.2.1 + command-line-usage: 7.0.3 + flatbuffers: 24.3.25 + json-bignum: 0.0.3 + tslib: 2.8.1 + arch@2.2.0: {} argparse@1.0.10: @@ -5367,6 +5663,10 @@ snapshots: argparse@2.0.1: {} + array-back@3.1.0: {} + + array-back@6.2.2: {} + arrify@1.0.1: {} babel-dead-code-elimination@1.0.6: @@ -5527,6 +5827,10 @@ snapshots: ccount@2.0.1: {} + chalk-template@0.4.0: + dependencies: + chalk: 4.1.2 + chalk@2.4.2: dependencies: ansi-styles: 3.2.1 @@ -5600,8 +5904,24 @@ snapshots: colorette@1.4.0: {} + colorette@2.0.20: {} + comma-separated-tokens@2.0.3: {} + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + + command-line-usage@7.0.3: + dependencies: + array-back: 6.2.2 + chalk-template: 0.4.0 + table-layout: 4.1.1 + typical: 7.3.0 + commander@12.1.0: {} commander@8.3.0: {} @@ -5934,6 +6254,10 @@ snapshots: dependencies: to-regex-range: 5.0.1 + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -5948,6 +6272,8 @@ snapshots: dependencies: semver-regex: 4.0.5 + flatbuffers@24.3.25: {} + fs.realpath@1.0.0: {} fsevents@2.3.3: @@ -6511,6 +6837,8 @@ snapshots: jsesc@2.5.2: {} + json-bignum@0.0.3: {} + json-buffer@3.0.1: {} json-parse-even-better-errors@2.3.1: {} @@ -6572,6 +6900,8 @@ snapshots: dependencies: p-locate: 5.0.0 + lodash.camelcase@4.3.0: {} + lodash.isequal@4.5.0: {} lodash.ismatch@4.4.0: {} @@ -6807,6 +7137,20 @@ snapshots: dependencies: arch: 2.2.0 + oxc-resolver@1.12.0: + optionalDependencies: + '@oxc-resolver/binding-darwin-arm64': 1.12.0 + '@oxc-resolver/binding-darwin-x64': 1.12.0 + '@oxc-resolver/binding-freebsd-x64': 1.12.0 + '@oxc-resolver/binding-linux-arm-gnueabihf': 1.12.0 + '@oxc-resolver/binding-linux-arm64-gnu': 1.12.0 + '@oxc-resolver/binding-linux-arm64-musl': 1.12.0 + '@oxc-resolver/binding-linux-x64-gnu': 1.12.0 + '@oxc-resolver/binding-linux-x64-musl': 1.12.0 + '@oxc-resolver/binding-wasm32-wasi': 1.12.0 + '@oxc-resolver/binding-win32-arm64-msvc': 1.12.0 + '@oxc-resolver/binding-win32-x64-msvc': 1.12.0 + p-cancelable@2.1.1: {} p-finally@1.0.0: {} @@ -7161,6 +7505,11 @@ snapshots: buffer-from: 1.1.2 source-map: 0.6.1 + source-map-support@0.5.21: + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + source-map@0.6.1: {} source-map@0.7.4: {} @@ -7252,6 +7601,11 @@ snapshots: tabbable@6.2.0: {} + table-layout@4.1.1: + dependencies: + array-back: 6.2.2 + wordwrapjs: 5.1.0 + test-exclude@6.0.0: dependencies: '@istanbuljs/schema': 0.1.3 @@ -7291,8 +7645,7 @@ snapshots: tslib@2.7.0: {} - tslib@2.8.1: - optional: true + tslib@2.8.1: {} tsx@4.19.0: dependencies: @@ -7358,6 +7711,10 @@ snapshots: typescript@5.6.2: {} + typical@4.0.0: {} + + typical@7.3.0: {} + uc.micro@2.1.0: {} uint8arrays@3.1.1: @@ -7545,6 +7902,8 @@ snapshots: dependencies: isexe: 2.0.0 + wordwrapjs@5.1.0: {} + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 diff --git a/tests/c1-integration/package.json b/tests/c1-integration/package.json index 384f34a..a4d0f86 100644 --- a/tests/c1-integration/package.json +++ b/tests/c1-integration/package.json @@ -14,13 +14,15 @@ "scripts": { "lint": "eslint src --fix", "build:clean": "del dist", + "build:types": "tsc --project tsconfig.json --emitDeclarationOnly --skipLibCheck", "build:js": "swc src -d ./dist --config-file ../../.swcrc --strip-leading-paths", - "build": "pnpm build:clean && pnpm build:js", - "test": "pnpm build && node --experimental-vm-modules ../../node_modules/jest/bin/jest.js --runInBand" + "build": "pnpm build:clean && pnpm build:types && pnpm build:js", + "test": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js --runInBand" }, "dependencies": { "@ceramic-sdk/events": "workspace:^", "@ceramic-sdk/identifiers": "workspace:^", + "@ceramic-sdk/flight-sql-client": "workspace:^", "@ceramic-sdk/http-client": "workspace:^", "@ceramic-sdk/model-client": "workspace:^", "@ceramic-sdk/model-handler": "workspace:^", @@ -29,6 +31,7 @@ "@ceramic-sdk/model-instance-protocol": "workspace:^", "@ceramic-sdk/model-protocol": "workspace:^", "@didtools/key-did": "^1.0.0", + "apache-arrow": "18.0.0", "@jest/environment": "^29.7.0", "@types/cross-spawn": "^6.0.0", "cross-spawn": "^7.0.5", diff --git a/tests/c1-integration/src/index.ts b/tests/c1-integration/src/index.ts index 8ae13c1..6c414af 100644 --- a/tests/c1-integration/src/index.ts +++ b/tests/c1-integration/src/index.ts @@ -39,7 +39,7 @@ export default class CeramicOneContainer { this.#container = container } - static async healthFn(port): Promise { + static async healthFn(port: number): Promise { try { const res = await fetch(`http://localhost:${port}/ceramic/version`) diff --git a/tests/c1-integration/test/flight.test.ts b/tests/c1-integration/test/flight.test.ts new file mode 100644 index 0000000..ea02ea8 --- /dev/null +++ b/tests/c1-integration/test/flight.test.ts @@ -0,0 +1,81 @@ +import { + type ClientOptions, + type FlightSqlClient, + createFlightSqlClient, +} from '@ceramic-sdk/flight-sql-client' +import { tableFromIPC } from 'apache-arrow' +import CeramicOneContainer from '../src' +import type { EnvironmentOptions } from '../src' + +const CONTAINER_OPTS: EnvironmentOptions = { + containerName: 'ceramic-test-flight', + apiPort: 5222, + flightSqlPort: 5223, + testPort: 5223, +} + +const OPTIONS: ClientOptions = { + headers: new Array(), + username: undefined, + password: undefined, + token: undefined, + tls: false, + host: '127.0.0.1', + port: CONTAINER_OPTS.flightSqlPort, +} + +async function getClient(): Promise { + return createFlightSqlClient(OPTIONS) +} + +describe('flight sql', () => { + let c1Container: CeramicOneContainer + + beforeAll(async () => { + c1Container = await CeramicOneContainer.startContainer(CONTAINER_OPTS) + }, 10000) + + test('makes query', async () => { + const client = await getClient() + const buffer = await client.query('SELECT * FROM conclusion_feed') + const data = tableFromIPC(buffer) + console.log(JSON.stringify(data)) + }) + + test('catalogs', async () => { + const client = await getClient() + const buffer = await client.getCatalogs() + const data = tableFromIPC(buffer) + console.log(JSON.stringify(data)) + }) + + test('schemas', async () => { + const client = await getClient() + const buffer = await client.getDbSchemas({}) + const data = tableFromIPC(buffer) + console.log(JSON.stringify(data)) + }) + + test('tables', async () => { + const client = await getClient() + const withSchema = await client.getTables({ includeSchema: true }) + const noSchema = await client.getTables({ includeSchema: false }) + console.log(JSON.stringify(tableFromIPC(withSchema))) + console.log(JSON.stringify(tableFromIPC(noSchema))) + expect(withSchema).not.toBe(noSchema) + }) + + // disabled until server support is implemented + test.skip('prepared stmt', async () => { + const client = await createFlightSqlClient(OPTIONS) + const data = await client.preparedStatement( + 'SELECT * from conclusion_feed where stream_type = $1', + new Array(['$1', '3']), + ) + console.log(data) + }) + + afterAll(async () => { + await c1Container.teardown() + }) +}) diff --git a/tests/c1-integration/tsconfig.json b/tests/c1-integration/tsconfig.json new file mode 100644 index 0000000..34756dd --- /dev/null +++ b/tests/c1-integration/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": ["src"] +}