Merge pull request #157 from drizzle-team/d1-http

D1 http

changelogs/drizzle-orm/0.17.4.md

@@ -0,0 +1,23 @@
+We have released [SQLite Proxy Driver](https://github.com/drizzle-team/drizzle-orm/tree/main/examples/sqlite-proxy)
+
+---
+
+Perfect way to setup custom logic for database calls instead of predefined drivers
+
+Should work well with serverless apps 🚀
+
+```typescript
+// Custom Proxy HTTP driver
+  const db = drizzle(async (sql, params, method) => {
+    try {
+      const rows = await axios.post('http://localhost:3000/query', { sql, params, method });
+
+      return { rows: rows.data };
+    } catch (e: any) {
+      console.error('Error from sqlite proxy server: ', e.response.data)
+      return { rows: [] };
+    }
+  });
+```
+
+> For more example you can check [full documentation](https://github.com/drizzle-team/drizzle-orm/tree/main/examples/sqlite-proxy)

drizzle-orm/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "drizzle-orm",
-	"version": "0.17.3",
+	"version": "0.17.4",
 	"description": "Drizzle ORM package for SQL databases",
 	"scripts": {
 		"build": "tsc && resolve-tspaths && cp ../README.md package.json dist/",

drizzle-orm/src/sqlite-core/README.md

@@ -19,6 +19,7 @@ Here you can find extensive docs for SQLite module.
 | [bun:sqlite](https://github.com/oven-sh/bun#bunsqlite-sqlite3-module) | ✅ | [Example](https://github.com/drizzle-team/drizzle-orm/tree/main/examples/bun-sqlite)| |
 | [Cloudflare D1](https://developers.cloudflare.com/d1/) | ✅ | [Example](https://github.com/drizzle-team/drizzle-orm/tree/main/examples/cloudflare-d1)| |
 | [Fly.io LiteFS](https://fly.io/docs/litefs/getting-started/) | ✅ | | |
+| [Custom proxy driver](https://github.com/drizzle-team/drizzle-orm/tree/main/examples/sqlite-proxy) | ✅ | | |
 
 ## 💾 Installation
 
@@ -71,6 +72,19 @@ import { drizzle, DrizzleD1Database } from 'drizzle-orm/d1';
 // env.DB from cloudflare worker environment
 const db: DrizzleD1Database = drizzle(env.DB);
 const result = await db.select(users).all() // pay attention this one is async
+
+// Custom Proxy HTTP driver
+  const db = drizzle(async (sql, params, method) => {
+    try {
+      const rows = await axios.post('http://localhost:3000/query', { sql, params, method });
+
+      return { rows: rows.data };
+    } catch (e: any) {
+      console.error('Error from sqlite proxy server: ', e.response.data)
+      return { rows: [] };
+    }
+  });
+// More example for proxy: https://github.com/drizzle-team/drizzle-orm/tree/main/examples/sqlite-proxy
 ```
 
 ## SQL schema declaration

drizzle-orm/src/sqlite-proxy/driver.ts

@@ -0,0 +1,26 @@
+import { Logger } from '~/logger';
+import { BaseSQLiteDatabase } from '~/sqlite-core/db';
+import { SQLiteAsyncDialect } from '~/sqlite-core/dialect';
+import { SQLiteRemoteSession } from './session';
+
+export interface DrizzleConfig {
+	logger?: Logger;
+}
+
+export interface SqliteRemoteResult<T = unknown> {
+	rows?: T[];
+}
+
+export type SqliteRemoteDatabase = BaseSQLiteDatabase<'async', SqliteRemoteResult>;
+
+export type RemoteCallback = (
+	sql: string,
+	params: any[],
+	method: 'run' | 'all' | 'values',
+) => Promise<{ rows: any[][] }>;
+
+export function drizzle(callback: RemoteCallback, config: DrizzleConfig = {}): SqliteRemoteDatabase {
+	const dialect = new SQLiteAsyncDialect();
+	const session = new SQLiteRemoteSession(callback, dialect, { logger: config.logger });
+	return new BaseSQLiteDatabase(dialect, session);
+}

drizzle-orm/src/sqlite-proxy/index.ts

@@ -0,0 +1,2 @@
+export * from './driver';
+export * from './session';

drizzle-orm/src/sqlite-proxy/migrator.ts

@@ -0,0 +1,39 @@
+import { MigrationConfig, readMigrationFiles } from '~/migrator';
+import { sql } from '~/sql';
+import { SqliteRemoteDatabase } from './driver';
+
+export type ProxyMigrator = (migrationQueries: string[]) => Promise<void>;
+
+export async function migrate(db: SqliteRemoteDatabase, callback: ProxyMigrator, config: string | MigrationConfig) {
+	const migrations = readMigrationFiles(config);
+
+	const migrationTableCreate = sql`CREATE TABLE IF NOT EXISTS "__drizzle_migrations" (
+		id SERIAL PRIMARY KEY,
+		hash text NOT NULL,
+		created_at numeric
+	)`;
+
+	await db.run(migrationTableCreate);
+
+	const dbMigrations = await db.values<[number, string, string]>(
+		sql`SELECT id, hash, created_at FROM "__drizzle_migrations" ORDER BY created_at DESC LIMIT 1`,
+	);
+
+	const lastDbMigration = dbMigrations[0] ?? undefined;
+
+	try {
+		const queriesToRun: string[] = [];
+		for (const migration of migrations) {
+			if (!lastDbMigration || parseInt(lastDbMigration[2], 10)! < migration.folderMillis) {
+				queriesToRun.push(migration.sql);
+				queriesToRun.push(
+					`INSERT INTO "__drizzle_migrations" ("hash", "created_at") VALUES('${migration.hash}', '${migration.folderMillis}')`,
+				);
+			}
+		}
+
+		await callback(queriesToRun);
+	} catch (e) {
+		throw e;
+	}
+}

drizzle-orm/src/sqlite-proxy/session.ts

@@ -0,0 +1,82 @@
+import { Logger, NoopLogger } from '~/logger';
+import { fillPlaceholders, Query } from '~/sql';
+import { SQLiteAsyncDialect } from '~/sqlite-core/dialect';
+import { SelectFieldsOrdered } from '~/sqlite-core/operations';
+import {
+	PreparedQuery as PreparedQueryBase,
+	PreparedQueryConfig as PreparedQueryConfigBase,
+	SQLiteSession,
+} from '~/sqlite-core/session';
+import { mapResultRow } from '~/utils';
+import { RemoteCallback, SqliteRemoteResult } from './driver';
+
+export interface SQLiteRemoteSessionOptions {
+	logger?: Logger;
+}
+
+type PreparedQueryConfig = Omit<PreparedQueryConfigBase, 'statement' | 'run'>;
+
+export class SQLiteRemoteSession extends SQLiteSession<'async', SqliteRemoteResult> {
+	private logger: Logger;
+
+	constructor(
+		private client: RemoteCallback,
+		dialect: SQLiteAsyncDialect,
+		options: SQLiteRemoteSessionOptions = {},
+	) {
+		super(dialect);
+		this.logger = options.logger ?? new NoopLogger();
+	}
+
+	exec(query: string): void {
+		throw Error('To implement: Proxy SQLite');
+		// await this.client.exec(query.sql);
+		// return this.client(this.queryString, params).then(({ rows }) => rows!)
+	}
+
+	prepareQuery<T extends Omit<PreparedQueryConfig, 'run'>>(query: Query, fields?: SelectFieldsOrdered): PreparedQuery<T> {
+		return new PreparedQuery(this.client, query.sql, query.params, this.logger, fields);
+	}
+}
+
+export class PreparedQuery<T extends PreparedQueryConfig = PreparedQueryConfig> extends PreparedQueryBase<
+	{ type: 'async'; run: SqliteRemoteResult; all: T['all']; get: T['get']; values: T['values'] }
+> {
+	constructor(
+		private client: RemoteCallback,
+		private queryString: string,
+		private params: unknown[],
+		private logger: Logger,
+		private fields: SelectFieldsOrdered | undefined,
+	) {
+		super();
+	}
+
+	async run(placeholderValues?: Record<string, unknown>): Promise<SqliteRemoteResult> {
+		const params = fillPlaceholders(this.params, placeholderValues ?? {});
+		this.logger.logQuery(this.queryString, params);
+		return await this.client(this.queryString, params, 'run');
+	}
+
+	async all(placeholderValues?: Record<string, unknown>): Promise<T['all']> {
+		const { fields } = this;
+		if (fields) {
+			return this.values(placeholderValues).then((values) => values.map((row) => mapResultRow(fields, row)));
+		}
+
+		const params = fillPlaceholders(this.params, placeholderValues ?? {});
+		this.logger.logQuery(this.queryString, params);
+		return this.client(this.queryString, params, 'all').then(({ rows }) => rows!);
+	}
+
+	async get(placeholderValues?: Record<string, unknown>): Promise<T['get']> {
+		return await this.all(placeholderValues).then((rows) => rows[0]);
+	}
+
+	async values<T extends any[] = unknown[]>(placeholderValues?: Record<string, unknown>): Promise<T[]> {
+		const params = fillPlaceholders(this.params, placeholderValues ?? {});
+		this.logger.logQuery(this.queryString, params);
+		const clientResult = await this.client(this.queryString, params, 'values');
+		return clientResult.rows as T[];
+	}
+}

examples/sqlite-proxy/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+.vscode
+*.db

examples/sqlite-proxy/README.md

@@ -0,0 +1,153 @@
+Example project for [Drizzle ORM SQLite Proxy package](https://github.com/drizzle-team/drizzle-orm/tree/main/drizzle-orm/src/sqlite-core)
+
+Subscribe to our updates on [Twitter](https://twitter.com/DrizzleOrm) and [Discord](https://discord.gg/MdXYZk5QtH)
+
+---
+
+**SQLite Proxy Driver** was designed to easily define custom drivers, https clients, rpc and much more. No need to wait until Drizzle ORM will create support for specific drivers you need. Just create it yourself! 🚀
+
+SQLite Proxy driver will do all the work except of 2 things, that you will be responsible for:
+
+1. Calls to database, http servers or any other way to communicate with database
+2. Mapping data from database to `{rows: string[][], ...additional db response params}` format. Only `row` field is required
+
+</br>
+This project has simple example of defining http proxy server, that will proxy all calls from drizzle orm to database and back. This example could perfectly fit for serverless applications
+
+---
+
+## Project structure
+
+1. `schema.ts` - drizzle orm schema file
+2. `index.ts` - basic script, that uses drizzle orm sqlite proxy driver to define logic for server to server communication over http
+3. `server.ts` - server implementation example
+
+### Database calls
+
+---
+
+#### All you need to do - is to setup drizzle database instance with http call implementation
+
+</br>
+
+> **Warning**:
+> You will be responsible for proper error handling in this part. Drizzle always waits for `{rows: string[][]}` so if any error was on http call(or any other call) - be sure, that you return at least empty array back
+
+</br>
+
+```typescript
+import axios from 'axios';
+import { drizzle } from 'drizzle-orm/sqlite-proxy';
+
+const db = drizzle(async (sql, params, method) => {
+  try {
+    const rows = await axios.post('http://localhost:3000/query', {
+      sql,
+      params,
+      method,
+    });
+
+    return { rows: rows.data };
+  } catch (e: any) {
+    console.error('Error from sqlite proxy server: ', e.response.data);
+    return { rows: [] };
+  }
+});
+```
+
+We have 3 params, that will be sent to server. It's your decision which of them and in which way should be used
+
+1. `sql` - SQL query (`SELECT * FROM users WHERE id = ?`)
+2. `params` - params, that should be sent on database call (For query above it could be: `[1]`)
+3. `method` - Method, that was executed (`run` | `all` | `values`). Hint for proxy server on which sqlite method to invoke
+
+### Migrations using SQLite Proxy
+
+---
+
+In current SQLite Proxy version - drizzle don't handle transactions for migrations. As for now we are sending an array of queries, that should be executed by user and user should do `commit` or `rollback` logic
+
+</br>
+
+> **Warning**:
+> You will be responsible for proper error handling in this part. Drizzle just finds migrations, that need to be executed on this iteration and if finds some -> provide `queries` array to callback
+
+</br>
+
+```typescript
+import axios from 'axios';
+import { migrate } from 'drizzle-orm/sqlite-proxy/migrator';
+
+
+await migrate(db, async (queries) => {
+    try {
+      await axios.post('http://localhost:3000/migrate', { queries });
+    } catch (e) {
+      console.log(e)
+      throw Error('Proxy server cannot run migrations')
+    }
+  }, { migrationsFolder: 'drizzle' });
+```
+
+1. `queries` - array of sql statements, that should be run on migration
+
+### Proxy Server implementation example
+
+---
+
+> **Note**:
+> It's just a suggestion on how proxy server could be set up and a simple example of params handling on `query` and `migration` calls
+
+```typescript
+import Database from 'better-sqlite3';
+import express from 'express';
+
+const app = express();
+app.use(express.json());
+const port = 3000;
+
+const db = new Database('./test.db');
+
+app.post('/query', (req, res) => {
+  const { sql: sqlBody, params, method } = req.body;
+
+  if (method === 'run') {
+    try {
+      const result = db.prepare(sqlBody).run(params);
+      res.send(result);
+    } catch (e: any) {
+            res.status(500).json({ error: e.message });
+    }
+  } else if (method === 'all' || method === 'values') {
+        try {
+      const rows = db.prepare(sqlBody).raw().all(params);
+            res.send(rows);
+    } catch (e: any) {
+            res.status(500).json({ error: e.message });
+    }
+  } else {
+    res.status(500).json({ error: 'Unkown method value' });
+  }
+});
+
+app.post('/migrate', (req, res) => {
+  const { queries } = req.body;
+
+  db.exec('BEGIN');
+  try {
+    for (const query of queries) {
+      db.exec(query);
+    }
+
+    db.exec('COMMIT');
+  } catch (e: any) {
+    db.exec('ROLLBACK');
+  }
+
+  res.send({});
+});
+
+app.listen(port, () => {
+  console.log(`Example app listening on port ${port}`);
+});
+```

examples/sqlite-proxy/drizzle/20230202162455/migration.sql

@@ -0,0 +1,5 @@
+CREATE TABLE users12 (
+	`id` integer PRIMARY KEY NOT NULL,
+	`name` text NOT NULL,
+	`email` text NOT NULL
+);