From f5f7c20eed01072fbb1960c2002906e28faca908 Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 19:25:21 -0500 Subject: [PATCH 01/28] Postgres-js: Added json and jsonb to the list of bypassed types on postgres.js driver --- drizzle-orm/src/postgres-js/driver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index ae1b48a21..2b2523e25 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -23,7 +23,7 @@ export function drizzle = Record val; // Override postgres.js default date parsers: https://github.com/porsager/postgres/discussions/761 - for (const type of ['1184', '1082', '1083', '1114']) { + for (const type of ['1184', '1082', '1083', '1114', '114', '3802']) { client.options.parsers[type as any] = transparentParser; client.options.serializers[type as any] = transparentParser; } From fcc8be7d8b27ea9a9299c3bdc24d31c53968c2e9 Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 19:26:29 -0500 Subject: [PATCH 02/28] [Pg] Added simple tests to pg and postgres-js integration tests for json and jsonb columns --- integration-tests/tests/pg.test.ts | 37 +++++++++++++++++++++ integration-tests/tests/postgres.js.test.ts | 37 +++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/integration-tests/tests/pg.test.ts b/integration-tests/tests/pg.test.ts index 3b31d7d60..e88a01833 100644 --- a/integration-tests/tests/pg.test.ts +++ b/integration-tests/tests/pg.test.ts @@ -49,6 +49,7 @@ import { intersect, intersectAll, interval, + json, jsonb, macaddr, macaddr8, @@ -2839,6 +2840,42 @@ test.serial('test mode string for timestamp with timezone in different timezone' await db.execute(sql`drop table if exists ${table}`); }); +test.serial('proper json and jsonb handling', async (t) => { + const { db } = t.context; + + const jsonTable = pgTable('json_table', { + json: json('json').$type<{ name: string; age: number }>(), + jsonb: jsonb('jsonb').$type<{ name: string; age: number }>(), + }); + + await db.execute(sql`drop table if exists ${jsonTable}`); + + db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + + await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); + + const result = await db.select().from(jsonTable); + + const justNames = await db.select({ + name1: sql`${jsonTable.json}->>'name'`.as('name1'), + name2: sql`${jsonTable.jsonb}->>'name'`.as('name2'), + }).from(jsonTable); + + t.deepEqual(result, [ + { + json: { name: 'Tom', age: 75 }, + jsonb: { name: 'Pete', age: 23 }, + }, + ]); + + t.deepEqual(justNames, [ + { + name1: 'Tom', + name2: 'Pete', + }, + ]); +}); + test.serial('orderBy with aliased column', (t) => { const { db } = t.context; diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index 0fd0c45ea..d23b294b4 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -31,6 +31,7 @@ import { getViewConfig, integer, interval, + json, jsonb, type PgColumn, pgEnum, @@ -1812,6 +1813,42 @@ test.serial('select from enum', async (t) => { await db.execute(sql`drop type ${name(categoryEnum.enumName)}`); }); +test.serial('proper json and jsonb handling', async (t) => { + const { db } = t.context; + + const jsonTable = pgTable('json_table', { + json: json('json').$type<{ name: string; age: number }>(), + jsonb: jsonb('jsonb').$type<{ name: string; age: number }>(), + }); + + await db.execute(sql`drop table if exists ${jsonTable}`); + + db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + + await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); + + const result = await db.select().from(jsonTable); + + const justNames = await db.select({ + name1: sql`${jsonTable.json}->>'name'`.as('name1'), + name2: sql`${jsonTable.jsonb}->>'name'`.as('name2'), + }).from(jsonTable); + + t.deepEqual(result, [ + { + json: { name: 'Tom', age: 75 }, + jsonb: { name: 'Pete', age: 23 }, + }, + ]); + + t.deepEqual(justNames, [ + { + name1: 'Tom', + name2: 'Pete', + }, + ]); +}); + test.serial('orderBy with aliased column', (t) => { const { db } = t.context; From dd5835868cf88f0e9c38359e186d1584a00aa7da Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 21:27:40 -0500 Subject: [PATCH 03/28] fix: bypassing the tranformation is only needed in the parser, not the serializer --- drizzle-orm/src/postgres-js/driver.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index 2b2523e25..2c4031c83 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -23,10 +23,12 @@ export function drizzle = Record val; // Override postgres.js default date parsers: https://github.com/porsager/postgres/discussions/761 - for (const type of ['1184', '1082', '1083', '1114', '114', '3802']) { - client.options.parsers[type as any] = transparentParser; + for (const type of ['1184', '1082', '1083', '1114']) { + if (type !== '114' && type !== '3802') client.options.parsers[type as any] = transparentParser; client.options.serializers[type as any] = transparentParser; } + client.options.parsers['114'] = transparentParser; + client.options.parsers['3802'] = transparentParser; const dialect = new PgDialect(); let logger; From ca792625fc64ca696cacafce1da32b0902b843ad Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 21:32:52 -0500 Subject: [PATCH 04/28] Added additional tests to postgres-js integration tests --- integration-tests/tests/postgres.js.test.ts | 127 ++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/integration-tests/tests/postgres.js.test.ts b/integration-tests/tests/postgres.js.test.ts index d23b294b4..3382f24e2 100644 --- a/integration-tests/tests/postgres.js.test.ts +++ b/integration-tests/tests/postgres.js.test.ts @@ -94,6 +94,12 @@ const orders = pgTable('orders', { quantity: integer('quantity').notNull(), }); +const jsonTestTable = pgTable('jsontest', { + id: serial('id').primaryKey(), + json: json('json').$type<{ string: string; number: number }>(), + jsonb: jsonb('jsonb').$type<{ string: string; number: number }>(), +}); + const usersMigratorTable = pgTable('users12', { id: serial('id').primaryKey(), name: text('name').notNull(), @@ -234,6 +240,15 @@ test.beforeEach(async (t) => { ) `, ); + await ctx.db.execute( + sql` + create table jsontest ( + id serial primary key, + json json, + jsonb jsonb + ) + `, + ); }); test.serial('select all fields', async (t) => { @@ -422,6 +437,118 @@ test.serial('json insert', async (t) => { t.deepEqual(result, [{ id: 1, name: 'John', jsonb: ['foo', 'bar'] }]); }); +test.serial('set json/jsonb fields with objects and retrieve with the ->> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + +test.serial('set json/jsonb fields with strings and retrieve with the ->> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + +test.serial('set json/jsonb fields with objects and retrieve with the -> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + +test.serial('set json/jsonb fields with strings and retrieve with the -> operator', async (t) => { + const { db } = t.context; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + t.deepEqual(result, [{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]); + + await db.execute(sql`drop table ${jsonTestTable}`); +}); + test.serial('insert with overridden default values', async (t) => { const { db } = t.context; From 562c25bb28cdd8243a28b1970b6c25b004b7d3d8 Mon Sep 17 00:00:00 2001 From: Angelelz Date: Wed, 10 Jan 2024 22:06:18 -0500 Subject: [PATCH 05/28] fixed parsing properly --- drizzle-orm/src/postgres-js/driver.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drizzle-orm/src/postgres-js/driver.ts b/drizzle-orm/src/postgres-js/driver.ts index 2c4031c83..7f44344e8 100644 --- a/drizzle-orm/src/postgres-js/driver.ts +++ b/drizzle-orm/src/postgres-js/driver.ts @@ -24,11 +24,11 @@ export function drizzle = Record Date: Sun, 21 Jul 2024 19:10:52 +0200 Subject: [PATCH 06/28] Fix isTable helper function by adding IsDrizzleTable Symbol to Table base class --- drizzle-orm/src/table.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drizzle-orm/src/table.ts b/drizzle-orm/src/table.ts index 3db9d5559..8632dd35c 100644 --- a/drizzle-orm/src/table.ts +++ b/drizzle-orm/src/table.ts @@ -107,6 +107,9 @@ export class Table implements SQLWrapper { /** @internal */ [IsAlias] = false; + /** @internal */ + [IsDrizzleTable] = true; + /** @internal */ [ExtraConfigBuilder]: ((self: any) => Record) | undefined = undefined; From ff3f1660106b366a4919fb7b4f51bf9604837bbc Mon Sep 17 00:00:00 2001 From: RemiPe Date: Thu, 25 Jul 2024 17:54:00 +0200 Subject: [PATCH 07/28] Update inArray and notInArray documentation remove the throws part since both methods do not throw anymore --- drizzle-orm/src/sql/expressions/conditions.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drizzle-orm/src/sql/expressions/conditions.ts b/drizzle-orm/src/sql/expressions/conditions.ts index 0a911e1ff..ba0e21fbc 100644 --- a/drizzle-orm/src/sql/expressions/conditions.ts +++ b/drizzle-orm/src/sql/expressions/conditions.ts @@ -256,11 +256,6 @@ export const lte: BinaryOperator = (left: SQLWrapper, right: unknown): SQL => { * Test whether the first parameter, a column or expression, * has a value from a list passed as the second argument. * - * ## Throws - * - * The argument passed in the second array can't be empty: - * if an empty is provided, this method will throw. - * * ## Examples * * ```ts @@ -302,11 +297,6 @@ export function inArray( * has a value that is not present in a list passed as the * second argument. * - * ## Throws - * - * The argument passed in the second array can't be empty: - * if an empty is provided, this method will throw. - * * ## Examples * * ```ts From 5510112de88b5550f911e4d236a4d5a38b487f39 Mon Sep 17 00:00:00 2001 From: Karibash Date: Thu, 1 Aug 2024 23:13:36 +0900 Subject: [PATCH 08/28] bugfix: Fix a bug with default values for columns of type string --- drizzle-kit/src/serializer/mysqlSerializer.ts | 2 +- drizzle-kit/tests/introspect/mysql.test.ts | 44 ++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/drizzle-kit/src/serializer/mysqlSerializer.ts b/drizzle-kit/src/serializer/mysqlSerializer.ts index eb18743fb..14e867128 100644 --- a/drizzle-kit/src/serializer/mysqlSerializer.ts +++ b/drizzle-kit/src/serializer/mysqlSerializer.ts @@ -481,7 +481,7 @@ export const fromDatabase = async ( default: columnDefault === null ? undefined : /^-?[\d.]+(?:e-?\d+)?$/.test(columnDefault) - && !columnType.startsWith('decimal') + && !['decimal', 'char', 'varchar'].some((type) => columnType.startsWith(type)) ? Number(columnDefault) : isDefaultAnExpression ? clearDefaults(columnDefault, collation) diff --git a/drizzle-kit/tests/introspect/mysql.test.ts b/drizzle-kit/tests/introspect/mysql.test.ts index 23cd28a16..e35b34f40 100644 --- a/drizzle-kit/tests/introspect/mysql.test.ts +++ b/drizzle-kit/tests/introspect/mysql.test.ts @@ -1,6 +1,6 @@ import Docker from 'dockerode'; import { SQL, sql } from 'drizzle-orm'; -import { int, mysqlTable, text } from 'drizzle-orm/mysql-core'; +import { char, int, mysqlTable, text, varchar } from 'drizzle-orm/mysql-core'; import * as fs from 'fs'; import getPort from 'get-port'; import { Connection, createConnection } from 'mysql2/promise'; @@ -123,3 +123,45 @@ test('generated always column virtual: link to another column', async () => { await client.query(`drop table users;`); }); + +test('Default value of character type column: char', async () => { + const schema = { + users: mysqlTable('users', { + id: int('id'), + sortKey: char('sortKey', { length: 255 }).default('0'), + }), + }; + + const { statements, sqlStatements } = await introspectMySQLToFile( + client, + schema, + 'default-value-char-column', + 'drizzle', + ); + + expect(statements.length).toBe(0); + expect(sqlStatements.length).toBe(0); + + await client.query(`drop table users;`); +}); + +test('Default value of character type column: varchar', async () => { + const schema = { + users: mysqlTable('users', { + id: int('id'), + sortKey: varchar('sortKey', { length: 255 }).default('0'), + }), + }; + + const { statements, sqlStatements } = await introspectMySQLToFile( + client, + schema, + 'default-value-varchar-column', + 'drizzle', + ); + + expect(statements.length).toBe(0); + expect(sqlStatements.length).toBe(0); + + await client.query(`drop table users;`); +}); From 19c8926a98e64b012803c8a14e9b42e46b0e005e Mon Sep 17 00:00:00 2001 From: Karibash Date: Thu, 1 Aug 2024 23:25:49 +0900 Subject: [PATCH 09/28] bugfix: Fix a bug that import statements for columns of type double are not inserted --- drizzle-kit/src/introspect-mysql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/drizzle-kit/src/introspect-mysql.ts b/drizzle-kit/src/introspect-mysql.ts index fb1c71428..21be79654 100644 --- a/drizzle-kit/src/introspect-mysql.ts +++ b/drizzle-kit/src/introspect-mysql.ts @@ -153,6 +153,7 @@ export const schemaToTypeScript = ( patched = patched.startsWith('datetime(') ? 'datetime' : patched; patched = patched.startsWith('varbinary(') ? 'varbinary' : patched; patched = patched.startsWith('int(') ? 'int' : patched; + patched = patched.startsWith('double(') ? 'double' : patched; return patched; }) .filter((type) => { From ce1643f56c9f09b55f12aef91f0983d9c79ce545 Mon Sep 17 00:00:00 2001 From: Mario564 Date: Mon, 5 Aug 2024 23:27:13 -0700 Subject: [PATCH 10/28] Improve default value generation for array columns in PG --- drizzle-kit/src/serializer/pgSerializer.ts | 44 ++++ drizzle-kit/tests/pg-array.test.ts | 255 +++++++++++++++++++++ 2 files changed, 299 insertions(+) create mode 100644 drizzle-kit/tests/pg-array.test.ts diff --git a/drizzle-kit/src/serializer/pgSerializer.ts b/drizzle-kit/src/serializer/pgSerializer.ts index 4ab37a0ae..4c4831e8f 100644 --- a/drizzle-kit/src/serializer/pgSerializer.ts +++ b/drizzle-kit/src/serializer/pgSerializer.ts @@ -75,6 +75,43 @@ function stringFromDatabaseIdentityProperty(field: any): string | undefined { : String(field); } +function buildArrayString(array: any[], sqlType: string): string { + sqlType = sqlType.split('[')[0]; + const values = array + .map((value) => { + if (typeof value === 'number' || typeof value === 'bigint') { + return value.toString(); + } else if (typeof value === 'boolean') { + return value ? 'true' : 'false'; + } else if (Array.isArray(value)) { + return buildArrayString(value, sqlType); + } else if (value instanceof Date) { + if (sqlType === 'date') { + return `"${value.toISOString().split('T')[0]}"`; + } else if (sqlType === 'timestamp') { + return `"${ + value.toISOString() + .replace('T', ' ') + .slice(0, 23) + }"`; + } else { + return `"${value.toISOString()}"`; + } + } else if (typeof value === 'object') { + return `"${ + JSON + .stringify(value) + .replaceAll('"', '\\"') + }"`; + } + + return `"${value}"`; + }) + .join(','); + + return `{${values}}`; +} + export const generatePgSnapshot = ( tables: AnyPgTable[], enums: PgEnum[], @@ -226,6 +263,13 @@ export const generatePgSnapshot = ( } else { columnToSet.default = `'${column.default.toISOString()}'`; } + } else if (sqlTypeLowered.match(/.*\[\d*\].*|.*\[\].*/g) !== null && Array.isArray(column.default)) { + columnToSet.default = `'${ + buildArrayString( + column.default, + sqlTypeLowered, + ) + }'::${sqlTypeLowered}`; } else { // Should do for all types // columnToSet.default = `'${column.default}'::${sqlTypeLowered}`; diff --git a/drizzle-kit/tests/pg-array.test.ts b/drizzle-kit/tests/pg-array.test.ts new file mode 100644 index 000000000..273d7cfcc --- /dev/null +++ b/drizzle-kit/tests/pg-array.test.ts @@ -0,0 +1,255 @@ +import { bigint, boolean, date, integer, json, pgTable, serial, text, timestamp } from 'drizzle-orm/pg-core'; +import { expect, test } from 'vitest'; +import { diffTestSchemas } from './schemaDiffer'; + +test('array #1: empty array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: integer('values').array().default([]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { name: 'values', type: 'integer[]', primaryKey: false, notNull: false, default: "'{}'::integer[]" }, + }); +}); + +test('array #2: integer array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: integer('values').array().default([1, 2, 3]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { name: 'values', type: 'integer[]', primaryKey: false, notNull: false, default: "'{1,2,3}'::integer[]" }, + }); +}); + +test('array #3: bigint array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: bigint('values', { mode: 'bigint' }).array().default([BigInt(1), BigInt(2), BigInt(3)]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { name: 'values', type: 'bigint[]', primaryKey: false, notNull: false, default: "'{1,2,3}'::bigint[]" }, + }); +}); + +test('array #4: boolean array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: boolean('values').array().default([true, false, true]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'boolean[]', + primaryKey: false, + notNull: false, + default: "'{true,false,true}'::boolean[]", + }, + }); +}); + +test('array #5: multi-dimensional array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: integer('values').array().array().default([[1, 2], [3, 4]]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'integer[][]', + primaryKey: false, + notNull: false, + default: "'{{1,2},{3,4}}'::integer[][]", + }, + }); +}); + +test('array #6: date array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: date('values').array().default(['2024-08-06', '2024-08-07']), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'date[]', + primaryKey: false, + notNull: false, + default: '\'{"2024-08-06","2024-08-07"}\'::date[]', + }, + }); +}); + +test('array #7: timestamp array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: timestamp('values').array().default([new Date('2024-08-06'), new Date('2024-08-07')]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'timestamp[]', + primaryKey: false, + notNull: false, + default: '\'{"2024-08-06 00:00:00.000","2024-08-07 00:00:00.000"}\'::timestamp[]', + }, + }); +}); + +test('array #8: json array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: json('values').array().default([{ a: 1 }, { b: 2 }]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'json[]', + primaryKey: false, + notNull: false, + default: '\'{"{\\"a\\":1}","{\\"b\\":2}"}\'::json[]', + }, + }); +}); + +test('array #9: text array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: text('values').array().default(['abc', 'def']), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'text[]', + primaryKey: false, + notNull: false, + default: '\'{"abc","def"}\'::text[]', + }, + }); +}); From b290f42b4f00d469b5825f3c33fa194ed77c87e6 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Tue, 6 Aug 2024 14:27:46 +0300 Subject: [PATCH 11/28] Add tests to pg-common --- integration-tests/tests/pg/pg-common.ts | 169 +++++++++++++++++++++++- 1 file changed, 162 insertions(+), 7 deletions(-) diff --git a/integration-tests/tests/pg/pg-common.ts b/integration-tests/tests/pg/pg-common.ts index fb69c5877..df1582bea 100644 --- a/integration-tests/tests/pg/pg-common.ts +++ b/integration-tests/tests/pg/pg-common.ts @@ -70,6 +70,7 @@ import { uniqueKeyName, uuid as pgUuid, varchar, + json, } from 'drizzle-orm/pg-core'; import getPort from 'get-port'; import { v4 as uuidV4 } from 'uuid'; @@ -199,6 +200,12 @@ const users2MySchemaTable = mySchema.table('users2', { cityId: integer('city_id').references(() => citiesTable.id), }); +const jsonTestTable = pgTable('jsontest', { + id: serial('id').primaryKey(), + json: json('json').$type<{ string: string; number: number }>(), + jsonb: jsonb('jsonb').$type<{ string: string; number: number }>(), +}); + let pgContainer: Docker.Container; export async function createDockerDB(): Promise<{ connectionString: string; container: Docker.Container }> { @@ -358,6 +365,16 @@ export function tests() { ) `, ); + + await db.execute( + sql` + create table jsontest ( + id serial primary key, + json json, + jsonb jsonb + ) + `, + ); }); async function setupSetOperationTest(db: PgDatabase) { @@ -2347,9 +2364,8 @@ export function tests() { await db.execute(sql`drop type if exists ${sql.identifier(categoryEnum.enumName)}`); await db.execute( - sql`create type ${ - sql.identifier(muscleEnum.enumName) - } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, + sql`create type ${sql.identifier(muscleEnum.enumName) + } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, ); await db.execute( sql`create type ${sql.identifier(forceEnum.enumName)} as enum ('isometric', 'isotonic', 'isokinetic')`, @@ -2359,9 +2375,8 @@ export function tests() { ); await db.execute(sql`create type ${sql.identifier(mechanicEnum.enumName)} as enum ('compound', 'isolation')`); await db.execute( - sql`create type ${ - sql.identifier(equipmentEnum.enumName) - } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, + sql`create type ${sql.identifier(equipmentEnum.enumName) + } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, ); await db.execute( sql`create type ${sql.identifier(categoryEnum.enumName)} as enum ('upper_body', 'lower_body', 'full_body')`, @@ -4481,5 +4496,145 @@ export function tests() { expect(users.length).toBeGreaterThan(0); }); + + test('proper json and jsonb handling', async (ctx) => { + const { db } = ctx.pg; + + const jsonTable = pgTable('json_table', { + json: json('json').$type<{ name: string; age: number }>(), + jsonb: jsonb('jsonb').$type<{ name: string; age: number }>(), + }); + + await db.execute(sql`drop table if exists ${jsonTable}`); + + db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + + await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); + + const result = await db.select().from(jsonTable); + + const justNames = await db.select({ + name1: sql`${jsonTable.json}->>'name'`.as('name1'), + name2: sql`${jsonTable.jsonb}->>'name'`.as('name2'), + }).from(jsonTable); + + expect(result).toStrictEqual([ + { + json: { name: 'Tom', age: 75 }, + jsonb: { name: 'Pete', age: 23 }, + }, + ]); + + expect(justNames).toStrictEqual([ + { + name1: 'Tom', + name2: 'Pete', + }, + ]); + }); + + test('set json/jsonb fields with objects and retrieve with the ->> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]) + }); + + test('set json/jsonb fields with strings and retrieve with the ->> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->>'string'`, + jsonNumberField: sql`${jsonTestTable.json}->>'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->>'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->>'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: String(testNumber), + jsonbStringField: testString, + jsonbNumberField: String(testNumber), + }]) + }); + + test('set json/jsonb fields with objects and retrieve with the -> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: obj, + jsonb: obj, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]) + }); + + test('set json/jsonb fields with strings and retrieve with the -> operator', async (ctx) => { + const { db } = ctx.pg; + + const obj = { string: 'test', number: 123 }; + const { string: testString, number: testNumber } = obj; + + await db.insert(jsonTestTable).values({ + json: sql`${JSON.stringify(obj)}`, + jsonb: sql`${JSON.stringify(obj)}`, + }); + + const result = await db.select({ + jsonStringField: sql`${jsonTestTable.json}->'string'`, + jsonNumberField: sql`${jsonTestTable.json}->'number'`, + jsonbStringField: sql`${jsonTestTable.jsonb}->'string'`, + jsonbNumberField: sql`${jsonTestTable.jsonb}->'number'`, + }).from(jsonTestTable); + + expect(result).toStrictEqual([{ + jsonStringField: testString, + jsonNumberField: testNumber, + jsonbStringField: testString, + jsonbNumberField: testNumber, + }]) + }); }); -} +} \ No newline at end of file From 0accd97a853294f7ed9916673aee974f06419e2e Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Tue, 6 Aug 2024 14:51:08 +0300 Subject: [PATCH 12/28] Use dprint --- integration-tests/tests/pg/pg-common.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/integration-tests/tests/pg/pg-common.ts b/integration-tests/tests/pg/pg-common.ts index df1582bea..0b2b48769 100644 --- a/integration-tests/tests/pg/pg-common.ts +++ b/integration-tests/tests/pg/pg-common.ts @@ -49,6 +49,7 @@ import { intersect, intersectAll, interval, + json, jsonb, macaddr, macaddr8, @@ -70,7 +71,6 @@ import { uniqueKeyName, uuid as pgUuid, varchar, - json, } from 'drizzle-orm/pg-core'; import getPort from 'get-port'; import { v4 as uuidV4 } from 'uuid'; @@ -2364,8 +2364,9 @@ export function tests() { await db.execute(sql`drop type if exists ${sql.identifier(categoryEnum.enumName)}`); await db.execute( - sql`create type ${sql.identifier(muscleEnum.enumName) - } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, + sql`create type ${ + sql.identifier(muscleEnum.enumName) + } as enum ('abdominals', 'hamstrings', 'adductors', 'quadriceps', 'biceps', 'shoulders', 'chest', 'middle_back', 'calves', 'glutes', 'lower_back', 'lats', 'triceps', 'traps', 'forearms', 'neck', 'abductors')`, ); await db.execute( sql`create type ${sql.identifier(forceEnum.enumName)} as enum ('isometric', 'isotonic', 'isokinetic')`, @@ -2375,8 +2376,9 @@ export function tests() { ); await db.execute(sql`create type ${sql.identifier(mechanicEnum.enumName)} as enum ('compound', 'isolation')`); await db.execute( - sql`create type ${sql.identifier(equipmentEnum.enumName) - } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, + sql`create type ${ + sql.identifier(equipmentEnum.enumName) + } as enum ('barbell', 'dumbbell', 'bodyweight', 'machine', 'cable', 'kettlebell')`, ); await db.execute( sql`create type ${sql.identifier(categoryEnum.enumName)} as enum ('upper_body', 'lower_body', 'full_body')`, @@ -4507,7 +4509,7 @@ export function tests() { await db.execute(sql`drop table if exists ${jsonTable}`); - db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); + await db.execute(sql`create table ${jsonTable} (json json, jsonb jsonb)`); await db.insert(jsonTable).values({ json: { name: 'Tom', age: 75 }, jsonb: { name: 'Pete', age: 23 } }); @@ -4556,7 +4558,7 @@ export function tests() { jsonNumberField: String(testNumber), jsonbStringField: testString, jsonbNumberField: String(testNumber), - }]) + }]); }); test('set json/jsonb fields with strings and retrieve with the ->> operator', async (ctx) => { @@ -4582,7 +4584,7 @@ export function tests() { jsonNumberField: String(testNumber), jsonbStringField: testString, jsonbNumberField: String(testNumber), - }]) + }]); }); test('set json/jsonb fields with objects and retrieve with the -> operator', async (ctx) => { @@ -4608,7 +4610,7 @@ export function tests() { jsonNumberField: testNumber, jsonbStringField: testString, jsonbNumberField: testNumber, - }]) + }]); }); test('set json/jsonb fields with strings and retrieve with the -> operator', async (ctx) => { @@ -4634,7 +4636,7 @@ export function tests() { jsonNumberField: testNumber, jsonbStringField: testString, jsonbNumberField: testNumber, - }]) + }]); }); }); -} \ No newline at end of file +} From f74f8f3cf3bc172cdf5a3d3e95be92f0bc798954 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Tue, 6 Aug 2024 18:26:31 +0100 Subject: [PATCH 13/28] Add tests for SQLite using `type: boolean` with prepared statements --- integration-tests/tests/sqlite/sqlite-common.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/integration-tests/tests/sqlite/sqlite-common.ts b/integration-tests/tests/sqlite/sqlite-common.ts index 49c609941..4f84f7111 100644 --- a/integration-tests/tests/sqlite/sqlite-common.ts +++ b/integration-tests/tests/sqlite/sqlite-common.ts @@ -845,12 +845,12 @@ export function tests() { const { db } = ctx.sqlite; const stmt = db.insert(usersTable).values({ - verified: true, + verified: sql.placeholder("verified"), name: sql.placeholder('name'), }).prepare(); for (let i = 0; i < 10; i++) { - await stmt.run({ name: `John ${i}` }); + await stmt.run({ name: `John ${i}`, verified: i % 2 === 0 }); } const result = await db.select({ @@ -861,15 +861,15 @@ export function tests() { expect(result).toEqual([ { id: 1, name: 'John 0', verified: true }, - { id: 2, name: 'John 1', verified: true }, + { id: 2, name: 'John 1', verified: false }, { id: 3, name: 'John 2', verified: true }, - { id: 4, name: 'John 3', verified: true }, + { id: 4, name: 'John 3', verified: false }, { id: 5, name: 'John 4', verified: true }, - { id: 6, name: 'John 5', verified: true }, + { id: 6, name: 'John 5', verified: false }, { id: 7, name: 'John 6', verified: true }, - { id: 8, name: 'John 7', verified: true }, + { id: 8, name: 'John 7', verified: false }, { id: 9, name: 'John 8', verified: true }, - { id: 10, name: 'John 9', verified: true }, + { id: 10, name: 'John 9', verified: false }, ]); }); From 9ff80929afcbe1642e5a97fe7412a7005f4abbd8 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Tue, 6 Aug 2024 21:39:26 +0300 Subject: [PATCH 14/28] Do not sort pks in sqlite --- .../src/serializer/sqliteSerializer.ts | 107 ++++++++---------- 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/drizzle-kit/src/serializer/sqliteSerializer.ts b/drizzle-kit/src/serializer/sqliteSerializer.ts index c673daafb..2dfdedcea 100644 --- a/drizzle-kit/src/serializer/sqliteSerializer.ts +++ b/drizzle-kit/src/serializer/sqliteSerializer.ts @@ -65,8 +65,8 @@ export const generateSqliteSnapshot = ( as: is(generated.as, SQL) ? `(${dialect.sqlToQuery(generated.as as SQL, 'indexes').sql})` : typeof generated.as === 'function' - ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` - : `(${generated.as as any})`, + ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` + : `(${generated.as as any})`, type: generated.mode ?? 'virtual', } : undefined, @@ -79,9 +79,9 @@ export const generateSqliteSnapshot = ( columnToSet.default = typeof column.default === 'string' ? `'${column.default}'` : typeof column.default === 'object' - || Array.isArray(column.default) - ? `'${JSON.stringify(column.default)}'` - : column.default; + || Array.isArray(column.default) + ? `'${JSON.stringify(column.default)}'` + : column.default; } } columnsObject[column.name] = columnToSet; @@ -90,24 +90,19 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[column.uniqueName!]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${ - withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${ - chalk.underline.blue( - tableName, - ) + `\n${withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${chalk.underline.blue( + tableName, + ) } table. - The unique constraint ${ - chalk.underline.blue( - column.uniqueName, - ) - } on the ${ - chalk.underline.blue( - column.name, - ) - } column is confilcting with a unique constraint name already defined for ${ - chalk.underline.blue( - existingUnique.columns.join(','), - ) + The unique constraint ${chalk.underline.blue( + column.uniqueName, + ) + } on the ${chalk.underline.blue( + column.name, + ) + } column is confilcting with a unique constraint name already defined for ${chalk.underline.blue( + existingUnique.columns.join(','), + ) } columns\n`) }`, ); @@ -202,26 +197,21 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[name]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${ - withStyle.errorWarning( - `We\'ve found duplicated unique constraint names in ${ - chalk.underline.blue( - tableName, - ) - } table. \nThe unique constraint ${ - chalk.underline.blue( - name, - ) - } on the ${ - chalk.underline.blue( - columnNames.join(','), - ) - } columns is confilcting with a unique constraint name already defined for ${ - chalk.underline.blue( - existingUnique.columns.join(','), - ) - } columns\n`, + `\n${withStyle.errorWarning( + `We\'ve found duplicated unique constraint names in ${chalk.underline.blue( + tableName, + ) + } table. \nThe unique constraint ${chalk.underline.blue( + name, + ) + } on the ${chalk.underline.blue( + columnNames.join(','), ) + } columns is confilcting with a unique constraint name already defined for ${chalk.underline.blue( + existingUnique.columns.join(','), + ) + } columns\n`, + ) }`, ); process.exit(1); @@ -237,7 +227,7 @@ export const generateSqliteSnapshot = ( primaryKeys.forEach((it) => { if (it.columns.length > 1) { primaryKeysObject[it.getName()] = { - columns: it.columns.map((it) => it.name).sort(), + columns: it.columns.map((it) => it.name), name: it.getName(), }; } else { @@ -464,26 +454,26 @@ export const fromDatabase = async ( default: columnDefault === null ? undefined : /^-?[\d.]+(?:e-?\d+)?$/.test(columnDefault) - ? Number(columnDefault) - : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( + ? Number(columnDefault) + : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( columnDefault, ) - ? `(${columnDefault})` - : columnDefault === 'false' - ? false - : columnDefault === 'true' - ? true - : columnDefault.startsWith("'") && columnDefault.endsWith("'") - ? columnDefault - // ? columnDefault.substring(1, columnDefault.length - 1) - : `(${columnDefault})`, + ? `(${columnDefault})` + : columnDefault === 'false' + ? false + : columnDefault === 'true' + ? true + : columnDefault.startsWith("'") && columnDefault.endsWith("'") + ? columnDefault + // ? columnDefault.substring(1, columnDefault.length - 1) + : `(${columnDefault})`, autoincrement: isAutoincrement, name: columnName, type: mapSqlToSqliteType(columnType), primaryKey: false, notNull: isNotNull, generated: tableToGeneratedColumnsInfo[tableName] - && tableToGeneratedColumnsInfo[tableName][columnName] + && tableToGeneratedColumnsInfo[tableName][columnName] ? { type: tableToGeneratedColumnsInfo[tableName][columnName].type, as: tableToGeneratedColumnsInfo[tableName][columnName].expression, @@ -580,11 +570,10 @@ export const fromDatabase = async ( const columnsTo = fkByTableName[`${tableName}_${id}`].columnsTo; fkByTableName[ `${tableName}_${id}` - ].name = `${tableName}_${ - columnsFrom.join( - '_', - ) - }_${refTableName}_${columnsTo.join('_')}_fk`; + ].name = `${tableName}_${columnsFrom.join( + '_', + ) + }_${refTableName}_${columnsTo.join('_')}_fk`; } for (const idx of Object.keys(fkByTableName)) { From 39c1419298c1f8b33d8436709e0fc9735f10c64e Mon Sep 17 00:00:00 2001 From: Mario564 Date: Tue, 6 Aug 2024 19:00:47 -0700 Subject: [PATCH 15/28] Add more tests --- drizzle-kit/src/serializer/pgSerializer.ts | 4 +- drizzle-kit/src/utils.ts | 4 + drizzle-kit/tests/pg-array.test.ts | 115 ++++++++++++++++++++- drizzle-kit/tests/push/pg.test.ts | 74 +++++++++++++ 4 files changed, 194 insertions(+), 3 deletions(-) diff --git a/drizzle-kit/src/serializer/pgSerializer.ts b/drizzle-kit/src/serializer/pgSerializer.ts index 4c4831e8f..6470cf1ff 100644 --- a/drizzle-kit/src/serializer/pgSerializer.ts +++ b/drizzle-kit/src/serializer/pgSerializer.ts @@ -30,7 +30,7 @@ import type { Table, UniqueConstraint, } from '../serializer/pgSchema'; -import type { DB } from '../utils'; +import { type DB, isPgArrayType } from '../utils'; import { sqlToStr } from '.'; const dialect = new PgDialect(); @@ -263,7 +263,7 @@ export const generatePgSnapshot = ( } else { columnToSet.default = `'${column.default.toISOString()}'`; } - } else if (sqlTypeLowered.match(/.*\[\d*\].*|.*\[\].*/g) !== null && Array.isArray(column.default)) { + } else if (isPgArrayType(sqlTypeLowered) && Array.isArray(column.default)) { columnToSet.default = `'${ buildArrayString( column.default, diff --git a/drizzle-kit/src/utils.ts b/drizzle-kit/src/utils.ts index 279520ea6..6a7faff45 100644 --- a/drizzle-kit/src/utils.ts +++ b/drizzle-kit/src/utils.ts @@ -327,3 +327,7 @@ export const normaliseSQLiteUrl = ( assertUnreachable(type); }; + +export function isPgArrayType(sqlType: string) { + return sqlType.match(/.*\[\d*\].*|.*\[\].*/g) !== null; +} diff --git a/drizzle-kit/tests/pg-array.test.ts b/drizzle-kit/tests/pg-array.test.ts index 273d7cfcc..a35411adb 100644 --- a/drizzle-kit/tests/pg-array.test.ts +++ b/drizzle-kit/tests/pg-array.test.ts @@ -1,4 +1,16 @@ -import { bigint, boolean, date, integer, json, pgTable, serial, text, timestamp } from 'drizzle-orm/pg-core'; +import { + bigint, + boolean, + date, + integer, + json, + pgEnum, + pgTable, + serial, + text, + timestamp, + uuid, +} from 'drizzle-orm/pg-core'; import { expect, test } from 'vitest'; import { diffTestSchemas } from './schemaDiffer'; @@ -253,3 +265,104 @@ test('array #9: text array default', async (t) => { }, }); }); + +test('array #10: uuid array default', async (t) => { + const from = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: uuid('values').array().default([ + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'b0eebc99-9c0b-4ef8-bb6d-cbb9bd380a11', + ]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'uuid[]', + primaryKey: false, + notNull: false, + default: '\'{"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11","b0eebc99-9c0b-4ef8-bb6d-cbb9bd380a11"}\'::uuid[]', + }, + }); +}); + +test('array #11: enum array default', async (t) => { + const testEnum = pgEnum('test_enum', ['a', 'b', 'c']); + + const from = { + enum: testEnum, + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + enum: testEnum, + test: pgTable('test', { + id: serial('id').primaryKey(), + values: testEnum('values').array().default(['a', 'b', 'c']), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'test_enum[]', + primaryKey: false, + notNull: false, + default: '\'{"a","b","c"}\'::test_enum[]', + }, + }); +}); + +test('array #12: enum empty array default', async (t) => { + const testEnum = pgEnum('test_enum', ['a', 'b', 'c']); + + const from = { + enum: testEnum, + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const to = { + enum: testEnum, + test: pgTable('test', { + id: serial('id').primaryKey(), + values: testEnum('values').array().default([]), + }), + }; + + const { statements } = await diffTestSchemas(from, to, []); + + expect(statements.length).toBe(1); + expect(statements[0]).toStrictEqual({ + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { + name: 'values', + type: 'test_enum[]', + primaryKey: false, + notNull: false, + default: "'{}'::test_enum[]", + }, + }); +}); diff --git a/drizzle-kit/tests/push/pg.test.ts b/drizzle-kit/tests/push/pg.test.ts index 1439d864e..cd5908bad 100644 --- a/drizzle-kit/tests/push/pg.test.ts +++ b/drizzle-kit/tests/push/pg.test.ts @@ -2162,3 +2162,77 @@ test('add identity to column - few params', async () => { // await client.query(st); // } }); + +test('add array column - empty array default', async () => { + const client = new PGlite(); + + const schema1 = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const schema2 = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: integer('values').array().default([]), + }), + }; + + const { statements, sqlStatements } = await diffTestSchemasPush( + client, + schema1, + schema2, + [], + false, + ['public'], + ); + + expect(statements).toStrictEqual([ + { + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { name: 'values', type: 'integer[]', primaryKey: false, notNull: false, default: "'{}'::integer[]" }, + }, + ]); + expect(sqlStatements).toStrictEqual([ + 'ALTER TABLE "test" ADD COLUMN "values" integer[] DEFAULT \'{}\'::integer[];', + ]); +}); + +test('add array column - default', async () => { + const client = new PGlite(); + + const schema1 = { + test: pgTable('test', { + id: serial('id').primaryKey(), + }), + }; + const schema2 = { + test: pgTable('test', { + id: serial('id').primaryKey(), + values: integer('values').array().default([1, 2, 3]), + }), + }; + + const { statements, sqlStatements } = await diffTestSchemasPush( + client, + schema1, + schema2, + [], + false, + ['public'], + ); + + expect(statements).toStrictEqual([ + { + type: 'alter_table_add_column', + tableName: 'test', + schema: '', + column: { name: 'values', type: 'integer[]', primaryKey: false, notNull: false, default: "'{1,2,3}'::integer[]" }, + }, + ]); + expect(sqlStatements).toStrictEqual([ + 'ALTER TABLE "test" ADD COLUMN "values" integer[] DEFAULT \'{1,2,3}\'::integer[];', + ]); +}); From 06107e4932f3e642a642926abae9bd51efafa479 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:36:44 +0100 Subject: [PATCH 16/28] Fix placeholders being mapped to it's driver value instead of the value --- drizzle-orm/src/sql/sql.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drizzle-orm/src/sql/sql.ts b/drizzle-orm/src/sql/sql.ts index 244a95d5d..3814c9aaa 100644 --- a/drizzle-orm/src/sql/sql.ts +++ b/drizzle-orm/src/sql/sql.ts @@ -203,7 +203,11 @@ export class SQL implements SQLWrapper { } if (is(chunk, Param)) { - const mappedValue = (chunk.value === null) ? null : chunk.encoder.mapToDriverValue(chunk.value); + if (is(chunk.value, Placeholder)) { + return { sql: escapeParam(paramStartIndex.value++, chunk), params: [chunk], typings: ['none'] }; + } + + const mappedValue = chunk.value === null ? null : chunk.encoder.mapToDriverValue(chunk.value); if (is(mappedValue, SQL)) { return this.buildQueryFromSourceParams([mappedValue], config); @@ -583,9 +587,18 @@ export function fillPlaceholders(params: unknown[], values: Record Date: Wed, 7 Aug 2024 12:51:32 +0100 Subject: [PATCH 17/28] style: use tabs instead of spaces --- integration-tests/tests/sqlite/sqlite-common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/sqlite/sqlite-common.ts b/integration-tests/tests/sqlite/sqlite-common.ts index 4f84f7111..7a2485582 100644 --- a/integration-tests/tests/sqlite/sqlite-common.ts +++ b/integration-tests/tests/sqlite/sqlite-common.ts @@ -845,7 +845,7 @@ export function tests() { const { db } = ctx.sqlite; const stmt = db.insert(usersTable).values({ - verified: sql.placeholder("verified"), + verified: sql.placeholder("verified"), name: sql.placeholder('name'), }).prepare(); From cc8d53bfed5a5bfda040f1c5ebf33e33d503d3b9 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:54:17 +0100 Subject: [PATCH 18/28] style: use ' instead of " --- integration-tests/tests/sqlite/sqlite-common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/sqlite/sqlite-common.ts b/integration-tests/tests/sqlite/sqlite-common.ts index 7a2485582..8a40f3df9 100644 --- a/integration-tests/tests/sqlite/sqlite-common.ts +++ b/integration-tests/tests/sqlite/sqlite-common.ts @@ -845,7 +845,7 @@ export function tests() { const { db } = ctx.sqlite; const stmt = db.insert(usersTable).values({ - verified: sql.placeholder("verified"), + verified: sql.placeholder('verified'), name: sql.placeholder('name'), }).prepare(); From 94cf6e90c3b66058cfe22f8fe76ad07b65e36026 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 14:51:40 +0100 Subject: [PATCH 19/28] revert sqlite prepared statement test --- .../tests/sqlite/sqlite-common.ts | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/integration-tests/tests/sqlite/sqlite-common.ts b/integration-tests/tests/sqlite/sqlite-common.ts index 8a40f3df9..3e25247a8 100644 --- a/integration-tests/tests/sqlite/sqlite-common.ts +++ b/integration-tests/tests/sqlite/sqlite-common.ts @@ -844,32 +844,28 @@ export function tests() { test('prepared statement reuse', async (ctx) => { const { db } = ctx.sqlite; - const stmt = db.insert(usersTable).values({ - verified: sql.placeholder('verified'), - name: sql.placeholder('name'), - }).prepare(); + const stmt = db.insert(usersTable).values({ name: sql.placeholder('name') }).prepare(); for (let i = 0; i < 10; i++) { - await stmt.run({ name: `John ${i}`, verified: i % 2 === 0 }); + await stmt.run({ name: `John ${i}` }); } const result = await db.select({ id: usersTable.id, name: usersTable.name, - verified: usersTable.verified, }).from(usersTable).all(); expect(result).toEqual([ - { id: 1, name: 'John 0', verified: true }, - { id: 2, name: 'John 1', verified: false }, - { id: 3, name: 'John 2', verified: true }, - { id: 4, name: 'John 3', verified: false }, - { id: 5, name: 'John 4', verified: true }, - { id: 6, name: 'John 5', verified: false }, - { id: 7, name: 'John 6', verified: true }, - { id: 8, name: 'John 7', verified: false }, - { id: 9, name: 'John 8', verified: true }, - { id: 10, name: 'John 9', verified: false }, + { id: 1, name: 'John 0' }, + { id: 2, name: 'John 1' }, + { id: 3, name: 'John 2' }, + { id: 4, name: 'John 3' }, + { id: 5, name: 'John 4' }, + { id: 6, name: 'John 5' }, + { id: 7, name: 'John 6' }, + { id: 8, name: 'John 7' }, + { id: 9, name: 'John 8' }, + { id: 10, name: 'John 9' }, ]); }); From b988d36da5343920fc704f56f587e7f1a63ce40f Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Wed, 7 Aug 2024 17:10:40 +0300 Subject: [PATCH 20/28] Up pg snapshots starting from v5 to latest(v7) --- drizzle-kit/src/api.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drizzle-kit/src/api.ts b/drizzle-kit/src/api.ts index 06f6dc1c0..9334c84b6 100644 --- a/drizzle-kit/src/api.ts +++ b/drizzle-kit/src/api.ts @@ -12,7 +12,7 @@ import { } from './cli/commands/migrate'; import { pgPushIntrospect } from './cli/commands/pgIntrospect'; import { pgSuggestions } from './cli/commands/pgPushUtils'; -import { updateUpToV6 as upPgV6 } from './cli/commands/pgUp'; +import { updateUpToV6 as upPgV6, updateUpToV7 as upPgV7 } from './cli/commands/pgUp'; import { sqlitePushIntrospect } from './cli/commands/sqliteIntrospect'; import { logSuggestionsAndReturn } from './cli/commands/sqlitePushUtils'; import { originUUID } from './global'; @@ -194,7 +194,7 @@ export const pushSQLiteSchema = async ( }, run: async (query: string) => { return Promise.resolve(drizzleInstance.run(sql.raw(query))).then( - () => {}, + () => { }, ); }, }; @@ -341,5 +341,11 @@ export const pushMySQLSchema = async ( }; export const upPgSnapshot = (snapshot: Record) => { - return upPgV6(snapshot); + if (snapshot.version === '5') { + return upPgV7(upPgV6(snapshot)) + } + if (snapshot.version === '6') { + return upPgV7(snapshot); + } + return snapshot; }; From ab77a45fcc729f3ad9da7be36ddc370e1f92cf07 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:25:21 +0100 Subject: [PATCH 21/28] add 'insert: placeholders on columns with encoder' tests --- integration-tests/tests/mysql/mysql-common.ts | 24 +++++++++++++++++++ integration-tests/tests/pg/pg-common.ts | 24 ++++++++++++++++++- .../tests/sqlite/sqlite-common.ts | 22 +++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/integration-tests/tests/mysql/mysql-common.ts b/integration-tests/tests/mysql/mysql-common.ts index 47a60c7c8..ce459e1b7 100644 --- a/integration-tests/tests/mysql/mysql-common.ts +++ b/integration-tests/tests/mysql/mysql-common.ts @@ -1155,6 +1155,30 @@ export function tests(driver?: string) { expect(result).toEqual([{ id: 1, name: 'John' }]); }); + test('insert: placeholders on columns with encoder', async (ctx) => { + const { db } = ctx.mysql; + + const date = new Date(); + + const statement = db.insert(usersTable).values({ + name: 'John', + createdAt: sql.placeholder('createdAt'), + }).prepare(); + + await statement.execute({ createdAt: date }); + + const result = await db + .select({ + id: usersTable.id, + createdAt: usersTable.createdAt, + }) + .from(usersTable); + + expect(result).toEqual([ + { id: 1, createdAt: date }, + ]); + }); + test('prepared statement reuse', async (ctx) => { const { db } = ctx.mysql; diff --git a/integration-tests/tests/pg/pg-common.ts b/integration-tests/tests/pg/pg-common.ts index fb69c5877..6332894e7 100644 --- a/integration-tests/tests/pg/pg-common.ts +++ b/integration-tests/tests/pg/pg-common.ts @@ -246,7 +246,7 @@ export function tests() { create table users ( id serial primary key, name text not null, - verified boolean not null default false, + verified boolean not null default false, jsonb jsonb, created_at timestamptz not null default now() ) @@ -1116,6 +1116,28 @@ export function tests() { expect(result).toEqual([{ id: 1, name: 'John' }]); }); + test('insert: placeholders on columns with encoder', async (ctx) => { + const { db } = ctx.pg; + + const statement = db.insert(usersTable).values({ + name: 'John', + jsonb: sql.placeholder('jsonb'), + }).prepare('encoder_statement'); + + await statement.execute({ jsonb: ['foo', 'bar'] }); + + const result = await db + .select({ + id: usersTable.id, + jsonb: usersTable.jsonb, + }) + .from(usersTable); + + expect(result).toEqual([ + { id: 1, jsonb: ['foo', 'bar'] }, + ]); + }); + test('prepared statement reuse', async (ctx) => { const { db } = ctx.pg; diff --git a/integration-tests/tests/sqlite/sqlite-common.ts b/integration-tests/tests/sqlite/sqlite-common.ts index 3e25247a8..be452bcf1 100644 --- a/integration-tests/tests/sqlite/sqlite-common.ts +++ b/integration-tests/tests/sqlite/sqlite-common.ts @@ -869,6 +869,28 @@ export function tests() { ]); }); + test('insert: placeholders on columns with encoder', async (ctx) => { + const { db } = ctx.sqlite; + + const stmt = db.insert(usersTable).values({ + name: 'John', + verified: sql.placeholder('verified'), + }).prepare(); + + await stmt.run({ verified: true }); + await stmt.run({ verified: false }); + + const result = await db.select({ + id: usersTable.id, + verified: usersTable.verified, + }).from(usersTable).all(); + + expect(result).toEqual([ + { id: 1, verified: true }, + { id: 2, verified: false }, + ]); + }); + test('prepared statement with placeholder in .where', async (ctx) => { const { db } = ctx.sqlite; From 75fb0e3e52ea78ed8310f0a652271fa8147d3ede Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:35:13 +0100 Subject: [PATCH 22/28] specify date epoch for mysql placeholder test --- integration-tests/tests/mysql/mysql-common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/mysql/mysql-common.ts b/integration-tests/tests/mysql/mysql-common.ts index ce459e1b7..70cc0e10c 100644 --- a/integration-tests/tests/mysql/mysql-common.ts +++ b/integration-tests/tests/mysql/mysql-common.ts @@ -1158,7 +1158,7 @@ export function tests(driver?: string) { test('insert: placeholders on columns with encoder', async (ctx) => { const { db } = ctx.mysql; - const date = new Date(); + const date = new Date(1723041271); const statement = db.insert(usersTable).values({ name: 'John', From 497e9da587afe734d69512120d4bfb833646e712 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:42:01 +0100 Subject: [PATCH 23/28] remove precision from date in placeholder encoder mysql test --- integration-tests/tests/mysql/mysql-common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/mysql/mysql-common.ts b/integration-tests/tests/mysql/mysql-common.ts index 70cc0e10c..07dab7dda 100644 --- a/integration-tests/tests/mysql/mysql-common.ts +++ b/integration-tests/tests/mysql/mysql-common.ts @@ -1158,7 +1158,7 @@ export function tests(driver?: string) { test('insert: placeholders on columns with encoder', async (ctx) => { const { db } = ctx.mysql; - const date = new Date(1723041271); + const date = new Date("2024-08-07T15:30:00Z"); const statement = db.insert(usersTable).values({ name: 'John', From df9e5962b301b3bac1554b389edb8c10720c8abd Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Wed, 7 Aug 2024 17:42:17 +0300 Subject: [PATCH 24/28] Format with dprint --- drizzle-kit/src/api.ts | 4 +- .../src/serializer/sqliteSerializer.ts | 105 ++++++++++-------- 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/drizzle-kit/src/api.ts b/drizzle-kit/src/api.ts index 9334c84b6..00cdb1b61 100644 --- a/drizzle-kit/src/api.ts +++ b/drizzle-kit/src/api.ts @@ -194,7 +194,7 @@ export const pushSQLiteSchema = async ( }, run: async (query: string) => { return Promise.resolve(drizzleInstance.run(sql.raw(query))).then( - () => { }, + () => {}, ); }, }; @@ -342,7 +342,7 @@ export const pushMySQLSchema = async ( export const upPgSnapshot = (snapshot: Record) => { if (snapshot.version === '5') { - return upPgV7(upPgV6(snapshot)) + return upPgV7(upPgV6(snapshot)); } if (snapshot.version === '6') { return upPgV7(snapshot); diff --git a/drizzle-kit/src/serializer/sqliteSerializer.ts b/drizzle-kit/src/serializer/sqliteSerializer.ts index 2dfdedcea..da4492278 100644 --- a/drizzle-kit/src/serializer/sqliteSerializer.ts +++ b/drizzle-kit/src/serializer/sqliteSerializer.ts @@ -65,8 +65,8 @@ export const generateSqliteSnapshot = ( as: is(generated.as, SQL) ? `(${dialect.sqlToQuery(generated.as as SQL, 'indexes').sql})` : typeof generated.as === 'function' - ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` - : `(${generated.as as any})`, + ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` + : `(${generated.as as any})`, type: generated.mode ?? 'virtual', } : undefined, @@ -79,9 +79,9 @@ export const generateSqliteSnapshot = ( columnToSet.default = typeof column.default === 'string' ? `'${column.default}'` : typeof column.default === 'object' - || Array.isArray(column.default) - ? `'${JSON.stringify(column.default)}'` - : column.default; + || Array.isArray(column.default) + ? `'${JSON.stringify(column.default)}'` + : column.default; } } columnsObject[column.name] = columnToSet; @@ -90,19 +90,24 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[column.uniqueName!]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${chalk.underline.blue( - tableName, - ) + `\n${ + withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${ + chalk.underline.blue( + tableName, + ) } table. - The unique constraint ${chalk.underline.blue( - column.uniqueName, - ) - } on the ${chalk.underline.blue( - column.name, - ) - } column is confilcting with a unique constraint name already defined for ${chalk.underline.blue( - existingUnique.columns.join(','), - ) + The unique constraint ${ + chalk.underline.blue( + column.uniqueName, + ) + } on the ${ + chalk.underline.blue( + column.name, + ) + } column is confilcting with a unique constraint name already defined for ${ + chalk.underline.blue( + existingUnique.columns.join(','), + ) } columns\n`) }`, ); @@ -197,21 +202,26 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[name]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${withStyle.errorWarning( - `We\'ve found duplicated unique constraint names in ${chalk.underline.blue( - tableName, - ) - } table. \nThe unique constraint ${chalk.underline.blue( - name, - ) - } on the ${chalk.underline.blue( - columnNames.join(','), + `\n${ + withStyle.errorWarning( + `We\'ve found duplicated unique constraint names in ${ + chalk.underline.blue( + tableName, + ) + } table. \nThe unique constraint ${ + chalk.underline.blue( + name, + ) + } on the ${ + chalk.underline.blue( + columnNames.join(','), + ) + } columns is confilcting with a unique constraint name already defined for ${ + chalk.underline.blue( + existingUnique.columns.join(','), + ) + } columns\n`, ) - } columns is confilcting with a unique constraint name already defined for ${chalk.underline.blue( - existingUnique.columns.join(','), - ) - } columns\n`, - ) }`, ); process.exit(1); @@ -454,26 +464,26 @@ export const fromDatabase = async ( default: columnDefault === null ? undefined : /^-?[\d.]+(?:e-?\d+)?$/.test(columnDefault) - ? Number(columnDefault) - : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( + ? Number(columnDefault) + : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( columnDefault, ) - ? `(${columnDefault})` - : columnDefault === 'false' - ? false - : columnDefault === 'true' - ? true - : columnDefault.startsWith("'") && columnDefault.endsWith("'") - ? columnDefault - // ? columnDefault.substring(1, columnDefault.length - 1) - : `(${columnDefault})`, + ? `(${columnDefault})` + : columnDefault === 'false' + ? false + : columnDefault === 'true' + ? true + : columnDefault.startsWith("'") && columnDefault.endsWith("'") + ? columnDefault + // ? columnDefault.substring(1, columnDefault.length - 1) + : `(${columnDefault})`, autoincrement: isAutoincrement, name: columnName, type: mapSqlToSqliteType(columnType), primaryKey: false, notNull: isNotNull, generated: tableToGeneratedColumnsInfo[tableName] - && tableToGeneratedColumnsInfo[tableName][columnName] + && tableToGeneratedColumnsInfo[tableName][columnName] ? { type: tableToGeneratedColumnsInfo[tableName][columnName].type, as: tableToGeneratedColumnsInfo[tableName][columnName].expression, @@ -570,10 +580,11 @@ export const fromDatabase = async ( const columnsTo = fkByTableName[`${tableName}_${id}`].columnsTo; fkByTableName[ `${tableName}_${id}` - ].name = `${tableName}_${columnsFrom.join( - '_', - ) - }_${refTableName}_${columnsTo.join('_')}_fk`; + ].name = `${tableName}_${ + columnsFrom.join( + '_', + ) + }_${refTableName}_${columnsTo.join('_')}_fk`; } for (const idx of Object.keys(fkByTableName)) { From 6158d8ba6baa08fa0a8a780de7910164b35983c6 Mon Sep 17 00:00:00 2001 From: veloii <85405932+veloii@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:45:58 +0100 Subject: [PATCH 25/28] lint --- integration-tests/tests/mysql/mysql-common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/mysql/mysql-common.ts b/integration-tests/tests/mysql/mysql-common.ts index 07dab7dda..58f7a1e2c 100644 --- a/integration-tests/tests/mysql/mysql-common.ts +++ b/integration-tests/tests/mysql/mysql-common.ts @@ -1158,7 +1158,7 @@ export function tests(driver?: string) { test('insert: placeholders on columns with encoder', async (ctx) => { const { db } = ctx.mysql; - const date = new Date("2024-08-07T15:30:00Z"); + const date = new Date('2024-08-07T15:30:00Z'); const statement = db.insert(usersTable).values({ name: 'John', From 86a8714f97ebde2d249952faf43e072717fab71c Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 8 Aug 2024 16:03:10 +0300 Subject: [PATCH 26/28] Add release notes --- changelogs/drizzle-kit/0.24.0.md | 24 ++++ changelogs/drizzle-orm/0.33.0.md | 60 ++++++++++ drizzle-kit/package.json | 2 +- .../src/serializer/sqliteSerializer.ts | 106 ++++++++---------- drizzle-orm/package.json | 2 +- 5 files changed, 133 insertions(+), 61 deletions(-) create mode 100644 changelogs/drizzle-kit/0.24.0.md create mode 100644 changelogs/drizzle-orm/0.33.0.md diff --git a/changelogs/drizzle-kit/0.24.0.md b/changelogs/drizzle-kit/0.24.0.md new file mode 100644 index 000000000..b766e018b --- /dev/null +++ b/changelogs/drizzle-kit/0.24.0.md @@ -0,0 +1,24 @@ +## Breaking changes (for SQLite users) + +#### Fixed [Composite primary key order is not consistent](https://github.com/drizzle-team/drizzle-kit-mirror/issues/342) by removing `sort` in SQLite and to be consistant with the same logic in PostgreSQL and MySQL + +The issue that may arise for SQLite users with any driver using composite primary keys is that the order in the database may differ from the Drizzle schema. + +- If you are using `push`, you **MAY** be prompted to update your table with a new order of columns in the composite primary key. You will need to either change it manually in the database or push the changes, but this may lead to data loss, etc. + +- If you are using `generate`, you **MAY** also be prompted to update your table with a new order of columns in the composite primary key. You can either keep that migration or skip it by emptying the SQL migration file. + +If nothing works for you and you are blocked, please reach out to me @AndriiSherman. I will try to help you! + + +## Bug fixes + +- [[BUG] When using double type columns, import is not inserted](https://github.com/drizzle-team/drizzle-kit-mirror/issues/403) - thanks @Karibash +- [[BUG] A number value is specified as the default for a column of type char](https://github.com/drizzle-team/drizzle-kit-mirror/issues/404) - thanks @Karibash +- [[BUG]: Array default in migrations are wrong](https://github.com/drizzle-team/drizzle-orm/issues/2621) - thanks @L-Mario564 +- [[FEATURE]: Simpler default array fields](https://github.com/drizzle-team/drizzle-orm/issues/2709) - thanks @L-Mario564 +- [[BUG]: drizzle-kit generate succeeds but generates invalid SQL for default([]) - Postgres](https://github.com/drizzle-team/drizzle-orm/issues/2432) - thanks @L-Mario564 +- [[BUG]: Incorrect type for array column default value](https://github.com/drizzle-team/drizzle-orm/issues/2334) - thanks @L-Mario564 +- [[BUG]: error: column is of type integer[] but default expression is of type integer](https://github.com/drizzle-team/drizzle-orm/issues/2224) - thanks @L-Mario564 +- [[BUG]: Default value in array generating wrong migration file](https://github.com/drizzle-team/drizzle-orm/issues/1003) - thanks @L-Mario564 +- [[BUG]: enum as array, not possible?](https://github.com/drizzle-team/drizzle-orm/issues/1564) - thanks @L-Mario564 \ No newline at end of file diff --git a/changelogs/drizzle-orm/0.33.0.md b/changelogs/drizzle-orm/0.33.0.md new file mode 100644 index 000000000..e8fa5a6c8 --- /dev/null +++ b/changelogs/drizzle-orm/0.33.0.md @@ -0,0 +1,60 @@ +## Breaking changes (for some of postgres.js users) + +#### Bugs fixed for this breaking change + +- [Open +[BUG]: jsonb always inserted as a json string when using postgres-js](https://github.com/drizzle-team/drizzle-orm/issues/724) +- [[BUG]: jsonb type on postgres implement incorrectly](https://github.com/drizzle-team/drizzle-orm/issues/1511) + +If you were using `postgres-js` with `jsonb` fields, you might have seen stringified objects in your database, while drizzle insert and select operations were working as expected. + +You need to convert those fields from strings to actual JSON objects. To do this, you can use the following query to update your database: + +**if you are using jsonb:** +```sql +update table_name +set jsonb_column = (jsonb_column #>> '{}')::jsonb; +``` + +**if you are using json:** +```sql +update table_name +set json_column = (json_column #>> '{}')::json; +``` + +We've tested it in several cases, and it worked well, but only if all stringified objects are arrays or objects. If you have primitives like strings, numbers, booleans, etc., you can use this query to update all the fields + +**if you are using jsonb:** +```sql +UPDATE table_name +SET jsonb_column = CASE + -- Convert to JSONB if it is a valid JSON object or array + WHEN jsonb_column #>> '{}' LIKE '{%' OR jsonb_column #>> '{}' LIKE '[%' THEN + (jsonb_column #>> '{}')::jsonb + ELSE + jsonb_column +END +WHERE + jsonb_column IS NOT NULL; +``` + +**if you are using json:** +```sql +UPDATE table_name +SET json_column = CASE + -- Convert to JSON if it is a valid JSON object or array + WHEN json_column #>> '{}' LIKE '{%' OR json_column #>> '{}' LIKE '[%' THEN + (json_column #>> '{}')::json + ELSE + json_column +END +WHERE json_column IS NOT NULL; +``` + +If nothing works for you and you are blocked, please reach out to me @AndriiSherman. I will try to help you! + +## Bug Fixes + +- [[BUG]: boolean mode not working with prepared statements (bettersqlite)](https://github.com/drizzle-team/drizzle-orm/issues/2568) - thanks @veloii +- [[BUG]: isTable helper function is not working](https://github.com/drizzle-team/drizzle-orm/issues/2672) - thanks @hajek-raven +- [[BUG]: Documentation is outdated on inArray and notInArray Methods](https://github.com/drizzle-team/drizzle-orm/issues/2690) - thanks @RemiPeruto \ No newline at end of file diff --git a/drizzle-kit/package.json b/drizzle-kit/package.json index c3f885722..25297e5b9 100644 --- a/drizzle-kit/package.json +++ b/drizzle-kit/package.json @@ -1,6 +1,6 @@ { "name": "drizzle-kit", - "version": "0.23.2", + "version": "0.24.0", "homepage": "https://orm.drizzle.team", "keywords": [ "drizzle", diff --git a/drizzle-kit/src/serializer/sqliteSerializer.ts b/drizzle-kit/src/serializer/sqliteSerializer.ts index c673daafb..d9fcc63f7 100644 --- a/drizzle-kit/src/serializer/sqliteSerializer.ts +++ b/drizzle-kit/src/serializer/sqliteSerializer.ts @@ -65,8 +65,8 @@ export const generateSqliteSnapshot = ( as: is(generated.as, SQL) ? `(${dialect.sqlToQuery(generated.as as SQL, 'indexes').sql})` : typeof generated.as === 'function' - ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` - : `(${generated.as as any})`, + ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` + : `(${generated.as as any})`, type: generated.mode ?? 'virtual', } : undefined, @@ -79,9 +79,9 @@ export const generateSqliteSnapshot = ( columnToSet.default = typeof column.default === 'string' ? `'${column.default}'` : typeof column.default === 'object' - || Array.isArray(column.default) - ? `'${JSON.stringify(column.default)}'` - : column.default; + || Array.isArray(column.default) + ? `'${JSON.stringify(column.default)}'` + : column.default; } } columnsObject[column.name] = columnToSet; @@ -90,24 +90,19 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[column.uniqueName!]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${ - withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${ - chalk.underline.blue( - tableName, - ) + `\n${withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${chalk.underline.blue( + tableName, + ) } table. - The unique constraint ${ - chalk.underline.blue( - column.uniqueName, - ) - } on the ${ - chalk.underline.blue( - column.name, - ) - } column is confilcting with a unique constraint name already defined for ${ - chalk.underline.blue( - existingUnique.columns.join(','), - ) + The unique constraint ${chalk.underline.blue( + column.uniqueName, + ) + } on the ${chalk.underline.blue( + column.name, + ) + } column is confilcting with a unique constraint name already defined for ${chalk.underline.blue( + existingUnique.columns.join(','), + ) } columns\n`) }`, ); @@ -202,26 +197,21 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[name]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${ - withStyle.errorWarning( - `We\'ve found duplicated unique constraint names in ${ - chalk.underline.blue( - tableName, - ) - } table. \nThe unique constraint ${ - chalk.underline.blue( - name, - ) - } on the ${ - chalk.underline.blue( - columnNames.join(','), - ) - } columns is confilcting with a unique constraint name already defined for ${ - chalk.underline.blue( - existingUnique.columns.join(','), - ) - } columns\n`, + `\n${withStyle.errorWarning( + `We\'ve found duplicated unique constraint names in ${chalk.underline.blue( + tableName, + ) + } table. \nThe unique constraint ${chalk.underline.blue( + name, + ) + } on the ${chalk.underline.blue( + columnNames.join(','), ) + } columns is confilcting with a unique constraint name already defined for ${chalk.underline.blue( + existingUnique.columns.join(','), + ) + } columns\n`, + ) }`, ); process.exit(1); @@ -237,7 +227,7 @@ export const generateSqliteSnapshot = ( primaryKeys.forEach((it) => { if (it.columns.length > 1) { primaryKeysObject[it.getName()] = { - columns: it.columns.map((it) => it.name).sort(), + columns: it.columns.map((it) => it.name), name: it.getName(), }; } else { @@ -464,26 +454,26 @@ export const fromDatabase = async ( default: columnDefault === null ? undefined : /^-?[\d.]+(?:e-?\d+)?$/.test(columnDefault) - ? Number(columnDefault) - : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( + ? Number(columnDefault) + : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( columnDefault, ) - ? `(${columnDefault})` - : columnDefault === 'false' - ? false - : columnDefault === 'true' - ? true - : columnDefault.startsWith("'") && columnDefault.endsWith("'") - ? columnDefault - // ? columnDefault.substring(1, columnDefault.length - 1) - : `(${columnDefault})`, + ? `(${columnDefault})` + : columnDefault === 'false' + ? false + : columnDefault === 'true' + ? true + : columnDefault.startsWith("'") && columnDefault.endsWith("'") + ? columnDefault + // ? columnDefault.substring(1, columnDefault.length - 1) + : `(${columnDefault})`, autoincrement: isAutoincrement, name: columnName, type: mapSqlToSqliteType(columnType), primaryKey: false, notNull: isNotNull, generated: tableToGeneratedColumnsInfo[tableName] - && tableToGeneratedColumnsInfo[tableName][columnName] + && tableToGeneratedColumnsInfo[tableName][columnName] ? { type: tableToGeneratedColumnsInfo[tableName][columnName].type, as: tableToGeneratedColumnsInfo[tableName][columnName].expression, @@ -509,7 +499,6 @@ export const fromDatabase = async ( for (const [key, value] of Object.entries(tableToPk)) { if (value.length > 1) { - value.sort(); result[key].compositePrimaryKeys = { [`${key}_${value.join('_')}_pk`]: { columns: value, @@ -580,10 +569,9 @@ export const fromDatabase = async ( const columnsTo = fkByTableName[`${tableName}_${id}`].columnsTo; fkByTableName[ `${tableName}_${id}` - ].name = `${tableName}_${ - columnsFrom.join( - '_', - ) + ].name = `${tableName}_${columnsFrom.join( + '_', + ) }_${refTableName}_${columnsTo.join('_')}_fk`; } diff --git a/drizzle-orm/package.json b/drizzle-orm/package.json index 9b0db78bc..888f7efcb 100644 --- a/drizzle-orm/package.json +++ b/drizzle-orm/package.json @@ -1,6 +1,6 @@ { "name": "drizzle-orm", - "version": "0.32.2", + "version": "0.33.0", "description": "Drizzle ORM package for SQL databases", "type": "module", "scripts": { From 6205f018f5667092bc0daec56cd0e8c7131048c3 Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 8 Aug 2024 16:12:11 +0300 Subject: [PATCH 27/28] Use dprint --- .../src/serializer/sqliteSerializer.ts | 103 ++++++++++-------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/drizzle-kit/src/serializer/sqliteSerializer.ts b/drizzle-kit/src/serializer/sqliteSerializer.ts index d9fcc63f7..ce544235b 100644 --- a/drizzle-kit/src/serializer/sqliteSerializer.ts +++ b/drizzle-kit/src/serializer/sqliteSerializer.ts @@ -65,8 +65,8 @@ export const generateSqliteSnapshot = ( as: is(generated.as, SQL) ? `(${dialect.sqlToQuery(generated.as as SQL, 'indexes').sql})` : typeof generated.as === 'function' - ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` - : `(${generated.as as any})`, + ? `(${dialect.sqlToQuery(generated.as() as SQL, 'indexes').sql})` + : `(${generated.as as any})`, type: generated.mode ?? 'virtual', } : undefined, @@ -79,9 +79,9 @@ export const generateSqliteSnapshot = ( columnToSet.default = typeof column.default === 'string' ? `'${column.default}'` : typeof column.default === 'object' - || Array.isArray(column.default) - ? `'${JSON.stringify(column.default)}'` - : column.default; + || Array.isArray(column.default) + ? `'${JSON.stringify(column.default)}'` + : column.default; } } columnsObject[column.name] = columnToSet; @@ -90,19 +90,24 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[column.uniqueName!]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${chalk.underline.blue( - tableName, - ) + `\n${ + withStyle.errorWarning(`We\'ve found duplicated unique constraint names in ${ + chalk.underline.blue( + tableName, + ) } table. - The unique constraint ${chalk.underline.blue( - column.uniqueName, - ) - } on the ${chalk.underline.blue( - column.name, - ) - } column is confilcting with a unique constraint name already defined for ${chalk.underline.blue( - existingUnique.columns.join(','), - ) + The unique constraint ${ + chalk.underline.blue( + column.uniqueName, + ) + } on the ${ + chalk.underline.blue( + column.name, + ) + } column is confilcting with a unique constraint name already defined for ${ + chalk.underline.blue( + existingUnique.columns.join(','), + ) } columns\n`) }`, ); @@ -197,21 +202,26 @@ export const generateSqliteSnapshot = ( const existingUnique = indexesObject[name]; if (typeof existingUnique !== 'undefined') { console.log( - `\n${withStyle.errorWarning( - `We\'ve found duplicated unique constraint names in ${chalk.underline.blue( - tableName, - ) - } table. \nThe unique constraint ${chalk.underline.blue( - name, - ) - } on the ${chalk.underline.blue( - columnNames.join(','), + `\n${ + withStyle.errorWarning( + `We\'ve found duplicated unique constraint names in ${ + chalk.underline.blue( + tableName, + ) + } table. \nThe unique constraint ${ + chalk.underline.blue( + name, + ) + } on the ${ + chalk.underline.blue( + columnNames.join(','), + ) + } columns is confilcting with a unique constraint name already defined for ${ + chalk.underline.blue( + existingUnique.columns.join(','), + ) + } columns\n`, ) - } columns is confilcting with a unique constraint name already defined for ${chalk.underline.blue( - existingUnique.columns.join(','), - ) - } columns\n`, - ) }`, ); process.exit(1); @@ -454,26 +464,26 @@ export const fromDatabase = async ( default: columnDefault === null ? undefined : /^-?[\d.]+(?:e-?\d+)?$/.test(columnDefault) - ? Number(columnDefault) - : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( + ? Number(columnDefault) + : ['CURRENT_TIME', 'CURRENT_DATE', 'CURRENT_TIMESTAMP'].includes( columnDefault, ) - ? `(${columnDefault})` - : columnDefault === 'false' - ? false - : columnDefault === 'true' - ? true - : columnDefault.startsWith("'") && columnDefault.endsWith("'") - ? columnDefault - // ? columnDefault.substring(1, columnDefault.length - 1) - : `(${columnDefault})`, + ? `(${columnDefault})` + : columnDefault === 'false' + ? false + : columnDefault === 'true' + ? true + : columnDefault.startsWith("'") && columnDefault.endsWith("'") + ? columnDefault + // ? columnDefault.substring(1, columnDefault.length - 1) + : `(${columnDefault})`, autoincrement: isAutoincrement, name: columnName, type: mapSqlToSqliteType(columnType), primaryKey: false, notNull: isNotNull, generated: tableToGeneratedColumnsInfo[tableName] - && tableToGeneratedColumnsInfo[tableName][columnName] + && tableToGeneratedColumnsInfo[tableName][columnName] ? { type: tableToGeneratedColumnsInfo[tableName][columnName].type, as: tableToGeneratedColumnsInfo[tableName][columnName].expression, @@ -569,9 +579,10 @@ export const fromDatabase = async ( const columnsTo = fkByTableName[`${tableName}_${id}`].columnsTo; fkByTableName[ `${tableName}_${id}` - ].name = `${tableName}_${columnsFrom.join( - '_', - ) + ].name = `${tableName}_${ + columnsFrom.join( + '_', + ) }_${refTableName}_${columnsTo.join('_')}_fk`; } From f2a2b5f17b5a01819cb0dcbe319a0678064331bc Mon Sep 17 00:00:00 2001 From: AndriiSherman Date: Thu, 8 Aug 2024 16:53:05 +0300 Subject: [PATCH 28/28] Update release notes --- changelogs/drizzle-orm/0.33.0.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelogs/drizzle-orm/0.33.0.md b/changelogs/drizzle-orm/0.33.0.md index e8fa5a6c8..0093c9d05 100644 --- a/changelogs/drizzle-orm/0.33.0.md +++ b/changelogs/drizzle-orm/0.33.0.md @@ -6,6 +6,10 @@ [BUG]: jsonb always inserted as a json string when using postgres-js](https://github.com/drizzle-team/drizzle-orm/issues/724) - [[BUG]: jsonb type on postgres implement incorrectly](https://github.com/drizzle-team/drizzle-orm/issues/1511) +> As we are doing with other drivers, we've changed the behavior of PostgreSQL-JS to pass raw JSON values, the same as you see them in the database. So if you are using the PostgreSQL-JS driver and passing data to Drizzle elsewhere, please check the new behavior of the client after it is passed to Drizzle. + +> We will update it to ensure it does not override driver behaviors, but this will be done as a complex task for everything in Drizzle in other releases + If you were using `postgres-js` with `jsonb` fields, you might have seen stringified objects in your database, while drizzle insert and select operations were working as expected. You need to convert those fields from strings to actual JSON objects. To do this, you can use the following query to update your database: