diff --git a/.changeset/cyan-doors-tap.md b/.changeset/cyan-doors-tap.md new file mode 100644 index 0000000..9de2bba --- /dev/null +++ b/.changeset/cyan-doors-tap.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-sync-rules': minor +'@powersync/service-core': patch +'powersync-open-service': patch +--- + +Support expressions on request parameters in parameter queries. diff --git a/.github/workflows/development_image_release.yaml b/.github/workflows/development_image_release.yaml index cc4ff2a..466aba3 100644 --- a/.github/workflows/development_image_release.yaml +++ b/.github/workflows/development_image_release.yaml @@ -35,7 +35,7 @@ jobs: with: node-version-file: '.nvmrc' - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: 9 diff --git a/.github/workflows/development_packages_release.yaml b/.github/workflows/development_packages_release.yaml index 9b67b4c..a02abcb 100644 --- a/.github/workflows/development_packages_release.yaml +++ b/.github/workflows/development_packages_release.yaml @@ -21,7 +21,7 @@ jobs: with: node-version-file: '.nvmrc' - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: 9 diff --git a/.github/workflows/packages_release.yaml b/.github/workflows/packages_release.yaml index a7c73ce..224c745 100644 --- a/.github/workflows/packages_release.yaml +++ b/.github/workflows/packages_release.yaml @@ -20,7 +20,7 @@ jobs: with: node-version-file: '.nvmrc' - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: 9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2712005..9c2b757 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: with: node-version-file: '.nvmrc' - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 name: Install pnpm with: version: 9 diff --git a/packages/sync-rules/README.md b/packages/sync-rules/README.md index 7b8b07a..3e13a81 100644 --- a/packages/sync-rules/README.md +++ b/packages/sync-rules/README.md @@ -1,3 +1,129 @@ # powersync-sync-rules A library containing logic for PowerSync sync rules. + +This is not intended to be used directly by users of PowerSync. If you are interested in the internals, read on. + +# Overview + +A core design constraint is that sync rules define two operations: + +1. Given a data row, compute a list of buckets that it belongs to. +2. Given an authenticated user, return a list of buckets for the user. + +This implementation of sync rules use SQL queries to declaratively define those operations using familiar SQL operations. + +We define (1) using data queries, and (2) using parameter queries. + +Example: + +```yaml +bucket_definitions: + by_org: + # parameter query + # This defines bucket parameters are `bucket.org_id` + parameters: select org_id from users where id = token_parameters.user_id + # data query + data: + - select * from documents where org_id = bucket.org_id +``` + +For the above example, a document with `org_id: 'org1'` will belong to a single bucket `by_org["org1"]`. Similarly, a user with `org_id: 'org1'` will sync the bucket `by_org["org1"]`. + +An important aspect is that none of these SQL queries are actually executed against any SQL database. Instead, it is used to pre-process data before storing the data in a format for efficient sync operations. + +When data is replicated from the source database to PowerSync, we do two things for each row: + +1. Evaluate data queries on the row: `syncRules.evaluateRow(row)`. +2. Evaluate parameter queries on the row: `syncRules.evaluateParameterRow(row)`. + +Data queries also have the option to transform the row instead of just using `select *`. We store the transformed data for each of the buckets it belongs to. + +# Query Structure + +## Data queries + +A data query is turned into a function `(row) => Array<{bucket, data}>`. The main implementation is in the `SqlDataQuery` class. + +The main clauses in a data query are the ones comparing bucket parameters, for example `WHERE documents.document_org_id = bucket.bucket_org_id`. In this case, a document with `document_org_id: 'org1'` will have a bucket parameter of `bucket_org_id: 'org1'`. + +A data query must match each bucket parameter. To be able to always compute the bucket ids, there are major limitations on the operators supported with bucket parameters, as well as how expressions can be combined using AND and OR. + +The WHERE clause of a data query is compiled into a `ParameterMatchClause`. + +Query clauses are structured as follows: + +```SQL +'literal' -- StaticValueClause +mytable.column -- RowValueClause +fn(mytable.column) -- RowValueClause. This includes most operators. +bucket.param -- ParameterValueClause +fn(bucket.param) -- Error: not allowed + +mytable.column = mytable.other_column -- RowValueClause +mytable.column = bucket.param -- ParameterMatchClause +bucket.param IN mytable.some_array -- ParameterMatchClause +(mytable.column1 = bucket.param1) AND (mytable.column2 = bucket.param2) -- ParameterMatchClause +(mytable.column1 = bucket.param) OR (mytable.column2 = bucket.param) -- ParameterMatchClause +``` + +## Parameter Queries + +There are two types of parameter queries: + +1. Queries without tables. These just operate on request parameters. Example: `select token_parameters.user_id`. Thes are implemented in the `StaticSqlParameterQuery` class. +2. Queries with tables. Example: `select org_id from users where id = token_parameters.user_id`. These use parameter tables, and are implemented in `SqlParameterQuery`. These are used to pre-process rows in the parameter tables for efficient lookup later. + +### StaticSqlParameterQuery + +These are effecitively just a function of `(request) => Array[{bucket}]`. These queries can select values from request parameters, and apply filters from request parameters. + +The WHERE filter is a ParameterMatchClause that operates on the request parameters. +The bucket parameters are each a RowValueClause that operates on the request parameters. + +Compiled expression clauses are structured as follows: + +```SQL +'literal' -- StaticValueClause +token_parameters.param -- RowValueClause +fn(token_parameters.param) -- RowValueClause. This includes most operators. +``` + +The implementation may be refactored to be more consistent with `SqlParameterQuery` in the future - using `RowValueClause` for request parameters is not ideal. + +### SqlParameterQuery + +These queries pre-process parameter tables to effectively create an "index" for efficient queries when syncing. + +For a parameter query `select org_id from users where users.org_id = token_parameters.org_id and lower(users.email) = token_parameters.email`, this would effectively create an index on `users.org_id, lower(users.email)`. These indexes are referred to as "lookup" values. Only direct equality lookups are supported on these indexes currently (including the IN operator). Support for more general queries such as "greater than" operators may be added later. + +A SqlParameterQuery defines the following operations: + +1. `evaluateParameterRow(row)`: Given a parameter row, compute the lookup index entries. +2. `getLookups(request)`: Given request parameters, compute the lookup index entries we need to find. +3. `queryBucketIds(request)`: Uses `getLookups(request)`, combined with a database lookup, to compute bucket ids from request parameters. + +The compiled query is based on the following: + +1. WHERE clause compiled into a `ParameterMatchClause`. This computes the lookup index. +2. `lookup_extractors`: Set of `RowValueClause`. Each of these represent a SELECT clause based on a row value, e.g. `SELECT users.org_id`. These are evaluated during the `evaluateParameterRow` call. +3. `static_extractors`. Set of `RowValueClause`. Each of these represent a SELECT clause based on a request parameter, e.g. `SELECT token_parameters.user_id`. These are evaluated during the `queryBucketIds` call. + +Compiled expression clauses are structured as follows: + +```SQL +'literal' -- StaticValueClause +mytable.column -- RowValueClause +fn(mytable.column) -- RowValueClause. This includes most operators. +token_parameters.param -- ParameterValueClause +fn(token_parameters.param) -- ParameterValueClause +fn(mytable.column, token_parameters.param) -- Error: not allowed + +mytable.column = mytable.other_column -- RowValueClause +mytable.column = token_parameters.param -- ParameterMatchClause +token_parameters.param IN mytable.some_array -- ParameterMatchClause +mytable.some_value IN token_parameters.some_array -- ParameterMatchClause + +(mytable.column1 = token_parameters.param1) AND (mytable.column2 = token_parameters.param2) -- ParameterMatchClause +(mytable.column1 = token_parameters.param) OR (mytable.column2 = token_parameters.param) -- ParameterMatchClause +``` diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index 91bba07..ac7ef3c 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -78,16 +78,18 @@ export class SqlDataQuery { }); const filter = tools.compileWhereClause(where); - const allParams = new Set([...filter.bucketParameters!, ...bucket_parameters.map((p) => `bucket.${p}`)]); + const inputParameterNames = filter.inputParameters!.map((p) => p.key); + const bucketParameterNames = bucket_parameters.map((p) => `bucket.${p}`); + const allParams = new Set([...inputParameterNames, ...bucketParameterNames]); if ( - (!filter.error && allParams.size != filter.bucketParameters!.length) || + (!filter.error && allParams.size != filter.inputParameters!.length) || allParams.size != bucket_parameters.length ) { rows.errors.push( new SqlRuleError( - `Query must cover all bucket parameters: ${JSONBig.stringify(bucket_parameters)} != ${JSONBig.stringify( - filter.bucketParameters - )}`, + `Query must cover all bucket parameters. Expected: ${JSONBig.stringify( + bucketParameterNames + )} Got: ${JSONBig.stringify(inputParameterNames)}`, sql, q._location ) @@ -196,7 +198,7 @@ export class SqlDataQuery { evaluateRow(table: SourceTableInterface, row: SqliteRow): EvaluationResult[] { try { const tables = { [this.table!]: this.addSpecialParameters(table, row) }; - const bucketParameters = this.filter!.filter(tables); + const bucketParameters = this.filter!.filterRow(tables); const bucketIds = bucketParameters.map((params) => getBucketId(this.descriptor_name!, this.bucket_parameters!, params) ); diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 784dc5e..ed9e280 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -2,6 +2,8 @@ import { parse, SelectedColumn } from 'pgsql-ast-parser'; import { EvaluatedParameters, EvaluatedParametersResult, + FilterParameters, + InputParameter, ParameterMatchClause, QueryBucketIdOptions, QuerySchema, @@ -9,7 +11,7 @@ import { SqliteJsonRow, SqliteJsonValue, SqliteRow, - StaticRowValueClause, + RowValueClause, SyncParameters } from './types.js'; import { SqlRuleError } from './errors.js'; @@ -89,6 +91,7 @@ export class SqlParameterQuery { parameter_tables: ['token_parameters', 'user_parameters'], sql, supports_expanding_parameters: true, + supports_parameter_expressions: true, schema: querySchema }); const where = q.where; @@ -101,8 +104,8 @@ export class SqlParameterQuery { rows.filter = filter; rows.descriptor_name = descriptor_name; rows.bucket_parameters = bucket_parameters; - rows.input_parameters = filter.bucketParameters!; - const expandedParams = rows.input_parameters.filter((param) => param.endsWith('[*]')); + rows.input_parameters = filter.inputParameters!; + const expandedParams = rows.input_parameters!.filter((param) => param.expands); if (expandedParams.length > 1) { rows.errors.push(new SqlRuleError('Cannot have multiple array input parameters', sql)); } @@ -152,17 +155,30 @@ export class SqlParameterQuery { lookup_columns?: SelectedColumn[]; static_columns?: SelectedColumn[]; - lookup_extractors: Record = {}; - static_extractors: Record = {}; + /** + * Example: SELECT *user.id* FROM users WHERE ... + */ + lookup_extractors: Record = {}; + + /** + * Example: SELECT *token_parameters.user_id* + */ + static_extractors: Record = {}; filter?: ParameterMatchClause; descriptor_name?: string; + /** _Input_ token / user parameters */ - input_parameters?: string[]; + input_parameters?: InputParameter[]; - expanded_input_parameter?: string; + /** If specified, an input parameter that expands to an array. */ + expanded_input_parameter?: InputParameter; - /** _Output_ bucket parameters */ + /** + * _Output_ bucket parameters. + * + * Each one of these will be present in either lookup_extractors or static_extractors. + */ bucket_parameters?: string[]; id?: string; @@ -182,13 +198,13 @@ export class SqlParameterQuery { [this.table!]: row }; try { - const filterParameters = this.filter!.filter(tables); + const filterParameters = this.filter!.filterRow(tables); let result: EvaluatedParametersResult[] = []; for (let filterParamSet of filterParameters) { let lookup: SqliteJsonValue[] = [this.descriptor_name!, this.id!]; lookup.push( ...this.input_parameters!.map((param) => { - return filterParamSet[param]; + return param.filteredRowToLookupValue(filterParamSet); }) ); @@ -216,6 +232,9 @@ export class SqlParameterQuery { return [result]; } + /** + * Given partial parameter rows, turn into bucket ids. + */ resolveBucketIds(bucketParameters: SqliteJsonRow[], parameters: SyncParameters): string[] { const tables = { token_parameters: parameters.token_parameters, user_parameters: parameters.user_parameters }; @@ -246,28 +265,36 @@ export class SqlParameterQuery { .filter((lookup) => lookup != null) as string[]; } - lookupParam(param: string, parameters: SyncParameters) { - const [table, column] = param.split('.'); - const pt: SqliteJsonRow | undefined = (parameters as any)[table]; - return pt?.[column] ?? null; - } - + /** + * Given sync parameters, get lookups we need to perform on the database. + * + * Each lookup is [bucket definition name, parameter query index, ...lookup values] + */ getLookups(parameters: SyncParameters): SqliteJsonValue[][] { if (!this.expanded_input_parameter) { let lookup: SqliteJsonValue[] = [this.descriptor_name!, this.id!]; + let valid = true; lookup.push( ...this.input_parameters!.map((param): SqliteJsonValue => { // Scalar value - return this.lookupParam(param, parameters); + const value = param.parametersToLookupValue(parameters); + + if (isJsonValue(value)) { + return value; + } else { + valid = false; + return null; + } }) ); + if (!valid) { + return []; + } return [lookup]; } else { - const arrayString = this.lookupParam( - this.expanded_input_parameter.substring(0, this.expanded_input_parameter.length - 3), - parameters - ); + const arrayString = this.expanded_input_parameter.parametersToLookupValue(parameters); + if (arrayString == null || typeof arrayString != 'string') { return []; } @@ -281,26 +308,46 @@ export class SqlParameterQuery { return []; } - return values.map((expandedValue): SqliteJsonValue[] => { - let lookup: SqliteJsonValue[] = [this.descriptor_name!, this.id!]; - - lookup.push( - ...this.input_parameters!.map((param): SqliteJsonValue => { - if (param == this.expanded_input_parameter) { - // Expand array value - return expandedValue; - } else { - // Scalar value - return this.lookupParam(param, parameters); - } - }) - ); + return values + .map((expandedValue) => { + let lookup: SqliteJsonValue[] = [this.descriptor_name!, this.id!]; + let valid = true; + lookup.push( + ...this.input_parameters!.map((param): SqliteJsonValue => { + if (param == this.expanded_input_parameter) { + // Expand array value + return expandedValue; + } else { + // Scalar value + const value = param.parametersToLookupValue(parameters); + + if (isJsonValue(value)) { + return value; + } else { + valid = false; + return null; + } + } + }) + ); + if (!valid) { + return null; + } - return lookup; - }); + return lookup; + }) + .filter((lookup) => lookup != null) as SqliteJsonValue[][]; } } + /** + * Given sync parameters (token and user parameters), return bucket ids. + * + * This is done in three steps: + * 1. Given the parameters, get lookups we need to perform on the database. + * 2. Perform the lookups, returning parameter sets (partial rows). + * 3. Given the parameter sets, resolve bucket ids. + */ async queryBucketIds(options: QueryBucketIdOptions): Promise { let lookups = this.getLookups(options.parameters); if (lookups.length == 0) { diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 8a38255..ab3b041 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -1,5 +1,5 @@ import { SelectedColumn, SelectFromStatement } from 'pgsql-ast-parser'; -import { ParameterMatchClause, SqliteJsonValue, StaticRowValueClause, SyncParameters } from './types.js'; +import { ParameterMatchClause, SqliteJsonValue, RowValueClause, SyncParameters } from './types.js'; import { SqlTools } from './sql_filters.js'; import { getBucketId, isJsonValue } from './utils.js'; import { SqlRuleError } from './errors.js'; @@ -51,7 +51,7 @@ export class StaticSqlParameterQuery { sql?: string; columns?: SelectedColumn[]; - static_extractors: Record = {}; + static_extractors: Record = {}; descriptor_name?: string; /** _Output_ bucket parameters */ bucket_parameters?: string[]; @@ -64,7 +64,7 @@ export class StaticSqlParameterQuery { getStaticBucketIds(parameters: SyncParameters): string[] { const tables = { token_parameters: parameters.token_parameters, user_parameters: parameters.user_parameters }; - const filtered = this.filter!.filter(tables); + const filtered = this.filter!.filterRow(tables); if (filtered.length == 0) { return []; } diff --git a/packages/sync-rules/src/sql_filters.ts b/packages/sync-rules/src/sql_filters.ts index f095748..8a0a8fe 100644 --- a/packages/sync-rules/src/sql_filters.ts +++ b/packages/sync-rules/src/sql_filters.ts @@ -1,32 +1,51 @@ import { Expr, ExprRef, NodeLocation, SelectedColumn } from 'pgsql-ast-parser'; +import { nil } from 'pgsql-ast-parser/src/utils.js'; +import { ExpressionType, SqliteType, TYPE_NONE } from './ExpressionType.js'; import { SqlRuleError } from './errors.js'; -import { BASIC_OPERATORS, cast, CAST_TYPES, jsonExtract, SQL_FUNCTIONS, sqliteTypeOf } from './sql_functions.js'; import { - ClauseError, - CompiledClause, - ParameterMatchClause, - QueryParameters, - QuerySchema, - SqliteValue, - StaticRowValueClause, - TrueIfParametersMatch -} from './types.js'; -import { nil } from 'pgsql-ast-parser/src/utils.js'; -import { isJsonValue } from './utils.js'; + BASIC_OPERATORS, + CAST_TYPES, + OPERATOR_IS_NOT_NULL, + OPERATOR_IS_NULL, + OPERATOR_JSON_EXTRACT_JSON, + OPERATOR_JSON_EXTRACT_SQL, + OPERATOR_NOT, + SQL_FUNCTIONS, + SqlFunction, + cast, + castOperator, + sqliteTypeOf +} from './sql_functions.js'; import { + SQLITE_FALSE, + SQLITE_TRUE, andFilters, compileStaticOperator, + getOperatorFunction, isClauseError, isParameterMatchClause, isParameterValueClause, - isStaticRowValueClause, + isRowValueClause, + isStaticValueClause, orFilters, - SQLITE_FALSE, - SQLITE_TRUE, sqliteNot, toBooleanParameterSetClause } from './sql_support.js'; -import { ExpressionType, SqliteType, TYPE_NONE } from './ExpressionType.js'; +import { + ClauseError, + CompiledClause, + InputParameter, + ParameterMatchClause, + ParameterValueClause, + QueryParameters, + QuerySchema, + SqliteJsonRow, + SqliteValue, + RowValueClause, + StaticValueClause, + TrueIfParametersMatch +} from './types.js'; +import { isJsonValue } from './utils.js'; export const MATCH_CONST_FALSE: TrueIfParametersMatch = []; export const MATCH_CONST_TRUE: TrueIfParametersMatch = [{}]; @@ -70,17 +89,28 @@ export interface SqlToolsOptions { */ supports_expanding_parameters?: boolean; + /** + * true if expressions on parameters are supported, e.g. upper(token_parameters.user_id) + */ + supports_parameter_expressions?: boolean; + schema?: QuerySchema; } export class SqlTools { default_table?: string; value_tables: string[]; + /** + * ['bucket'] for data queries + * ['token_parameters', 'user_parameters'] for parameter queries + */ parameter_tables: string[]; sql: string; errors: SqlRuleError[] = []; supports_expanding_parameters: boolean; + supports_parameter_expressions: boolean; + schema?: QuerySchema; constructor(options: SqlToolsOptions) { @@ -97,6 +127,7 @@ export class SqlTools { this.parameter_tables = options.parameter_tables ?? []; this.sql = options.sql; this.supports_expanding_parameters = options.supports_expanding_parameters ?? false; + this.supports_parameter_expressions = options.supports_parameter_expressions ?? false; } error(message: string, expr: NodeLocation | Expr | undefined): ClauseError { @@ -121,9 +152,9 @@ export class SqlTools { return toBooleanParameterSetClause(base); } - compileStaticExtractor(expr: Expr | nil): StaticRowValueClause | ClauseError { + compileStaticExtractor(expr: Expr | nil): RowValueClause | ClauseError { const clause = this.compileClause(expr); - if (!isStaticRowValueClause(clause) && !isClauseError(clause)) { + if (!isRowValueClause(clause) && !isClauseError(clause)) { throw new SqlRuleError('Bucket parameters are not allowed here', this.sql, expr ?? undefined); } return clause; @@ -134,20 +165,10 @@ export class SqlTools { */ compileClause(expr: Expr | nil): CompiledClause { if (expr == null) { - return { - evaluate: () => SQLITE_TRUE, - getType() { - return ExpressionType.INTEGER; - } - }; + return staticValueClause(SQLITE_TRUE); } else if (isStatic(expr)) { const value = staticValue(expr); - return { - evaluate: () => value, - getType() { - return ExpressionType.fromTypeText(sqliteTypeOf(value)); - } - }; + return staticValueClause(value); } else if (expr.type == 'ref') { const column = expr.name; if (column == '*') { @@ -157,8 +178,7 @@ export class SqlTools { return this.error(`Schema is not supported in column references`, expr); } if (this.isParameterRef(expr)) { - const param = this.getParameterRef(expr)!; - return { bucketParameter: param }; + return this.getParameterRefClause(expr); } else if (this.isTableRef(expr)) { const table = this.getTableName(expr); this.checkRef(table, expr); @@ -169,7 +189,7 @@ export class SqlTools { getType(schema) { return schema.getType(table, column); } - }; + } satisfies RowValueClause; } else { const ref = [(expr as ExprRef).table?.schema, (expr as ExprRef).table?.name, (expr as ExprRef).name] .filter((e) => e != null) @@ -181,7 +201,7 @@ export class SqlTools { const leftFilter = this.compileClause(left); const rightFilter = this.compileClause(right); if (isClauseError(leftFilter) || isClauseError(rightFilter)) { - return { error: true } as ClauseError; + return { error: true } satisfies ClauseError; } if (op == 'AND') { @@ -202,31 +222,33 @@ export class SqlTools { // 2. static, parameterValue // 3. static true, parameterMatch - not supported yet - let staticFilter1: StaticRowValueClause; + let staticFilter1: RowValueClause; let otherFilter1: CompiledClause; - if (!isStaticRowValueClause(leftFilter) && !isStaticRowValueClause(rightFilter)) { + if (!isRowValueClause(leftFilter) && !isRowValueClause(rightFilter)) { return this.error(`Cannot have bucket parameters on both sides of = operator`, expr); - } else if (isStaticRowValueClause(leftFilter)) { + } else if (isRowValueClause(leftFilter)) { staticFilter1 = leftFilter; otherFilter1 = rightFilter; } else { - staticFilter1 = rightFilter as StaticRowValueClause; + staticFilter1 = rightFilter as RowValueClause; otherFilter1 = leftFilter; } const staticFilter = staticFilter1; const otherFilter = otherFilter1; - if (isStaticRowValueClause(otherFilter)) { - // 1. static, static - return compileStaticOperator(op, leftFilter as StaticRowValueClause, rightFilter as StaticRowValueClause); + if (isRowValueClause(otherFilter)) { + // 1. static = static + return compileStaticOperator(op, leftFilter as RowValueClause, rightFilter as RowValueClause); } else if (isParameterValueClause(otherFilter)) { - // 2. static, parameterValue + // 2. static = parameterValue + const inputParam = basicInputParameter(otherFilter); + return { error: false, - bucketParameters: [otherFilter.bucketParameter], + inputParameters: [inputParam], unbounded: false, - filter(tables: QueryParameters): TrueIfParametersMatch { + filterRow(tables: QueryParameters): TrueIfParametersMatch { const value = staticFilter.evaluate(tables); if (value == null) { // null never matches on = @@ -237,18 +259,15 @@ export class SqlTools { // Cannot persist this, e.g. BLOB return MATCH_CONST_FALSE; } - return [{ [otherFilter.bucketParameter]: value }]; + + return [{ [inputParam.key]: value }]; } - }; + } satisfies ParameterMatchClause; } else if (isParameterMatchClause(otherFilter)) { // 3. static, parameterMatch // (bucket.param = 'something') = staticValue // To implement this, we need to ensure the static value here can only be true. - return this.error( - `Bucket parameter clauses cannot currently be combined with other operators`, - - expr - ); + return this.error(`Parameter match clauses cannot be used here`, expr); } else { throw new Error('Unexpected'); } @@ -257,15 +276,19 @@ export class SqlTools { // static IN static // parameterValue IN static - if (isStaticRowValueClause(leftFilter) && isStaticRowValueClause(rightFilter)) { + if (isRowValueClause(leftFilter) && isRowValueClause(rightFilter)) { + // static1 IN static2 return compileStaticOperator(op, leftFilter, rightFilter); - } else if (isParameterValueClause(leftFilter) && isStaticRowValueClause(rightFilter)) { - const param = leftFilter.bucketParameter; + } else if (isParameterValueClause(leftFilter) && isRowValueClause(rightFilter)) { + // token_parameters.value IN table.some_array + // bucket.param IN table.some_array + const inputParam = basicInputParameter(leftFilter); + return { error: false, - bucketParameters: [param], + inputParameters: [inputParam], unbounded: true, - filter(tables: QueryParameters): TrueIfParametersMatch { + filterRow(tables: QueryParameters): TrueIfParametersMatch { const aValue = rightFilter.evaluate(tables); if (aValue == null) { return MATCH_CONST_FALSE; @@ -275,182 +298,102 @@ export class SqlTools { throw new Error('Not an array'); } return values.map((value) => { - return { [param]: value }; + return { [inputParam.key]: value }; }); } - }; + } satisfies ParameterMatchClause; } else if ( this.supports_expanding_parameters && - isStaticRowValueClause(leftFilter) && + isRowValueClause(leftFilter) && isParameterValueClause(rightFilter) ) { - const param = `${rightFilter.bucketParameter}[*]`; + // table.some_value IN token_parameters.some_array + // This expands into "table_some_value = " for each value of the array. + // We only support one such filter per query + const key = `${rightFilter.key}[*]`; + + const inputParam: InputParameter = { + key: key, + expands: true, + filteredRowToLookupValue: (filterParameters) => { + return filterParameters[key]; + }, + parametersToLookupValue: (parameters) => { + return rightFilter.lookupParameterValue(parameters); + } + }; + return { error: false, - bucketParameters: [param], + inputParameters: [inputParam], unbounded: false, - filter(tables: QueryParameters): TrueIfParametersMatch { + filterRow(tables: QueryParameters): TrueIfParametersMatch { const value = leftFilter.evaluate(tables); if (!isJsonValue(value)) { // Cannot persist, e.g. BLOB return MATCH_CONST_FALSE; } - return [{ [param]: value }]; + return [{ [inputParam.key]: value }]; } - }; + } satisfies ParameterMatchClause; } else { return this.error(`Unsupported usage of IN operator`, expr); } } else if (BASIC_OPERATORS.has(op)) { - if (!isStaticRowValueClause(leftFilter) || !isStaticRowValueClause(rightFilter)) { - return this.error(`Operator ${op} is not supported on bucket parameters`, expr); - } - return compileStaticOperator(op, leftFilter, rightFilter); + const fnImpl = getOperatorFunction(op); + return this.composeFunction(fnImpl, [leftFilter, rightFilter], [left, right]); } else { return this.error(`Operator not supported: ${op}`, expr); } } else if (expr.type == 'unary') { if (expr.op == 'NOT') { - const filter = this.compileClause(expr.operand); - if (isClauseError(filter)) { - return filter; - } else if (!isStaticRowValueClause(filter)) { - return this.error('Cannot use NOT on bucket parameter filters', expr); - } - - return { - evaluate: (tables) => { - const value = filter.evaluate(tables); - return sqliteNot(value); - }, - getType() { - return ExpressionType.INTEGER; - } - }; + const clause = this.compileClause(expr.operand); + return this.composeFunction(OPERATOR_NOT, [clause], [expr.operand]); } else if (expr.op == 'IS NULL') { - const leftFilter = this.compileClause(expr.operand); - if (isClauseError(leftFilter)) { - return leftFilter; - } else if (isStaticRowValueClause(leftFilter)) { - // 1. static IS NULL - const nullValue: StaticRowValueClause = { - evaluate: () => null, - getType() { - return ExpressionType.INTEGER; - } - }; - return compileStaticOperator('IS', leftFilter, nullValue); - } else if (isParameterValueClause(leftFilter)) { - // 2. param IS NULL - return { - error: false, - bucketParameters: [leftFilter.bucketParameter], - unbounded: false, - filter(tables: QueryParameters): TrueIfParametersMatch { - return [{ [leftFilter.bucketParameter]: null }]; - } - }; - } else { - return this.error(`Cannot use IS NULL here`, expr); - } + const clause = this.compileClause(expr.operand); + return this.composeFunction(OPERATOR_IS_NULL, [clause], [expr.operand]); } else if (expr.op == 'IS NOT NULL') { - const leftFilter = this.compileClause(expr.operand); - if (isClauseError(leftFilter)) { - return leftFilter; - } else if (isStaticRowValueClause(leftFilter)) { - // 1. static IS NULL - const nullValue: StaticRowValueClause = { - evaluate: () => null, - getType() { - return ExpressionType.INTEGER; - } - }; - return compileStaticOperator('IS NOT', leftFilter, nullValue); - } else { - return this.error(`Cannot use IS NOT NULL here`, expr); - } + const clause = this.compileClause(expr.operand); + return this.composeFunction(OPERATOR_IS_NOT_NULL, [clause], [expr.operand]); } else { return this.error(`Operator ${expr.op} is not supported`, expr); } } else if (expr.type == 'call' && expr.function?.name != null) { const fn = expr.function.name; - const fnImpl = SQL_FUNCTIONS[fn]; - if (fnImpl == null) { - return this.error(`Function '${fn}' is not defined`, expr); - } - - let error = false; - const argExtractors = expr.args.map((arg) => { - const clause = this.compileClause(arg); - - if (isClauseError(clause)) { - error = true; - } else if (!isStaticRowValueClause(clause)) { - error = true; - return this.error(`Bucket parameters are not supported in function call arguments`, arg); + if (expr.function.schema == null) { + const fnImpl = SQL_FUNCTIONS[fn]; + if (fnImpl == null) { + return this.error(`Function '${fn}' is not defined`, expr); } - return clause; - }) as StaticRowValueClause[]; - if (error) { - return { error: true }; + const argClauses = expr.args.map((arg) => this.compileClause(arg)); + const composed = this.composeFunction(fnImpl, argClauses, expr.args); + return composed; + } else { + return this.error(`Function '${expr.function.schema}.${fn}' is not defined`, expr); } - - return { - evaluate: (tables) => { - const args = argExtractors.map((e) => e.evaluate(tables)); - return fnImpl.call(...args); - }, - getType(schema) { - const argTypes = argExtractors.map((e) => e.getType(schema)); - return fnImpl.getReturnType(argTypes); - } - }; } else if (expr.type == 'member') { const operand = this.compileClause(expr.operand); - if (isClauseError(operand)) { - return operand; - } else if (!isStaticRowValueClause(operand)) { - return this.error(`Bucket parameters are not supported in member lookups`, expr.operand); + + if (!(typeof expr.member == 'string' && (expr.op == '->>' || expr.op == '->'))) { + return this.error(`Unsupported member operation ${expr.op}`, expr); } - if (typeof expr.member == 'string' && (expr.op == '->>' || expr.op == '->')) { - return { - evaluate: (tables) => { - const containerString = operand.evaluate(tables); - return jsonExtract(containerString, expr.member, expr.op); - }, - getType() { - return ExpressionType.ANY_JSON; - } - }; + const debugArgs: Expr[] = [expr.operand, expr]; + const args: CompiledClause[] = [operand, staticValueClause(expr.member)]; + if (expr.op == '->') { + return this.composeFunction(OPERATOR_JSON_EXTRACT_JSON, args, debugArgs); } else { - return this.error(`Unsupported member operation ${expr.op}`, expr); + return this.composeFunction(OPERATOR_JSON_EXTRACT_SQL, args, debugArgs); } } else if (expr.type == 'cast') { const operand = this.compileClause(expr.operand); - if (isClauseError(operand)) { - return operand; - } else if (!isStaticRowValueClause(operand)) { - return this.error(`Bucket parameters are not supported in cast expressions`, expr.operand); - } const to = (expr.to as any)?.name?.toLowerCase() as string | undefined; - if (CAST_TYPES.has(to!)) { - return { - evaluate: (tables) => { - const value = operand.evaluate(tables); - if (value == null) { - return null; - } - return cast(value, to!); - }, - getType() { - return ExpressionType.fromTypeText(to as SqliteType); - } - }; - } else { + const castFn = castOperator(to); + if (castFn == null) { return this.error(`CAST not supported for '${to}'`, expr); } + return this.composeFunction(castFn, [operand], [expr.operand]); } else { return this.error(`${expr.type} not supported here`, expr); } @@ -519,10 +462,16 @@ export class SqlTools { } } - getParameterRef(expr: Expr) { - if (this.isParameterRef(expr)) { - return `${expr.table!.name}.${expr.name}`; - } + getParameterRefClause(expr: ExprRef): ParameterValueClause { + const table = expr.table!.name; + const column = expr.name; + return { + key: `${table}.${column}`, + lookupParameterValue: (parameters) => { + const pt: SqliteJsonRow | undefined = (parameters as any)[table]; + return pt?.[column] ?? null; + } + } satisfies ParameterValueClause; } refHasSchema(ref: ExprRef) { @@ -548,6 +497,96 @@ export class SqlTools { throw new SqlRuleError(`Undefined table ${ref.table?.name}`, this.sql, ref); } } + + /** + * Given a function, compile a clause with the function over compiled arguments. + * + * For functions with multiple arguments, the following combinations are supported: + * fn(StaticValueClause, StaticValueClause) => StaticValueClause + * fn(ParameterValueClause, ParameterValueClause) => ParameterValueClause + * fn(RowValueClause, RowValueClause) => RowValueClause + * fn(ParameterValueClause, StaticValueClause) => ParameterValueClause + * fn(RowValueClause, StaticValueClause) => RowValueClause + * + * This is not supported, and will likely never be supported: + * fn(ParameterValueClause, RowValueClause) => error + * + * @param fnImpl The function or operator implementation + * @param argClauses The compiled argument clauses + * @param debugArgExpressions The original parsed expressions, for debug info only + * @returns a compiled function clause + */ + composeFunction(fnImpl: SqlFunction, argClauses: CompiledClause[], debugArgExpressions: Expr[]): CompiledClause { + let argsType: 'static' | 'row' | 'param' = 'static'; + for (let i = 0; i < argClauses.length; i++) { + const debugArg = debugArgExpressions[i]; + const clause = argClauses[i]; + if (isClauseError(clause)) { + // Return immediately on error + return clause; + } else if (isStaticValueClause(clause)) { + // argsType unchanged + } else if (isParameterValueClause(clause)) { + if (!this.supports_parameter_expressions) { + return this.error(`Cannot use bucket parameters in expressions`, debugArg); + } + if (argsType == 'static' || argsType == 'param') { + argsType = 'param'; + } else { + return this.error(`Cannot use table values and parameters in the same clauses`, debugArg); + } + } else if (isRowValueClause(clause)) { + if (argsType == 'static' || argsType == 'row') { + argsType = 'row'; + } else { + return this.error(`Cannot use table values and parameters in the same clauses`, debugArg); + } + } else { + return this.error(`Parameter match clauses cannot be used here`, debugArg); + } + } + + if (argsType == 'row' || argsType == 'static') { + return { + evaluate: (tables) => { + const args = argClauses.map((e) => (e as RowValueClause).evaluate(tables)); + return fnImpl.call(...args); + }, + getType(schema) { + const argTypes = argClauses.map((e) => (e as RowValueClause).getType(schema)); + return fnImpl.getReturnType(argTypes); + } + } satisfies RowValueClause; + } else if (argsType == 'param') { + const argStrings = argClauses.map((e) => { + if (isParameterValueClause(e)) { + return e.key; + } else if (isStaticValueClause(e)) { + return e.value; + } else { + throw new Error('unreachable condition'); + } + }); + const name = `${fnImpl.debugName}(${argStrings.join(',')})`; + return { + key: name, + lookupParameterValue: (parameters) => { + const args = argClauses.map((e) => { + if (isParameterValueClause(e)) { + return e.lookupParameterValue(parameters); + } else if (isStaticValueClause(e)) { + return e.value; + } else { + throw new Error('unreachable condition'); + } + }); + return fnImpl.call(...args); + } + } satisfies ParameterValueClause; + } else { + throw new Error('unreachable condition'); + } + } } function isStatic(expr: Expr) { @@ -563,3 +602,26 @@ function staticValue(expr: Expr): SqliteValue { return (expr as any).value; } } + +function staticValueClause(value: SqliteValue): StaticValueClause { + return { + value: value, + evaluate: () => value, + getType() { + return ExpressionType.fromTypeText(sqliteTypeOf(value)); + } + }; +} + +function basicInputParameter(clause: ParameterValueClause): InputParameter { + return { + key: clause.key, + expands: false, + filteredRowToLookupValue: (filterParameters) => { + return filterParameters[clause.key]; + }, + parametersToLookupValue: (parameters) => { + return clause.lookupParameterValue(parameters); + } + }; +} diff --git a/packages/sync-rules/src/sql_functions.ts b/packages/sync-rules/src/sql_functions.ts index 753af22..7ca6df7 100644 --- a/packages/sync-rules/src/sql_functions.ts +++ b/packages/sync-rules/src/sql_functions.ts @@ -1,12 +1,12 @@ import { JSONBig } from '@powersync/service-jsonbig'; -import { SQLITE_FALSE, SQLITE_TRUE, sqliteBool } from './sql_support.js'; +import { SQLITE_FALSE, SQLITE_TRUE, sqliteBool, sqliteNot } from './sql_support.js'; import { SqliteValue } from './types.js'; import { jsonValueToSqlite } from './utils.js'; // Declares @syncpoint/wkx module // This allows for consumers of this lib to resolve types correctly /// import wkx from '@syncpoint/wkx'; -import { ExpressionType, TYPE_INTEGER } from './ExpressionType.js'; +import { ExpressionType, SqliteType, TYPE_INTEGER } from './ExpressionType.js'; export const BASIC_OPERATORS = new Set([ '=', @@ -33,12 +33,14 @@ export interface FunctionParameter { } export interface SqlFunction { + readonly debugName: string; parameters: FunctionParameter[]; call: (...args: SqliteValue[]) => SqliteValue; getReturnType(args: ExpressionType[]): ExpressionType; } const upper: SqlFunction = { + debugName: 'upper', call(value: SqliteValue) { const text = castAsText(value); return text?.toUpperCase() ?? null; @@ -50,6 +52,7 @@ const upper: SqlFunction = { }; const lower: SqlFunction = { + debugName: 'lower', call(value: SqliteValue) { const text = castAsText(value); return text?.toLowerCase() ?? null; @@ -61,6 +64,7 @@ const lower: SqlFunction = { }; const hex: SqlFunction = { + debugName: 'hex', call(value: SqliteValue) { const binary = castAsBlob(value); if (binary == null) { @@ -75,6 +79,7 @@ const hex: SqlFunction = { }; const length: SqlFunction = { + debugName: 'length', call(value: SqliteValue) { if (value == null) { return null; @@ -92,6 +97,7 @@ const length: SqlFunction = { }; const base64: SqlFunction = { + debugName: 'base64', call(value: SqliteValue) { const binary = castAsBlob(value); if (binary == null) { @@ -106,6 +112,7 @@ const base64: SqlFunction = { }; const fn_typeof: SqlFunction = { + debugName: 'typeof', call(value: SqliteValue) { return sqliteTypeOf(value); }, @@ -116,6 +123,7 @@ const fn_typeof: SqlFunction = { }; const ifnull: SqlFunction = { + debugName: 'ifnull', call(x: SqliteValue, y: SqliteValue) { if (x == null) { return y; @@ -139,6 +147,7 @@ const ifnull: SqlFunction = { }; const json_extract: SqlFunction = { + debugName: 'json_extract', call(json: SqliteValue, path: SqliteValue) { return jsonExtract(json, path, 'json_extract'); }, @@ -152,6 +161,7 @@ const json_extract: SqlFunction = { }; const json_array_length: SqlFunction = { + debugName: 'json_array_length', call(json: SqliteValue, path?: SqliteValue) { if (path != null) { json = json_extract.call(json, path); @@ -177,6 +187,7 @@ const json_array_length: SqlFunction = { }; const json_valid: SqlFunction = { + debugName: 'json_valid', call(json: SqliteValue) { const jsonString = castAsText(json); if (jsonString == null) { @@ -196,6 +207,7 @@ const json_valid: SqlFunction = { }; const unixepoch: SqlFunction = { + debugName: 'unixepoch', call(value?: SqliteValue, specifier?: SqliteValue, specifier2?: SqliteValue) { if (value == null) { return null; @@ -240,6 +252,7 @@ const unixepoch: SqlFunction = { }; const datetime: SqlFunction = { + debugName: 'datetime', call(value?: SqliteValue, specifier?: SqliteValue, specifier2?: SqliteValue) { if (value == null) { return null; @@ -285,6 +298,7 @@ const datetime: SqlFunction = { }; const st_asgeojson: SqlFunction = { + debugName: 'st_asgeojson', call(geometry?: SqliteValue) { const geo = parseGeometry(geometry); if (geo == null) { @@ -299,6 +313,7 @@ const st_asgeojson: SqlFunction = { }; const st_astext: SqlFunction = { + debugName: 'st_astext', call(geometry?: SqliteValue) { const geo = parseGeometry(geometry); if (geo == null) { @@ -312,6 +327,7 @@ const st_astext: SqlFunction = { } }; const st_x: SqlFunction = { + debugName: 'st_x', call(geometry?: SqliteValue) { const geo = parseGeometry(geometry); if (geo == null) { @@ -329,6 +345,7 @@ const st_x: SqlFunction = { }; const st_y: SqlFunction = { + debugName: 'st_y', call(geometry?: SqliteValue) { const geo = parseGeometry(geometry); if (geo == null) { @@ -696,6 +713,86 @@ export function jsonExtract(sourceValue: SqliteValue, path: SqliteValue, operato } } +export const OPERATOR_JSON_EXTRACT_JSON: SqlFunction = { + debugName: 'operator->', + call(json: SqliteValue, path: SqliteValue) { + return jsonExtract(json, path, '->'); + }, + parameters: [ + { name: 'json', type: ExpressionType.ANY, optional: false }, + { name: 'path', type: ExpressionType.ANY, optional: false } + ], + getReturnType(args) { + return ExpressionType.ANY_JSON; + } +}; + +export const OPERATOR_JSON_EXTRACT_SQL: SqlFunction = { + debugName: 'operator->>', + call(json: SqliteValue, path: SqliteValue) { + return jsonExtract(json, path, '->>'); + }, + parameters: [ + { name: 'json', type: ExpressionType.ANY, optional: false }, + { name: 'path', type: ExpressionType.ANY, optional: false } + ], + getReturnType(_args) { + return ExpressionType.ANY_JSON; + } +}; + +export const OPERATOR_IS_NULL: SqlFunction = { + debugName: 'operator_is_null', + call(value: SqliteValue) { + return evaluateOperator('IS', value, null); + }, + parameters: [{ name: 'value', type: ExpressionType.ANY, optional: false }], + getReturnType(_args) { + return ExpressionType.INTEGER; + } +}; + +export const OPERATOR_IS_NOT_NULL: SqlFunction = { + debugName: 'operator_is_not_null', + call(value: SqliteValue) { + return evaluateOperator('IS NOT', value, null); + }, + parameters: [{ name: 'value', type: ExpressionType.ANY, optional: false }], + getReturnType(_args) { + return ExpressionType.INTEGER; + } +}; + +export const OPERATOR_NOT: SqlFunction = { + debugName: 'operator_not', + call(value: SqliteValue) { + return sqliteNot(value); + }, + parameters: [{ name: 'value', type: ExpressionType.ANY, optional: false }], + getReturnType(_args) { + return ExpressionType.INTEGER; + } +}; + +export function castOperator(castTo: string | undefined): SqlFunction | null { + if (castTo == null || !CAST_TYPES.has(castTo)) { + return null; + } + return { + debugName: `operator_cast_${castTo}`, + call(value: SqliteValue) { + if (value == null) { + return null; + } + return cast(value, castTo!); + }, + parameters: [{ name: 'value', type: ExpressionType.ANY, optional: false }], + getReturnType(_args) { + return ExpressionType.fromTypeText(castTo as SqliteType); + } + }; +} + export interface ParseDateFlags { /** * True if input is unixepoch instead of julien days diff --git a/packages/sync-rules/src/sql_support.ts b/packages/sync-rules/src/sql_support.ts index 938e909..99edd51 100644 --- a/packages/sync-rules/src/sql_support.ts +++ b/packages/sync-rules/src/sql_support.ts @@ -2,30 +2,36 @@ import { ClauseError, CompiledClause, FilterParameters, + InputParameter, ParameterMatchClause, ParameterValueClause, QueryParameters, SqliteValue, - StaticRowValueClause, + RowValueClause, + StaticValueClause, TrueIfParametersMatch } from './types.js'; import { MATCH_CONST_FALSE, MATCH_CONST_TRUE } from './sql_filters.js'; -import { evaluateOperator, getOperatorReturnType } from './sql_functions.js'; +import { SqlFunction, evaluateOperator, getOperatorReturnType } from './sql_functions.js'; import { SelectFromStatement } from 'pgsql-ast-parser'; import { SqlRuleError } from './errors.js'; import { ExpressionType } from './ExpressionType.js'; export function isParameterMatchClause(clause: CompiledClause): clause is ParameterMatchClause { - return Array.isArray((clause as ParameterMatchClause).bucketParameters); + return Array.isArray((clause as ParameterMatchClause).inputParameters); } -export function isStaticRowValueClause(clause: CompiledClause): clause is StaticRowValueClause { - return typeof (clause as StaticRowValueClause).evaluate == 'function'; +export function isRowValueClause(clause: CompiledClause): clause is RowValueClause { + return typeof (clause as RowValueClause).evaluate == 'function'; +} + +export function isStaticValueClause(clause: CompiledClause): clause is StaticValueClause { + return isRowValueClause(clause) && typeof (clause as StaticValueClause).value != 'undefined'; } export function isParameterValueClause(clause: CompiledClause): clause is ParameterValueClause { // noinspection SuspiciousTypeOfGuard - return typeof (clause as ParameterValueClause).bucketParameter == 'string'; + return typeof (clause as ParameterValueClause).key == 'string'; } export function isClauseError(clause: CompiledClause): clause is ClauseError { @@ -53,11 +59,7 @@ export function sqliteNot(value: SqliteValue | boolean) { return sqliteBool(!sqliteBool(value)); } -export function compileStaticOperator( - op: string, - left: StaticRowValueClause, - right: StaticRowValueClause -): StaticRowValueClause { +export function compileStaticOperator(op: string, left: RowValueClause, right: RowValueClause): RowValueClause { return { evaluate: (tables) => { const leftValue = left.evaluate(tables); @@ -72,8 +74,24 @@ export function compileStaticOperator( }; } +export function getOperatorFunction(op: string): SqlFunction { + return { + debugName: `operator${op}`, + call(...args: SqliteValue[]) { + return evaluateOperator(op, args[0], args[1]); + }, + getReturnType(args) { + return getOperatorReturnType(op, args[0], args[1]); + }, + parameters: [ + { name: 'left', type: ExpressionType.ANY, optional: false }, + { name: 'right', type: ExpressionType.ANY, optional: false } + ] + }; +} + export function andFilters(a: CompiledClause, b: CompiledClause): CompiledClause { - if (isStaticRowValueClause(a) && isStaticRowValueClause(b)) { + if (isRowValueClause(a) && isRowValueClause(b)) { // Optimization return { evaluate(tables: QueryParameters): SqliteValue { @@ -84,29 +102,29 @@ export function andFilters(a: CompiledClause, b: CompiledClause): CompiledClause getType() { return ExpressionType.INTEGER; } - }; + } satisfies RowValueClause; } const aFilter = toBooleanParameterSetClause(a); const bFilter = toBooleanParameterSetClause(b); - const aParams = aFilter.bucketParameters; - const bParams = bFilter.bucketParameters; + const aParams = aFilter.inputParameters; + const bParams = bFilter.inputParameters; if (aFilter.unbounded && bFilter.unbounded) { // This could explode the number of buckets for the row throw new Error('Cannot have multiple IN expressions on bucket parameters'); } - const combined = new Set([...aParams, ...bParams]); + const combinedMap = new Map([...aParams, ...bParams].map((p) => [p.key, p])); return { error: aFilter.error || bFilter.error, - bucketParameters: [...combined], + inputParameters: [...combinedMap.values()], unbounded: aFilter.unbounded || bFilter.unbounded, // result count = a.count * b.count - filter: (tables) => { - const aResult = aFilter.filter(tables); - const bResult = bFilter.filter(tables); + filterRow: (tables) => { + const aResult = aFilter.filterRow(tables); + const bResult = bFilter.filterRow(tables); let results: FilterParameters[] = []; for (let result1 of aResult) { @@ -126,11 +144,11 @@ export function andFilters(a: CompiledClause, b: CompiledClause): CompiledClause } return results; } - }; + } satisfies ParameterMatchClause; } export function orFilters(a: CompiledClause, b: CompiledClause): CompiledClause { - if (isStaticRowValueClause(a) && isStaticRowValueClause(b)) { + if (isRowValueClause(a) && isRowValueClause(b)) { // Optimization return { evaluate(tables: QueryParameters): SqliteValue { @@ -141,7 +159,7 @@ export function orFilters(a: CompiledClause, b: CompiledClause): CompiledClause getType() { return ExpressionType.INTEGER; } - }; + } satisfies RowValueClause; } const aFilter = toBooleanParameterSetClause(a); @@ -150,12 +168,12 @@ export function orFilters(a: CompiledClause, b: CompiledClause): CompiledClause } export function orParameterSetClauses(a: ParameterMatchClause, b: ParameterMatchClause): ParameterMatchClause { - const aParams = a.bucketParameters; - const bParams = b.bucketParameters; + const aParams = a.inputParameters; + const bParams = b.inputParameters; // This gives the guaranteed set of parameters matched against. - const allParams = new Set([...aParams, ...bParams]); - if (allParams.size != aParams.length || allParams.size != bParams.length) { + const combinedMap = new Map([...aParams, ...bParams].map((p) => [p.key, p])); + if (combinedMap.size != aParams.length || combinedMap.size != bParams.length) { throw new Error( `Left and right sides of OR must use the same parameters, or split into separate queries. ${JSON.stringify( aParams @@ -163,7 +181,7 @@ export function orParameterSetClauses(a: ParameterMatchClause, b: ParameterMatch ); } - const parameters = [...allParams]; + const parameters = [...combinedMap.values()]; // assets.region_id = bucket.region_id AND bucket.user_id IN assets.user_ids // OR bucket.region_id IN assets.region_ids AND bucket.user_id = assets.user_id @@ -171,16 +189,16 @@ export function orParameterSetClauses(a: ParameterMatchClause, b: ParameterMatch const unbounded = a.unbounded || b.unbounded; return { error: a.error || b.error, - bucketParameters: parameters, + inputParameters: parameters, unbounded, // result count = a.count + b.count - filter: (tables) => { - const aResult = a.filter(tables); - const bResult = b.filter(tables); + filterRow: (tables) => { + const aResult = a.filterRow(tables); + const bResult = b.filterRow(tables); let results: FilterParameters[] = [...aResult, ...bResult]; return results; } - }; + } satisfies ParameterMatchClause; } /** @@ -191,36 +209,49 @@ export function orParameterSetClauses(a: ParameterMatchClause, b: ParameterMatch export function toBooleanParameterSetClause(clause: CompiledClause): ParameterMatchClause { if (isParameterMatchClause(clause)) { return clause; - } else if (isStaticRowValueClause(clause)) { + } else if (isRowValueClause(clause)) { return { error: false, - bucketParameters: [], + inputParameters: [], unbounded: false, - filter(tables: QueryParameters): TrueIfParametersMatch { + filterRow(tables: QueryParameters): TrueIfParametersMatch { const value = sqliteBool(clause.evaluate(tables)); return value ? MATCH_CONST_TRUE : MATCH_CONST_FALSE; } - }; + } satisfies ParameterMatchClause; } else if (isClauseError(clause)) { return { error: true, - bucketParameters: [], + inputParameters: [], unbounded: false, - filter(tables: QueryParameters): TrueIfParametersMatch { + filterRow(tables: QueryParameters): TrueIfParametersMatch { throw new Error('invalid clause'); } - }; + } satisfies ParameterMatchClause; } else { // Equivalent to `bucket.param = true` - const param = clause.bucketParameter; + const key = clause.key; + + const inputParam: InputParameter = { + key: key, + expands: false, + filteredRowToLookupValue: (filterParameters) => { + return filterParameters[key]; + }, + parametersToLookupValue: (parameters) => { + const inner = clause.lookupParameterValue(parameters); + return sqliteBool(inner); + } + }; + return { error: false, - bucketParameters: [param], + inputParameters: [inputParam], unbounded: false, - filter(tables: QueryParameters): TrueIfParametersMatch { - return [{ [param]: SQLITE_TRUE }]; + filterRow(tables: QueryParameters): TrueIfParametersMatch { + return [{ [key]: SQLITE_TRUE }]; } - }; + } satisfies ParameterMatchClause; } } diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 47f385c..6dc749a 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -120,16 +120,56 @@ export type QueryParameters = { [table: string]: SqliteRow }; * A single set of parameters that would make a WHERE filter true. * * Each parameter is prefixed with a table name, e.g. 'bucket.param'. + * + * Data queries: this is converted into a bucket id, given named bucket parameters. + * + * Parameter queries: this is converted into a lookup array. */ export type FilterParameters = { [parameter: string]: SqliteJsonValue }; +export interface InputParameter { + /** + * An unique identifier per parameter in a query. + * + * This is used to identify the same parameters used in a query multiple times. + * + * The value itself does not necessarily have any specific meaning. + */ + key: string; + + /** + * True if the parameter expands to an array. This means parametersToLookupValue() can + * return a JSON array. This is different from `unbounded` on the clause. + */ + expands: boolean; + + /** + * Given FilterParameters from a data row, return the associated value. + * + * Only relevant for parameter queries. + */ + filteredRowToLookupValue(filterParameters: FilterParameters): SqliteJsonValue; + + /** + * Given SyncParamters, return the associated value to lookup. + * + * Only relevant for parameter queries. + */ + parametersToLookupValue(parameters: SyncParameters): SqliteValue; +} + export interface EvaluateRowOptions { sourceTable: SourceTableInterface; record: SqliteRow; } /** - * Given a row, produces a set of parameters that would make the clause evaluate to true. + * This is a clause that matches row and parameter values. + * + * Example: + * [WHERE] users.org_id = bucket.org_id + * + * For a given a row, this produces a set of parameters that would make the clause evaluate to true. */ export interface ParameterMatchClause { error: boolean; @@ -141,10 +181,11 @@ export interface ParameterMatchClause { * * These parameters are always matched by this clause, and no additional parameters are matched. */ - bucketParameters: string[]; + inputParameters: InputParameter[]; /** - * True if the filter depends on an unbounded array column. + * True if the filter depends on an unbounded array column. This means filterRow can return + * multiple items. * * We restrict filters to only allow a single unbounded column for bucket parameters, otherwise the number of * bucketParameter combinations could grow too much. @@ -154,20 +195,33 @@ export interface ParameterMatchClause { /** * Given a data row, give a set of filter parameters that would make the filter be true. * + * For StaticSqlParameterQuery, the tables are token_parameters and user_parameters. + * For others, it is the table of the data or parameter query. + * * @param tables - {table => row} * @return The filter parameters */ - filter(tables: QueryParameters): TrueIfParametersMatch; + filterRow(tables: QueryParameters): TrueIfParametersMatch; } /** - * Given a row, produces a set of parameters that would make the clause evaluate to true. + * This is a clause that operates on request or bucket parameters. */ export interface ParameterValueClause { /** - * The parameter fields used for this, e.g. 'bucket.region_id' + * An unique key for the clause. + * + * For bucket parameters, this is `bucket.${name}`. + * For expressions, the exact format is undefined. + */ + key: string; + + /** + * Given SyncParamters, return the associated value to lookup. + * + * Only relevant for parameter queries. */ - bucketParameter: string; + lookupParameterValue(parameters: SyncParameters): SqliteValue; } export interface QuerySchema { @@ -176,18 +230,30 @@ export interface QuerySchema { } /** - * Only needs row values as input, producing a static value as output. + * A clause that uses row values as input. + * + * For parameter queries, that is the parameter table being queried. + * For data queries, that is the data table being queried. */ -export interface StaticRowValueClause { +export interface RowValueClause { evaluate(tables: QueryParameters): SqliteValue; getType(schema: QuerySchema): ExpressionType; } +/** + * Completely static value. + * + * Extends RowValueClause to simplify code in some places. + */ +export interface StaticValueClause extends RowValueClause { + readonly value: SqliteValue; +} + export interface ClauseError { error: true; } -export type CompiledClause = StaticRowValueClause | ParameterMatchClause | ParameterValueClause | ClauseError; +export type CompiledClause = RowValueClause | ParameterMatchClause | ParameterValueClause | ClauseError; /** * true if any of the filter parameter sets match diff --git a/packages/sync-rules/test/src/data_queries.test.ts b/packages/sync-rules/test/src/data_queries.test.ts new file mode 100644 index 0000000..0aff50e --- /dev/null +++ b/packages/sync-rules/test/src/data_queries.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, test } from 'vitest'; +import { ExpressionType, SqlDataQuery } from '../../src/index.js'; +import { ASSETS, BASIC_SCHEMA } from './util.js'; + +describe('data queries', () => { + test('bucket parameters = query', function () { + const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; + const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql); + expect(query.errors).toEqual([]); + + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' })).toEqual([ + { + bucket: 'mybucket["org1"]', + table: 'assets', + id: 'asset1', + data: { id: 'asset1', org_id: 'org1' } + } + ]); + + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]); + }); + + test('bucket parameters IN query', function () { + const sql = 'SELECT * FROM assets WHERE bucket.category IN assets.categories'; + const query = SqlDataQuery.fromSql('mybucket', ['category'], sql); + expect(query.errors).toEqual([]); + + expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) })).toMatchObject([ + { + bucket: 'mybucket["red"]', + table: 'assets', + id: 'asset1' + }, + { + bucket: 'mybucket["green"]', + table: 'assets', + id: 'asset1' + } + ]); + + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]); + }); + + test('table alias', function () { + const sql = 'SELECT * FROM assets as others WHERE others.org_id = bucket.org_id'; + const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql); + expect(query.errors).toEqual([]); + + expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: 'org1' })).toEqual([ + { + bucket: 'mybucket["org1"]', + table: 'others', + id: 'asset1', + data: { id: 'asset1', org_id: 'org1' } + } + ]); + }); + + test('types', () => { + const schema = BASIC_SCHEMA; + + const q1 = SqlDataQuery.fromSql('q1', ['user_id'], `SELECT * FROM assets WHERE owner_id = bucket.user_id`); + expect(q1.getColumnOutputs(schema)).toEqual([ + { + name: 'assets', + columns: [ + { name: 'id', type: ExpressionType.TEXT }, + { name: 'name', type: ExpressionType.TEXT }, + { name: 'count', type: ExpressionType.INTEGER }, + { name: 'owner_id', type: ExpressionType.TEXT } + ] + } + ]); + + const q2 = SqlDataQuery.fromSql( + 'q1', + ['user_id'], + ` + SELECT id :: integer as id, + upper(name) as name_upper, + hex('test') as hex, + count + 2 as count2, + count * 3.0 as count3, + count * '4' as count4, + name ->> '$.attr' as json_value, + ifnull(name, 2.0) as maybe_name + FROM assets WHERE owner_id = bucket.user_id` + ); + expect(q2.getColumnOutputs(schema)).toEqual([ + { + name: 'assets', + columns: [ + { name: 'id', type: ExpressionType.INTEGER }, + { name: 'name_upper', type: ExpressionType.TEXT }, + { name: 'hex', type: ExpressionType.TEXT }, + { name: 'count2', type: ExpressionType.INTEGER }, + { name: 'count3', type: ExpressionType.REAL }, + { name: 'count4', type: ExpressionType.NUMERIC }, + { name: 'json_value', type: ExpressionType.ANY_JSON }, + { name: 'maybe_name', type: ExpressionType.TEXT.or(ExpressionType.REAL) } + ] + } + ]); + }); + + test('validate columns', () => { + const schema = BASIC_SCHEMA; + const q1 = SqlDataQuery.fromSql( + 'q1', + ['user_id'], + 'SELECT id, name, count FROM assets WHERE owner_id = bucket.user_id', + schema + ); + expect(q1.errors).toEqual([]); + + const q2 = SqlDataQuery.fromSql( + 'q2', + ['user_id'], + 'SELECT id, upper(description) as d FROM assets WHERE other_id = bucket.user_id', + schema + ); + expect(q2.errors).toMatchObject([ + { + message: `Column not found: other_id`, + type: 'warning' + }, + { + message: `Column not found: description`, + type: 'warning' + } + ]); + + const q3 = SqlDataQuery.fromSql( + 'q3', + ['user_id'], + 'SELECT id, description, * FROM nope WHERE other_id = bucket.user_id', + schema + ); + expect(q3.errors).toMatchObject([ + { + message: `Table public.nope not found`, + type: 'warning' + } + ]); + }); + + test('invalid query - invalid IN', function () { + const sql = 'SELECT * FROM assets WHERE assets.category IN bucket.categories'; + const query = SqlDataQuery.fromSql('mybucket', ['categories'], sql); + expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Unsupported usage of IN operator' }]); + }); + + test('invalid query - not all parameters used', function () { + const sql = 'SELECT * FROM assets WHERE 1'; + const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql); + expect(query.errors).toMatchObject([ + { type: 'fatal', message: 'Query must cover all bucket parameters. Expected: ["bucket.org_id"] Got: []' } + ]); + }); + + test('invalid query - parameter not defined', function () { + const sql = 'SELECT * FROM assets WHERE assets.org_id = bucket.org_id'; + const query = SqlDataQuery.fromSql('mybucket', [], sql); + expect(query.errors).toMatchObject([ + { type: 'fatal', message: 'Query must cover all bucket parameters. Expected: [] Got: ["bucket.org_id"]' } + ]); + }); + + test('invalid query - function on parameter (1)', function () { + const sql = 'SELECT * FROM assets WHERE assets.org_id = upper(bucket.org_id)'; + const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql); + expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Cannot use bucket parameters in expressions' }]); + }); + + test('invalid query - function on parameter (2)', function () { + const sql = 'SELECT * FROM assets WHERE assets.org_id = upper(bucket.org_id)'; + const query = SqlDataQuery.fromSql('mybucket', [], sql); + expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Cannot use bucket parameters in expressions' }]); + }); +}); diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts new file mode 100644 index 0000000..eb85ad2 --- /dev/null +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -0,0 +1,520 @@ +import { describe, expect, test } from 'vitest'; +import { SqlParameterQuery, normalizeTokenParameters } from '../../src/index.js'; +import { BASIC_SCHEMA } from './util.js'; + +describe('parameter queries', () => { + test('token_parameters IN query', function () { + const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + expect(query.evaluateParameterRow({ id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) })).toEqual([ + { + lookup: ['mybucket', '1', 'user1'], + bucket_parameters: [ + { + group_id: 'group1' + } + ] + }, + { + lookup: ['mybucket', '1', 'user2'], + bucket_parameters: [ + { + group_id: 'group1' + } + ] + } + ]); + expect( + query.getLookups( + normalizeTokenParameters({ + user_id: 'user1' + }) + ) + ).toEqual([['mybucket', '1', 'user1']]); + }); + + test('IN token_parameters query', function () { + const sql = 'SELECT id as region_id FROM regions WHERE name IN token_parameters.region_names'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ + { + lookup: ['mybucket', '1', 'colorado'], + bucket_parameters: [ + { + region_id: 'region1' + } + ] + } + ]); + expect( + query.getLookups( + normalizeTokenParameters({ + region_names: JSON.stringify(['colorado', 'texas']) + }) + ) + ).toEqual([ + ['mybucket', '1', 'colorado'], + ['mybucket', '1', 'texas'] + ]); + }); + + test('queried numeric parameters', () => { + const sql = + 'SELECT users.int1, users.float1, users.float2 FROM users WHERE users.int1 = token_parameters.int1 AND users.float1 = token_parameters.float1 AND users.float2 = token_parameters.float2'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. + expect(query.evaluateParameterRow({ int1: 314n, float1: 3.14, float2: 314 })).toEqual([ + { + lookup: ['mybucket', '1', 314n, 3.14, 314], + + bucket_parameters: [{ int1: 314n, float1: 3.14, float2: 314 }] + } + ]); + + // Similarly, we don't need to worry about the types here. + // This test just checks the current behavior. + expect(query.getLookups(normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ + ['mybucket', '1', 314n, 3.14, 314n] + ]); + + // We _do_ need to care about the bucket string representation. + expect(query.resolveBucketIds([{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({}))).toEqual([ + 'mybucket[314,3.14,314]' + ]); + + expect(query.resolveBucketIds([{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({}))).toEqual([ + 'mybucket[314,3.14,314]' + ]); + }); + + test('plain token_parameter (baseline)', () => { + const sql = 'SELECT id from users WHERE filter_param = token_parameters.user_id'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + { + lookup: ['mybucket', undefined, 'test_param'], + + bucket_parameters: [{ id: 'test_id' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([['mybucket', undefined, 'test']]); + }); + + test('function on token_parameter', () => { + const sql = 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + { + lookup: ['mybucket', undefined, 'test_param'], + + bucket_parameters: [{ id: 'test_id' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([['mybucket', undefined, 'TEST']]); + }); + + test('token parameter member operator', () => { + const sql = "SELECT id from users WHERE filter_param = token_parameters.some_param ->> 'description'"; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + { + lookup: ['mybucket', undefined, 'test_param'], + + bucket_parameters: [{ id: 'test_id' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ some_param: { description: 'test_description' } }))).toEqual([ + ['mybucket', undefined, 'test_description'] + ]); + }); + + test('token parameter and binary operator', () => { + const sql = 'SELECT id from users WHERE filter_param = token_parameters.some_param + 2'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.getLookups(normalizeTokenParameters({ some_param: 3 }))).toEqual([['mybucket', undefined, 5n]]); + }); + + test('token parameter IS NULL as filter', () => { + const sql = 'SELECT id from users WHERE filter_param = (token_parameters.some_param IS NULL)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.getLookups(normalizeTokenParameters({ some_param: null }))).toEqual([['mybucket', undefined, 1n]]); + expect(query.getLookups(normalizeTokenParameters({ some_param: 'test' }))).toEqual([['mybucket', undefined, 0n]]); + }); + + test('direct token parameter', () => { + const sql = 'SELECT FROM users WHERE token_parameters.some_param'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', undefined, 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ['mybucket', undefined, 0n] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ['mybucket', undefined, 1n] + ]); + }); + + test('token parameter IS NULL', () => { + const sql = 'SELECT FROM users WHERE token_parameters.some_param IS NULL'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', undefined, 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ['mybucket', undefined, 1n] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ['mybucket', undefined, 0n] + ]); + }); + + test('token parameter IS NOT NULL', () => { + const sql = 'SELECT FROM users WHERE token_parameters.some_param IS NOT NULL'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', undefined, 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ['mybucket', undefined, 0n] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ['mybucket', undefined, 1n] + ]); + }); + + test('token parameter NOT', () => { + const sql = 'SELECT FROM users WHERE NOT token_parameters.is_admin'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', undefined, 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + ['mybucket', undefined, 1n] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: 123 }))).toEqual([ + ['mybucket', undefined, 0n] + ]); + }); + + test('row filter and token parameter IS NULL', () => { + const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.some_param IS NULL'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', undefined, 'user1', 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ['mybucket', undefined, 'user1', 1n] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ['mybucket', undefined, 'user1', 0n] + ]); + }); + + test('row filter and direct token parameter', () => { + const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.some_param'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', undefined, 'user1', 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: 123 }))).toEqual([ + ['mybucket', undefined, 'user1', 1n] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', some_param: null }))).toEqual([ + ['mybucket', undefined, 'user1', 0n] + ]); + }); + + test('cast', () => { + const sql = 'SELECT FROM users WHERE users.id = cast(token_parameters.user_id as text)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ + ['mybucket', undefined, 'user1'] + ]); + expect(query.getLookups(normalizeTokenParameters({ user_id: 123 }))).toEqual([['mybucket', undefined, '123']]); + }); + + test('IS NULL row filter', () => { + const sql = 'SELECT id FROM users WHERE role IS NULL'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'user1', role: null })).toEqual([ + { + lookup: ['mybucket', undefined], + bucket_parameters: [{ id: 'user1' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([['mybucket', undefined]]); + }); + + test('token filter (1)', () => { + // Also supported: token_parameters.is_admin = true + // Not supported: token_parameters.is_admin != false + // Support could be added later. + const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', '1', 'user1', 1n], + bucket_parameters: [{}] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + ['mybucket', '1', 'user1', 1n] + ]); + // Would not match any actual lookups + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ + ['mybucket', '1', 'user1', 0n] + ]); + }); + + test('token filter (2)', () => { + const sql = + 'SELECT users.id AS user_id, token_parameters.is_admin as is_admin FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + + expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ + { + lookup: ['mybucket', '1', 'user1', 1n], + + bucket_parameters: [{ user_id: 'user1' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + ['mybucket', '1', 'user1', 1n] + ]); + + expect( + query.resolveBucketIds([{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true })) + ).toEqual(['mybucket["user1",1]']); + }); + + test('case-sensitive parameter queries (1)', () => { + const sql = 'SELECT users."userId" AS user_id FROM users WHERE users."userId" = token_parameters.user_id'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + + expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([ + { + lookup: ['mybucket', '1', 'user1'], + + bucket_parameters: [{ user_id: 'user1' }] + } + ]); + }); + + test('case-sensitive parameter queries (2)', () => { + // Note: This documents current behavior. + // This may change in the future - we should check against expected behavior for + // Postgres and/or SQLite. + const sql = 'SELECT users.userId AS user_id FROM users WHERE users.userId = token_parameters.user_id'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + + expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([]); + expect(query.evaluateParameterRow({ userid: 'user1' })).toEqual([ + { + lookup: ['mybucket', '1', 'user1'], + + bucket_parameters: [{ user_id: 'user1' }] + } + ]); + }); + + test('dynamic global parameter query', () => { + const sql = "SELECT workspaces.id AS workspace_id FROM workspaces WHERE visibility = 'public'"; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + query.id = '1'; + + expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'public' })).toEqual([ + { + lookup: ['mybucket', '1'], + + bucket_parameters: [{ workspace_id: 'workspace1' }] + } + ]); + + expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'private' })).toEqual([]); + }); + + test('multiple different functions on token_parameter with AND', () => { + // This is treated as two separate lookup index values + const sql = + 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id) AND filter_param = lower(token_parameters.user_id)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'test_id', filter_param: 'test_param' })).toEqual([ + { + lookup: ['mybucket', undefined, 'test_param', 'test_param'], + + bucket_parameters: [{ id: 'test_id' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([ + ['mybucket', undefined, 'TEST', 'test'] + ]); + }); + + test('multiple same functions on token_parameter with OR', () => { + // This is treated as the same index lookup value, can use OR with the two clauses + const sql = + 'SELECT id from users WHERE filter_param1 = upper(token_parameters.user_id) OR filter_param2 = upper(token_parameters.user_id)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors).toEqual([]); + + expect(query.evaluateParameterRow({ id: 'test_id', filter_param1: 'test1', filter_param2: 'test2' })).toEqual([ + { + lookup: ['mybucket', undefined, 'test1'], + bucket_parameters: [{ id: 'test_id' }] + }, + { + lookup: ['mybucket', undefined, 'test2'], + bucket_parameters: [{ id: 'test_id' }] + } + ]); + + expect(query.getLookups(normalizeTokenParameters({ user_id: 'test' }))).toEqual([['mybucket', undefined, 'TEST']]); + }); + + test('invalid OR in parameter queries', () => { + // Supporting this case is more tricky. We can do this by effectively denormalizing the OR clause + // into separate queries, but it's a significant change. For now, developers should do that manually. + const sql = + "SELECT workspaces.id AS workspace_id FROM workspaces WHERE workspaces.user_id = token_parameters.user_id OR visibility = 'public'"; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/must use the same parameters/); + }); + + test('invalid OR in parameter queries (2)', () => { + const sql = + 'SELECT id from users WHERE filter_param = upper(token_parameters.user_id) OR filter_param = lower(token_parameters.user_id)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/must use the same parameters/); + }); + + test('invalid parameter match clause (1)', () => { + const sql = 'SELECT FROM users WHERE (id = token_parameters.user_id) = false'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/Parameter match clauses cannot be used here/); + }); + + test('invalid parameter match clause (2)', () => { + const sql = 'SELECT FROM users WHERE NOT (id = token_parameters.user_id)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/Parameter match clauses cannot be used here/); + }); + + test('invalid parameter match clause (3)', () => { + // May be supported in the future + const sql = 'SELECT FROM users WHERE token_parameters.start_at < users.created_at'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/Cannot use table values and parameters in the same clauses/); + }); + + test('invalid parameter match clause (4)', () => { + const sql = 'SELECT FROM users WHERE json_extract(users.description, token_parameters.path)'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/Cannot use table values and parameters in the same clauses/); + }); + + test('invalid function schema', () => { + const sql = 'SELECT FROM users WHERE something.length(users.id) = 0'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; + expect(query.errors[0].message).toMatch(/Function 'something.length' is not defined/); + }); + + test('validate columns', () => { + const schema = BASIC_SCHEMA; + + const q1 = SqlParameterQuery.fromSql( + 'q4', + 'SELECT id FROM assets WHERE owner_id = token_parameters.user_id', + schema + ); + expect(q1.errors).toMatchObject([]); + + const q2 = SqlParameterQuery.fromSql( + 'q5', + 'SELECT id as asset_id FROM assets WHERE other_id = token_parameters.user_id', + schema + ); + + expect(q2.errors).toMatchObject([ + { + message: 'Column not found: other_id', + type: 'warning' + } + ]); + }); +}); diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts new file mode 100644 index 0000000..d682744 --- /dev/null +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from 'vitest'; +import { SqlParameterQuery, normalizeTokenParameters } from '../../src/index.js'; +import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js'; + +describe('static parameter queries', () => { + test('basic query', function () { + const sql = 'SELECT token_parameters.user_id'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery; + expect(query.errors).toEqual([]); + expect(query.bucket_parameters!).toEqual(['user_id']); + expect(query.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1' }))).toEqual(['mybucket["user1"]']); + }); + + test('global query', function () { + const sql = 'SELECT'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery; + expect(query.errors).toEqual([]); + expect(query.bucket_parameters!).toEqual([]); + expect(query.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1' }))).toEqual(['mybucket[]']); + }); + + test('query with filter', function () { + const sql = 'SELECT token_parameters.user_id WHERE token_parameters.is_admin'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery; + expect(query.errors).toEqual([]); + expect(query.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ + 'mybucket["user1"]' + ]); + expect(query.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([]); + }); + + test('function in select clause', function () { + const sql = 'SELECT upper(token_parameters.user_id) as upper_id'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery; + expect(query.errors).toEqual([]); + expect(query.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1' }))).toEqual(['mybucket["USER1"]']); + expect(query.bucket_parameters!).toEqual(['upper_id']); + }); + + test('function in filter clause', function () { + const sql = "SELECT WHERE upper(token_parameters.role) = 'ADMIN'"; + const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery; + expect(query.errors).toEqual([]); + expect(query.getStaticBucketIds(normalizeTokenParameters({ role: 'admin' }))).toEqual(['mybucket[]']); + expect(query.getStaticBucketIds(normalizeTokenParameters({ role: 'user' }))).toEqual([]); + }); + + test('comparison in filter clause', function () { + const sql = 'SELECT WHERE token_parameters.id1 = token_parameters.id2'; + const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery; + expect(query.errors).toEqual([]); + expect(query.getStaticBucketIds(normalizeTokenParameters({ id1: 't1', id2: 't1' }))).toEqual(['mybucket[]']); + expect(query.getStaticBucketIds(normalizeTokenParameters({ id1: 't1', id2: 't2' }))).toEqual([]); + }); +}); diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index af9f51c..d796165 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,49 +1,14 @@ +import { describe, expect, test } from 'vitest'; import { DEFAULT_SCHEMA, DEFAULT_TAG, DartSchemaGenerator, - ExpressionType, JsSchemaGenerator, - SourceTableInterface, - SqlDataQuery, - SqlParameterQuery, SqlSyncRules, StaticSchema, normalizeTokenParameters } from '../../src/index.js'; -import { describe, expect, test } from 'vitest'; - -class TestSourceTable implements SourceTableInterface { - readonly connectionTag = DEFAULT_TAG; - readonly schema = DEFAULT_SCHEMA; - - constructor(public readonly table: string) {} -} - -const ASSETS = new TestSourceTable('assets'); -const USERS = new TestSourceTable('users'); - -const BASIC_SCHEMA = new StaticSchema([ - { - tag: DEFAULT_TAG, - schemas: [ - { - name: DEFAULT_SCHEMA, - tables: [ - { - name: 'assets', - columns: [ - { name: 'id', pg_type: 'uuid' }, - { name: 'name', pg_type: 'text' }, - { name: 'count', pg_type: 'int4' }, - { name: 'owner_id', pg_type: 'uuid' } - ] - } - ] - } - ] - } -]); +import { ASSETS, BASIC_SCHEMA, TestSourceTable, USERS } from './util.js'; describe('sync rules', () => { test('parse empty sync rules', () => { @@ -90,8 +55,8 @@ bucket_definitions: expect(bucket.bucket_parameters).toEqual([]); const param_query = bucket.global_parameter_queries[0]; - expect(param_query.filter!.filter({ token_parameters: { is_admin: 1n } })).toEqual([{}]); - expect(param_query.filter!.filter({ token_parameters: { is_admin: 0n } })).toEqual([]); + expect(param_query.filter!.filterRow({ token_parameters: { is_admin: 1n } })).toEqual([{}]); + expect(param_query.filter!.filterRow({ token_parameters: { is_admin: 0n } })).toEqual([]); expect(rules.getStaticBucketIds(normalizeTokenParameters({ is_admin: true }))).toEqual(['mybucket[]']); expect(rules.getStaticBucketIds(normalizeTokenParameters({ is_admin: false }))).toEqual([]); expect(rules.getStaticBucketIds(normalizeTokenParameters({}))).toEqual([]); @@ -442,7 +407,6 @@ bucket_definitions: } ]); - // TODO: Deduplicate somewhere expect( rules.evaluateRow({ sourceTable: ASSETS, record: { id: 'asset2', description: 'test', role: 'normal' } }) ).toEqual([ @@ -487,63 +451,6 @@ bucket_definitions: expect(rules.getStaticBucketIds(normalizeTokenParameters({ is_admin: true }))).toEqual(['mybucket[1]']); }); - test('token_parameters IN query', function () { - const sql = 'SELECT id as group_id FROM groups WHERE token_parameters.user_id IN groups.user_ids'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - expect(query.evaluateParameterRow({ id: 'group1', user_ids: JSON.stringify(['user1', 'user2']) })).toEqual([ - { - lookup: ['mybucket', '1', 'user1'], - bucket_parameters: [ - { - group_id: 'group1' - } - ] - }, - { - lookup: ['mybucket', '1', 'user2'], - bucket_parameters: [ - { - group_id: 'group1' - } - ] - } - ]); - expect( - query.getLookups( - normalizeTokenParameters({ - user_id: 'user1' - }) - ) - ).toEqual([['mybucket', '1', 'user1']]); - }); - - test('IN token_parameters query', function () { - const sql = 'SELECT id as region_id FROM regions WHERE name IN token_parameters.region_names'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - expect(query.evaluateParameterRow({ id: 'region1', name: 'colorado' })).toEqual([ - { - lookup: ['mybucket', '1', 'colorado'], - bucket_parameters: [ - { - region_id: 'region1' - } - ] - } - ]); - expect( - query.getLookups( - normalizeTokenParameters({ - region_names: JSON.stringify(['colorado', 'texas']) - }) - ) - ).toEqual([ - ['mybucket', '1', 'colorado'], - ['mybucket', '1', 'texas'] - ]); - }); - test('some math', () => { const rules = SqlSyncRules.fromYaml(` bucket_definitions: @@ -595,81 +502,15 @@ bucket_definitions: ]); }); - test('bucket with queried numeric parameters', () => { - const sql = - 'SELECT users.int1, users.float1, users.float2 FROM users WHERE users.int1 = token_parameters.int1 AND users.float1 = token_parameters.float1 AND users.float2 = token_parameters.float2'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - // Note: We don't need to worry about numeric vs decimal types in the lookup - JSONB handles normalization for us. - expect(query.evaluateParameterRow({ int1: 314n, float1: 3.14, float2: 314 })).toEqual([ - { - lookup: ['mybucket', '1', 314n, 3.14, 314], - - bucket_parameters: [{ int1: 314n, float1: 3.14, float2: 314 }] - } - ]); - - // Similarly, we don't need to worry about the types here. - // This test just checks the current behavior. - expect(query.getLookups(normalizeTokenParameters({ int1: 314n, float1: 3.14, float2: 314 }))).toEqual([ - ['mybucket', '1', 314n, 3.14, 314n] - ]); - - // We _do_ need to care about the bucket string representation. - expect(query.resolveBucketIds([{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({}))).toEqual([ - 'mybucket[314,3.14,314]' - ]); - - expect(query.resolveBucketIds([{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({}))).toEqual([ - 'mybucket[314,3.14,314]' - ]); - }); - - test('parameter query with token filter (1)', () => { - // Also supported: token_parameters.is_admin = true - // Not supported: token_parameters.is_admin != false - // Support could be added later. - const sql = 'SELECT FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ - { - lookup: ['mybucket', '1', 'user1', 1n], - bucket_parameters: [{}] - } - ]); - - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ['mybucket', '1', 'user1', 1n] - ]); - // Would not match any actual lookups - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual([ - ['mybucket', '1', 'user1', 0n] - ]); - }); - - test('parameter query with token filter (2)', () => { - const sql = - 'SELECT users.id AS user_id, token_parameters.is_admin as is_admin FROM users WHERE users.id = token_parameters.user_id AND token_parameters.is_admin'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - - expect(query.evaluateParameterRow({ id: 'user1' })).toEqual([ - { - lookup: ['mybucket', '1', 'user1', 1n], - - bucket_parameters: [{ user_id: 'user1' }] - } - ]); - - expect(query.getLookups(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - ['mybucket', '1', 'user1', 1n] - ]); - - expect( - query.resolveBucketIds([{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true })) - ).toEqual(['mybucket["user1",1]']); + test('static parameter query with function on token_parameter', () => { + const rules = SqlSyncRules.fromYaml(` +bucket_definitions: + mybucket: + parameters: SELECT upper(token_parameters.user_id) as upper + data: [] + `); + expect(rules.errors).toEqual([]); + expect(rules.getStaticBucketIds(normalizeTokenParameters({ user_id: 'test' }))).toEqual(['mybucket["TEST"]']); }); test('custom table and id', () => { @@ -818,172 +659,7 @@ bucket_definitions: ]); }); - test('case-sensitive parameter queries (1)', () => { - const sql = 'SELECT users."userId" AS user_id FROM users WHERE users."userId" = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - - expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([ - { - lookup: ['mybucket', '1', 'user1'], - - bucket_parameters: [{ user_id: 'user1' }] - } - ]); - }); - - test('case-sensitive parameter queries (2)', () => { - // Note: This documents current behavior. - // This may change in the future - we should check against expected behavior for - // Postgres and/or SQLite. - const sql = 'SELECT users.userId AS user_id FROM users WHERE users.userId = token_parameters.user_id'; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - - expect(query.evaluateParameterRow({ userId: 'user1' })).toEqual([]); - expect(query.evaluateParameterRow({ userid: 'user1' })).toEqual([ - { - lookup: ['mybucket', '1', 'user1'], - - bucket_parameters: [{ user_id: 'user1' }] - } - ]); - }); - - test('dynamic global parameter query', () => { - const sql = "SELECT workspaces.id AS workspace_id FROM workspaces WHERE visibility = 'public'"; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - query.id = '1'; - - expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'public' })).toEqual([ - { - lookup: ['mybucket', '1'], - - bucket_parameters: [{ workspace_id: 'workspace1' }] - } - ]); - - expect(query.evaluateParameterRow({ id: 'workspace1', visibility: 'private' })).toEqual([]); - }); - - test('invalid OR in parameter queries', () => { - // Supporting this case is more tricky. We can do this by effectively denormalizing the OR clause - // into separate queries, but it's a significant change. For now, developers should do that manually. - const sql = - "SELECT workspaces.id AS workspace_id FROM workspaces WHERE workspaces.user_id = token_parameters.user_id OR visibility = 'public'"; - const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery; - expect(query.errors[0].message).toMatch(/must use the same parameters/); - }); - - test('types', () => { - const schema = BASIC_SCHEMA; - - const q1 = SqlDataQuery.fromSql('q1', ['user_id'], `SELECT * FROM assets WHERE owner_id = bucket.user_id`); - expect(q1.getColumnOutputs(schema)).toEqual([ - { - name: 'assets', - columns: [ - { name: 'id', type: ExpressionType.TEXT }, - { name: 'name', type: ExpressionType.TEXT }, - { name: 'count', type: ExpressionType.INTEGER }, - { name: 'owner_id', type: ExpressionType.TEXT } - ] - } - ]); - - const q2 = SqlDataQuery.fromSql( - 'q1', - ['user_id'], - ` - SELECT id :: integer as id, - upper(name) as name_upper, - hex('test') as hex, - count + 2 as count2, - count * 3.0 as count3, - count * '4' as count4, - name ->> '$.attr' as json_value, - ifnull(name, 2.0) as maybe_name - FROM assets WHERE owner_id = bucket.user_id` - ); - expect(q2.getColumnOutputs(schema)).toEqual([ - { - name: 'assets', - columns: [ - { name: 'id', type: ExpressionType.INTEGER }, - { name: 'name_upper', type: ExpressionType.TEXT }, - { name: 'hex', type: ExpressionType.TEXT }, - { name: 'count2', type: ExpressionType.INTEGER }, - { name: 'count3', type: ExpressionType.REAL }, - { name: 'count4', type: ExpressionType.NUMERIC }, - { name: 'json_value', type: ExpressionType.ANY_JSON }, - { name: 'maybe_name', type: ExpressionType.TEXT.or(ExpressionType.REAL) } - ] - } - ]); - }); - - test('validate columns', () => { - const schema = BASIC_SCHEMA; - const q1 = SqlDataQuery.fromSql( - 'q1', - ['user_id'], - 'SELECT id, name, count FROM assets WHERE owner_id = bucket.user_id', - schema - ); - expect(q1.errors).toEqual([]); - - const q2 = SqlDataQuery.fromSql( - 'q2', - ['user_id'], - 'SELECT id, upper(description) as d FROM assets WHERE other_id = bucket.user_id', - schema - ); - expect(q2.errors).toMatchObject([ - { - message: `Column not found: other_id`, - type: 'warning' - }, - { - message: `Column not found: description`, - type: 'warning' - } - ]); - - const q3 = SqlDataQuery.fromSql( - 'q3', - ['user_id'], - 'SELECT id, description, * FROM nope WHERE other_id = bucket.user_id', - schema - ); - expect(q3.errors).toMatchObject([ - { - message: `Table public.nope not found`, - type: 'warning' - } - ]); - - const q4 = SqlParameterQuery.fromSql( - 'q4', - 'SELECT id FROM assets WHERE owner_id = token_parameters.user_id', - schema - ); - expect(q4.errors).toMatchObject([]); - - const q5 = SqlParameterQuery.fromSql( - 'q5', - 'SELECT id as asset_id FROM assets WHERE other_id = token_parameters.user_id', - schema - ); - - expect(q5.errors).toMatchObject([ - { - message: 'Column not found: other_id', - type: 'warning' - } - ]); - }); - - test('progate parameter schema errors', () => { + test('propagate parameter schema errors', () => { const rules = SqlSyncRules.fromYaml( ` bucket_definitions: diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts new file mode 100644 index 0000000..049dfa2 --- /dev/null +++ b/packages/sync-rules/test/src/util.ts @@ -0,0 +1,33 @@ +import { DEFAULT_SCHEMA, DEFAULT_TAG, SourceTableInterface, StaticSchema } from '../../src/index.js'; + +export class TestSourceTable implements SourceTableInterface { + readonly connectionTag = DEFAULT_TAG; + readonly schema = DEFAULT_SCHEMA; + + constructor(public readonly table: string) {} +} + +export const ASSETS = new TestSourceTable('assets'); +export const USERS = new TestSourceTable('users'); + +export const BASIC_SCHEMA = new StaticSchema([ + { + tag: DEFAULT_TAG, + schemas: [ + { + name: DEFAULT_SCHEMA, + tables: [ + { + name: 'assets', + columns: [ + { name: 'id', pg_type: 'uuid' }, + { name: 'name', pg_type: 'text' }, + { name: 'count', pg_type: 'int4' }, + { name: 'owner_id', pg_type: 'uuid' } + ] + } + ] + } + ] + } +]);