From 4616699a8a8334ab26a80fb19c752a16e68c22ca Mon Sep 17 00:00:00 2001 From: Shaya Potter Date: Thu, 6 Jun 2024 14:25:43 +0300 Subject: [PATCH 1/3] add ISEMPTY field support to FT.CREATE first part of supporting what to do with "missing" values that has been implemented in RedisSearch --- packages/search/lib/commands/CREATE.spec.ts | 126 +++++++++++++++++--- packages/search/lib/commands/CREATE.ts | 31 ++++- 2 files changed, 140 insertions(+), 17 deletions(-) diff --git a/packages/search/lib/commands/CREATE.spec.ts b/packages/search/lib/commands/CREATE.spec.ts index dad983c003..4e57ba9215 100644 --- a/packages/search/lib/commands/CREATE.spec.ts +++ b/packages/search/lib/commands/CREATE.spec.ts @@ -69,24 +69,74 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'WITHSUFFIXTRIE'] ); }); + + it('with ISEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TEXT, + MISSING_VAUES: { + IS_EMPTY: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'ISEMPTY'] + ); + }); }); - it('NUMERIC', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: SCHEMA_FIELD_TYPE.NUMERIC - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'NUMERIC'] - ); + describe('NUMERIC', () => { + describe('transformArguments', () => { + it('no options', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: SCHEMA_FIELD_TYPE.NUMERIC + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'NUMERIC'] + ); + }); + + it('with ISEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.NUMERIC, + MISSING_VAUES: { + IS_EMPTY: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'NUMERIC', 'ISEMPTY'] + ); + }); + }); }); - it('GEO', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: SCHEMA_FIELD_TYPE.GEO - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEO'] - ); + describe('GEO', () => { + describe('transformArguments', () => { + it('no options', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: SCHEMA_FIELD_TYPE.GEO + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEO'] + ); + }); + + it('with ISEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.GEO, + MISSING_VAUES: { + IS_EMPTY: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEO', 'ISEMPTY'] + ); + }); + }); }); describe('TAG', () => { @@ -147,6 +197,20 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TAG', 'WITHSUFFIXTRIE'] ); }); + + it('with ISEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TAG, + MISSING_VAUES: { + IS_EMPTY: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TAG', 'ISEMPTY'] + ); + }); }); describe('VECTOR', () => { @@ -193,6 +257,25 @@ describe('FT.CREATE', () => { ] ); }); + + it('with ISEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.VECTOR, + ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.HNSW, + TYPE: 'FLOAT32', + DIM: 2, + DISTANCE_METRIC: 'L2', + MISSING_VAUES: { + IS_EMPTY: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'VECTOR', 'HNSW', '7', 'TYPE', + 'FLOAT32', 'DIM', '2', 'DISTANCE_METRIC', 'L2', 'ISEMPTY'] + ); + }); }); describe('GEOSHAPE', () => { @@ -229,6 +312,21 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEOSHAPE', 'COORD_SYSTEM', 'SPHERICAL'] ); }); + + it('with ISEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.GEOSHAPE, + MISSING_VAUES: { + IS_EMPTY: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEOSHAPE', 'ISEMPTY'] + ); + }); + }); it('with AS', () => { diff --git a/packages/search/lib/commands/CREATE.ts b/packages/search/lib/commands/CREATE.ts index f6b66085f2..20eb20917b 100644 --- a/packages/search/lib/commands/CREATE.ts +++ b/packages/search/lib/commands/CREATE.ts @@ -12,12 +12,17 @@ export const SCHEMA_FIELD_TYPE = { export type SchemaFieldType = typeof SCHEMA_FIELD_TYPE[keyof typeof SCHEMA_FIELD_TYPE]; +export interface MissingValues { + IS_EMPTY?: boolean; +} + type SchemaField< T extends SchemaFieldType, E = Record > = T | ({ type: T; AS?: RedisArgument; + MISSING_VAUES?: MissingValues; } & E); type SchemaCommonField< @@ -108,6 +113,16 @@ export interface RediSearchSchema { ); } +function pushMissingValues(args: CommandArguments, missingValues?: MissingValues) { + if (!missingValues) { + return; + } + + if (missingValues.IS_EMPTY) { + args.push("ISEMPTY"); + } +} + export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { for (const [field, fieldOptions] of Object.entries(schema)) { args.push(field); @@ -141,11 +156,14 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { args.push('WITHSUFFIXTRIE'); } + pushMissingValues(args, fieldOptions.MISSING_VAUES); + break; - // case SchemaFieldTypes.NUMERIC: - // case SchemaFieldTypes.GEO: - // break; + case SCHEMA_FIELD_TYPE.NUMERIC: + case SCHEMA_FIELD_TYPE.GEO: + pushMissingValues(args, fieldOptions.MISSING_VAUES); + break; case SCHEMA_FIELD_TYPE.TAG: if (fieldOptions.SEPARATOR) { @@ -160,6 +178,8 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { args.push('WITHSUFFIXTRIE'); } + pushMissingValues(args, fieldOptions.MISSING_VAUES); + break; case SCHEMA_FIELD_TYPE.VECTOR: @@ -200,6 +220,9 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { break; } + + pushMissingValues(args, fieldOptions.MISSING_VAUES); + args[lengthIndex] = (args.length - lengthIndex - 1).toString(); continue; // vector fields do not contain SORTABLE and NOINDEX options @@ -209,6 +232,8 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { args.push('COORD_SYSTEM', fieldOptions.COORD_SYSTEM); } + pushMissingValues(args, fieldOptions.MISSING_VAUES); + continue; // geo shape fields do not contain SORTABLE and NOINDEX options } From 9c91c5c3374ba8722ca09610354a47caeee2a383 Mon Sep 17 00:00:00 2001 From: Shaya Potter Date: Thu, 6 Jun 2024 21:06:54 +0300 Subject: [PATCH 2/3] move from ISEMPTY to ISMISSING/ISNULL --- packages/search/lib/commands/CREATE.spec.ts | 120 +++++--------------- packages/search/lib/commands/CREATE.ts | 11 +- 2 files changed, 38 insertions(+), 93 deletions(-) diff --git a/packages/search/lib/commands/CREATE.spec.ts b/packages/search/lib/commands/CREATE.spec.ts index 4e57ba9215..5a506fbf42 100644 --- a/packages/search/lib/commands/CREATE.spec.ts +++ b/packages/search/lib/commands/CREATE.spec.ts @@ -69,20 +69,6 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'WITHSUFFIXTRIE'] ); }); - - it('with ISEMPTY', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.TEXT, - MISSING_VAUES: { - IS_EMPTY: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'ISEMPTY'] - ); - }); }); describe('NUMERIC', () => { @@ -95,20 +81,6 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'NUMERIC'] ); }); - - it('with ISEMPTY', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.NUMERIC, - MISSING_VAUES: { - IS_EMPTY: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'NUMERIC', 'ISEMPTY'] - ); - }); }); }); @@ -122,20 +94,6 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEO'] ); }); - - it('with ISEMPTY', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.GEO, - MISSING_VAUES: { - IS_EMPTY: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEO', 'ISEMPTY'] - ); - }); }); }); @@ -197,20 +155,6 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TAG', 'WITHSUFFIXTRIE'] ); }); - - it('with ISEMPTY', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.TAG, - MISSING_VAUES: { - IS_EMPTY: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TAG', 'ISEMPTY'] - ); - }); }); describe('VECTOR', () => { @@ -257,25 +201,6 @@ describe('FT.CREATE', () => { ] ); }); - - it('with ISEMPTY', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.VECTOR, - ALGORITHM: SCHEMA_VECTOR_FIELD_ALGORITHM.HNSW, - TYPE: 'FLOAT32', - DIM: 2, - DISTANCE_METRIC: 'L2', - MISSING_VAUES: { - IS_EMPTY: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'VECTOR', 'HNSW', '7', 'TYPE', - 'FLOAT32', 'DIM', '2', 'DISTANCE_METRIC', 'L2', 'ISEMPTY'] - ); - }); }); describe('GEOSHAPE', () => { @@ -312,21 +237,6 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEOSHAPE', 'COORD_SYSTEM', 'SPHERICAL'] ); }); - - it('with ISEMPTY', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.GEOSHAPE, - MISSING_VAUES: { - IS_EMPTY: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'GEOSHAPE', 'ISEMPTY'] - ); - }); - }); it('with AS', () => { @@ -536,6 +446,36 @@ describe('FT.CREATE', () => { ); }); }); + + describe('Missing Values', () => { + it('with ISNULL', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TEXT, + MISSING_VAUES: { + IS_NULL: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'ISNULL'] + ); + }); + + it('with ISMISSING', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TEXT, + MISSING_VAUES: { + IS_MISSING: true + } + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'ISMISSING'] + ); + }); + }); }); testUtils.testWithClient('client.ft.create', async client => { diff --git a/packages/search/lib/commands/CREATE.ts b/packages/search/lib/commands/CREATE.ts index 20eb20917b..23b10a8391 100644 --- a/packages/search/lib/commands/CREATE.ts +++ b/packages/search/lib/commands/CREATE.ts @@ -13,7 +13,8 @@ export const SCHEMA_FIELD_TYPE = { export type SchemaFieldType = typeof SCHEMA_FIELD_TYPE[keyof typeof SCHEMA_FIELD_TYPE]; export interface MissingValues { - IS_EMPTY?: boolean; + IS_NULL?: boolean; + IS_MISSING?: boolean; } type SchemaField< @@ -118,8 +119,12 @@ function pushMissingValues(args: CommandArguments, missingValues?: MissingValues return; } - if (missingValues.IS_EMPTY) { - args.push("ISEMPTY"); + if (missingValues.IS_MISSING) { + args.push("ISMISSING"); + } + + if (missingValues.IS_NULL) { + args.push("ISNULL"); } } From 7cfa9e71f28916ea9a09fe9c0f0efba10e3bb84b Mon Sep 17 00:00:00 2001 From: Shaya Potter Date: Thu, 15 Aug 2024 11:53:54 +0300 Subject: [PATCH 3/3] sync with changes leibele wanted for v4 release --- packages/search/lib/commands/CREATE.spec.ts | 65 ++++++++-------- packages/search/lib/commands/CREATE.ts | 85 ++++++++++----------- 2 files changed, 76 insertions(+), 74 deletions(-) diff --git a/packages/search/lib/commands/CREATE.spec.ts b/packages/search/lib/commands/CREATE.spec.ts index 5a506fbf42..5a622f1fd8 100644 --- a/packages/search/lib/commands/CREATE.spec.ts +++ b/packages/search/lib/commands/CREATE.spec.ts @@ -69,6 +69,17 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'WITHSUFFIXTRIE'] ); }); + it('with INDEXEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TEXT, + INDEXEMPTY: true + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'INDEXEMPTY'] + ); + }); }); describe('NUMERIC', () => { @@ -155,6 +166,18 @@ describe('FT.CREATE', () => { ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TAG', 'WITHSUFFIXTRIE'] ); }); + + it('with INDEXEMPTY', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TAG, + INDEXEMPTY: true + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TAG', 'INDEXEMPTY'] + ); + }); }); describe('VECTOR', () => { @@ -290,6 +313,18 @@ describe('FT.CREATE', () => { }); }); + it('with INDEXMISSING', () => { + assert.deepEqual( + CREATE.transformArguments('index', { + field: { + type: SCHEMA_FIELD_TYPE.TEXT, + INDEXMISSING: true + } + }), + ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'INDEXMISSING'] + ); + }); + it('with ON', () => { assert.deepEqual( CREATE.transformArguments('index', {}, { @@ -446,36 +481,6 @@ describe('FT.CREATE', () => { ); }); }); - - describe('Missing Values', () => { - it('with ISNULL', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.TEXT, - MISSING_VAUES: { - IS_NULL: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'ISNULL'] - ); - }); - - it('with ISMISSING', () => { - assert.deepEqual( - CREATE.transformArguments('index', { - field: { - type: SCHEMA_FIELD_TYPE.TEXT, - MISSING_VAUES: { - IS_MISSING: true - } - } - }), - ['FT.CREATE', 'index', 'SCHEMA', 'field', 'TEXT', 'ISMISSING'] - ); - }); - }); }); testUtils.testWithClient('client.ft.create', async client => { diff --git a/packages/search/lib/commands/CREATE.ts b/packages/search/lib/commands/CREATE.ts index 23b10a8391..1094adc7e5 100644 --- a/packages/search/lib/commands/CREATE.ts +++ b/packages/search/lib/commands/CREATE.ts @@ -12,31 +12,42 @@ export const SCHEMA_FIELD_TYPE = { export type SchemaFieldType = typeof SCHEMA_FIELD_TYPE[keyof typeof SCHEMA_FIELD_TYPE]; -export interface MissingValues { - IS_NULL?: boolean; - IS_MISSING?: boolean; -} - type SchemaField< T extends SchemaFieldType, E = Record > = T | ({ type: T; AS?: RedisArgument; - MISSING_VAUES?: MissingValues; + INDEXMISSING?: boolean; } & E); +type CommonFieldArguments = { + SORTABLE?: boolean | 'UNF'; + NOINDEX?: boolean; +}; + type SchemaCommonField< T extends SchemaFieldType, E = Record > = SchemaField< - T, - ({ - SORTABLE?: boolean | 'UNF'; - NOINDEX?: boolean; - } & E) + T, + (CommonFieldArguments & E) >; +function pushCommonFieldArguments(args: CommandArguments, fieldOptions: CommonFieldArguments) { + if (fieldOptions.SORTABLE) { + args.push('SORTABLE'); + + if (fieldOptions.SORTABLE === 'UNF') { + args.push('UNF'); + } + } + + if (fieldOptions.NOINDEX) { + args.push('NOINDEX'); + } +} + export const SCHEMA_TEXT_FIELD_PHONETIC = { DM_EN: 'dm:en', DM_FR: 'dm:fr', @@ -51,6 +62,7 @@ type SchemaTextField = SchemaCommonField; type SchemaNumericField = SchemaCommonField; @@ -61,6 +73,7 @@ type SchemaTagField = SchemaCommonField; export const SCHEMA_VECTOR_FIELD_ALGORITHM = { @@ -98,7 +111,7 @@ export const SCHEMA_GEO_SHAPE_COORD_SYSTEM = { export type SchemaGeoShapeFieldCoordSystem = typeof SCHEMA_GEO_SHAPE_COORD_SYSTEM[keyof typeof SCHEMA_GEO_SHAPE_COORD_SYSTEM]; -type SchemaGeoShapeField = SchemaField; @@ -114,20 +127,6 @@ export interface RediSearchSchema { ); } -function pushMissingValues(args: CommandArguments, missingValues?: MissingValues) { - if (!missingValues) { - return; - } - - if (missingValues.IS_MISSING) { - args.push("ISMISSING"); - } - - if (missingValues.IS_NULL) { - args.push("ISNULL"); - } -} - export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { for (const [field, fieldOptions] of Object.entries(schema)) { args.push(field); @@ -161,13 +160,17 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { args.push('WITHSUFFIXTRIE'); } - pushMissingValues(args, fieldOptions.MISSING_VAUES); + pushCommonFieldArguments(args, fieldOptions); + + if (fieldOptions.INDEXEMPTY) { + args.push('INDEXEMPTY'); + } break; case SCHEMA_FIELD_TYPE.NUMERIC: case SCHEMA_FIELD_TYPE.GEO: - pushMissingValues(args, fieldOptions.MISSING_VAUES); + pushCommonFieldArguments(args, fieldOptions); break; case SCHEMA_FIELD_TYPE.TAG: @@ -183,7 +186,11 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { args.push('WITHSUFFIXTRIE'); } - pushMissingValues(args, fieldOptions.MISSING_VAUES); + pushCommonFieldArguments(args, fieldOptions); + + if (fieldOptions.INDEXEMPTY) { + args.push('INDEXEMPTY'); + } break; @@ -226,32 +233,22 @@ export function pushSchema(args: CommandArguments, schema: RediSearchSchema) { break; } - pushMissingValues(args, fieldOptions.MISSING_VAUES); - args[lengthIndex] = (args.length - lengthIndex - 1).toString(); - continue; // vector fields do not contain SORTABLE and NOINDEX options + break; case SCHEMA_FIELD_TYPE.GEOSHAPE: if (fieldOptions.COORD_SYSTEM !== undefined) { args.push('COORD_SYSTEM', fieldOptions.COORD_SYSTEM); } - pushMissingValues(args, fieldOptions.MISSING_VAUES); - - continue; // geo shape fields do not contain SORTABLE and NOINDEX options - } - - if (fieldOptions.SORTABLE) { - args.push('SORTABLE'); + pushCommonFieldArguments(args, fieldOptions); - if (fieldOptions.SORTABLE === 'UNF') { - args.push('UNF'); - } + break; // geo shape fields do not contain SORTABLE and NOINDEX options } - if (fieldOptions.NOINDEX) { - args.push('NOINDEX'); + if (fieldOptions.INDEXMISSING) { + args.push('INDEXMISSING'); } } }