From 748ef42c7c25972ac8ec3556d5674ed04c853bea Mon Sep 17 00:00:00 2001 From: Ola Okelola <10857143+lolopinto@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:06:13 -0700 Subject: [PATCH] More Entquery filters (#1635) --- ts/src/core/base.ts | 9 + ts/src/core/clause.ts | 53 +- ts/src/core/ent.ts | 36 +- ts/src/core/loaders/assoc_edge_loader.test.ts | 184 +++++- ts/src/core/loaders/assoc_edge_loader.ts | 25 + ts/src/core/query/assoc_query.ts | 127 +++- ts/src/core/query/query.ts | 91 ++- ts/src/core/query/shared_assoc_test.ts | 598 +++++++++++++++++- ts/src/core/query/shared_test.ts | 68 +- ts/src/core/query_impl.ts | 12 +- ts/src/parse_schema/parse.test.ts | 2 +- ts/src/testutils/builder.ts | 2 +- ts/src/testutils/query.ts | 70 ++ 13 files changed, 1145 insertions(+), 132 deletions(-) create mode 100644 ts/src/testutils/query.ts diff --git a/ts/src/core/base.ts b/ts/src/core/base.ts index 03e8955ef..7bdab2037 100644 --- a/ts/src/core/base.ts +++ b/ts/src/core/base.ts @@ -172,6 +172,14 @@ export interface QueryableDataOptions extends SelectBaseDataOptions, QueryDataOptions {} +// for now, no complicated joins or no need to support multiple joins +// just one simple join +interface JoinOptions { + tableName: string; + alias?: string; + clause: clause.Clause; +} + export interface QueryDataOptions { distinct?: boolean; clause: clause.Clause; @@ -179,6 +187,7 @@ export interface QueryDataOptions { groupby?: K; limit?: number; disableTransformations?: boolean; + join?:JoinOptions; } // For loading data from database diff --git a/ts/src/core/clause.ts b/ts/src/core/clause.ts index 76ef088bf..61323bc67 100644 --- a/ts/src/core/clause.ts +++ b/ts/src/core/clause.ts @@ -105,6 +105,8 @@ class simpleClause implements Clause { } } +// NB: we're not using alias in this class in clause method +// if we end up with a subclass that does, we need to handle it class queryClause implements Clause { constructor( protected dependentQueryOptions: QueryableDataOptions, // private value: any, // private op: string, // private handleNull?: Clause, @@ -143,20 +145,6 @@ class existsQueryClause extends queryClause { } } -class columnInQueryClause extends queryClause< - T, - K -> { - constructor( - protected dependentQueryOptions: QueryableDataOptions, - protected col: K, - ) { - // TODO renderCol needed here... - //TODO cal just kill this - super(dependentQueryOptions, `${col} IN`); - } -} - class isNullClause implements Clause { constructor(protected col: K) {} @@ -205,6 +193,30 @@ class isNotNullClause implements Clause { } } +class simpleExpression implements Clause { + constructor(protected expression: string) {} + + clause(idx: number, alias?: string): string { + return this.expression; + } + + columns(): K[] { + return []; + } + + values(): any[] { + return []; + } + + logValues(): any[] { + return []; + } + + instanceKey(): string { + return `${this.expression}`; + } +} + class arraySimpleClause implements Clause { constructor(protected col: K, private value: any, private op: string) {} @@ -835,13 +847,6 @@ export function DBTypeNotIn( return new notInClause(col, values, typ); } -export function ColInQuery( - col: K, - queryOptions: QueryableDataOptions, -): Clause { - return new columnInQueryClause(queryOptions, col); -} - interface TsQuery { // todo lang ::reconfig language: "english" | "french" | "german" | "simple"; @@ -1212,3 +1217,9 @@ export function getCombinedClause( } return cls; } + +export function Expression( + expression: string, +): Clause { + return new simpleExpression(expression); +} \ No newline at end of file diff --git a/ts/src/core/ent.ts b/ts/src/core/ent.ts index 49e5e5377..c119e6077 100644 --- a/ts/src/core/ent.ts +++ b/ts/src/core/ent.ts @@ -1373,7 +1373,7 @@ function defaultEdgeQueryOptions( id1: ID, edgeType: string, id2?: ID, -): EdgeQueryableDataOptions { +): Required> { let cls = clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType)); if (id2) { cls = clause.And(cls, clause.Eq("id2", id2)); @@ -1424,7 +1424,7 @@ export async function loadCustomEdges( fields, defaultOptions, tableName, - } = await loadEgesInfo(options); + } = await loadEdgesInfo(options); const rows = await loadRows({ tableName, @@ -1439,7 +1439,7 @@ export async function loadCustomEdges( }); } -async function loadEgesInfo( +async function loadEdgesInfo( options: loadCustomEdgesOptions, id2?: ID, ) { @@ -1450,7 +1450,7 @@ async function loadEgesInfo( } const defaultOptions = defaultEdgeQueryOptions(id1, edgeType, id2); - let cls = defaultOptions.clause!; + let cls = defaultOptions.clause; if (options.queryOptions?.clause) { cls = clause.And(cls, options.queryOptions.clause); } @@ -1543,11 +1543,11 @@ export async function loadEdgeForID2( cls: actualClause, fields, tableName, - } = await loadEgesInfo(options, options.id2); + } = await loadEdgesInfo(options, options.id2); const row = await loadRow({ tableName, - fields: fields, + fields, clause: actualClause, context: options.context, }); @@ -1556,6 +1556,30 @@ export async function loadEdgeForID2( } } +export async function loadTwoWayEdges( + opts: loadCustomEdgesOptions, +): Promise { + const { cls: actualClause, fields, tableName } = await loadEdgesInfo(opts); + + const rows = await loadRows({ + tableName, + alias: "t1", + fields, + clause: actualClause, + context: opts.context, + join: { + tableName, + alias: "t2", + clause: clause.And( + // these are not values so need this to not think they're values... + clause.Expression("t1.id1 = t2.id2"), + clause.Expression("t1.id2 = t2.id1"), + ), + }, + }); + return rows as T[]; +} + export async function loadNodesByEdge( viewer: Viewer, id1: ID, diff --git a/ts/src/core/loaders/assoc_edge_loader.test.ts b/ts/src/core/loaders/assoc_edge_loader.test.ts index c84f9e122..c42b5832d 100644 --- a/ts/src/core/loaders/assoc_edge_loader.test.ts +++ b/ts/src/core/loaders/assoc_edge_loader.test.ts @@ -27,13 +27,11 @@ import { tempDBTables, verifyUserToContactEdges, createEdges, + createTestUser, + addEdge, } from "../../testutils/fake_data/test_helpers"; -import { - AssocEdgeLoader, - AssocEdgeLoaderFactory, - AssocLoader, -} from "./assoc_edge_loader"; +import { AssocEdgeLoaderFactory, AssocLoader } from "./assoc_edge_loader"; import { testEdgeGlobalSchema } from "../../testutils/test_edge_global_schema"; import { SimpleAction } from "../../testutils/builder"; import { convertDate } from "../convert"; @@ -43,14 +41,14 @@ const ml = new MockLogs(); let ctx: TestContext; -const getNewLoader = (context: boolean = true) => { +const getNewContactsLoader = (context: boolean = true) => { return new AssocEdgeLoaderFactory( EdgeType.UserToContacts, AssocEdge, ).createLoader(context ? ctx : undefined); }; -const getConfigurableLoader = ( +const getConfigurableContactsLoader = ( context: boolean, options: EdgeQueryableDataOptions, ) => { @@ -60,6 +58,16 @@ const getConfigurableLoader = ( ).createConfigurableLoader(options, context ? ctx : undefined); }; +const getConfigurableFollowingLoader = ( + context: boolean, + options: EdgeQueryableDataOptions, +) => { + return new AssocEdgeLoaderFactory( + EdgeType.UserToFollowing, + CustomEdge, + ).createConfigurableLoader(options, context ? ctx : undefined); +}; + describe("postgres", () => { let tdb: TempDB; @@ -174,7 +182,7 @@ describe("sqlite global ", () => { function commonTests() { test("multi-ids. with context", async () => { await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), (ids) => { expect(ml.logs.length).toBe(1); expect(ml.logs[0].query).toMatch(/^SELECT * /); @@ -186,7 +194,7 @@ function commonTests() { test("multi-ids. with context and deletion", async () => { await testWithDeleteMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), (ids) => { expect(ml.logs.length).toBe(1); expect(ml.logs[0].query).toMatch(/^SELECT * /); @@ -203,7 +211,7 @@ function commonTests() { test("multi-ids. without context", async () => { await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, ); @@ -211,7 +219,7 @@ function commonTests() { test("multi-ids. without context and deletion", async () => { await testWithDeleteMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, ); @@ -219,14 +227,14 @@ function commonTests() { test("multi-ids. with context, offset", async () => { await testMultiQueryDataOffset( - (options) => getConfigurableLoader(true, options), + (options) => getConfigurableContactsLoader(true, options), true, ); }); test("multi-ids. without context, offset", async () => { await testMultiQueryDataOffset((options) => - getConfigurableLoader(false, options), + getConfigurableContactsLoader(false, options), ); }); @@ -235,7 +243,7 @@ function commonTests() { // initial. default limit await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), (ids) => { expect(ml.logs.length).toBe(1); expect(ml.logs[0].query).toMatch(/^SELECT * /); @@ -248,7 +256,7 @@ function commonTests() { // query again with same data and same limit and it should all be cache hits await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), verifyGroupedCacheHit, verifyGroupedCacheHit, undefined, @@ -258,7 +266,7 @@ function commonTests() { // change slice e.g. first N and now we hit the db again await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), (ids) => { expect(ml.logs.length).toBe(1); expect(ml.logs[0].query).toMatch(/^SELECT * /); @@ -271,7 +279,7 @@ function commonTests() { // query for first 3 again and all hits await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), verifyGroupedCacheHit, verifyGroupedCacheHit, 3, @@ -280,7 +288,7 @@ function commonTests() { // change slice e.g. first N and now we hit the db again await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), (ids) => { expect(ml.logs.length).toBe(1); expect(ml.logs[0].query).toMatch(/^SELECT * /); @@ -297,7 +305,7 @@ function commonTests() { // initial. default limit await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, undefined, @@ -306,7 +314,7 @@ function commonTests() { // // query again with same data and same limit and still we refetch it all await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, undefined, @@ -315,7 +323,7 @@ function commonTests() { // change slice e.g. first N and now we hit the db again await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, 3, @@ -324,7 +332,7 @@ function commonTests() { // refetch for 3. hit db again await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, 3, @@ -333,7 +341,7 @@ function commonTests() { // change slice e.g. first N and now we hit the db again await testMultiQueryDataAvail( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, 3, @@ -344,7 +352,7 @@ function commonTests() { test("with context. cache hit single id", async () => { const [user, contacts] = await createAllContacts(); ml.clear(); - const loader = getNewLoader(); + const loader = getNewContactsLoader(); const edges = await loader.load(user.id); verifyUserToContactEdges(user, edges, contacts.reverse()); verifyMultiCountQueryCacheMiss([user.id]); @@ -379,7 +387,7 @@ function commonTests() { test("without context. cache hit single id", async () => { const [user, contacts] = await createAllContacts(); ml.clear(); - const loader = getNewLoader(false); + const loader = getNewContactsLoader(false); const edges = await loader.load(user.id); verifyUserToContactEdges(user, edges, contacts.reverse()); verifyMultiCountQueryCacheMiss([user.id]); @@ -414,7 +422,7 @@ function commonTests() { test("with context. cache miss single id", async () => { const id = uuidv4(); ml.clear(); - const loader = getNewLoader(); + const loader = getNewContactsLoader(); const edges = await loader.load(id); expect(edges.length).toBe(0); verifyMultiCountQueryCacheMiss([id]); @@ -429,7 +437,7 @@ function commonTests() { test("without context. cache miss single id", async () => { const id = uuidv4(); ml.clear(); - const loader = getNewLoader(false); + const loader = getNewContactsLoader(false); const edges = await loader.load(id); expect(edges.length).toBe(0); verifyMultiCountQueryCacheMiss([id]); @@ -440,12 +448,120 @@ function commonTests() { verifyMultiCountQueryCacheMiss([id]); }); + + async function verifyTwoWayEdges( + loaderFn: (opts: EdgeQueryableDataOptions) => AssocLoader, + ) { + // create loader first so we can pass context to createTestUser + const loader = loaderFn({}); + + const user = await createTestUser({}, loader.context); + let twowayIds: ID[] = []; + for (let i = 0; i < 10; i++) { + const user2 = await createTestUser({}, loader.context); + await addEdge( + user, + FakeUserSchema, + EdgeType.UserToFollowing, + false, + user2, + ); + if (i % 2 == 0) { + twowayIds.push(user2.id); + await addEdge( + user2, + FakeUserSchema, + EdgeType.UserToFollowing, + false, + user, + ); + } + } + const edges = await loader.load(user.id); + const twoWay = await loader.loadTwoWay(user.id); + expect(twowayIds.sort()).toEqual(twoWay.map((e) => e.id2).sort()); + expect(edges.length).toBe(10); + expect(twoWay.length).toBe(5); + + const action = new SimpleAction( + user.viewer, + FakeUserSchema, + new Map(), + WriteOperation.Edit, + user, + ); + let i = 0; + ml.clear(); + + twowayIds = []; + for await (const edge of twoWay) { + if (i % 2 === 0) { + twowayIds.push(edge.id2); + } else { + action.builder.orchestrator.removeOutboundEdge( + edge.id2, + EdgeType.UserToFollowing, + ); + } + i++; + } + await action.saveX(); + user.viewer.context?.cache?.clearCache(); + loader.clearAll(); + // TODO why isn't this done automatically... + + const edges2 = await loader.load(user.id); + const twoWay2 = await loader.loadTwoWay(user.id); + + expect(twowayIds.sort()).toEqual(twoWay2.map((e) => e.id2).sort()); + + // deleted some things here which shouldn't show up here + expect(edges2.length).toBe(8); + expect(twoWay2.length).toBe(3); + + const hasGlobal = __hasGlobalSchema(); + + const loader2 = loaderFn({ + disableTransformations: true, + }); + const edges3 = await loader2.load(user.id); + const twoWay3 = await loader2.loadTwoWay(user.id); + if (!hasGlobal) { + expect(edges3.length).toBe(8); + expect(twoWay3.length).toBe(3); + + // same ids as second time + expect(twoWay2.map((e) => e.id1).sort()).toEqual( + twoWay3.map((e) => e.id1).sort(), + ); + } else { + expect(edges3.length).toBe(10); + expect(twoWay3.length).toBe(5); + + // same ids as first time + expect(twoWay.map((e) => e.id1).sort()).toEqual( + twoWay3.map((e) => e.id1).sort(), + ); + } + } + + test("two way edges with context", async () => { + await verifyTwoWayEdges((opts) => + getConfigurableFollowingLoader(true, opts), + ); + }); + + test("two way edges without context", async () => { + await verifyTwoWayEdges((opts) => + getConfigurableFollowingLoader(false, opts), + ); + }); } function globalTests() { test("multi-ids. with context and reload with deleted", async () => { await testWithDeleteMultiQueryDataLoadDeleted( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), (ids) => { expect(ml.logs.length).toBe(1); expect(ml.logs[0].query).toMatch(/^SELECT * /); @@ -462,7 +578,7 @@ function globalTests() { test("multi-ids. without context and reload with deleted", async () => { await testWithDeleteMultiQueryDataLoadDeleted( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, ); @@ -470,7 +586,7 @@ function globalTests() { test("single id. with context and reload with deleted", async () => { await testWithDeleteSingleQueryDataLoadDeleted( - (opts) => getConfigurableLoader(true, opts), + (opts) => getConfigurableContactsLoader(true, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, ); @@ -478,7 +594,7 @@ function globalTests() { test("single id. without context and reload with deleted", async () => { await testWithDeleteSingleQueryDataLoadDeleted( - (opts) => getConfigurableLoader(false, opts), + (opts) => getConfigurableContactsLoader(false, opts), verifyMultiCountQueryCacheMiss, verifyMultiCountQueryCacheMiss, ); @@ -538,11 +654,11 @@ function globalTests() { } test("load id2 with context", async () => { - await verifyLoadID2((opts) => getConfigurableLoader(true, opts)); + await verifyLoadID2((opts) => getConfigurableContactsLoader(true, opts)); }); test("load id2 without context", async () => { - await verifyLoadID2((opts) => getConfigurableLoader(false, opts)); + await verifyLoadID2((opts) => getConfigurableContactsLoader(false, opts)); }); } @@ -770,7 +886,7 @@ async function testWithDeleteSingleQueryDataLoadDeleted( ml.clear(); loader.clearAll(); - const loader2 = getConfigurableLoader(true, { + const loader2 = getConfigurableContactsLoader(true, { disableTransformations: true, }); diff --git a/ts/src/core/loaders/assoc_edge_loader.ts b/ts/src/core/loaders/assoc_edge_loader.ts index a43a7d6f8..3570b7d83 100644 --- a/ts/src/core/loaders/assoc_edge_loader.ts +++ b/ts/src/core/loaders/assoc_edge_loader.ts @@ -17,12 +17,15 @@ import { buildGroupQuery, AssocEdgeData, getEdgeClauseAndFields, + loadTwoWayEdges, } from "../ent"; import * as clause from "../clause"; import { logEnabled } from "../logger"; import { CacheMap, getCustomLoader } from "./loader"; import memoizee from "memoizee"; +// any loader created here or in places like this doesn't get context cache cleared... +// so any manual createLoader needs to be changed and added to ContextCache so that ContextCache.clearAll() works function createLoader( options: EdgeQueryableDataOptions, edgeType: string, @@ -102,6 +105,8 @@ function createLoader( export interface AssocLoader extends Loader { loadEdgeForID2(id: ID, id2: ID): Promise; + + loadTwoWay(id: ID): Promise; } export class AssocEdgeLoader implements Loader { @@ -135,6 +140,16 @@ export class AssocEdgeLoader implements Loader { return loader.load(id); } + async loadTwoWay(id: ID): Promise { + return loadTwoWayEdges({ + ctr: this.edgeCtr, + id1: id, + edgeType: this.edgeType, + context: this.context, + queryOptions: this.options, + }); + } + // maybe eventually optimize this async loadEdgeForID2(id: ID, id2: ID) { return loadEdgeForID2({ @@ -172,6 +187,16 @@ export class AssocDirectEdgeLoader }); } + async loadTwoWay(id: ID): Promise { + return loadTwoWayEdges({ + ctr: this.edgeCtr, + id1: id, + edgeType: this.edgeType, + context: this.context, + queryOptions: this.options, + }); + } + async loadEdgeForID2(id: ID, id2: ID) { return loadEdgeForID2({ id1: id, diff --git a/ts/src/core/query/assoc_query.ts b/ts/src/core/query/assoc_query.ts index 144bac1f3..9982a7182 100644 --- a/ts/src/core/query/assoc_query.ts +++ b/ts/src/core/query/assoc_query.ts @@ -38,6 +38,8 @@ export abstract class AssocEdgeQueryBase< extends BaseEdgeQuery implements EdgeQuery { + private loadTwoWay: boolean = false; + constructor( public viewer: TViewer, public src: EdgeQuerySource, @@ -192,7 +194,9 @@ export abstract class AssocEdgeQueryBase< // so only makes sense if one of these... // Id2 needs to be an option - const edges = await loader.load(info.id); + const edges = this.loadTwoWay + ? await loader.loadTwoWay(info.id) + : await loader.load(info.id); this.edges.set(info.id, edges); }), ); @@ -242,10 +246,52 @@ export abstract class AssocEdgeQueryBase< // start is inclusive, end is exclusive __withinBeta(start: Date, end: Date) { - this.__assertNoFiltersBETA("after"); + this.__assertNoFiltersBETA("within"); this.__addCustomFilterBETA(new WithinFilter(start, end)); return this; } + + /** + * intersect multiple queries together with this one to get candidate edges + * the edges returned will always be from the originating edge query + * + * @param others list of other queries to intersect with the source edge + */ + __intersect(...others: AssocEdgeQueryBase[]) { + // TODO I don't really see a reason why we can't chain first or something first before this + // but for now let's not support it + // when we do this correctly, we'll allow chaining + this.__assertNoFiltersBETA("intersect"); + this.__addCustomFilterBETA(new IntersectFilter(others)); + return this; + } + + /** + * union multiple queries together with this one to get candidate edges + * if the edge exists in the source query, that's the edge returned + * if the edge doesn't exist, the first edge in the list of queries that has the edge is returned + * @param others list of other queries to union with the source edge + */ + __union>( + ...others: AssocEdgeQueryBase[] + ) { + // same chain comment from intersect... + this.__assertNoFiltersBETA("union"); + this.__addCustomFilterBETA(new UnionFilter(others)); + return this; + } + + /** + * this fetches edges where there's a two way connection between both sets of edges + * e.g. in a social networking system, where the source and dest are both following each other + * + * will not work in the future when there's sharding... + */ + __twoWay() { + this.__assertNoFiltersBETA("twoWay"); + this.loadTwoWay = true; + return this; + } } export interface EdgeQueryCtr< @@ -297,3 +343,80 @@ class WithinFilter implements EdgeQueryFilter { return options; } } + +class IntersectFilter< + TDest extends Ent, + TEdge extends AssocEdge, + TViewer extends Viewer = Viewer, +> implements EdgeQueryFilter +{ + private edges: Array> = []; + constructor( + private queries: AssocEdgeQueryBase[], + ) {} + + async fetch(): Promise { + let i = 0; + // maybe future optimization. instead of this, use a SQL query if edge_types are the same + for await (const query of this.queries) { + const edges = await query.queryEdges(); + const set = new Set(); + edges.forEach((edge) => { + set.add(edge.id2); + }); + this.edges[i] = set; + i++; + } + } + + filter(_id: ID, edges: TEdge[]): TEdge[] { + return edges.filter((edge) => { + return this.edges.every((set) => { + return set.has(edge.id2); + }); + }); + } +} + +class UnionFilter< + TDest extends Ent, + TEdge extends AssocEdge, + TViewer extends Viewer = Viewer, +> implements EdgeQueryFilter +{ + private edges: Array = []; + constructor( + private queries: AssocEdgeQueryBase[], + ) {} + + async fetch(): Promise { + let i = 0; + // maybe future optimization. instead of this, use a SQL query if edge_types are the same + for await (const query of this.queries) { + const edges = await query.queryEdges(); + this.edges[i] = edges; + i++; + } + } + + filter(_id: ID, edges: TEdge[]): TEdge[] { + const set = new Set(); + const result: TEdge[] = []; + for (const edge of edges) { + set.add(edge.id2); + result.push(edge); + } + + for (const edges of this.edges) { + for (const edge of edges) { + if (set.has(edge.id2)) { + continue; + } + result.push(edge); + set.add(edge.id2); + } + } + + return result; + } +} diff --git a/ts/src/core/query/query.ts b/ts/src/core/query/query.ts index e3e0ec6cb..ece594da9 100644 --- a/ts/src/core/query/query.ts +++ b/ts/src/core/query/query.ts @@ -56,6 +56,10 @@ export interface EdgeQuery< //maybe id2 shouldn't return EdgeQuery but a different object from which you can query edge. the ent you don't need to query since you can just query that on your own. export interface EdgeQueryFilter { + // filter that needs to fetch data before it can be applied + // should not be used in conjunction with query + fetch?(): Promise; + // this is a filter that does the processing in TypeScript instead of (or in addition to) the SQL layer filter?(id: ID, edges: T[]): T[]; @@ -135,6 +139,7 @@ class FirstFilter implements EdgeQueryFilter { private sortCol: string; private edgeQuery: BaseEdgeQuery; private pageMap: Map = new Map(); + private usedQuery = false; constructor(private options: FirstFilterOptions) { assertPositive(options.limit); @@ -146,18 +151,59 @@ class FirstFilter implements EdgeQueryFilter { this.edgeQuery = options.query; } + private setPageMap(ret: T[], id: ID, hasNextPage: boolean) { + this.pageMap.set(id, { + hasNextPage, + // hasPreviousPage always false even if there's a previous page because + // we shouldn't be querying in both directions at the same + hasPreviousPage: false, + startCursor: this.edgeQuery.getCursor(ret[0]), + endCursor: this.edgeQuery.getCursor(ret[ret.length - 1]), + }); + } + filter(id: ID, edges: T[]): T[] { if (edges.length > this.options.limit) { - const ret = edges.slice(0, this.options.limit); - this.pageMap.set(id, { - hasNextPage: true, - // hasPreviousPage always false even if there's a previous page because - // we shouldn't be querying in both directions at the same - hasPreviousPage: false, - startCursor: this.edgeQuery.getCursor(ret[0]), - endCursor: this.edgeQuery.getCursor(ret[ret.length - 1]), - }); - return ret; + // we need a way to know where the cursor is and if we used it and if not need to do this in TypeScript and not in SQL + // so can't filter from 0 but specific item + + // if we used the query or we're querying the first N, slice from 0 + if (this.usedQuery || !this.offset) { + const ret = edges.slice(0, this.options.limit); + this.setPageMap(ret, id, true); + return ret; + } else if (this.offset) { + const ret: T[] = []; + let found = false; + let i = 0; + let hasNextPage = false; + for (const edge of edges) { + const id = edge[this.options.cursorCol]; + if (id === this.offset) { + // found! + found = true; + hasNextPage = true; + continue; + } + if (found) { + ret.push(edge); + if (ret.length === this.options.limit) { + if (i === ret.length - 1) { + hasNextPage = false; + } + break; + } + } + i++; + } + if (hasNextPage && ret.length < this.options.limit) { + hasNextPage = false; + } + if (ret.length) { + this.setPageMap(ret, id, hasNextPage); + } + return ret; + } } // TODO: in the future, when we have caching for edges // we'll want to hit that cache instead of passing the limit down to the @@ -170,6 +216,8 @@ class FirstFilter implements EdgeQueryFilter { async query( options: EdgeQueryableDataOptions, ): Promise { + this.usedQuery = true; + // we fetch an extra one to see if we're at the end const limit = this.options.limit + 1; @@ -621,8 +669,24 @@ export abstract class BaseEdgeQuery< // may need to bring sql mode or something back for (const filter of this.filters) { if (filter.query) { + if (filter.fetch) { + throw new Error( + `a filter that augments the query cannot currently fetch`, + ); + } let res = filter.query(options); options = isPromise(res) ? await res : res; + } else { + // if we've seen a filter that doesn't have a query, we can't do anything in SQL + // TODO figure out filter interactions https://github.com/lolopinto/ent/issues/685 + + // this is a scenario where if we have the first N filters that can modify the query, + // we do that in SQL and then we do the rest in code + // but once we have something doing it in code (e.g. intersect()), we can't do anything else in SQL + + // and then have to differentiate between filters that augment (limit or add to) the items returned + // and those that reorder... + break; } } @@ -633,6 +697,13 @@ export abstract class BaseEdgeQuery< return this.edges; } + // fetch anything we need to filter the query + for await (const filter of this.filters) { + if (filter.fetch) { + await filter.fetch(); + } + } + // filter as needed for (let [id, edges] of this.edges) { this.filters.forEach((filter) => { diff --git a/ts/src/core/query/shared_assoc_test.ts b/ts/src/core/query/shared_assoc_test.ts index a5f6e0b2c..be1d5c86f 100644 --- a/ts/src/core/query/shared_assoc_test.ts +++ b/ts/src/core/query/shared_assoc_test.ts @@ -22,6 +22,8 @@ import { UserToIncomingFriendRequestsQuery, ViewerWithAccessToken, FakeUserSchema, + FakeEventSchema, + EventToAttendeesQuery, } from "../../testutils/fake_data/index"; import { inputs, @@ -33,6 +35,8 @@ import { createTestEvent, getEventInput, createUserPlusFriendRequests, + addEdge, + createEdges, } from "../../testutils/fake_data/test_helpers"; import { MockLogs } from "../../testutils/mock_log"; import { And, Clause, Eq, Greater, GreaterEq, Less } from "../clause"; @@ -40,10 +44,17 @@ import { SimpleAction } from "../../testutils/builder"; import { DateTime } from "luxon"; import { convertDate } from "../convert"; import { TestContext } from "../../testutils/context/test_context"; +import { getVerifyAfterEachCursorGeneric } from "../../testutils/query"; export function assocTests(ml: MockLogs, global = false) { ml.mock(); + // beforeAll(async () => { + // // TODO this is needed when we do only in a test here + // // need to figure this out + // await createEdges(); + // }); + describe("custom edge", () => { let user1, user2: FakeUser; @@ -127,12 +138,12 @@ export function assocTests(ml: MockLogs, global = false) { logsStart = 0, direction = "DESC", }: verifyQueryProps) { - const clses: Clause[] = [Eq("id1", ""), Eq("edge_type", "")]; + const clauses: Clause[] = [Eq("id1", ""), Eq("edge_type", "")]; if (extraClause) { - clses.push(extraClause); + clauses.push(extraClause); } if (global) { - clses.push(Eq("deleted_at", null)); + clauses.push(Eq("deleted_at", null)); } expect(ml.logs.length).toBe(length); for (let i = logsStart; i < numQueries; i++) { @@ -141,7 +152,7 @@ export function assocTests(ml: MockLogs, global = false) { expect(whereClause, `${i}`).toBe( // default limit - `${And(...clses).clause( + `${And(...clauses).clause( 1, )} ORDER BY time ${direction}, id2 ${direction} LIMIT ${expLimit}`, ); @@ -231,7 +242,7 @@ export function assocTests(ml: MockLogs, global = false) { for (let i = 0; i < this.dataz.length; i++) { let data = this.dataz[i]; - expect(countMap.get(data[0].id)).toStrictEqual(inputs.length); + expect(countMap.get(data[0].id)).toBe(inputs.length); } verifyCountQuery({ numQueries: 3, length: 3 }); } @@ -244,7 +255,7 @@ export function assocTests(ml: MockLogs, global = false) { for (let i = 0; i < this.dataz.length; i++) { let data = this.dataz[i]; - expect(countMap.get(data[0].id)).toStrictEqual(data[1].length); + expect(countMap.get(data[0].id)).toBe(data[1].length); } verifyQuery({ length: this.dataz.length, @@ -403,6 +414,43 @@ export function assocTests(ml: MockLogs, global = false) { }); }); + function validateQueryIntersectOrUnion( + ml: MockLogs, + user1: FakeUser, + user2: FakeUser, + ) { + return function (_query, _cursor: string | undefined) { + // 2 queries for user because privacy + // 2 queries to fetch edges. + expect(ml.logs.length).toBe(4); + + const where1 = getWhereClause(ml.logs[1]); + const clauses1 = [ + Eq("id1", user1.id), + Eq("edge_type", EdgeType.UserToFriends), + ]; + const clauses2 = [ + Eq("id1", user2.id), + Eq("edge_type", EdgeType.UserToFriends), + ]; + if (global) { + clauses1.push(Eq("deleted_at", null)); + clauses2.push(Eq("deleted_at", null)); + } + const clause1 = And(...clauses1); + expect(where1).toBe(`${clause1.clause(1)} ORDER BY time DESC LIMIT 1000`); + expect(ml.logs[1].values).toStrictEqual(clause1.values()); + + const where2 = getWhereClause(ml.logs[3]); + const clause2 = And(...clauses2); + // TODO 1001 vs 1000 here + expect(where2).toBe( + `${clause2.clause(1)} ORDER BY time DESC, id2 DESC LIMIT 1001`, + ); + expect(ml.logs[3].values).toStrictEqual(clause2.values()); + }; + } + class ChainTestQueryFilter { user: FakeUser; event: FakeEvent; @@ -1325,6 +1373,544 @@ export function assocTests(ml: MockLogs, global = false) { }); }); + describe("intersect", () => { + let users: FakeUser[] = []; + let user1: FakeUser; + let user2: FakeUser; + + beforeEach(async () => { + users = []; + for (let i = 0; i < 10; i++) { + const user = await createTestUser(); + users.push(user); + } + + for (let i = 0; i < 10; i++) { + const user = users[i]; + // decreasing number of friends for each user + const candidates = users + .slice(0, 10 - i) + .filter((u) => u.id != user.id); + await addEdge( + user, + FakeUserSchema, + EdgeType.UserToFriends, + false, + ...candidates, + ); + const count = await UserToFriendsQuery.query( + user.viewer, + user, + ).queryRawCount(); + expect(count, `${i}`).toBe(candidates.length); + } + + user1 = users[0]; + user2 = users[1]; + }); + + function getQuery() { + return UserToFriendsQuery.query(user1.viewer, user1.id).__intersect( + UserToFriendsQuery.query(user2.viewer, user2.id), + ); + } + + function getCandidateIDs() { + // not the first 2 since that's user1 and user2 + // not the last one since that's removed from user2's list of friends + return users.slice(2, users.length - 1).map((u) => u.id); + } + + test("ids", async () => { + const ids = await getQuery().queryIDs(); + const candidates = getCandidateIDs(); + expect(ids.length).toBe(candidates.length); + expect(ids.sort()).toEqual(candidates.sort()); + }); + + test("count", async () => { + const count = await getQuery().queryCount(); + const candidates = getCandidateIDs(); + expect(count).toBe(candidates.length); + }); + + test("raw_count", async () => { + const count = await getQuery().queryRawCount(); + // raw count doesn't include the intersection + expect(count).toBe(users.length - 1); + }); + + test("edges", async () => { + const edges = await getQuery().queryEdges(); + const candidates = getCandidateIDs(); + expect(edges.length).toBe(candidates.length); + // for an intersect, the edge returned is always for the source id + for (const edge of edges) { + expect(edge.id1).toBe(user1.id); + } + }); + + test("ents", async () => { + const ents = await getQuery().queryEnts(); + const candidates = getCandidateIDs().sort(); + expect(ents.length).toBe(candidates.length); + expect(ents.map((u) => u.id).sort()).toEqual(candidates.sort()); + }); + + test("first", async () => { + const ents = await getQuery().first(2).queryEnts(); + expect(ents.length).toBe(2); + }); + + test("first. after each cursor", async () => { + const edges = await getQuery().queryEdges(); + + const { verify, getCursor } = getVerifyAfterEachCursorGeneric( + edges, + 2, + user1, + getQuery, + ml, + validateQueryIntersectOrUnion(ml, user1, user2), + ); + + // this one intentionally not generic so we know where to stop... + expect(edges.length).toBe(7); + await verify(0, true, true, undefined); + await verify(2, true, true, getCursor(edges[1])); + await verify(4, true, true, getCursor(edges[3])); + // 1 item, no nextPage + await verify(6, true, false, getCursor(edges[5])); + await verify(7, false, false, getCursor(edges[6])); + }); + + test("multiple intersections", async () => { + const query = UserToFriendsQuery.query( + user1.viewer, + user1.id, + ).__intersect( + UserToFriendsQuery.query(user2.viewer, user2.id), + UserToFriendsQuery.query(user2.viewer, users[2].id), + UserToFriendsQuery.query(user2.viewer, users[3].id), + ); + + // user0 => 0-9 - self + // user1 => 0-8 - self + // user2 => 0-7 - self + // user3 => 0-6 -self + // 7 users - 4 = 3 since you can't be friends with yourself + const candidates = [users[4], users[5], users[6]]; + const [ids, count, edges, ents] = await Promise.all([ + query.queryIDs(), + query.queryCount(), + query.queryEdges(), + query.queryEnts(), + ]); + expect(ids.length).toBe(candidates.length); + expect(count).toBe(candidates.length); + expect(edges.length).toBe(candidates.length); + expect(ents.length).toBe(candidates.length); + expect(edges.map((e) => e.id2).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + expect(ents.map((e) => e.id).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + }); + + test("multiple sources", async () => { + const query = UserToFriendsQuery.query(user1.viewer, [ + user1.id, + user2.id, + ]).__intersect( + UserToFriendsQuery.query(user2.viewer, users[2].id), + UserToFriendsQuery.query(user2.viewer, users[3].id), + ); + + // user0 => 0-9 - self + // user1 => 0-8 - self + + // user2 => 0-7 - self + // user3 => 0-6 -self + + // 7 users - 3 = 4 since you can't be friends with yourself + // subtracting 3 for each instead of 4 like above because user1 not intersecting with user2 + const candidates1 = [users[1], users[4], users[5], users[6]]; + const candidates2 = [users[0], users[4], users[5], users[6]]; + const [idsMap, countMap, edgesMap, entsMap] = await Promise.all([ + query.queryAllIDs(), + query.queryAllCount(), + query.queryAllEdges(), + query.queryAllEnts(), + ]); + + async function verify(source: FakeUser, candidates: FakeUser[]) { + const ids = idsMap.get(source.id) ?? []; + const count = countMap.get(source.id) ?? []; + const edges = edgesMap.get(source.id) ?? []; + const ents = entsMap.get(source.id) ?? []; + + expect(ids.length).toBe(candidates.length); + expect(count).toBe(candidates.length); + expect(edges.length).toBe(candidates.length); + expect(ents.length).toBe(candidates.length); + expect(edges.map((e) => e.id2).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + expect(ents.map((e) => e.id).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + } + + await verify(user1, candidates1); + await verify(user2, candidates2); + }); + + test("different edge types", async () => { + const event = await createTestEvent(user2); + for (let i = 0; i < 5; i++) { + const newUser = await createTestUser(); + await addEdge( + event, + FakeEventSchema, + EdgeType.EventToAttendees, + false, + newUser, + ); + } + + const query = UserToFriendsQuery.query( + user1.viewer, + user1.id, + ).__intersect(EventToAttendeesQuery.query(user1.viewer, event.id)); + + const count = await query.queryCount(); + expect(count).toBe(0); + + const candidates: FakeUser[] = []; + for (let i = 0; i < users.length; i++) { + const user = users[i]; + if (user.id !== user1.id && user.id !== user2.id) { + await addEdge( + event, + FakeEventSchema, + EdgeType.EventToAttendees, + false, + user, + ); + candidates.push(user); + } + } + + const query2 = UserToFriendsQuery.query( + user1.viewer, + user1.id, + ).__intersect(EventToAttendeesQuery.query(user1.viewer, event.id)); + + const count2 = await query2.queryCount(); + const ents = await query2.queryEnts(); + const ids = await query2.queryIDs(); + expect(count2).toBe(candidates.length); + expect(ids.length).toBe(candidates.length); + expect(ents.length).toBe(candidates.length); + expect(ents.map((e) => e.id).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + }); + }); + + describe("union", () => { + let users: FakeUser[] = []; + let user1: FakeUser; + let user2: FakeUser; + let friends: Map = new Map(); + + beforeEach(async () => { + users = []; + friends = new Map(); + for (let i = 0; i < 10; i++) { + const user = await createTestUser(); + users.push(user); + } + + for (let i = 0; i < 10; i++) { + const user = users[i]; + // decreasing number of friends for each user + + // we want little overlap so new users created. + const candidates = users + .slice(0, 10 - i) + .filter((u) => u.id != user.id); + + // add at least 5 more to each user + // every user should end up with 14 or 15 + // 9 + 5 = 14 + // 8 + 6 = 14 + // 7 + 7 = 14 + // 6 + 8 = 14 + // 5 + 9 = 14 + // 4 + 10 = 14 + // 3 + 11 = 14 + // 2 + 12 = 14 + // 1 + 13 = 14 + // 0 + 14 = 14 + + // 10 original + // 5 new for friend 1 + // 6 new for friend 2 + const newFriends = await Promise.all( + new Array(5 + i).fill(null).map(() => createTestUser()), + ); + candidates.push(...newFriends); + + await addEdge( + user, + FakeUserSchema, + EdgeType.UserToFriends, + false, + ...candidates, + ); + const count = await UserToFriendsQuery.query( + user.viewer, + user, + ).queryRawCount(); + expect(count, `${i}`).toBe(candidates.length); + friends.set(user.id, candidates); + } + + user1 = users[0]; + user2 = users[1]; + }); + + function getQuery() { + return UserToFriendsQuery.query(user1.viewer, user1.id).__union( + UserToFriendsQuery.query(user2.viewer, user2.id), + ); + } + + function getCandidateIDs() { + const set = new Set(); + friends.get(user1.id)?.forEach((u) => set.add(u)); + friends.get(user2.id)?.forEach((u) => set.add(u)); + + return Array.from(set.values()).map((u) => u.id); + } + + test("ids", async () => { + const ids = await getQuery().queryIDs(); + const candidates = getCandidateIDs(); + expect(ids.length).toBe(candidates.length); + expect(ids.sort()).toEqual(candidates.sort()); + }); + + test("count", async () => { + const count = await getQuery().queryCount(); + const candidates = getCandidateIDs(); + expect(count).toBe(candidates.length); + }); + + test("raw_count", async () => { + const count = await getQuery().queryRawCount(); + // raw count doesn't include the intersection + expect(count).toBe(friends.get(user1.id)?.length); + }); + + test("edges", async () => { + const edges = await getQuery().queryEdges(); + const candidates = getCandidateIDs(); + expect(edges.length).toBe(candidates.length); + const idMap = new Map(); + + for (const edge of edges) { + const ct = (idMap.get(edge.id1) ?? 0) + 1; + idMap.set(edge.id1, ct); + } + let id1count = friends.get(user1.id)?.length ?? 0; + expect(idMap.get(user1.id)).toBe(id1count); + expect(idMap.get(user2.id)).toBe(edges.length - id1count); + }); + + test("ents", async () => { + // only returns privacy aware which is friends + self... + const ents = await getQuery().queryEnts(); + const candidates = getCandidateIDs().sort(); + let id1count = friends.get(user1.id)?.length ?? 0; + const visible = id1count + 1; + + expect(ents.length).toBe(visible); + }); + + test("first", async () => { + const ents = await getQuery().first(2).queryEnts(); + expect(ents.length).toBe(2); + }); + + test("first. after each cursor", async () => { + const edges = await getQuery().queryEdges(); + + const { verify, getCursor } = getVerifyAfterEachCursorGeneric( + edges, + 3, + user1, + getQuery, + ml, + validateQueryIntersectOrUnion(ml, user1, user2), + ); + // hardcoded to test + expect(edges.length).toBe(21); + await verify(0, true, true, undefined); + await verify(3, true, true, getCursor(edges[2])); + await verify(6, true, true, getCursor(edges[5])); + await verify(9, true, true, getCursor(edges[8])); + await verify(12, true, true, getCursor(edges[11])); + await verify(15, true, true, getCursor(edges[14])); + await verify(18, true, true, getCursor(edges[17])); + await verify(21, false, false, getCursor(edges[20])); + }); + + test("multiple unions", async () => { + const user3 = users[2]; + const user4 = users[3]; + + const query = UserToFriendsQuery.query(user1.viewer, user1.id).__union( + UserToFriendsQuery.query(user1.viewer, user2.id), + UserToFriendsQuery.query(user1.viewer, user3.id), + UserToFriendsQuery.query(user1.viewer, user4.id), + ); + const candidates = Array.from( + new Set([ + ...(friends.get(user1.id) ?? []), // 0-9 (-self) + 5 friends + ...(friends.get(user2.id) ?? []), // 0-9 (-self) + 6 friends + ...(friends.get(user3.id) ?? []), // 0-9 (-self) + 7 friends + ...(friends.get(user4.id) ?? []), // 0-9 (-self) + 8 friends + ]).values(), + ); + // user1 can only see self + friends + const user1Visible = friends.get(user1.id) ?? []; + user1Visible.push(user1); + + // should be 36 + expect(candidates.length).toBe(10 + 5 + 6 + 7 + 8); + const [ids, count, edges, ents] = await Promise.all([ + query.queryIDs(), + query.queryCount(), + query.queryEdges(), + query.queryEnts(), + ]); + expect(ids.length).toBe(candidates.length); + expect(count).toBe(candidates.length); + expect(edges.length).toBe(candidates.length); + expect(ents.length).toBe(user1Visible.length); + expect(edges.map((e) => e.id2).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + expect(ents.map((e) => e.id).sort()).toEqual( + user1Visible.map((u) => u.id).sort(), + ); + }); + + test("multiple sources", async () => { + const user3 = users[2]; + const user4 = users[3]; + + const query = UserToFriendsQuery.query(user1.viewer, [ + user1.id, + user2.id, + ]).__union( + UserToFriendsQuery.query(user2.viewer, user3.id), + UserToFriendsQuery.query(user2.viewer, user4.id), + ); + + const candidates1 = Array.from( + new Set([ + ...(friends.get(user1.id) ?? []), // 0-9 (-self) + 5 friends + ...(friends.get(user3.id) ?? []), // 0-9 (-self) + 7 friends + ...(friends.get(user4.id) ?? []), // 0-9 (-self) + 8 friends + ]).values(), + ); + const candidates2 = Array.from( + new Set([ + ...(friends.get(user2.id) ?? []), // 0-9 (-self) + 6 friends + ...(friends.get(user3.id) ?? []), // 0-9 (-self) + 7 friends + ...(friends.get(user4.id) ?? []), // 0-9 (-self) + 8 friends + ]).values(), + ); + const [idsMap, countMap, edgesMap, entsMap] = await Promise.all([ + query.queryAllIDs(), + query.queryAllCount(), + query.queryAllEdges(), + query.queryAllEnts(), + ]); + // user1 can only see self + friends + const user1Visible = friends.get(user1.id) ?? []; + user1Visible.push(user1); + + // since the EntQuery uses user1's viewer + // can only see user2's friends that intersect with user1's friends + // and that's every user except for the last user since we don't add that user to user2's friends + const user2Visible = users.slice(0, users.length - 1); + + async function verify( + source: FakeUser, + candidates: FakeUser[], + visible: FakeUser[], + ) { + const ids = idsMap.get(source.id) ?? []; + const count = countMap.get(source.id) ?? []; + const edges = edgesMap.get(source.id) ?? []; + const ents = entsMap.get(source.id) ?? []; + + expect(ids.length).toBe(candidates.length); + expect(count).toBe(candidates.length); + expect(edges.length).toBe(candidates.length); + expect(ents.length).toBe(visible.length); + expect(edges.map((e) => e.id2).sort()).toEqual( + candidates.map((u) => u.id).sort(), + ); + expect(ents.map((e) => e.id).sort()).toEqual( + visible.map((u) => u.id).sort(), + ); + } + + await verify(user1, candidates1, user1Visible); + await verify(user2, candidates2, user2Visible); + }); + + test("different edge types", async () => { + const event = await createTestEvent(user2); + const newUsers: FakeUser[] = []; + for (let i = 0; i < 5; i++) { + const newUser = await createTestUser(); + await addEdge( + event, + FakeEventSchema, + EdgeType.EventToAttendees, + false, + newUser, + ); + newUsers.push(newUser); + } + + const query = UserToFriendsQuery.query(user1.viewer, user1.id).__union( + EventToAttendeesQuery.query(user1.viewer, event.id), + ); + + const candidates = [ + ...(friends.get(user1.id) ?? []).map((u) => u.id), + ...newUsers.map((u) => u.id), + ]; + + const count = await query.queryCount(); + expect(count).toBe(candidates.length); + + const ents = await query.queryEnts(); + const ids = await query.queryIDs(); + expect(ids.length).toBe(candidates.length); + // can only see friends + expect(ents.length).toBe(friends.get(user1.id)?.length); + }); + }); + if (!global) { return; } diff --git a/ts/src/core/query/shared_test.ts b/ts/src/core/query/shared_test.ts index 088a29f9f..972719bb7 100644 --- a/ts/src/core/query/shared_test.ts +++ b/ts/src/core/query/shared_test.ts @@ -32,6 +32,10 @@ import { WriteOperation } from "../../action"; import { MockLogs } from "../../testutils/mock_log"; import { Clause, PaginationMultipleColsSubQuery } from "../clause"; import { OrderBy, getOrderByPhrase, reverseOrderBy } from "../query_impl"; +import { + getVerifyAfterEachCursorGeneric, + getWhereClause, +} from "../../testutils/query"; interface options { newQuery: ( @@ -51,14 +55,6 @@ interface options { rawDataVerify?(user: FakeUser): Promise; } -function getWhereClause(query: any) { - const idx = (query.query as string).indexOf("WHERE"); - if (idx !== -1) { - return query.query.substr(idx + 6); - } - return null; -} - export const commonTests = (opts: options) => { setLogLevels(["query", "error"]); const ml = opts.ml; @@ -351,47 +347,21 @@ export const commonTests = (opts: options) => { pageLength: number, user: FakeUser, ) { - let query: EdgeQuery; - - async function verify( - i: number, - hasEdge: boolean, - hasNextPage: boolean, - cursor?: string, - ) { - ml.clear(); - query = opts.newQuery(getViewer(), user); - const newEdges = await query.first(pageLength, cursor).queryEdges(); - - const pagination = query.paginationInfo().get(user.id); - if (hasEdge) { - expect(newEdges[0], `${i}`).toStrictEqual(edges[i]); - expect(newEdges.length, `${i}`).toBe( - edges.length - i >= pageLength ? pageLength : edges.length - i, - ); - } else { - expect(newEdges.length, `${i}`).toBe(0); - } - - if (hasNextPage) { - expect(pagination?.hasNextPage, `${i}`).toBe(true); - expect(pagination?.hasPreviousPage, `${i}`).toBe(false); - } else { - expect(pagination?.hasNextPage, `${i}`).toBe(undefined); - expect(pagination?.hasNextPage, `${i}`).toBe(undefined); - } - - if (cursor) { - verifyFirstAfterCursorQuery(query!, 1, pageLength); - } else { - verifyQuery(query!, { orderby: opts.orderby, limit: pageLength }); - } - } - - function getCursor(edge: TData) { - return query.getCursor(edge); - } - return { verify, getCursor }; + return getVerifyAfterEachCursorGeneric( + edges, + pageLength, + user, + // @ts-ignore weirdness with import paths... + () => opts.newQuery(getViewer(), user), + ml, + (query, cursor) => { + if (cursor) { + verifyFirstAfterCursorQuery(query, 1, pageLength); + } else { + verifyQuery(query, { orderby: opts.orderby, limit: pageLength }); + } + }, + ); } function getVerifyBeforeEachCursor( diff --git a/ts/src/core/query_impl.ts b/ts/src/core/query_impl.ts index eb94b8eb8..07f0bdb3d 100644 --- a/ts/src/core/query_impl.ts +++ b/ts/src/core/query_impl.ts @@ -40,12 +40,20 @@ export function buildQuery(options: QueryableDataOptions): string { : options.fields.join(", "); // always start at 1 - const whereClause = options.clause.clause(1, options.alias); const parts: string[] = []; const tableName = options.alias ? `${options.tableName} AS ${options.alias}` : options.tableName; - parts.push(`SELECT ${fields} FROM ${tableName} WHERE ${whereClause}`); + parts.push(`SELECT ${fields} FROM ${tableName}`); + + let whereStart = 1; + if (options.join) { + const joinTable = options.join.alias ? `${options.join.tableName} ${options.join.alias}` : options.join.tableName; + parts.push(`JOIN ${joinTable} ON ${options.join.clause.clause(1)}`); + whereStart += options.join.clause.values().length; + } + + parts.push(`WHERE ${options.clause.clause(whereStart, options.alias)}`); if (options.groupby) { parts.push(`GROUP BY ${options.groupby}`); } diff --git a/ts/src/parse_schema/parse.test.ts b/ts/src/parse_schema/parse.test.ts index 48370bcb4..9a670f847 100644 --- a/ts/src/parse_schema/parse.test.ts +++ b/ts/src/parse_schema/parse.test.ts @@ -1,4 +1,4 @@ -import { FieldMap, Schema } from "src/schema"; +import { FieldMap, Schema } from "../schema"; import { StringType } from "../schema/field"; import { BaseEntSchema, EntSchema } from "../schema/base_schema"; import { parseSchema } from "./parse"; diff --git a/ts/src/testutils/builder.ts b/ts/src/testutils/builder.ts index cf393fa9d..55b8b4948 100644 --- a/ts/src/testutils/builder.ts +++ b/ts/src/testutils/builder.ts @@ -32,7 +32,7 @@ import { EntSchemaWithTZ, } from "../schema/base_schema"; import { FieldInfoMap, getStorageKey } from "../schema/schema"; -import { Clause } from "src/core/clause"; +import { Clause } from "../core/clause"; import { ChangesetOptions } from "../action/action"; export class BaseEnt { diff --git a/ts/src/testutils/query.ts b/ts/src/testutils/query.ts new file mode 100644 index 000000000..40c5ce3a8 --- /dev/null +++ b/ts/src/testutils/query.ts @@ -0,0 +1,70 @@ +import { Data, Ent, Viewer } from "../core/base"; +import { FakeUser } from "./fake_data"; +import { EdgeQuery } from "../core/query"; +import { MockLogs } from "./mock_log"; + +export function getVerifyAfterEachCursorGeneric< + TSource extends Ent, + TDest extends Ent, + TData extends Data, +>( + edges: TData[], + pageLength: number, + user: FakeUser, + getQuery: () => EdgeQuery, + ml: MockLogs, + verifyQuery?: ( + query: EdgeQuery, + cursor: string | undefined, + ) => void, +) { + let query: EdgeQuery; + + async function verify( + i: number, + hasEdge: boolean, + hasNextPage: boolean | undefined, + cursor?: string, + ) { + ml.clear(); + query = getQuery(); + const newEdges = await query.first(pageLength, cursor).queryEdges(); + + const pagination = query.paginationInfo().get(user.id); + if (hasEdge) { + expect(newEdges[0], `${i}`).toEqual(edges[i]); + expect(newEdges.length, `${i}`).toBe( + edges.length - i >= pageLength ? pageLength : edges.length - i, + ); + // verify items are the same in order + expect(newEdges, `${i}`).toEqual(edges.slice(i, i + newEdges.length)); + } else { + expect(newEdges.length, `${i}`).toBe(0); + } + + if (hasNextPage) { + expect(pagination?.hasNextPage, `${i}`).toBe(true); + expect(pagination?.hasPreviousPage, `${i}`).toBe(false); + } else { + expect(pagination?.hasPreviousPage, `${i}`).toBeFalsy(); + expect(pagination?.hasNextPage, `${i}`).toBeFalsy(); + } + + if (verifyQuery) { + verifyQuery(query!, cursor); + } + } + + function getCursor(edge: TData) { + return query.getCursor(edge); + } + return { verify, getCursor }; +} + +export function getWhereClause(query: any) { + const idx = (query.query as string).indexOf("WHERE"); + if (idx !== -1) { + return query.query.substr(idx + 6); + } + return null; +}