From 025f58dfd01740c43392dd228e9b8b629194e958 Mon Sep 17 00:00:00 2001 From: Artem Melnyk Date: Tue, 20 Feb 2024 14:37:40 +0100 Subject: [PATCH] feat: improve api for `PageIterator` and add new `ChunkPageIterator` --- endpoints/general.types.ts | 6 +- pagination.test.ts | 110 +++++++++++++---- pagination.ts | 245 +++++++++++++++++++++++++------------ 3 files changed, 255 insertions(+), 106 deletions(-) diff --git a/endpoints/general.types.ts b/endpoints/general.types.ts index 0444ce6..d73b418 100644 --- a/endpoints/general.types.ts +++ b/endpoints/general.types.ts @@ -46,12 +46,12 @@ export type CursorPagingObject = { /** * The cursor to use as key to find the next page of items. */ - after: string; + after?: string; /** * The cursor to use as key to find the previous page of items. */ - before: string; - }; + before?: string; + } | null; /** * The total number of items available to return. */ diff --git a/pagination.test.ts b/pagination.test.ts index 94b331d..a15f5df 100644 --- a/pagination.test.ts +++ b/pagination.test.ts @@ -1,52 +1,64 @@ -import { PageIterator } from "./pagination.ts"; +import { CursorPageIterator, PageIterator } from "./pagination.ts"; import { assertEquals } from "std/assert/mod.ts"; type MockArtist = { - id: number; + id: string; type: "artist"; name: string; }; const totalMockItems = 50; -const mockArtists: MockArtist[] = Array(totalMockItems).fill(0).map(( - _, - i, -) => ({ - id: i, - type: "artist", - name: "Radiohead", -})); +const mockArtists: MockArtist[] = Array(totalMockItems) + .fill(0) + .map((_, i) => ({ + id: i.toString(), + type: "artist", + name: "Radiohead", + })); Deno.test("PageIterator: asyncIterator", async () => { - const pageIterator = new PageIterator((opts) => { - const limit = opts.limit || 20; - const offset = opts.offset || 0; + const pageIterator = new PageIterator((offset) => { + const limit = 20; return Promise.resolve({ - href: "#", items: mockArtists.slice(offset, offset + limit), - offset, limit, next: offset + limit < totalMockItems ? "http://example.com" : null, - previous: offset > 0 ? "http://example.com" : null, total: totalMockItems, }); }); - const iter = pageIterator.asyncIterator(); + const iter1 = pageIterator[Symbol.asyncIterator](); for (let i = 0; i < totalMockItems; i++) { - assertEquals(await iter.next(), { done: false, value: mockArtists[i] }); + const result = await iter1.next(); + assertEquals(result, { + done: false, + value: mockArtists[i], + }); } - assertEquals(await iter.next(), { + assertEquals(await iter1.next(), { + done: true, + value: null, + }); + + const iter2 = pageIterator[Symbol.asyncIterator](-totalMockItems); + + for (let i = 0; i < totalMockItems; i++) { + const result = await iter2.next(); + assertEquals(result, { + done: false, + value: mockArtists[i], + }); + } + assertEquals(await iter2.next(), { done: true, value: null, }); }); Deno.test("PageIterator: collect", async () => { - const pageIterator = new PageIterator((opts) => { - const limit = opts.limit || 20; - const offset = opts.offset || 0; + const pageIterator = new PageIterator((offset) => { + const limit = 20; return Promise.resolve({ href: "#", @@ -61,4 +73,58 @@ Deno.test("PageIterator: collect", async () => { const items = await pageIterator.collect(); assertEquals(items, mockArtists); + + const items2 = await pageIterator.collect(10); + assertEquals(items2, mockArtists.slice(0, 10)); +}); + +Deno.test("CursorPageIterator: asyncIterator", async () => { + const cursorPageIterator = new CursorPageIterator((opts) => { + const limit = 20; + + if (opts.after === undefined) { + return Promise.resolve({ + items: mockArtists.slice(0, limit), + next: limit < totalMockItems ? "http://example.com" : null, + cursors: { + after: mockArtists.at(limit - 1)?.id, + }, + }); + } + + const afterArtistIndex = mockArtists.findIndex((artist) => + artist.id === opts.after + ); + if (afterArtistIndex === -1) { + throw new Error("Invalid cursor"); + } + + return Promise.resolve({ + items: mockArtists.slice( + afterArtistIndex + 1, + afterArtistIndex + 1 + limit, + ), + next: afterArtistIndex + limit < totalMockItems + ? "http://example.com" + : null, + cursors: { + after: mockArtists.at(afterArtistIndex + limit)?.id, + }, + }); + }); + + const iter = cursorPageIterator[Symbol.asyncIterator](); + + for (let i = 0; i < totalMockItems; i++) { + const result = await iter.next(); + console.log(result); + assertEquals(result, { + done: false, + value: mockArtists[i], + }); + } + assertEquals(await iter.next(), { + done: true, + value: null, + }); }); diff --git a/pagination.ts b/pagination.ts index 03e91e7..ee6521f 100644 --- a/pagination.ts +++ b/pagination.ts @@ -1,115 +1,198 @@ -import type { Prettify } from "./shared.ts"; -import type { PagingObject, PagingOptions } from "./endpoints/general.types.ts"; +export type PageIteratorOptions = { + /** + * The Spotify API does not allow you to use a negative offset, but you can do so with this property. This will be useful when, for example, you want to get the last 100 elements. + * + * Under the hood, it will first get the total number of items by fetching with an offset of `0` and then calculate the starting offset. + * + * @default 0 + */ + initialOffset?: number; +}; /** - * Represents the possible directions a paginator can take, where the values of "next" and "prev" indicate whether the iterator is navigating forward or backward. + * A helper class which allows you to iterate over items in a paginated API response with javascript async iterators. + * + * @example + * ```ts + * const playlistIter = new PageIterator((offset) => + * getPlaylistTracks(client, "SOME_PLAYLITS_ID", { offset, limit: 50 }) + * ); + * + * // Iterate over the playlist tracks + * for await (const track of playlistIter) { + * console.log(track); + * } + * + * // Collect all the tracks + * const tracks = await playlistIter.collect(); + * + * // Collect the last 100 tracks in playlist + * const lastHundredTracks = new PageIterator( + * (offset) => + * getPlaylistTracks(client, "SOME_PLAYLITS_ID", { offset, limit: 50 }), + * { initialOffset: -100 }, + * ).collect(); + * ``` */ -type PaginatorDirection = "next" | "prev"; - -type NextPageOptions = { - limit?: number; - setOffset?: (offset: number) => number; -}; +export class PageIterator { + private options: Required; -type PageIterOptions = Prettify< - PagingOptions & { - direction?: PaginatorDirection; + constructor( + private readonly fetcher: (offset: number) => Promise<{ + limit: number; + next: string | null; + total: number; + items: TItem[]; + }>, + options: PageIteratorOptions = {}, + ) { + this.options = { initialOffset: 0, ...options }; } ->; -const DEFAULTS: Required = { - direction: "next", - limit: 20, - offset: 0, -}; + async *[Symbol.asyncIterator]( + initialOffset?: number, + ): AsyncGenerator { + let offset = typeof initialOffset === "number" + ? initialOffset + : this.options.initialOffset; + + if (offset < 0) { + const page = await this.fetcher(0); + if (page.total === 0) { + return null; + } + offset = page.total + offset; + } -export class ChunkIterator { - private defaults: Required; + while (true) { + const page = await this.fetcher(offset); - constructor( - private fetcher: (opts: PagingOptions) => Promise>, - defaults: PageIterOptions = {}, - ) { - this.defaults = { ...DEFAULTS, ...defaults }; - } + for (let i = 0; i < page.items.length; i++) { + yield page.items[i]; + } - asyncIterator(): AsyncIterator< - TItem[], - TItem[], - NextPageOptions | undefined - > { - return this[Symbol.asyncIterator](); + if (!page.next) { + return null; + } + + offset = offset + page.limit; + } } - [Symbol.asyncIterator](): AsyncIterator< - TItem[], - TItem[], - NextPageOptions | undefined - > { - let done = false; - let { direction, limit, offset } = this.defaults; - - return { - next: async (opts = {}) => { - if (done) return { done, value: [] }; - limit = opts.limit ?? this.defaults.limit; - offset = opts.setOffset ? opts.setOffset(offset) : offset; - - const chunk = await this.fetcher({ limit, offset }); - - if ( - (direction === "next" && !chunk.next) || - (direction === "prev" && !chunk.previous) - ) { - done = true; - return { value: chunk.items, done: false }; - } - - offset = direction === "next" ? offset + limit : offset - limit; - return { value: chunk.items, done }; - }, - }; + /** + * @param limit The maximum number of items to collect. By default it set to `Infinity`, which means it will collect all items. + */ + async collect(limit = Infinity): Promise { + if (limit < 0) { + throw new RangeError( + `The limit must be a positive number, got ${limit}`, + ); + } + const items: TItem[] = []; + for await (const item of this) { + items.push(item); + if (items.length >= limit) { + break; + } + } + return items; } } -export class PageIterator { - private defaults: Required; +type Direction = "backward" | "forward"; - constructor( - private fetcher: (opts: PagingOptions) => Promise>, - defaults: PageIterOptions = {}, - ) { - this.defaults = { ...DEFAULTS, ...defaults }; +export type CursorPageIteratorOptions = + & { + direction?: TDirection; } + & (TDirection extends "forward" ? { initialAfter?: string } + : { initialBefore?: string }); - asyncIterator(): AsyncGenerator { - return this[Symbol.asyncIterator](); +/** + * A helper class which allows you to iterate over items in a cursor paginated API response with javascript async iterators. + * + * @example + * ```ts + * // get the first 100 followed artists + * const artists = await new CursorPageIterator( + * opts => getFollowedArtists(client, { limit: 50, after: opts.after}) + * ).collect(100); + * ``` + */ +export class CursorPageIterator< + TItem, + TDirection extends "backward" | "forward" = "forward", +> { + private options: CursorPageIteratorOptions & { + direction: TDirection; + }; + + constructor( + private readonly fetcher: ( + options: TDirection extends "forward" ? { after?: string } + : { before?: string }, + ) => Promise<{ + next: string | null; + cursors: { + after?: string; + before?: string; + } | null; + items: TItem[]; + }>, + options: CursorPageIteratorOptions = {}, + ) { + this.options = { direction: "forward", ...options } as + & CursorPageIteratorOptions + & { + direction: TDirection; + }; } - async *[Symbol.asyncIterator](): AsyncGenerator { - let { direction, limit, offset } = this.defaults; + async *[Symbol.asyncIterator](): AsyncGenerator { + const direction = this.options.direction; + let cursor = direction === "forward" + ? "initialAfter" in this.options ? this.options.initialAfter : undefined + : "initialBefore" in this.options + ? this.options.initialBefore + : undefined; while (true) { - const chunk = await this.fetcher({ limit, offset }); + const page = await this.fetcher( + (direction === "forward" ? { after: cursor } : { before: cursor }) as { + after?: string; + before?: string; + }, + ); + + for (let i = 0; i < page.items.length; i++) { + yield page.items[i]; + } - if ( - (direction === "next" && !chunk.next) || - (direction === "prev" && !chunk.previous) - ) { - for (const item of chunk.items) yield item; + if (!page.next) { return null; } - for (const item of chunk.items) yield item; - - offset = direction === "next" ? offset + limit : offset - limit; + cursor = direction === "forward" + ? page.cursors?.after + : page.cursors?.before; } } - async collect(): Promise { + /** + * @param limit The maximum number of items to collect. By default it set to `Infinity`, which means it will collect all items. + */ + async collect(limit = Infinity): Promise { + if (limit < 0) { + throw new RangeError( + `The limit must be a positive number, got ${limit}`, + ); + } const items: TItem[] = []; for await (const item of this) { items.push(item); + if (items.length >= limit) { + break; + } } return items; }