From 50457928e4851ecaf0c86cb0941eb803566b382d Mon Sep 17 00:00:00 2001 From: Jet Zhou Date: Mon, 12 Feb 2024 18:09:31 -0800 Subject: [PATCH 1/2] Create a third `options` argument and allow users to specify a `maxPages` setting --- README.md | 28 +++++++++++++++++++++++++++- src/index.ts | 1 + src/iterator.ts | 9 +++++++++ src/options.ts | 3 +++ src/paginate.ts | 3 +++ test/paginate.test.ts | 39 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 src/options.ts diff --git a/README.md b/README.md index 6fc6e6b..5434bc4 100644 --- a/README.md +++ b/README.md @@ -157,9 +157,35 @@ await octokit.graphql.paginate( ); ``` +### Options + +You can provide a third argument to `paginate` or `iterator` to modify the behavior of the pagination. + +`maxPages` will stop the iteration at the specified number of pages, useful when you don't need all the items in the response but still want to take advantage of the automatic merging. + +``` +const { repository } = await octokit.graphql.paginate( + `query paginate($cursor: String) { + repository(owner: "octokit", name: "rest.js") { + issues(first: 10, after: $cursor) { + nodes { + title + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`, + { }, + { maxPages: 10 }, +); +``` + ### Pagination Direction -You can control the pagination direction by the properties deinfed in the `pageInfo` resource. +You can control the pagination direction by the properties defined in the `pageInfo` resource. For a forward pagination, use: diff --git a/src/index.ts b/src/index.ts index cad271b..a443ac1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import { Octokit } from "@octokit/core"; import { createIterator } from "./iterator"; import { createPaginate } from "./paginate"; export type { PageInfoForward, PageInfoBackward } from "./page-info"; +export type { Options } from "./options"; export function paginateGraphql(octokit: Octokit) { octokit.graphql; diff --git a/src/iterator.ts b/src/iterator.ts index fdd9f4b..c6e1659 100644 --- a/src/iterator.ts +++ b/src/iterator.ts @@ -2,19 +2,28 @@ import { extractPageInfos } from "./extract-page-info"; import { Octokit } from "@octokit/core"; import { getCursorFrom, hasAnotherPage } from "./page-info"; import { MissingCursorChange } from "./errors"; +import type { Options } from "./options"; const createIterator = (octokit: Octokit) => { return ( query: string, initialParameters: Record = {}, + options: Options = {}, ) => { let nextPageExists = true; let parameters = { ...initialParameters }; + const { maxPages } = options; + let page = 0; return { [Symbol.asyncIterator]: () => ({ async next() { if (!nextPageExists) return { done: true, value: {} as ResponseType }; + if (maxPages && page >= maxPages) { + return { done: true, value: {} as ResponseType }; + } + + page += 1; const response = await octokit.graphql( query, diff --git a/src/options.ts b/src/options.ts new file mode 100644 index 0000000..58f7dcf --- /dev/null +++ b/src/options.ts @@ -0,0 +1,3 @@ +export type Options = { + maxPages?: number; +}; diff --git a/src/paginate.ts b/src/paginate.ts index 5c4f930..6d3eea9 100644 --- a/src/paginate.ts +++ b/src/paginate.ts @@ -1,17 +1,20 @@ import { Octokit } from "@octokit/core"; import { mergeResponses } from "./merge-responses"; import { createIterator } from "./iterator"; +import type { Options } from "./options"; const createPaginate = (octokit: Octokit) => { const iterator = createIterator(octokit); return async ( query: string, initialParameters: Record = {}, + options: Options = {}, ): Promise => { let mergedResponse: ResponseType = {} as ResponseType; for await (const response of iterator( query, initialParameters, + options, )) { mergedResponse = mergeResponses(mergedResponse, response); } diff --git a/test/paginate.test.ts b/test/paginate.test.ts index 0628cc0..7e530da 100644 --- a/test/paginate.test.ts +++ b/test/paginate.test.ts @@ -282,6 +282,45 @@ describe("pagination", () => { ]); }); + it(".paginate.iterator() allows users to pass `maxPages` parameter and stops at the right place.", async (): Promise => { + const responses = createResponsePages({ amount: 3 }); + + const { octokit, getCallCount, getPassedVariablesForCall } = MockOctokit({ + responses, + }); + + const actualResponse = await octokit.graphql.paginate( + ` + query paginate ($cursor: String) { + repository(owner: "octokit", name: "rest.js") { + issues(first: 10, after: $cursor) { + nodes { + title + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`, + {}, + { maxPages: 2 }, + ); + + expect(actualResponse).toEqual({ + repository: { + issues: { + nodes: [{ title: "Issue 1" }, { title: "Issue 2" }], + pageInfo: { hasNextPage: true, endCursor: "endCursor2" }, + }, + }, + }); + expect(getCallCount()).toBe(2); + expect(getPassedVariablesForCall(1)).toBeUndefined(); + expect(getPassedVariablesForCall(2)).toEqual({ cursor: "endCursor1" }); + }); + it("paginate() throws error with path and variable name if cursors do not change between calls.", async (): Promise => { const [responsePage1, responsePage2] = createResponsePages({ amount: 2 }); responsePage2.repository.issues.pageInfo = { From 807b43c9c0abde3adf8b37ed1e0317afbb8d94c2 Mon Sep 17 00:00:00 2001 From: Jet Zhou Date: Wed, 14 Feb 2024 16:48:08 -0800 Subject: [PATCH 2/2] Alternative implementation --- README.md | 48 +++++++++++++++++++++++---- src/index.ts | 1 - src/iterator.ts | 13 +++----- src/options.ts | 3 -- src/paginate.ts | 5 ++- test/paginate.test.ts | 77 +++++++++++++++++++++++++++++++++++-------- 6 files changed, 111 insertions(+), 36 deletions(-) delete mode 100644 src/options.ts diff --git a/README.md b/README.md index 5434bc4..c51d26b 100644 --- a/README.md +++ b/README.md @@ -157,15 +157,45 @@ await octokit.graphql.paginate( ); ``` -### Options +### Early stop when iterating -You can provide a third argument to `paginate` or `iterator` to modify the behavior of the pagination. +You can provide a third argument, a function, to `paginate` or `iterator` to stop the pagination earlier. The function will be called with two arguments, the first is the content of the most recent page, the second is a `done` function to stop the iteration. -`maxPages` will stop the iteration at the specified number of pages, useful when you don't need all the items in the response but still want to take advantage of the automatic merging. +For example, you can stop the iteration after a certain number of pages: +```js +const maxPages = 2; +let pages = 0; + +await octokit.graphql.paginate( + `query paginate ($cursor: String) { + repository(owner: "octokit", name: "rest.js") { + issues(first: 10, after: $cursor) { + nodes { + title + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`, + {}, + (_, done) => { + pages += 1; + if (pages >= maxPages) { + done(); + } + }, +); ``` -const { repository } = await octokit.graphql.paginate( - `query paginate($cursor: String) { + +Or, to stop after you find a certain item: + +```js +await octokit.graphql.paginate( + `query paginate ($cursor: String) { repository(owner: "octokit", name: "rest.js") { issues(first: 10, after: $cursor) { nodes { @@ -178,8 +208,12 @@ const { repository } = await octokit.graphql.paginate( } } }`, - { }, - { maxPages: 10 }, + {}, + (response, done) => { + if (response?.repository?.issues?.nodes?.[0].title === "Issue 2") { + done(); + } + }, ); ``` diff --git a/src/index.ts b/src/index.ts index a443ac1..cad271b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,7 +2,6 @@ import { Octokit } from "@octokit/core"; import { createIterator } from "./iterator"; import { createPaginate } from "./paginate"; export type { PageInfoForward, PageInfoBackward } from "./page-info"; -export type { Options } from "./options"; export function paginateGraphql(octokit: Octokit) { octokit.graphql; diff --git a/src/iterator.ts b/src/iterator.ts index c6e1659..1cbce30 100644 --- a/src/iterator.ts +++ b/src/iterator.ts @@ -2,34 +2,31 @@ import { extractPageInfos } from "./extract-page-info"; import { Octokit } from "@octokit/core"; import { getCursorFrom, hasAnotherPage } from "./page-info"; import { MissingCursorChange } from "./errors"; -import type { Options } from "./options"; const createIterator = (octokit: Octokit) => { return ( query: string, initialParameters: Record = {}, - options: Options = {}, + stopFunction?: (response: ResponseType, done: () => void) => void, ) => { let nextPageExists = true; + let stopEarly = false; let parameters = { ...initialParameters }; - const { maxPages } = options; - let page = 0; return { [Symbol.asyncIterator]: () => ({ async next() { - if (!nextPageExists) return { done: true, value: {} as ResponseType }; - if (maxPages && page >= maxPages) { + if (!nextPageExists || stopEarly) { return { done: true, value: {} as ResponseType }; } - page += 1; - const response = await octokit.graphql( query, parameters, ); + stopFunction?.(response, () => (stopEarly = true)); + const pageInfoContext = extractPageInfos(response); const nextCursorValue = getCursorFrom(pageInfoContext.pageInfo); nextPageExists = hasAnotherPage(pageInfoContext.pageInfo); diff --git a/src/options.ts b/src/options.ts deleted file mode 100644 index 58f7dcf..0000000 --- a/src/options.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type Options = { - maxPages?: number; -}; diff --git a/src/paginate.ts b/src/paginate.ts index 6d3eea9..d9284ad 100644 --- a/src/paginate.ts +++ b/src/paginate.ts @@ -1,20 +1,19 @@ import { Octokit } from "@octokit/core"; import { mergeResponses } from "./merge-responses"; import { createIterator } from "./iterator"; -import type { Options } from "./options"; const createPaginate = (octokit: Octokit) => { const iterator = createIterator(octokit); return async ( query: string, initialParameters: Record = {}, - options: Options = {}, + stopFunction?: (response: ResponseType, done: () => void) => void, ): Promise => { let mergedResponse: ResponseType = {} as ResponseType; for await (const response of iterator( query, initialParameters, - options, + stopFunction, )) { mergedResponse = mergeResponses(mergedResponse, response); } diff --git a/test/paginate.test.ts b/test/paginate.test.ts index 7e530da..8ffe05c 100644 --- a/test/paginate.test.ts +++ b/test/paginate.test.ts @@ -282,30 +282,79 @@ describe("pagination", () => { ]); }); - it(".paginate.iterator() allows users to pass `maxPages` parameter and stops at the right place.", async (): Promise => { + it(".paginate.iterator() allows users to pass `stopFunction` and stops at the right place.", async (): Promise => { const responses = createResponsePages({ amount: 3 }); const { octokit, getCallCount, getPassedVariablesForCall } = MockOctokit({ responses, }); + const maxPages = 2; + let pages = 0; + const actualResponse = await octokit.graphql.paginate( - ` - query paginate ($cursor: String) { - repository(owner: "octokit", name: "rest.js") { - issues(first: 10, after: $cursor) { - nodes { - title - } - pageInfo { - hasNextPage - endCursor - } + `query paginate ($cursor: String) { + repository(owner: "octokit", name: "rest.js") { + issues(first: 10, after: $cursor) { + nodes { + title + } + pageInfo { + hasNextPage + endCursor } } - }`, + } + }`, {}, - { maxPages: 2 }, + (_, done) => { + pages += 1; + if (pages >= maxPages) { + done(); + } + }, + ); + + expect(actualResponse).toEqual({ + repository: { + issues: { + nodes: [{ title: "Issue 1" }, { title: "Issue 2" }], + pageInfo: { hasNextPage: true, endCursor: "endCursor2" }, + }, + }, + }); + expect(getCallCount()).toBe(2); + expect(getPassedVariablesForCall(1)).toBeUndefined(); + expect(getPassedVariablesForCall(2)).toEqual({ cursor: "endCursor1" }); + }); + + it(".paginate.iterator() allows users to pass `stopFunction` and stops at the right place.", async (): Promise => { + const responses = createResponsePages({ amount: 3 }); + + const { octokit, getCallCount, getPassedVariablesForCall } = MockOctokit({ + responses, + }); + + const actualResponse = await octokit.graphql.paginate( + `query paginate ($cursor: String) { + repository(owner: "octokit", name: "rest.js") { + issues(first: 10, after: $cursor) { + nodes { + title + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`, + {}, + (response, done) => { + if (response?.repository?.issues?.nodes?.[0].title === "Issue 2") { + done(); + } + }, ); expect(actualResponse).toEqual({