From aee95d4201f66673190bb75b367453faab4c6eee Mon Sep 17 00:00:00 2001 From: coji Date: Mon, 6 Apr 2026 12:03:36 +0900 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20crawl=20=E3=81=AE=20shouldStop=20?= =?UTF-8?q?=E5=A2=83=E7=95=8C=E6=9D=A1=E4=BB=B6=E3=82=92=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E3=81=97=E6=AF=8E=E6=99=82=E3=81=AE=E7=84=A1=E9=A7=84=E3=81=AA?= =?UTF-8?q?=E5=86=8D=E3=83=95=E3=82=A7=E3=83=83=E3=83=81=E3=82=92=E8=A7=A3?= =?UTF-8?q?=E6=B6=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pullrequestList の shouldStop が `<` (strictly less than) だったため、 updatedAt == lastFetchedAt の PR が毎回フィルタを通過し、全 repo の最新 PR が 毎時再フェッチされていた。`<=` に修正して既にフェッチ済みの PR を除外する。 Co-Authored-By: Claude Opus 4.6 (1M context) --- batch/github/fetcher.test.ts | 88 +++++++++++++++++++++++++++++++++++- batch/github/fetcher.ts | 4 +- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/batch/github/fetcher.test.ts b/batch/github/fetcher.test.ts index 3a14dbf9..5967eae0 100644 --- a/batch/github/fetcher.test.ts +++ b/batch/github/fetcher.test.ts @@ -3,7 +3,7 @@ import { describe, expect, test, vi } from 'vitest' import type { ShapedTimelineItem } from './model' // 純粋関数なので直接 import してテスト -const { buildRequestedAtMap, createFetcher, shapeTagNode } = +const { buildRequestedAtMap, createFetcher, paginateGraphQL, shapeTagNode } = await import('./fetcher') describe('buildRequestedAtMap', () => { @@ -263,3 +263,89 @@ describe('shapeTagNode', () => { expect(result).toBeNull() }) }) + +describe('paginateGraphQL shouldStop', () => { + // ページネーション結果のモック生成ヘルパー + type Node = { number: number; updatedAt: string } + const makeGraphqlFn = (pages: Node[][]) => { + let callIndex = 0 + return (_vars: Record) => { + const nodes = pages[callIndex++] ?? [] + return { + pullRequests: { + nodes, + pageInfo: { + hasNextPage: callIndex < pages.length, + endCursor: `cursor-${callIndex}`, + }, + }, + } + } + } + type Result = + ReturnType>> extends Promise< + infer R + > + ? R + : never + + const extractConnection = (r: Result) => r.pullRequests + const processNode = (n: Node) => n + + test('excludes node with updatedAt equal to stopBefore', async () => { + const stopBefore = '2026-04-01T00:00:00Z' + const pages: Node[][] = [ + [ + { number: 3, updatedAt: '2026-04-02T00:00:00Z' }, + { number: 2, updatedAt: '2026-04-01T00:00:00Z' }, // == stopBefore → 除外 + { number: 1, updatedAt: '2026-03-31T00:00:00Z' }, + ], + ] + const result = await paginateGraphQL( + makeGraphqlFn(pages), + extractConnection, + processNode, + { shouldStop: (node) => node.updatedAt <= stopBefore }, + ) + expect(result).toEqual([{ number: 3, updatedAt: '2026-04-02T00:00:00Z' }]) + }) + + test('includes nodes newer than stopBefore', async () => { + const stopBefore = '2026-04-01T00:00:00Z' + const pages: Node[][] = [ + [ + { number: 5, updatedAt: '2026-04-03T00:00:00Z' }, + { number: 4, updatedAt: '2026-04-02T00:00:00Z' }, + { number: 3, updatedAt: '2026-03-31T00:00:00Z' }, // older → stop + ], + ] + const result = await paginateGraphQL( + makeGraphqlFn(pages), + extractConnection, + processNode, + { shouldStop: (node) => node.updatedAt <= stopBefore }, + ) + expect(result).toEqual([ + { number: 5, updatedAt: '2026-04-03T00:00:00Z' }, + { number: 4, updatedAt: '2026-04-02T00:00:00Z' }, + ]) + }) + + test('returns all nodes when shouldStop is not provided', async () => { + const pages: Node[][] = [ + [ + { number: 1, updatedAt: '2026-04-01T00:00:00Z' }, + { number: 2, updatedAt: '2026-03-01T00:00:00Z' }, + ], + ] + const result = await paginateGraphQL( + makeGraphqlFn(pages), + extractConnection, + processNode, + ) + expect(result).toEqual([ + { number: 1, updatedAt: '2026-04-01T00:00:00Z' }, + { number: 2, updatedAt: '2026-03-01T00:00:00Z' }, + ]) + }) +}) diff --git a/batch/github/fetcher.ts b/batch/github/fetcher.ts index b860b1d4..86c5f8d3 100644 --- a/batch/github/fetcher.ts +++ b/batch/github/fetcher.ts @@ -761,7 +761,7 @@ interface PaginateOptions { * extractConnection でレスポンスからノード配列と pageInfo を取り出し、 * processNode でノードをアイテムに変換する。 */ -async function paginateGraphQL( +export async function paginateGraphQL( graphqlFn: (variables: Record) => Promise, extractConnection: ( result: TResult, @@ -1157,7 +1157,7 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { label: 'pullrequestList()', // ISO 8601 UTC 文字列同士なので lexicographic 比較 = 時系列比較 shouldStop: stopBefore - ? (node) => node.updatedAt < stopBefore + ? (node) => node.updatedAt <= stopBefore : undefined, }, ) From a8b26c08ae7606ecdcff397338147b602a173faf Mon Sep 17 00:00:00 2001 From: coji Date: Mon, 6 Apr 2026 12:07:05 +0900 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E3=83=86=E3=82=B9=E3=83=88=E3=81=AE?= =?UTF-8?q?=E5=9E=8B=E3=82=A8=E3=83=A9=E3=83=BC=E3=82=92=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=EF=BC=88Promise.resolve=20=E3=81=A7=20graphqlFn=20=E3=81=AE?= =?UTF-8?q?=E6=88=BB=E3=82=8A=E5=80=A4=E3=82=92=E5=8C=85=E3=82=80=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- batch/github/fetcher.test.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/batch/github/fetcher.test.ts b/batch/github/fetcher.test.ts index 5967eae0..93e8dbbd 100644 --- a/batch/github/fetcher.test.ts +++ b/batch/github/fetcher.test.ts @@ -271,7 +271,7 @@ describe('paginateGraphQL shouldStop', () => { let callIndex = 0 return (_vars: Record) => { const nodes = pages[callIndex++] ?? [] - return { + return Promise.resolve({ pullRequests: { nodes, pageInfo: { @@ -279,15 +279,10 @@ describe('paginateGraphQL shouldStop', () => { endCursor: `cursor-${callIndex}`, }, }, - } + }) } } - type Result = - ReturnType>> extends Promise< - infer R - > - ? R - : never + type Result = Awaited>> const extractConnection = (r: Result) => r.pullRequests const processNode = (n: Node) => n From 7d72c261f2d650d95fb7ad5f73b6910c8abcbb46 Mon Sep 17 00:00:00 2001 From: coji Date: Mon, 6 Apr 2026 12:09:32 +0900 Subject: [PATCH 3/3] =?UTF-8?q?chore:=20=E5=86=97=E9=95=B7=E3=81=AA?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- batch/github/fetcher.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/batch/github/fetcher.test.ts b/batch/github/fetcher.test.ts index 93e8dbbd..d092e7bb 100644 --- a/batch/github/fetcher.test.ts +++ b/batch/github/fetcher.test.ts @@ -265,7 +265,6 @@ describe('shapeTagNode', () => { }) describe('paginateGraphQL shouldStop', () => { - // ページネーション結果のモック生成ヘルパー type Node = { number: number; updatedAt: string } const makeGraphqlFn = (pages: Node[][]) => { let callIndex = 0