From ed15723cea8442fa6882e2f56857ea3be35f5ca8 Mon Sep 17 00:00:00 2001 From: coji Date: Tue, 7 Apr 2026 10:41:58 +0900 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20crawl=20=E6=99=82=E3=81=AE=E3=83=AA?= =?UTF-8?q?=E3=83=9D=E3=82=B8=E3=83=88=E3=83=AA=E4=B8=8D=E5=9C=A8=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3?= =?UTF-8?q?=E3=82=B0=E6=94=B9=E5=96=84=20(#275)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - paginateGraphQL に isResourceMissing オプションを追加し、repository null をページサイズ縮小ループに入れずに即座に GraphQLResourceMissingError として throw - ラベルに owner/repo を含めてエラー特定を容易に - crawl ジョブの per-repo ループを try/catch で囲み、失敗リポはスキップ して他のリポの処理を継続。失敗情報は output.failedRepos に格納 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/services/jobs/crawl.server.ts | 216 ++++++++++++++++-------------- batch/github/fetcher.ts | 52 ++++++- 2 files changed, 160 insertions(+), 108 deletions(-) diff --git a/app/services/jobs/crawl.server.ts b/app/services/jobs/crawl.server.ts index 24c5de95..074d9bcb 100644 --- a/app/services/jobs/crawl.server.ts +++ b/app/services/jobs/crawl.server.ts @@ -24,6 +24,9 @@ export const crawlJob = defineJob({ output: z.object({ fetchedRepos: z.number(), pullCount: z.number(), + failedRepos: z + .array(z.object({ repoLabel: z.string(), error: z.string() })) + .default([]), }), run: async (step, input) => { const orgId = input.organizationId as OrganizationId @@ -56,6 +59,7 @@ export const crawlJob = defineJob({ }) const updatedPrNumbers = new Map>() + const failedRepos: Array<{ repoLabel: string; error: string }> = [] const FETCH_ALL_SENTINEL = '2000-01-01T00:00:00Z' @@ -71,117 +75,125 @@ export const crawlJob = defineJob({ const repo = targetRepos[i] const repoLabel = `${repo.owner}/${repo.repo}` - const store = createStore({ - organizationId: orgId, - repositoryId: repo.id, - }) - const fetcher = createFetcher({ - owner: repo.owner, - repo: repo.repo, - octokit, - }) - - if (repo.releaseDetectionMethod === 'tags') { - await step.run(`fetch-tags:${repoLabel}`, async () => { - step.progress(i + 1, repoCount, `Fetching tags: ${repoLabel}...`) - const allTags = await fetcher.tags() - await store.saveTags(allTags) - return { tagCount: allTags.length } + try { + const store = createStore({ + organizationId: orgId, + repositoryId: repo.id, + }) + const fetcher = createFetcher({ + owner: repo.owner, + repo: repo.repo, + octokit, }) - } - // Watermark bounds full-sweep progress; targeted fetches must not - // advance it (see computeAdvancedScanWatermark / #278). Already - // preloaded via getOrganization, so no extra round-trip here. - const scanWatermark = input.refresh - ? FETCH_ALL_SENTINEL - : (repo.scanWatermark ?? FETCH_ALL_SENTINEL) - - const prNumberSet = input.prNumbers ? new Set(input.prNumbers) : null - const prsToFetch: Array<{ number: number; updatedAt?: string }> = - prNumberSet - ? (input.prNumbers?.map((n) => ({ number: n })) ?? []) - : await step.run(`fetch-prs:${repoLabel}`, async () => { + if (repo.releaseDetectionMethod === 'tags') { + await step.run(`fetch-tags:${repoLabel}`, async () => { + step.progress(i + 1, repoCount, `Fetching tags: ${repoLabel}...`) + const allTags = await fetcher.tags() + await store.saveTags(allTags) + return { tagCount: allTags.length } + }) + } + + // Watermark bounds full-sweep progress; targeted fetches must not + // advance it (see computeAdvancedScanWatermark / #278). Already + // preloaded via getOrganization, so no extra round-trip here. + const scanWatermark = input.refresh + ? FETCH_ALL_SENTINEL + : (repo.scanWatermark ?? FETCH_ALL_SENTINEL) + + const prNumberSet = input.prNumbers ? new Set(input.prNumbers) : null + const prsToFetch: Array<{ number: number; updatedAt?: string }> = + prNumberSet + ? (input.prNumbers?.map((n) => ({ number: n })) ?? []) + : await step.run(`fetch-prs:${repoLabel}`, async () => { + step.progress( + i + 1, + repoCount, + `Fetching PR list: ${repoLabel}...`, + ) + const stopBefore = + input.refresh || scanWatermark === FETCH_ALL_SENTINEL + ? undefined + : scanWatermark + return await fetcher.pullrequestList(stopBefore) + }) + + const repoUpdated = new Set() + for (let j = 0; j < prsToFetch.length; j++) { + const pr = prsToFetch[j] + const saved = await step.run( + `fetch-pr:${repoLabel}:#${pr.number}`, + async () => { step.progress( - i + 1, - repoCount, - `Fetching PR list: ${repoLabel}...`, + j + 1, + prsToFetch.length, + `Fetching ${repoLabel}#${pr.number} (${j + 1}/${prsToFetch.length})...`, ) - const stopBefore = - input.refresh || scanWatermark === FETCH_ALL_SENTINEL - ? undefined - : scanWatermark - return await fetcher.pullrequestList(stopBefore) - }) - - const repoUpdated = new Set() - for (let j = 0; j < prsToFetch.length; j++) { - const pr = prsToFetch[j] - const saved = await step.run( - `fetch-pr:${repoLabel}:#${pr.number}`, - async () => { - step.progress( - j + 1, - prsToFetch.length, - `Fetching ${repoLabel}#${pr.number} (${j + 1}/${prsToFetch.length})...`, - ) - const fetchedAt = new Date().toISOString() - try { - const [ - prMetadata, - commits, - discussions, - reviews, - timelineItems, - files, - ] = await Promise.all([ - fetcher.pullrequest(pr.number), - fetcher.commits(pr.number), - fetcher.comments(pr.number), - fetcher.reviews(pr.number), - fetcher.timelineItems(pr.number), - fetcher.files(pr.number), - ]) - const prForSave = { ...prMetadata, files } - await store.savePrData( - prForSave, - { + const fetchedAt = new Date().toISOString() + try { + const [ + prMetadata, commits, - reviews, discussions, + reviews, timelineItems, - }, - fetchedAt, - ) - return { saved: true as const, number: pr.number } - } catch (e) { - step.log.warn( - `Failed to fetch ${repoLabel}#${pr.number}: ${e instanceof Error ? e.message : e}`, - ) - return { saved: false as const, number: pr.number } - } - }, - ) - if (saved.saved) { - repoUpdated.add(saved.number) + files, + ] = await Promise.all([ + fetcher.pullrequest(pr.number), + fetcher.commits(pr.number), + fetcher.comments(pr.number), + fetcher.reviews(pr.number), + fetcher.timelineItems(pr.number), + fetcher.files(pr.number), + ]) + const prForSave = { ...prMetadata, files } + await store.savePrData( + prForSave, + { + commits, + reviews, + discussions, + timelineItems, + }, + fetchedAt, + ) + return { saved: true as const, number: pr.number } + } catch (e) { + step.log.warn( + `Failed to fetch ${repoLabel}#${pr.number}: ${e instanceof Error ? e.message : e}`, + ) + return { saved: false as const, number: pr.number } + } + }, + ) + if (saved.saved) { + repoUpdated.add(saved.number) + } } - } - if (repoUpdated.size > 0) { - updatedPrNumbers.set(repo.id, repoUpdated) - } + if (repoUpdated.size > 0) { + updatedPrNumbers.set(repo.id, repoUpdated) + } - // Advance the scan watermark only after a fully successful full-sweep. - // See computeAdvancedScanWatermark for the invariants. - const nextWatermark = computeAdvancedScanWatermark({ - isTargetedFetch: prNumberSet !== null, - prsToFetch, - savedPrNumbers: repoUpdated, - }) - if (nextWatermark !== null) { - await step.run(`advance-scan-watermark:${repoLabel}`, async () => { - await store.setScanWatermark(nextWatermark) + // Advance the scan watermark only after a fully successful full-sweep. + // See computeAdvancedScanWatermark for the invariants. + const nextWatermark = computeAdvancedScanWatermark({ + isTargetedFetch: prNumberSet !== null, + prsToFetch, + savedPrNumbers: repoUpdated, }) + if (nextWatermark !== null) { + await step.run(`advance-scan-watermark:${repoLabel}`, async () => { + await store.setScanWatermark(nextWatermark) + }) + } + } catch (e) { + const message = e instanceof Error ? e.message : String(e) + step.log.error( + `Failed to crawl ${repoLabel}, skipping remaining steps for this repo: ${message}`, + ) + failedRepos.push({ repoLabel, error: message }) } } @@ -190,7 +202,7 @@ export const crawlJob = defineJob({ await step.run('finalize', () => { clearOrgCache(orgId) }) - return { fetchedRepos: repoCount, pullCount: 0 } + return { fetchedRepos: repoCount, pullCount: 0, failedRepos } } if (input.prNumbers && updatedPrNumbers.size === 0) { @@ -236,6 +248,6 @@ export const crawlJob = defineJob({ } }) - return { fetchedRepos: repoCount, pullCount } + return { fetchedRepos: repoCount, pullCount, failedRepos } }, }) diff --git a/batch/github/fetcher.ts b/batch/github/fetcher.ts index 86c5f8d3..2ec1750b 100644 --- a/batch/github/fetcher.ts +++ b/batch/github/fetcher.ts @@ -745,7 +745,19 @@ interface PageInfo { endCursor?: string | null } -interface PaginateOptions { +/** + * GraphQL のトップレベルリソース（例: `repository`）が null の場合に投げられるエラー。 + * ページサイズを下げてもリソース自体が存在しない問題は解決しないため、 + * paginateGraphQL は即座にこの例外を投げてリトライループを打ち切る。 + */ +export class GraphQLResourceMissingError extends Error { + constructor(message: string) { + super(message) + this.name = 'GraphQLResourceMissingError' + } +} + +interface PaginateOptions { /** ページネーション用の初期ページサイズ（デフォルト: 100） */ initialPageSize?: number /** handleGraphQLError の最小ページサイズ。指定するとエラー時にページサイズ削減+リトライする */ @@ -754,6 +766,12 @@ interface PaginateOptions { label?: string /** ノードに対する早期打ち切り判定。true を返すとそのノードでページング停止 */ shouldStop?: (node: TNode) => boolean + /** + * リソース（例: repository）が存在しないことを判定する。 + * true を返すと GraphQLResourceMissingError を throw してループを即座に打ち切る。 + * nodes が null でもリソース自体が null なのか空ページなのかを区別できる。 + */ + isResourceMissing?: (result: TResult) => boolean } /** @@ -767,13 +785,14 @@ export async function paginateGraphQL( result: TResult, ) => { nodes: (TNode | null)[] | null; pageInfo: PageInfo } | null, processNode: (node: TNode) => TItem | null, - options: PaginateOptions = {}, + options: PaginateOptions = {}, ): Promise { const { initialPageSize = 100, minPageSize, label = 'paginateGraphQL', shouldStop, + isResourceMissing, } = options const items: TItem[] = [] let cursor: string | null = null @@ -803,6 +822,12 @@ export async function paginateGraphQL( result = await graphqlFn(variables) } + if (isResourceMissing?.(result)) { + throw new GraphQLResourceMissingError( + `${label}: resource not found (cursor: ${cursor})`, + ) + } + const connection = extractConnection(result) const nodes = connection?.nodes if (!nodes) { @@ -1136,7 +1161,11 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { (vars) => graphqlWithTimeout(queryStr, { owner, repo, ...vars }), (r) => r?.repository?.pullRequests ?? null, (node) => shapePullRequestNode(node, owner, repo), - { minPageSize: 10, label: 'pullrequests()' }, + { + minPageSize: 10, + label: `pullrequests(${owner}/${repo})`, + isResourceMissing: (r) => r?.repository == null, + }, ) } @@ -1154,7 +1183,8 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { (node) => ({ number: node.number, updatedAt: node.updatedAt }), { minPageSize: 10, - label: 'pullrequestList()', + label: `pullrequestList(${owner}/${repo})`, + isResourceMissing: (r) => r?.repository == null, // ISO 8601 UTC 文字列同士なので lexicographic 比較 = 時系列比較 shouldStop: stopBefore ? (node) => node.updatedAt <= stopBefore @@ -1175,7 +1205,12 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { number: pullNumber, }) - const node = result?.repository?.pullRequest ?? null + if (result?.repository == null) { + throw new GraphQLResourceMissingError( + `pullrequest(${owner}/${repo}): repository not found`, + ) + } + const node = result.repository.pullRequest ?? null const shaped = shapePullRequestNode(node, owner, repo) if (!shaped) { throw new Error(`PR #${pullNumber} not found in ${owner}/${repo}`) @@ -1282,6 +1317,10 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { (vars) => graphqlWithTimeout(queryStr, { owner, repo, ...vars }), (r) => r.repository?.refs ?? null, shapeTagNode, + { + label: `tags(${owner}/${repo})`, + isResourceMissing: (r) => r?.repository == null, + }, ) } @@ -1387,7 +1426,8 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { { initialPageSize: 25, minPageSize: 5, - label: 'pullrequestsWithDetails()', + label: `pullrequestsWithDetails(${owner}/${repo})`, + isResourceMissing: (r) => r?.repository == null, }, ) } From 333c10ed3a8a4195c203a6853bd6a7029aa3b867 Mon Sep 17 00:00:00 2001 From: coji Date: Tue, 7 Apr 2026 10:55:52 +0900 Subject: [PATCH 2/4] =?UTF-8?q?test:=20paginateGraphQL=20=E3=81=AE=20isRes?= =?UTF-8?q?ourceMissing=20=E7=B5=8C=E8=B7=AF=E3=82=92=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- batch/github/fetcher.test.ts | 52 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/batch/github/fetcher.test.ts b/batch/github/fetcher.test.ts index d092e7bb..0d721c58 100644 --- a/batch/github/fetcher.test.ts +++ b/batch/github/fetcher.test.ts @@ -3,8 +3,13 @@ import { describe, expect, test, vi } from 'vitest' import type { ShapedTimelineItem } from './model' // 純粋関数なので直接 import してテスト -const { buildRequestedAtMap, createFetcher, paginateGraphQL, shapeTagNode } = - await import('./fetcher') +const { + buildRequestedAtMap, + createFetcher, + GraphQLResourceMissingError, + paginateGraphQL, + shapeTagNode, +} = await import('./fetcher') describe('buildRequestedAtMap', () => { test('returns empty map for empty items', () => { @@ -325,6 +330,49 @@ describe('paginateGraphQL shouldStop', () => { ]) }) + test('throws GraphQLResourceMissingError when isResourceMissing returns true', async () => { + const graphqlFn = vi.fn(() => Promise.resolve({ repository: null })) + await expect( + paginateGraphQL( + graphqlFn, + (r: { + repository: { + nodes: unknown[] + pageInfo: { hasNextPage: boolean; endCursor: string | null } + } | null + }) => r.repository, + (n: unknown) => n, + { + minPageSize: 10, + label: 'test(owner/repo)', + isResourceMissing: (r) => r?.repository == null, + }, + ), + ).rejects.toBeInstanceOf(GraphQLResourceMissingError) + // ページサイズを縮小しながらリトライせず、1回呼んだだけで throw + expect(graphqlFn).toHaveBeenCalledTimes(1) + }) + + test('throws GraphQLResourceMissingError including label/cursor', async () => { + const graphqlFn = () => Promise.resolve({ repository: null }) + await expect( + paginateGraphQL( + graphqlFn, + (r: { + repository: { + nodes: unknown[] + pageInfo: { hasNextPage: boolean; endCursor: string | null } + } | null + }) => r.repository, + (n: unknown) => n, + { + label: 'pullrequestList(acme/ghost)', + isResourceMissing: (r) => r?.repository == null, + }, + ), + ).rejects.toThrow(/pullrequestList\(acme\/ghost\)/) + }) + test('returns all nodes when shouldStop is not provided', async () => { const pages: Node[][] = [ [ From 42eebfa0ea33a9ada2feb79ccd2f12699a2973fd Mon Sep 17 00:00:00 2001 From: coji Date: Tue, 7 Apr 2026 11:09:02 +0900 Subject: [PATCH 3/4] =?UTF-8?q?refactor:=20simplify=20=E3=83=AC=E3=83=93?= =?UTF-8?q?=E3=83=A5=E3=83=BC=E6=8C=87=E6=91=98=E3=82=92=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crawl の per-repo ハンドリングで getErrorMessageForLog を使用 - GraphQLResourceMissingError は step.run 内で sentinel として返し、 外で再 throw する。durably による無駄なリトライを防ぐ - fetcher.ts の repositoryMissing 述語を共通化（4 箇所の重複削減） - テストの inline 型定義を共通化 - 不要なコメントを削除 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/services/jobs/crawl.server.ts | 79 +++++++++++++++++++++---------- batch/github/fetcher.test.ts | 47 +++++++----------- batch/github/fetcher.ts | 18 ++++--- 3 files changed, 85 insertions(+), 59 deletions(-) diff --git a/app/services/jobs/crawl.server.ts b/app/services/jobs/crawl.server.ts index 074d9bcb..f186cf43 100644 --- a/app/services/jobs/crawl.server.ts +++ b/app/services/jobs/crawl.server.ts @@ -1,5 +1,6 @@ import { defineJob } from '@coji/durably' import { z } from 'zod' +import { getErrorMessageForLog } from '~/app/libs/error-message' import { clearOrgCache } from '~/app/services/cache.server' import { assertOrgGithubAuthResolvable, @@ -9,7 +10,10 @@ import { processConcurrencyKey } from '~/app/services/jobs/concurrency-keys.serv import { shouldTriggerFullOrgProcessJob } from '~/app/services/jobs/crawl-process-handoff.server' import type { OrganizationId } from '~/app/types/organization' import { getOrganization } from '~/batch/db/queries' -import { createFetcher } from '~/batch/github/fetcher' +import { + createFetcher, + GraphQLResourceMissingError, +} from '~/batch/github/fetcher' import { createStore } from '~/batch/github/store' import { computeAdvancedScanWatermark } from './scan-watermark' @@ -86,38 +90,65 @@ export const crawlJob = defineJob({ octokit, }) + // GraphQLResourceMissingError は step.run の外で再 throw する。 + // step.run 内で raw に throw すると durably がリトライしてしまうため、 + // sentinel として返してから外で投げ直し、外側 try/catch で記録する。 if (repo.releaseDetectionMethod === 'tags') { - await step.run(`fetch-tags:${repoLabel}`, async () => { + const result = await step.run(`fetch-tags:${repoLabel}`, async () => { step.progress(i + 1, repoCount, `Fetching tags: ${repoLabel}...`) - const allTags = await fetcher.tags() - await store.saveTags(allTags) - return { tagCount: allTags.length } + try { + const allTags = await fetcher.tags() + await store.saveTags(allTags) + return { tagCount: allTags.length, missingMessage: null } + } catch (e) { + if (e instanceof GraphQLResourceMissingError) { + return { tagCount: 0, missingMessage: e.message } + } + throw e + } }) + if (result.missingMessage !== null) { + throw new GraphQLResourceMissingError(result.missingMessage) + } } // Watermark bounds full-sweep progress; targeted fetches must not - // advance it (see computeAdvancedScanWatermark / #278). Already - // preloaded via getOrganization, so no extra round-trip here. + // advance it (see computeAdvancedScanWatermark / #278). const scanWatermark = input.refresh ? FETCH_ALL_SENTINEL : (repo.scanWatermark ?? FETCH_ALL_SENTINEL) const prNumberSet = input.prNumbers ? new Set(input.prNumbers) : null - const prsToFetch: Array<{ number: number; updatedAt?: string }> = - prNumberSet - ? (input.prNumbers?.map((n) => ({ number: n })) ?? []) - : await step.run(`fetch-prs:${repoLabel}`, async () => { - step.progress( - i + 1, - repoCount, - `Fetching PR list: ${repoLabel}...`, - ) - const stopBefore = - input.refresh || scanWatermark === FETCH_ALL_SENTINEL - ? undefined - : scanWatermark - return await fetcher.pullrequestList(stopBefore) - }) + let prsToFetch: Array<{ number: number; updatedAt?: string }> + if (prNumberSet) { + prsToFetch = input.prNumbers?.map((n) => ({ number: n })) ?? [] + } else { + const result = await step.run(`fetch-prs:${repoLabel}`, async () => { + step.progress(i + 1, repoCount, `Fetching PR list: ${repoLabel}...`) + const stopBefore = + input.refresh || scanWatermark === FETCH_ALL_SENTINEL + ? undefined + : scanWatermark + try { + return { + prs: await fetcher.pullrequestList(stopBefore), + missingMessage: null as string | null, + } + } catch (e) { + if (e instanceof GraphQLResourceMissingError) { + return { + prs: [] as Array<{ number: number; updatedAt: string }>, + missingMessage: e.message, + } + } + throw e + } + }) + if (result.missingMessage !== null) { + throw new GraphQLResourceMissingError(result.missingMessage) + } + prsToFetch = result.prs + } const repoUpdated = new Set() for (let j = 0; j < prsToFetch.length; j++) { @@ -161,7 +192,7 @@ export const crawlJob = defineJob({ return { saved: true as const, number: pr.number } } catch (e) { step.log.warn( - `Failed to fetch ${repoLabel}#${pr.number}: ${e instanceof Error ? e.message : e}`, + `Failed to fetch ${repoLabel}#${pr.number}: ${getErrorMessageForLog(e)}`, ) return { saved: false as const, number: pr.number } } @@ -189,7 +220,7 @@ export const crawlJob = defineJob({ }) } } catch (e) { - const message = e instanceof Error ? e.message : String(e) + const message = getErrorMessageForLog(e) step.log.error( `Failed to crawl ${repoLabel}, skipping remaining steps for this repo: ${message}`, ) diff --git a/batch/github/fetcher.test.ts b/batch/github/fetcher.test.ts index 0d721c58..e791a0b9 100644 --- a/batch/github/fetcher.test.ts +++ b/batch/github/fetcher.test.ts @@ -330,24 +330,23 @@ describe('paginateGraphQL shouldStop', () => { ]) }) + type RepoResult = { + repository: { + nodes: unknown[] + pageInfo: { hasNextPage: boolean; endCursor: string | null } + } | null + } + const extractRepoConnection = (r: RepoResult) => r.repository + const isRepoMissing = (r: RepoResult) => r?.repository == null + test('throws GraphQLResourceMissingError when isResourceMissing returns true', async () => { const graphqlFn = vi.fn(() => Promise.resolve({ repository: null })) await expect( - paginateGraphQL( - graphqlFn, - (r: { - repository: { - nodes: unknown[] - pageInfo: { hasNextPage: boolean; endCursor: string | null } - } | null - }) => r.repository, - (n: unknown) => n, - { - minPageSize: 10, - label: 'test(owner/repo)', - isResourceMissing: (r) => r?.repository == null, - }, - ), + paginateGraphQL(graphqlFn, extractRepoConnection, (n: unknown) => n, { + minPageSize: 10, + label: 'test(owner/repo)', + isResourceMissing: isRepoMissing, + }), ).rejects.toBeInstanceOf(GraphQLResourceMissingError) // ページサイズを縮小しながらリトライせず、1回呼んだだけで throw expect(graphqlFn).toHaveBeenCalledTimes(1) @@ -356,20 +355,10 @@ describe('paginateGraphQL shouldStop', () => { test('throws GraphQLResourceMissingError including label/cursor', async () => { const graphqlFn = () => Promise.resolve({ repository: null }) await expect( - paginateGraphQL( - graphqlFn, - (r: { - repository: { - nodes: unknown[] - pageInfo: { hasNextPage: boolean; endCursor: string | null } - } | null - }) => r.repository, - (n: unknown) => n, - { - label: 'pullrequestList(acme/ghost)', - isResourceMissing: (r) => r?.repository == null, - }, - ), + paginateGraphQL(graphqlFn, extractRepoConnection, (n: unknown) => n, { + label: 'pullrequestList(acme/ghost)', + isResourceMissing: isRepoMissing, + }), ).rejects.toThrow(/pullrequestList\(acme\/ghost\)/) }) diff --git a/batch/github/fetcher.ts b/batch/github/fetcher.ts index 2ec1750b..f0cefe35 100644 --- a/batch/github/fetcher.ts +++ b/batch/github/fetcher.ts @@ -757,6 +757,12 @@ export class GraphQLResourceMissingError extends Error { } } +const repositoryMissing = (r: unknown): boolean => + r != null && + typeof r === 'object' && + 'repository' in r && + (r as { repository: unknown }).repository == null + interface PaginateOptions { /** ページネーション用の初期ページサイズ（デフォルト: 100） */ initialPageSize?: number @@ -1164,7 +1170,7 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { { minPageSize: 10, label: `pullrequests(${owner}/${repo})`, - isResourceMissing: (r) => r?.repository == null, + isResourceMissing: repositoryMissing, }, ) } @@ -1184,7 +1190,7 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { { minPageSize: 10, label: `pullrequestList(${owner}/${repo})`, - isResourceMissing: (r) => r?.repository == null, + isResourceMissing: repositoryMissing, // ISO 8601 UTC 文字列同士なので lexicographic 比較 = 時系列比較 shouldStop: stopBefore ? (node) => node.updatedAt <= stopBefore @@ -1205,12 +1211,12 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { number: pullNumber, }) - if (result?.repository == null) { + if (repositoryMissing(result)) { throw new GraphQLResourceMissingError( `pullrequest(${owner}/${repo}): repository not found`, ) } - const node = result.repository.pullRequest ?? null + const node = result?.repository?.pullRequest ?? null const shaped = shapePullRequestNode(node, owner, repo) if (!shaped) { throw new Error(`PR #${pullNumber} not found in ${owner}/${repo}`) @@ -1319,7 +1325,7 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { shapeTagNode, { label: `tags(${owner}/${repo})`, - isResourceMissing: (r) => r?.repository == null, + isResourceMissing: repositoryMissing, }, ) } @@ -1427,7 +1433,7 @@ export const createFetcher = ({ owner, repo, octokit }: createFetcherProps) => { initialPageSize: 25, minPageSize: 5, label: `pullrequestsWithDetails(${owner}/${repo})`, - isResourceMissing: (r) => r?.repository == null, + isResourceMissing: repositoryMissing, }, ) } From e7a5c15b0d5740a812dec6ceba99b34d4a4e54bd Mon Sep 17 00:00:00 2001 From: coji Date: Tue, 7 Apr 2026 11:11:00 +0900 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20crawl=20CLI=20=E3=81=A7=20failedRepo?= =?UTF-8?q?s=20=E3=82=92=E5=87=BA=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- batch/commands/crawl.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/batch/commands/crawl.ts b/batch/commands/crawl.ts index d47ded64..15af50d5 100644 --- a/batch/commands/crawl.ts +++ b/batch/commands/crawl.ts @@ -84,6 +84,13 @@ export async function crawlCommand({ consola.success( `Crawl completed. ${output.fetchedRepos} repos, ${output.pullCount} PRs fetched.`, ) + if (output.failedRepos.length > 0) { + consola.warn( + `Failed to crawl ${output.failedRepos.length} repo(s):\n${output.failedRepos + .map((f) => ` - ${f.repoLabel}: ${f.error}`) + .join('\n')}`, + ) + } } finally { await durably.stop() await shutdown()