From 5892a694a44c975e872791d8dbc3b2f7fe10cffa Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 30 Jul 2025 16:47:29 -0400 Subject: [PATCH 1/5] Add rate-limit --- src/lib/rate-limit.ts | 149 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 src/lib/rate-limit.ts diff --git a/src/lib/rate-limit.ts b/src/lib/rate-limit.ts new file mode 100644 index 00000000..310a01e2 --- /dev/null +++ b/src/lib/rate-limit.ts @@ -0,0 +1,149 @@ +import type { IncomingHttpHeaders } from 'node:http'; +import { RequestError, type CancelableRequest, type Response } from 'got'; +import { logger } from '../logger'; + +/** + * Get the rate limit from the headers + * + * @param headers - The headers to get the rate limit from + * @returns The rate limit + */ +export function getRateLimit(headers: IncomingHttpHeaders): { + /** Limit of requests at the time window */ + limit: number | undefined; + /** Remaining requests */ + remaining: number | undefined; + /** Reset time */ + reset: Date | undefined; +} { + return { + limit: + typeof headers['x-ratelimit-limit'] === 'string' + ? Number.parseInt(headers['x-ratelimit-limit'], 10) + : undefined, + remaining: + typeof headers['x-ratelimit-remaining'] === 'string' + ? Number.parseInt(headers['x-ratelimit-remaining'], 10) + : undefined, + reset: + typeof headers['x-ratelimit-reset'] === 'string' + ? new Date(Number.parseInt(headers['x-ratelimit-reset'], 10) * 1000) + : undefined, + }; +} + +/** + * Wrap repeated `got` calls with rate limit handling. + */ +export class RateLimitClient { + private static readonly MAX_WAIT_TIME_MS = 3 * 60 * 1000; + + /** Limit of requests at the time window */ + private limit: number | undefined; + + /** Remaining requests */ + private remaining: number | undefined; + + /** Reset time */ + private reset: Date | undefined; + + /** Last response */ + private lastResponse: Response | undefined; + + /** + * Create a new rate limit client + */ + constructor() { + this.limit = undefined; + this.remaining = undefined; + this.reset = undefined; + this.lastResponse = undefined; + } + + /** + * Format a number of milliseconds to a string of seconds + * + * @param ms - The number of milliseconds + * @returns The number of seconds + */ + static formatMs(ms: number): string { + return `${(ms / 1000).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}s`; + } + + /** + * Wrap a `got` call with a rate limit + * + * @param callback - The got call + * @returns The response + */ + async withRateLimit( + callback: () => CancelableRequest>, + ): Promise> { + if ( + this.reset && + this.reset > new Date() && + (this.lastResponse?.statusCode === 429 || + (this.remaining && this.remaining <= 0)) + ) { + const timeUntilResetMs = this.reset.getTime() - new Date().getTime(); + + // Throw if it's beyond the maximum wait time + if (timeUntilResetMs > RateLimitClient.MAX_WAIT_TIME_MS) { + throw new Error( + `The time until the rate limit resets (${RateLimitClient.formatMs( + timeUntilResetMs, + )})` + + ` is beyond the maximum wait time (${RateLimitClient.formatMs( + RateLimitClient.MAX_WAIT_TIME_MS, + )}). Try again in ${this.reset.toLocaleString()}${ + this.limit + ? `, when the rate limit resets to ${this.limit.toLocaleString()}.` + : '' + }.`, + { cause: this.lastResponse?.body }, + ); + } + + // Wait until it resets before trying the request + logger.warn( + `Waiting until the rate limit resets (${RateLimitClient.formatMs( + timeUntilResetMs, + )}).`, + ); + await new Promise((resolve) => { + setTimeout(resolve, timeUntilResetMs); + }); + } + + let response: Response; + try { + response = await callback(); + } catch (error) { + if (error instanceof RequestError && error.response?.statusCode === 429) { + const rateLimit = getRateLimit(error.response.headers); + this.limit = rateLimit.limit; + this.remaining = rateLimit.remaining; + this.reset = rateLimit.reset; + this.lastResponse = error.response; + + logger.warn('A rate limit was exceeded on this request. Retrying...', { + cause: error, + }); + + // If the error is a 429, we can retry the request + return this.withRateLimit(callback); + } + throw error; + } + + const rateLimit = getRateLimit(response.headers); + this.limit = rateLimit.limit; + this.remaining = rateLimit.remaining; + this.reset = rateLimit.reset; + this.lastResponse = response; + + return response; + } +} From a8c80cd897c15650d8675492d531e41761311ca7 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:15:14 -0400 Subject: [PATCH 2/5] test --- src/lib/{rate-limit.ts => rateLimit.ts} | 97 ++++++++++++++----------- src/lib/tests/rateLimit.test.ts | 38 ++++++++++ 2 files changed, 92 insertions(+), 43 deletions(-) rename src/lib/{rate-limit.ts => rateLimit.ts} (62%) create mode 100644 src/lib/tests/rateLimit.test.ts diff --git a/src/lib/rate-limit.ts b/src/lib/rateLimit.ts similarity index 62% rename from src/lib/rate-limit.ts rename to src/lib/rateLimit.ts index 310a01e2..dafdb288 100644 --- a/src/lib/rate-limit.ts +++ b/src/lib/rateLimit.ts @@ -1,35 +1,13 @@ -import type { IncomingHttpHeaders } from 'node:http'; import { RequestError, type CancelableRequest, type Response } from 'got'; import { logger } from '../logger'; -/** - * Get the rate limit from the headers - * - * @param headers - The headers to get the rate limit from - * @returns The rate limit - */ -export function getRateLimit(headers: IncomingHttpHeaders): { - /** Limit of requests at the time window */ +interface RateLimitInfo { + /** Total number of requests available per reset */ limit: number | undefined; - /** Remaining requests */ + /** Remaining number of requests until reset */ remaining: number | undefined; - /** Reset time */ + /** Time at which the limit will next reset */ reset: Date | undefined; -} { - return { - limit: - typeof headers['x-ratelimit-limit'] === 'string' - ? Number.parseInt(headers['x-ratelimit-limit'], 10) - : undefined, - remaining: - typeof headers['x-ratelimit-remaining'] === 'string' - ? Number.parseInt(headers['x-ratelimit-remaining'], 10) - : undefined, - reset: - typeof headers['x-ratelimit-reset'] === 'string' - ? new Date(Number.parseInt(headers['x-ratelimit-reset'], 10) * 1000) - : undefined, - }; } /** @@ -38,20 +16,20 @@ export function getRateLimit(headers: IncomingHttpHeaders): { export class RateLimitClient { private static readonly MAX_WAIT_TIME_MS = 3 * 60 * 1000; - /** Limit of requests at the time window */ + /** Total number of requests available per reset */ private limit: number | undefined; - /** Remaining requests */ + /** Remaining number of requests until reset */ private remaining: number | undefined; - /** Reset time */ + /** Time at which the limit will next reset */ private reset: Date | undefined; /** Last response */ private lastResponse: Response | undefined; /** - * Create a new rate limit client + * Create a new rate limit client. Best to create a new instance for each batch of calls to an endpoint. */ constructor() { this.limit = undefined; @@ -60,6 +38,29 @@ export class RateLimitClient { this.lastResponse = undefined; } + /** + * Get the rate limit information from the headers + * + * @param headers - The headers from the got Response + * @returns The rate limit information + */ + static getRateLimitInfo(headers: Response['headers']): RateLimitInfo { + return { + limit: + typeof headers['x-ratelimit-limit'] === 'string' + ? Number.parseInt(headers['x-ratelimit-limit'], 10) + : undefined, + remaining: + typeof headers['x-ratelimit-remaining'] === 'string' + ? Number.parseInt(headers['x-ratelimit-remaining'], 10) + : undefined, + reset: + typeof headers['x-ratelimit-reset'] === 'string' + ? new Date(Number.parseInt(headers['x-ratelimit-reset'], 10) * 1000) + : undefined, + }; + } + /** * Format a number of milliseconds to a string of seconds * @@ -72,6 +73,22 @@ export class RateLimitClient { })}s`; } + /** + * Update the rate limit information and last response + * + * @param rateLimit - The rate limit information + * @param lastResponse - The last response + */ + protected updateRateLimitInfo( + rateLimit: RateLimitInfo, + lastResponse?: Response, + ): void { + this.limit = rateLimit.limit; + this.remaining = rateLimit.remaining; + this.reset = rateLimit.reset; + this.lastResponse = lastResponse; + } + /** * Wrap a `got` call with a rate limit * @@ -122,15 +139,12 @@ export class RateLimitClient { response = await callback(); } catch (error) { if (error instanceof RequestError && error.response?.statusCode === 429) { - const rateLimit = getRateLimit(error.response.headers); - this.limit = rateLimit.limit; - this.remaining = rateLimit.remaining; - this.reset = rateLimit.reset; - this.lastResponse = error.response; + const rateLimit = RateLimitClient.getRateLimitInfo( + error.response.headers, + ); + this.updateRateLimitInfo(rateLimit, error.response); - logger.warn('A rate limit was exceeded on this request. Retrying...', { - cause: error, - }); + logger.warn('A rate limit was exceeded on this request. Retrying...'); // If the error is a 429, we can retry the request return this.withRateLimit(callback); @@ -138,11 +152,8 @@ export class RateLimitClient { throw error; } - const rateLimit = getRateLimit(response.headers); - this.limit = rateLimit.limit; - this.remaining = rateLimit.remaining; - this.reset = rateLimit.reset; - this.lastResponse = response; + const rateLimit = RateLimitClient.getRateLimitInfo(response.headers); + this.updateRateLimitInfo(rateLimit, response); return response; } diff --git a/src/lib/tests/rateLimit.test.ts b/src/lib/tests/rateLimit.test.ts new file mode 100644 index 00000000..2179ce54 --- /dev/null +++ b/src/lib/tests/rateLimit.test.ts @@ -0,0 +1,38 @@ +import got from 'got'; +import { RateLimitClient } from '../rateLimit'; + +/** + * A mock of the RateLimitClient class for testing + */ +class RateLimitClientMock extends RateLimitClient { + /** + * Update the rate limit information and last response. + * Forcing this to be public to allow testing. + * + * @param args - The arguments to pass to the parent class + */ + public override updateRateLimitInfo( + ...args: Parameters + ): ReturnType { + super.updateRateLimitInfo(...args); + } +} + +const rateLimiter = new RateLimitClientMock(); +rateLimiter.updateRateLimitInfo({ + limit: 10, + remaining: 10, + reset: new Date(Date.now() + 1000), +}); + +for (const r of ['my', 'array']) { + const response = await rateLimiter.withRateLimit(() => + got.get(`https://example.com.com/${r}`, { + isStream: false, + resolveBodyOnly: false, + responseType: 'buffer', + }), + ); + + console.log(response); +} From c1aa08ac295444ba12c32326f3363cae676dee30 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:39:16 -0400 Subject: [PATCH 3/5] test structure --- src/lib/rateLimit.ts | 27 +++++++++++++++++++---- src/lib/tests/rateLimit.test.ts | 39 +++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/lib/rateLimit.ts b/src/lib/rateLimit.ts index dafdb288..7b3f67fb 100644 --- a/src/lib/rateLimit.ts +++ b/src/lib/rateLimit.ts @@ -92,11 +92,30 @@ export class RateLimitClient { /** * Wrap a `got` call with a rate limit * - * @param callback - The got call - * @returns The response + * @param callback - The call to `got`. It should return a Response, so do NOT chain `.json()` or `.text()`—use responseType instead. + * @param options - Options for rate limit handling + * @returns The `got` Response object + * @example + * ```ts + * for (const page of [1, 2, 3]) { + * const response = await rateLimiter.withRateLimit(() => + * got.get<{ id: number; title: string }>(`https://example.com/posts?page=${page}`, { responseType: 'json' }), + * ); + * console.log(response.body.title); + * } + * ``` */ async withRateLimit( callback: () => CancelableRequest>, + { + maxWaitTimeMs = RateLimitClient.MAX_WAIT_TIME_MS, + }: { + /** + * The number of milliseconds to wait until the rate limit resets. + * If not provided, the default maximum wait time will be used. + */ + maxWaitTimeMs?: number; + } = {}, ): Promise> { if ( this.reset && @@ -107,13 +126,13 @@ export class RateLimitClient { const timeUntilResetMs = this.reset.getTime() - new Date().getTime(); // Throw if it's beyond the maximum wait time - if (timeUntilResetMs > RateLimitClient.MAX_WAIT_TIME_MS) { + if (timeUntilResetMs > maxWaitTimeMs) { throw new Error( `The time until the rate limit resets (${RateLimitClient.formatMs( timeUntilResetMs, )})` + ` is beyond the maximum wait time (${RateLimitClient.formatMs( - RateLimitClient.MAX_WAIT_TIME_MS, + maxWaitTimeMs, )}). Try again in ${this.reset.toLocaleString()}${ this.limit ? `, when the rate limit resets to ${this.limit.toLocaleString()}.` diff --git a/src/lib/tests/rateLimit.test.ts b/src/lib/tests/rateLimit.test.ts index 2179ce54..48efc76a 100644 --- a/src/lib/tests/rateLimit.test.ts +++ b/src/lib/tests/rateLimit.test.ts @@ -1,3 +1,4 @@ +import { describe, it, beforeEach, expect } from 'vitest'; import got from 'got'; import { RateLimitClient } from '../rateLimit'; @@ -21,18 +22,34 @@ class RateLimitClientMock extends RateLimitClient { const rateLimiter = new RateLimitClientMock(); rateLimiter.updateRateLimitInfo({ limit: 10, - remaining: 10, + remaining: 0, reset: new Date(Date.now() + 1000), }); -for (const r of ['my', 'array']) { - const response = await rateLimiter.withRateLimit(() => - got.get(`https://example.com.com/${r}`, { - isStream: false, - resolveBodyOnly: false, - responseType: 'buffer', - }), - ); +describe('rateLimit', () => { + beforeEach(() => { + rateLimiter.updateRateLimitInfo({ + limit: 10, + remaining: 0, + reset: new Date(Date.now() + 1000), + }); + }); - console.log(response); -} + it('should wait until the rate limit resets', async () => { + const timeToWait = 10000; + rateLimiter.updateRateLimitInfo({ + limit: 10, + remaining: 0, + reset: new Date(Date.now() + timeToWait), + }); + const before = Date.now(); + const response = await rateLimiter.withRateLimit(() => + got.get('https://api.transcend.io/info', { + throwHttpErrors: false, + }), + ); + const after = Date.now(); + + expect(after - before).toBeGreaterThan(timeToWait); + }); +}); From 9d78cab0d536e5316d0905f5a59b01df14ff0c73 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:39:50 -0400 Subject: [PATCH 4/5] test structure --- src/lib/rateLimit.ts | 1 + src/lib/tests/rateLimit.test.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/rateLimit.ts b/src/lib/rateLimit.ts index 7b3f67fb..2b3ef6ce 100644 --- a/src/lib/rateLimit.ts +++ b/src/lib/rateLimit.ts @@ -117,6 +117,7 @@ export class RateLimitClient { maxWaitTimeMs?: number; } = {}, ): Promise> { + console.log(this.reset, this.reset > new Date()); if ( this.reset && this.reset > new Date() && diff --git a/src/lib/tests/rateLimit.test.ts b/src/lib/tests/rateLimit.test.ts index 48efc76a..7ff9f3b0 100644 --- a/src/lib/tests/rateLimit.test.ts +++ b/src/lib/tests/rateLimit.test.ts @@ -43,7 +43,7 @@ describe('rateLimit', () => { reset: new Date(Date.now() + timeToWait), }); const before = Date.now(); - const response = await rateLimiter.withRateLimit(() => + await rateLimiter.withRateLimit(() => got.get('https://api.transcend.io/info', { throwHttpErrors: false, }), From 6ef19f77f3d4fb6f4a47950d0215540b3c418655 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 30 Jul 2025 18:23:20 -0400 Subject: [PATCH 5/5] RateLimitClient --- src/lib/rateLimit.ts | 13 +++--- src/lib/tests/rateLimit.test.ts | 73 +++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/src/lib/rateLimit.ts b/src/lib/rateLimit.ts index 2b3ef6ce..4e7e6543 100644 --- a/src/lib/rateLimit.ts +++ b/src/lib/rateLimit.ts @@ -17,16 +17,16 @@ export class RateLimitClient { private static readonly MAX_WAIT_TIME_MS = 3 * 60 * 1000; /** Total number of requests available per reset */ - private limit: number | undefined; + public limit: number | undefined; /** Remaining number of requests until reset */ - private remaining: number | undefined; + public remaining: number | undefined; /** Time at which the limit will next reset */ - private reset: Date | undefined; + public reset: Date | undefined; /** Last response */ - private lastResponse: Response | undefined; + protected lastResponse: Response | undefined; /** * Create a new rate limit client. Best to create a new instance for each batch of calls to an endpoint. @@ -117,12 +117,11 @@ export class RateLimitClient { maxWaitTimeMs?: number; } = {}, ): Promise> { - console.log(this.reset, this.reset > new Date()); if ( - this.reset && + this.reset !== undefined && this.reset > new Date() && (this.lastResponse?.statusCode === 429 || - (this.remaining && this.remaining <= 0)) + (this.remaining !== undefined && this.remaining <= 0)) ) { const timeUntilResetMs = this.reset.getTime() - new Date().getTime(); diff --git a/src/lib/tests/rateLimit.test.ts b/src/lib/tests/rateLimit.test.ts index 7ff9f3b0..340f928f 100644 --- a/src/lib/tests/rateLimit.test.ts +++ b/src/lib/tests/rateLimit.test.ts @@ -19,37 +19,74 @@ class RateLimitClientMock extends RateLimitClient { } } -const rateLimiter = new RateLimitClientMock(); -rateLimiter.updateRateLimitInfo({ - limit: 10, - remaining: 0, - reset: new Date(Date.now() + 1000), -}); - describe('rateLimit', () => { + let rateLimiter: RateLimitClientMock; beforeEach(() => { - rateLimiter.updateRateLimitInfo({ - limit: 10, - remaining: 0, - reset: new Date(Date.now() + 1000), - }); + rateLimiter = new RateLimitClientMock(); }); it('should wait until the rate limit resets', async () => { - const timeToWait = 10000; + const timeToWait = 1000; rateLimiter.updateRateLimitInfo({ limit: 10, remaining: 0, reset: new Date(Date.now() + timeToWait), }); const before = Date.now(); - await rateLimiter.withRateLimit(() => - got.get('https://api.transcend.io/info', { - throwHttpErrors: false, + try { + await rateLimiter.withRateLimit(() => got.get('does/not/matter')); + } catch { + // Doesn't matter + } + const after = Date.now(); + + expect(after - before).toBeGreaterThanOrEqual(timeToWait); + }); + + it('should return the response from the API', async () => { + const response = await rateLimiter.withRateLimit(() => + got.get<{ + /** The pokemon's height in decimeters */ + height: number; + }>('https://pokeapi.co/api/v2/pokemon/ditto', { + responseType: 'json', }), ); - const after = Date.now(); - expect(after - before).toBeGreaterThan(timeToWait); + expect(response.statusCode).toBe(200); + expect(response.body.height).toBe(3); + }); + + it('should decrement rate limit remaining', async () => { + // This public API supports the 'x-ratelimit-remaining' header + const url = 'https://www.reddit.com/r/javascript.json'; + + // Run an initial request to hydrate the rate limit info + const res = await rateLimiter.withRateLimit(() => got.get(url)); + let prevRemaining: number = rateLimiter.remaining!; + expect(rateLimiter.remaining).toBeDefined(); + + // Test corner case: wait for a reset if we're 3 seconds away from resetting the `remaining` counter + const reset = res.headers['x-ratelimit-reset']; + if (typeof reset === 'string') { + // `x-ratelimit-reset` is in seconds on the Reddit API (and not a timestamp like our RateLimitClient expects) + const resetSeconds = Number.parseInt(reset, 10); + if (resetSeconds <= 3) { + // Wait for the reset to happen + await new Promise((resolve) => { + setTimeout(resolve, (resetSeconds + 1) * 1000); + }); + // Re-run the initial request to hydrate the rate limit info + await rateLimiter.withRateLimit(() => got.get(url)); + prevRemaining = rateLimiter.remaining!; + } + } + + // The actual test: confirm that it decrements properly with each request + for (let i = 0; i < 3; i += 1) { + await rateLimiter.withRateLimit(() => got.get(url)); + expect(prevRemaining - 1).toBe(rateLimiter.remaining!); + prevRemaining = rateLimiter.remaining!; + } }); });