From 075de921aecfc168808ab58b3c95b5e4a4e62598 Mon Sep 17 00:00:00 2001 From: Eugene Boruhov Date: Tue, 23 Sep 2025 15:56:08 +0200 Subject: [PATCH 1/2] Add `RequestQueue` for fetching queue, retries and errors --- CLAUDE.md | 50 +++++++++++++++++++ src/index.js | 91 ++++++++++++++++++++++------------- src/request-queue.js | 112 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 32 deletions(-) create mode 100644 CLAUDE.md create mode 100644 src/request-queue.js diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..3b5f1bb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,50 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Commands + +### Development +- `yarn dev` - Watch and compile code continuously +- `yarn storybook` - Run Storybook on port 6001 +- `yarn tdd` - Run Jest tests in watch mode +- `yarn start` - Run dev, storybook, and tdd in parallel +- `yarn test` - Run all Jest tests +- `yarn test:smoke` - Run smoke tests with open handles detection +- `yarn update-schema` - Update GraphQL introspection schema +- `yarn prepare` - Build the package using package-prepare +- `npm publish` - Transpile and publish to NPM + +### Testing +Tests are located in the `tests/` directory. To run a specific test file: +```bash +yarn test tests/[filename].test.js +``` + +## Architecture + +This package is a GraphQL content layer for fetching and processing conference content from GraphCMS. It: + +1. **Fetches data** from GraphCMS using GraphQL queries through multiple fetch modules (`fetch-*.js`) +2. **Processes content** through a post-processing layer that merges talks, Q&A sessions, and populates speaker activities +3. **Exposes content** via the `getContent` async function for consumption +4. **Generates Storybook** for visualizing both CMS and content layers + +### Key Components + +- **Entry point**: `src/index.js` - Creates GraphQL client and orchestrates all content fetching +- **Content fetchers**: `src/fetch-*.js` files - Each handles a specific content type (speakers, talks, sponsors, etc.) +- **Post-processing**: `src/postprocess.js` - Merges and enriches content relationships +- **Configuration**: Requires `CMS_ENDPOINT` and `CMS_TOKEN` environment variables for GraphCMS connection +- **Conference settings**: Must be passed to `getContent()` with conference-specific data including `conferenceTitle`, `eventYear`, `tagColors`, and `speakerAvatar` dimensions + +### Content Flow +1. Conference settings are passed to `getContent(conferenceSettings)` +2. All fetch modules run in parallel via Promise.all +3. Content pieces are merged with conflict resolution for duplicate keys +4. Post-processing enriches the content (populates speaker talks, merges Q&A sessions) +5. Schedule items are sorted chronologically +6. Final processed content is returned + +### GraphQL Schema +The GraphQL schema is stored in `schema.graphql` and can be updated using `yarn update-schema`. The schema endpoint is configured in `.graphqlconfig`. \ No newline at end of file diff --git a/src/index.js b/src/index.js index 1332996..a416fdf 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,7 @@ const { GraphQLClient } = require('graphql-request'); const { credentials } = require('./config'); +const RequestQueue = require('./request-queue'); const textContent = require('./fetch-texts'); const pageContent = require('./fetch-pages'); const brandContent = require('./fetch-brand'); @@ -42,40 +43,65 @@ const getQueriesData = (content, conferenceSettings) => { }; const getContent = async conferenceSettings => { - const fetchAll = [ - textContent, - pageContent, - brandContent, - speakerContent, - advisersContent, - performanceContent, - sponsorContent, - talksContent, - workshopContent, - mcContent, - faqContent, - extContent, - jobsContent, - committeeContent, - diversityContent, - latestLinksContent, - ].map(async content => { - try { - getQueriesData(content, conferenceSettings); - const getVarsFromSettings = content.selectSettings || (() => undefined); - const { conferenceTitle, eventYear } = conferenceSettings; - return await content.fetchData(client, { - conferenceTitle, - eventYear, - ...getVarsFromSettings(conferenceSettings), - }); - } catch (err) { - console.error(err); - process.exit(1); - } + const queue = new RequestQueue({ + concurrency: 5, + retryAttempts: 3, + retryDelay: 2000, + maxRetryDelay: 30000, + timeout: 60000, + }); + + const contentModules = [ + { module: textContent, name: 'texts' }, + { module: pageContent, name: 'pages' }, + { module: brandContent, name: 'brand' }, + { module: speakerContent, name: 'speakers' }, + { module: advisersContent, name: 'advisers' }, + { module: performanceContent, name: 'performance' }, + { module: sponsorContent, name: 'sponsors' }, + { module: talksContent, name: 'talks' }, + { module: workshopContent, name: 'workshops' }, + { module: mcContent, name: 'mc' }, + { module: faqContent, name: 'faq' }, + { module: extContent, name: 'extended' }, + { module: jobsContent, name: 'jobs' }, + { module: committeeContent, name: 'committee' }, + { module: diversityContent, name: 'diversity' }, + { module: latestLinksContent, name: 'landings' }, + ]; + + const fetchPromises = contentModules.map(({ module: content, name }) => { + return queue.add(async () => { + try { + getQueriesData(content, conferenceSettings); + const getVarsFromSettings = content.selectSettings || (() => undefined); + const { conferenceTitle, eventYear } = conferenceSettings; + + // eslint-disable-next-line no-console + console.log(`Fetching ${name} for ${conferenceTitle} ${eventYear}`); + + const result = await content.fetchData(client, { + conferenceTitle, + eventYear, + ...getVarsFromSettings(conferenceSettings), + }); + + // eslint-disable-next-line no-console + console.log( + `Successfully fetched ${name} for ${conferenceTitle} ${eventYear}`, + ); + return result; + } catch (err) { + console.error( + `Failed to fetch ${name} for ${conferenceSettings.conferenceTitle}:`, + err, + ); + throw err; + } + }, name); }); - const contentArray = await Promise.all(fetchAll); + const contentArray = await Promise.all(fetchPromises); const contentMap = contentArray.reduce((content, piece) => { try { const newKeys = Object.keys(piece); @@ -86,6 +112,7 @@ const getContent = async conferenceSettings => { piece[k] = { ...content[k], ...piece[k] }; }); } catch (err) { + // eslint-disable-next-line no-console console.log('content, piece', piece); console.error(err); } diff --git a/src/request-queue.js b/src/request-queue.js new file mode 100644 index 0000000..cd69b8d --- /dev/null +++ b/src/request-queue.js @@ -0,0 +1,112 @@ +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +class RequestQueue { + constructor(options = {}) { + this.concurrency = options.concurrency || 5; + this.retryAttempts = options.retryAttempts || 3; + this.retryDelay = options.retryDelay || 1000; + this.maxRetryDelay = options.maxRetryDelay || 30000; + this.timeout = options.timeout || 60000; + + this.queue = []; + this.running = 0; + this.results = new Map(); + } + + async add(fn, id) { + return new Promise((resolve, reject) => { + this.queue.push({ fn, id, resolve, reject, attempts: 0 }); + this.process(); + }); + } + + async process() { + while (this.running < this.concurrency && this.queue.length > 0) { + const task = this.queue.shift(); + this.running++; + this.executeTask(task); + } + } + + async executeTask(task) { + const { fn, id, resolve, reject, attempts } = task; + + try { + const timeoutPromise = new Promise((_, reject) => + setTimeout( + () => reject(new Error(`Request timeout after ${this.timeout}ms`)), + this.timeout, + ), + ); + + const result = await Promise.race([fn(), timeoutPromise]); + + resolve(result); + if (id) { + this.results.set(id, { success: true, data: result }); + } + } catch (error) { + const isRetriableError = this.isRetriable(error); + const nextAttempt = attempts + 1; + + if (isRetriableError && nextAttempt < this.retryAttempts) { + const delay = Math.min( + this.retryDelay * Math.pow(2, attempts), + this.maxRetryDelay, + ); + + console.warn( + `Request failed (attempt ${nextAttempt}/${this.retryAttempts}), retrying in ${delay}ms...`, + { + error: error.message || error, + id, + }, + ); + + await sleep(delay); + + task.attempts = nextAttempt; + this.queue.unshift(task); + } else { + console.error(`Request failed after ${nextAttempt} attempts`, { + error: error.message || error, + id, + }); + + if (id) { + this.results.set(id, { success: false, error }); + } + reject(error); + } + } finally { + this.running--; + this.process(); + } + } + + isRetriable(error) { + const errorMessage = error.message || ''; + const errorCode = error.code || ''; + const statusCode = error.response && error.response.status; + + if (statusCode === 429) return true; + + if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') return true; + + if ( + errorMessage.includes('timeout') || + errorMessage.includes('ETIMEDOUT') || + errorMessage.includes('429') || + errorMessage.includes('Too Many Requests') || + errorMessage.includes('rate limit') + ) { + return true; + } + + if (statusCode >= 500 && statusCode < 600) return true; + + return false; + } +} + +module.exports = RequestQueue; From f9c6afc4e6b93f62c00858f35f08660031d81de8 Mon Sep 17 00:00:00 2001 From: Denis Urban Date: Wed, 24 Sep 2025 18:09:25 +0300 Subject: [PATCH 2/2] bump up package version --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4f0afb0..3d48f5c 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@focus-reactive/graphql-content-layer", - "version": "3.2.6", + "version": "3.2.7", "private": false, "main": "dist/index.js", "scripts": { @@ -94,4 +94,4 @@ "gitnation", "conference" ] -} +} \ No newline at end of file