From 0865d914fc3474762527e26df205ac3c34b0e06d Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Wed, 17 Dec 2025 14:28:05 +0530 Subject: [PATCH 1/7] request method now works with multiple APIs, added Google Search API, Amazon Search API Changed the request method such that it can work with all of our APIs and the test was successful. The get, post, googleSearch, amazonSearch methods are working correctly. So now we just need to add the rest of the APIs. --- .DS_Store | Bin 6148 -> 6148 bytes dist/index.d.ts | 49 +++++++++++++- dist/index.js | 88 ++++++++++++++++++------- dist/version.d.ts | 2 +- dist/version.js | 2 +- package-lock.json | 4 +- src/index.ts | 159 ++++++++++++++++++++++++++++++++++++++-------- test-manual.js | 119 ++++++++++++++++++++++++++++++++++ 8 files changed, 369 insertions(+), 54 deletions(-) create mode 100644 test-manual.js diff --git a/.DS_Store b/.DS_Store index db7e98729ba38ef0ebe7917393218062ef0e316e..2803a9f70b43c3c6feb4f2c69970dee45f074349 100644 GIT binary patch literal 6148 zcmeHK%We}f6unMTI-w~F5=h-3jl{MMp=yy3nk^RBAozsdH zOicwSR39gFOpoaqoltEh+Eq9OoC5!v0{rb(=!hOtlb+(c{r&w8DSDL1UYN)bF}y)E zk!nAb!NZ0@5yg~IZpD4E;wqGqL~NhBG*sS%5|pxKp15W4uG5GD>d`s!0pj<-lB!&z z_ZU}FYW48R>Y-xwu#WfL;+5$XN;IiWPk=Q%wWUR2U@?k#o-y^7pGPV7c|kD~6;88} z(e-OMO5!Z9*T0EEv9xhz)7$j6yf>Y9a@xs*e3G?-@hdJpmof=2{UCf1_ov%i%&-=)2Ivf{aG?8NvnCdq7}A+iz|5_;<@9h0I0c*nmIAy# z_(+TmgL92)>p-Ti0KgKemBHs<1?Kn$8wTeZ(E<~i3e;3#t{6hoQSY0$hQYZ;O($V4 zAHs|*%ne1T(b2!J!bvnVy4)$?6j)ZEsEbuT|Bu(-|CdRw

lU{8tKyQoGY`;g!tU zy7F>-*7`_qk=QtHu2E5tnd4YFd=&2>DMOpj4zOWxt`Rjb_amTWaG6u!uPX2pUPsaK delta 291 zcmZoMXfc=|#>B)qu~2NHo}wrd0|Nsi1A_nqgDyh~LncEJLn=e^#=_-{lMO^z9QYVY zfc#=0i6jf;; @@ -42,10 +73,24 @@ export interface SpbConfig { retries?: number; timeout?: number; } +export interface GoogleSearchConfig { + search: string; + params?: GoogleSearchParams; + retries?: number; + timeout?: number; +} +export interface AmazonSearchConfig { + query: string; + params?: AmazonSearchParams; + retries?: number; + timeout?: number; +} export declare class ScrapingBeeClient { readonly api_key: string; constructor(api_key: string); private request; - get(config: SpbConfig): AxiosPromise; - post(config: SpbConfig): AxiosPromise; + get(config: SpbConfig): AxiosPromise; + post(config: SpbConfig): AxiosPromise; + googleSearch(config: GoogleSearchConfig): AxiosPromise; + amazonSearch(config: AmazonSearchConfig): AxiosPromise; } diff --git a/dist/index.js b/dist/index.js index d94f9c0..8beea85 100644 --- a/dist/index.js +++ b/dist/index.js @@ -7,44 +7,86 @@ exports.ScrapingBeeClient = void 0; const axios_1 = __importDefault(require("axios")); const axios_retry_1 = __importDefault(require("axios-retry")); const utils_1 = require("./utils"); -const API_URL = 'https://app.scrapingbee.com/api/v1/'; +const API_URL = 'https://app.scrapingbee.com/api/v1/'; // WILL BE REMOVED IN A FUTURE VERSION +const GOOGLE_API_URL = 'https://app.scrapingbee.com/api/v1/store/google'; +const AMAZON_SEARCH_API_URL = 'https://app.scrapingbee.com/api/v1/amazon/search'; class ScrapingBeeClient { constructor(api_key) { this.api_key = api_key; } - request(method, config) { - var _a, _b; - let params = config.params || {}; - // Headers - let headers = utils_1.process_headers(config.headers); - if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { - params.forward_headers = true; - } - // Cookies - params.cookies = config.cookies; - // Other query params - params['api_key'] = this.api_key; - params['url'] = config.url; - params = utils_1.process_params(params); - let axios_params = { - method: method, - headers: headers, - params: params, + request(config) { + var _a; + config.params['api_key'] = this.api_key; + const axiosConfig = { + method: config.method, + url: config.endpoint, + params: config.params, + headers: config.headers, data: config.data, responseType: 'arraybuffer', - timeout: (_b = config.timeout) !== null && _b !== void 0 ? _b : 0, + timeout: (_a = config.timeout) !== null && _a !== void 0 ? _a : 0, }; // Retry policy if (config.retries !== undefined) { axios_retry_1.default(axios_1.default, { retries: config.retries }); } - return axios_1.default(API_URL, axios_params); + return axios_1.default(axiosConfig); } + // WILL BE REMOVED IN A FUTURE VERSION get(config) { - return this.request('GET', config); + var _a; + let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); + let headers = utils_1.process_headers(config.headers); + if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { + params.forward_headers = true; + } + return this.request({ + method: 'GET', + endpoint: API_URL, + params: utils_1.process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); } + // WILL BE REMOVED IN A FUTURE VERSION post(config) { - return this.request('POST', config); + var _a; + let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); + let headers = utils_1.process_headers(config.headers); + if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { + params.forward_headers = true; + } + return this.request({ + method: 'POST', + endpoint: API_URL, + params: utils_1.process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); + } + googleSearch(config) { + const params = Object.assign({ search: config.search }, config.params); + return this.request({ + method: 'GET', + endpoint: GOOGLE_API_URL, + params: utils_1.process_params(params), + retries: config.retries, + timeout: config.timeout, + }); + } + amazonSearch(config) { + const params = Object.assign({ query: config.query }, config.params); + return this.request({ + method: 'GET', + endpoint: AMAZON_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); } } exports.ScrapingBeeClient = ScrapingBeeClient; diff --git a/dist/version.d.ts b/dist/version.d.ts index a25b917..0cee7dd 100644 --- a/dist/version.d.ts +++ b/dist/version.d.ts @@ -1 +1 @@ -export declare const LIB_VERSION = "1.7.5"; +export declare const LIB_VERSION = "1.7.6"; diff --git a/dist/version.js b/dist/version.js index 28c05b2..1dafffc 100644 --- a/dist/version.js +++ b/dist/version.js @@ -1,4 +1,4 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.LIB_VERSION = void 0; -exports.LIB_VERSION = "1.7.5"; +exports.LIB_VERSION = "1.7.6"; diff --git a/package-lock.json b/package-lock.json index 4a43328..1fb39c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "scrapingbee", - "version": "1.7.5", + "version": "1.7.6", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "scrapingbee", - "version": "1.7.5", + "version": "1.7.6", "license": "ISC", "dependencies": { "axios": "^1.7", diff --git a/src/index.ts b/src/index.ts index b18019a..0ae7e52 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,8 +3,11 @@ import axiosRetry from 'axios-retry'; import { process_params, process_headers } from './utils'; -const API_URL: string = 'https://app.scrapingbee.com/api/v1/'; +const API_URL: string = 'https://app.scrapingbee.com/api/v1/'; // WILL BE REMOVED IN A FUTURE VERSION +const GOOGLE_API_URL: string = 'https://app.scrapingbee.com/api/v1/store/google'; +const AMAZON_SEARCH_API_URL: string = 'https://app.scrapingbee.com/api/v1/amazon/search'; +// WILL BE REMOVED IN A FUTURE VERSION export type SpbParams = { block_ads?: boolean; block_resources?: boolean; @@ -38,6 +41,40 @@ export type SpbParams = { [key: string]: any; }; +export type GoogleSearchParams = { + add_html?: boolean; + country_code?: string; + device?: string; + extra_params?: string; + language?: string; + light_request?: boolean; + nfpr?: boolean; + page?: number; + search_type?: string; +} & { + [key: string]: any; +}; + +export type AmazonSearchParams = { + add_html?: boolean; + category_id?: string; + country?: string; + currency?: string; + device?: string; + domain?: string; + language?: string; + light_request?: boolean; + merchant_id?: string; + pages?: number; + screenshot?: boolean; + sort_by?: string; + start_page?: number; + zip_code?: string; +} & { + [key: string]: any; +}; + +// WILL BE REMOVED IN A FUTURE VERSION export interface SpbConfig { url: string; headers?: Record; @@ -48,6 +85,20 @@ export interface SpbConfig { timeout?: number; } +export interface GoogleSearchConfig { + search: string; + params?: GoogleSearchParams; + retries?: number; + timeout?: number; +} + +export interface AmazonSearchConfig { + query: string; + params?: AmazonSearchParams; + retries?: number; + timeout?: number; +} + export class ScrapingBeeClient { readonly api_key: string; @@ -55,45 +106,103 @@ export class ScrapingBeeClient { this.api_key = api_key; } - private request(method: string, config: SpbConfig): AxiosPromise { - let params = config.params || {}; + private request(config: Record): AxiosPromise { + config.params['api_key'] = this.api_key; + + const axiosConfig: AxiosRequestConfig = { + method: config.method as Method, + url: config.endpoint, + params: config.params, + headers: config.headers, + data: config.data, + responseType: 'arraybuffer', + timeout: config.timeout ?? 0, + }; + + // Retry policy + if (config.retries !== undefined) { + axiosRetry(axios, { retries: config.retries }); + } + + return axios(axiosConfig); + } + + // WILL BE REMOVED IN A FUTURE VERSION + public get(config: SpbConfig): AxiosPromise { + let params: Record = { + ...config.params, + url: config.url, + cookies: config.cookies + }; - // Headers let headers = process_headers(config.headers); if (Object.keys(config.headers ?? {}).length > 0) { params.forward_headers = true; } - // Cookies - params.cookies = config.cookies; - - // Other query params - params['api_key'] = this.api_key; - params['url'] = config.url; - params = process_params(params); - - let axios_params: AxiosRequestConfig = { - method: method as Method, + return this.request({ + method: 'GET', + endpoint: API_URL, + params: process_params(params), headers: headers, - params: params, data: config.data, - responseType: 'arraybuffer', - timeout: config.timeout ?? 0, + retries: config.retries, + timeout: config.timeout, + }); + } + + // WILL BE REMOVED IN A FUTURE VERSION + public post(config: SpbConfig): AxiosPromise { + let params: Record = { + ...config.params, + url: config.url, + cookies: config.cookies }; - // Retry policy - if (config.retries !== undefined) { - axiosRetry(axios, { retries: config.retries }); + let headers = process_headers(config.headers); + if (Object.keys(config.headers ?? {}).length > 0) { + params.forward_headers = true; } - return axios(API_URL, axios_params); + return this.request({ + method: 'POST', + endpoint: API_URL, + params: process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); } - public get(config: SpbConfig) { - return this.request('GET', config); + public googleSearch(config: GoogleSearchConfig): AxiosPromise { + const params: Record = { + search: config.search, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: GOOGLE_API_URL, + params: process_params(params), + retries: config.retries, + timeout: config.timeout, + }); } - public post(config: SpbConfig) { - return this.request('POST', config); + public amazonSearch(config: AmazonSearchConfig): AxiosPromise { + const params: Record = { + query: config.query, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: AMAZON_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); } + } diff --git a/test-manual.js b/test-manual.js new file mode 100644 index 0000000..30f9ee7 --- /dev/null +++ b/test-manual.js @@ -0,0 +1,119 @@ +const { ScrapingBeeClient } = require('./dist/index'); + +const API_KEY = process.env.SCRAPINGBEE_API_KEY; + +const client = new ScrapingBeeClient(API_KEY); + +async function testHtmlGet() { + console.log('=== Testing HTML API - GET ==='); + try { + const response = await client.get({ + url: 'https://httpbin.org/get', + params: { + render_js: false + } + }); + console.log('Status:', response.status); + console.log('Response:', response.data.toString().substring(0, 200) + '...'); + console.log('✅ HTML GET test passed!\n'); + } catch (error) { + console.log('❌ HTML GET test failed:', error.message); + } +} + +async function testHtmlPost() { + console.log('=== Testing HTML API - POST ==='); + try { + const response = await client.post({ + url: 'https://httpbin.org/post', + params: { + render_js: false + }, + data: 'test=data' + }); + console.log('Status:', response.status); + console.log('Response:', response.data.toString().substring(0, 200) + '...'); + console.log('✅ HTML POST test passed!\n'); + } catch (error) { + console.log('❌ HTML POST test failed:', error.message); + } +} + +async function testGoogleSearch() { + console.log('=== Testing Google Search API ==='); + try { + const response = await client.googleSearch({ + search: 'scrapingbee', + params: { + language: 'en', + country_code: 'us' + } + }); + console.log('Status:', response.status); + console.log('Response type:', typeof response.data); + + // Parse JSON response + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Results found:', data.organic_results?.length || 0); + console.log('✅ Google Search test passed!\n'); + } catch (error) { + console.log('❌ Google Search test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testAmazonSearch() { + console.log('=== Testing Amazon Search API ==='); + try { + const response = await client.amazonSearch({ + query: 'laptop', + params: { + domain: 'com', + pages: 1 + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Results found:', data.products?.length || 0); + console.log('✅ Amazon Search test passed!\n'); + } catch (error) { + console.log('❌ Amazon Search test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function runTests() { + console.log('\n🚀 Starting ScrapingBee SDK Tests\n'); + + await testHtmlGet(); + await testHtmlPost(); + await testGoogleSearch(); + await testAmazonSearch(); + + console.log('🏁 All tests completed!\n'); +} + +runTests(); \ No newline at end of file From 168d16427ab590dc98f490bcfd213004d2a53184 Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Wed, 17 Dec 2025 20:45:45 +0530 Subject: [PATCH 2/7] Added all APIs - Added all APIs. Now it supports HTML API, Google Search API, Amazon Search API, Amazon Product API, Walmart Search API, Walmart Product API, ChatGPT API, YouTube Search API, YouTube Metadata API, YouTube Trainability API, YouTube Transcript API - Added a method to call our usage endpoint - Create a new method for accessing our HTML API. So that in future, we can deprecate the get and post methods. --- dist/index.d.ts | 173 ++++++++++++++++++++--- dist/index.js | 125 ++++++++++++++++- src/index.ts | 359 +++++++++++++++++++++++++++++++++++++++++++++--- test-manual.js | 320 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 935 insertions(+), 42 deletions(-) diff --git a/dist/index.d.ts b/dist/index.d.ts index 1a94919..2504b39 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,5 +1,8 @@ import { AxiosPromise } from 'axios'; -export declare type SpbParams = { +export declare type HtmlApiParams = { + ai_extract_rules?: object | string; + ai_query?: string; + ai_selector?: string; block_ads?: boolean; block_resources?: boolean; cookies?: string | Record; @@ -7,9 +10,6 @@ export declare type SpbParams = { custom_google?: boolean; device?: string; extract_rules?: object | string; - ai_extract_rules?: { - summary: string; - }; forward_headers?: boolean; forward_headers_pure?: boolean; js_scenario?: object | string; @@ -17,7 +17,10 @@ export declare type SpbParams = { own_proxy?: string; premium_proxy?: boolean; render_js?: boolean; + return_page_markdown?: boolean; return_page_source?: boolean; + return_page_text?: boolean; + scraping_config?: string; screenshot?: boolean; screenshot_full_page?: boolean; screenshot_selector?: string; @@ -26,13 +29,23 @@ export declare type SpbParams = { timeout?: number; transparent_status_code?: boolean; wait?: number; - wait_browser?: string | Array; + wait_browser?: string; wait_for?: string; window_height?: number; window_width?: number; } & { [key: string]: any; }; +export interface HtmlApiConfig { + url: string; + method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS'; + headers?: Record; + cookies?: string | Record; + params?: HtmlApiParams; + data?: any; + retries?: number; + timeout?: number; +} export declare type GoogleSearchParams = { add_html?: boolean; country_code?: string; @@ -46,6 +59,12 @@ export declare type GoogleSearchParams = { } & { [key: string]: any; }; +export interface GoogleSearchConfig { + search: string; + params?: GoogleSearchParams; + retries?: number; + timeout?: number; +} export declare type AmazonSearchParams = { add_html?: boolean; category_id?: string; @@ -64,24 +83,132 @@ export declare type AmazonSearchParams = { } & { [key: string]: any; }; -export interface SpbConfig { - url: string; - headers?: Record; - cookies?: string | Record; - params?: SpbParams; - data?: any; +export interface AmazonSearchConfig { + query: string; + params?: AmazonSearchParams; retries?: number; timeout?: number; } -export interface GoogleSearchConfig { - search: string; - params?: GoogleSearchParams; +export declare type AmazonProductParams = { + add_html?: boolean; + autoselect_variant?: boolean; + country?: string; + currency?: string; + device?: string; + domain?: string; + language?: string; + light_request?: boolean; + screenshot?: boolean; + zip_code?: string; +} & { + [key: string]: any; +}; +export interface AmazonProductConfig { + query: string; + params?: AmazonProductParams; retries?: number; timeout?: number; } -export interface AmazonSearchConfig { +export declare type WalmartSearchParams = { + add_html?: boolean; + delivery_zip?: string; + device?: string; + domain?: string; + fulfillment_speed?: string; + fulfillment_type?: string; + light_request?: boolean; + max_price?: number; + min_price?: number; + screenshot?: boolean; + sort_by?: string; + store_id?: string; +} & { + [key: string]: any; +}; +export interface WalmartSearchConfig { query: string; - params?: AmazonSearchParams; + params?: WalmartSearchParams; + retries?: number; + timeout?: number; +} +export declare type WalmartProductParams = { + add_html?: boolean; + delivery_zip?: string; + device?: string; + domain?: string; + light_request?: boolean; + screenshot?: boolean; + store_id?: string; +} & { + [key: string]: any; +}; +export interface WalmartProductConfig { + product_id: string; + params?: WalmartProductParams; + retries?: number; + timeout?: number; +} +export declare type ChatGPTParams = { + add_html?: boolean; + country_code?: string; + search?: boolean; +} & { + [key: string]: any; +}; +export interface ChatGPTConfig { + prompt: string; + params?: ChatGPTParams; + retries?: number; + timeout?: number; +} +export declare type YouTubeSearchParams = { + '360'?: boolean; + '3d'?: boolean; + '4k'?: boolean; + creative_commons?: boolean; + duration?: string; + hd?: boolean; + hdr?: boolean; + live?: boolean; + location?: boolean; + purchased?: boolean; + sort_by?: string; + subtitles?: boolean; + type?: string; + upload_date?: string; + vr180?: boolean; +} & { + [key: string]: any; +}; +export interface YouTubeSearchConfig { + search: string; + params?: YouTubeSearchParams; + retries?: number; + timeout?: number; +} +export interface YouTubeMetadataConfig { + video_id: string; + retries?: number; + timeout?: number; +} +export declare type YouTubeTranscriptParams = { + language?: string; + transcript_origin?: string; +} & { + [key: string]: any; +}; +export interface YouTubeTranscriptConfig { + video_id: string; + params?: YouTubeTranscriptParams; + retries?: number; + timeout?: number; +} +export interface YouTubeTrainabilityConfig { + video_id: string; + retries?: number; + timeout?: number; +} +export interface UsageConfig { retries?: number; timeout?: number; } @@ -89,8 +216,18 @@ export declare class ScrapingBeeClient { readonly api_key: string; constructor(api_key: string); private request; - get(config: SpbConfig): AxiosPromise; - post(config: SpbConfig): AxiosPromise; + get(config: HtmlApiConfig): AxiosPromise; + post(config: HtmlApiConfig): AxiosPromise; googleSearch(config: GoogleSearchConfig): AxiosPromise; amazonSearch(config: AmazonSearchConfig): AxiosPromise; + amazonProduct(config: AmazonProductConfig): AxiosPromise; + walmartSearch(config: WalmartSearchConfig): AxiosPromise; + walmartProduct(config: WalmartProductConfig): AxiosPromise; + chatGPT(config: ChatGPTConfig): AxiosPromise; + youtubeSearch(config: YouTubeSearchConfig): AxiosPromise; + youtubeMetadata(config: YouTubeMetadataConfig): AxiosPromise; + youtubeTranscript(config: YouTubeTranscriptConfig): AxiosPromise; + youtubeTrainability(config: YouTubeTrainabilityConfig): AxiosPromise; + htmlApi(config: HtmlApiConfig): AxiosPromise; + usage(config?: UsageConfig): AxiosPromise; } diff --git a/dist/index.js b/dist/index.js index 8beea85..7b5a565 100644 --- a/dist/index.js +++ b/dist/index.js @@ -7,9 +7,18 @@ exports.ScrapingBeeClient = void 0; const axios_1 = __importDefault(require("axios")); const axios_retry_1 = __importDefault(require("axios-retry")); const utils_1 = require("./utils"); -const API_URL = 'https://app.scrapingbee.com/api/v1/'; // WILL BE REMOVED IN A FUTURE VERSION +const HTML_API_URL = 'https://app.scrapingbee.com/api/v1/'; const GOOGLE_API_URL = 'https://app.scrapingbee.com/api/v1/store/google'; const AMAZON_SEARCH_API_URL = 'https://app.scrapingbee.com/api/v1/amazon/search'; +const AMAZON_PRODUCT_API_URL = 'https://app.scrapingbee.com/api/v1/amazon/product'; +const WALMART_SEARCH_API_URL = 'https://app.scrapingbee.com/api/v1/walmart/search'; +const WALMART_PRODUCT_API_URL = 'https://app.scrapingbee.com/api/v1/walmart/product'; +const CHATGPT_API_URL = 'https://app.scrapingbee.com/api/v1/chatgpt'; +const YOUTUBE_SEARCH_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/search'; +const YOUTUBE_METADATA_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/metadata'; +const YOUTUBE_TRANSCRIPT_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/transcript'; +const YOUTUBE_TRAINABILITY_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/trainability'; +const USAGE_API_URL = 'https://app.scrapingbee.com/api/v1/usage'; class ScrapingBeeClient { constructor(api_key) { this.api_key = api_key; @@ -42,7 +51,7 @@ class ScrapingBeeClient { } return this.request({ method: 'GET', - endpoint: API_URL, + endpoint: HTML_API_URL, params: utils_1.process_params(params), headers: headers, data: config.data, @@ -60,7 +69,7 @@ class ScrapingBeeClient { } return this.request({ method: 'POST', - endpoint: API_URL, + endpoint: HTML_API_URL, params: utils_1.process_params(params), headers: headers, data: config.data, @@ -88,5 +97,115 @@ class ScrapingBeeClient { timeout: config.timeout, }); } + amazonProduct(config) { + const params = Object.assign({ query: config.query }, config.params); + return this.request({ + method: 'GET', + endpoint: AMAZON_PRODUCT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + walmartSearch(config) { + const params = Object.assign({ query: config.query }, config.params); + return this.request({ + method: 'GET', + endpoint: WALMART_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + walmartProduct(config) { + const params = Object.assign({ product_id: config.product_id }, config.params); + return this.request({ + method: 'GET', + endpoint: WALMART_PRODUCT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + chatGPT(config) { + const params = Object.assign({ prompt: config.prompt }, config.params); + return this.request({ + method: 'GET', + endpoint: CHATGPT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + youtubeSearch(config) { + const params = Object.assign({ search: config.search }, config.params); + return this.request({ + method: 'GET', + endpoint: YOUTUBE_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + youtubeMetadata(config) { + const params = { + video_id: config.video_id, + }; + return this.request({ + method: 'GET', + endpoint: YOUTUBE_METADATA_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + youtubeTranscript(config) { + const params = Object.assign({ video_id: config.video_id }, config.params); + return this.request({ + method: 'GET', + endpoint: YOUTUBE_TRANSCRIPT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + youtubeTrainability(config) { + const params = { + video_id: config.video_id, + }; + return this.request({ + method: 'GET', + endpoint: YOUTUBE_TRAINABILITY_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + htmlApi(config) { + var _a; + let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); + let headers = utils_1.process_headers(config.headers); + if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { + params.forward_headers = true; + } + return this.request({ + method: config.method || 'GET', + endpoint: HTML_API_URL, + params: utils_1.process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); + } + usage(config = {}) { + return this.request({ + method: 'GET', + endpoint: USAGE_API_URL, + params: {}, + retries: config.retries, + timeout: config.timeout, + }); + } } exports.ScrapingBeeClient = ScrapingBeeClient; diff --git a/src/index.ts b/src/index.ts index 0ae7e52..5d9d95b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,12 +3,25 @@ import axiosRetry from 'axios-retry'; import { process_params, process_headers } from './utils'; -const API_URL: string = 'https://app.scrapingbee.com/api/v1/'; // WILL BE REMOVED IN A FUTURE VERSION +const HTML_API_URL: string = 'https://app.scrapingbee.com/api/v1/'; const GOOGLE_API_URL: string = 'https://app.scrapingbee.com/api/v1/store/google'; const AMAZON_SEARCH_API_URL: string = 'https://app.scrapingbee.com/api/v1/amazon/search'; +const AMAZON_PRODUCT_API_URL: string = 'https://app.scrapingbee.com/api/v1/amazon/product'; +const WALMART_SEARCH_API_URL: string = 'https://app.scrapingbee.com/api/v1/walmart/search'; +const WALMART_PRODUCT_API_URL: string = 'https://app.scrapingbee.com/api/v1/walmart/product'; +const CHATGPT_API_URL: string = 'https://app.scrapingbee.com/api/v1/chatgpt'; +const YOUTUBE_SEARCH_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/search'; +const YOUTUBE_METADATA_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/metadata'; +const YOUTUBE_TRANSCRIPT_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/transcript'; +const YOUTUBE_TRAINABILITY_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/trainability'; +const USAGE_API_URL: string = 'https://app.scrapingbee.com/api/v1/usage'; -// WILL BE REMOVED IN A FUTURE VERSION -export type SpbParams = { + +// HTML API +export type HtmlApiParams = { + ai_extract_rules?: object | string; + ai_query?: string; + ai_selector?: string; block_ads?: boolean; block_resources?: boolean; cookies?: string | Record; @@ -16,7 +29,6 @@ export type SpbParams = { custom_google?: boolean; device?: string; extract_rules?: object | string; - ai_extract_rules?: { summary: string }; forward_headers?: boolean; forward_headers_pure?: boolean; js_scenario?: object | string; @@ -24,7 +36,10 @@ export type SpbParams = { own_proxy?: string; premium_proxy?: boolean; render_js?: boolean; + return_page_markdown?: boolean; return_page_source?: boolean; + return_page_text?: boolean; + scraping_config?: string; screenshot?: boolean; screenshot_full_page?: boolean; screenshot_selector?: string; @@ -33,7 +48,7 @@ export type SpbParams = { timeout?: number; transparent_status_code?: boolean; wait?: number; - wait_browser?: string | Array; + wait_browser?: string; wait_for?: string; window_height?: number; window_width?: number; @@ -41,6 +56,19 @@ export type SpbParams = { [key: string]: any; }; +export interface HtmlApiConfig { + url: string; + method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS'; // Some of these methods are not supported by HTML API + headers?: Record; + cookies?: string | Record; + params?: HtmlApiParams; + data?: any; + retries?: number; + timeout?: number; +} + +// GOOGLE + export type GoogleSearchParams = { add_html?: boolean; country_code?: string; @@ -55,6 +83,15 @@ export type GoogleSearchParams = { [key: string]: any; }; +export interface GoogleSearchConfig { + search: string; + params?: GoogleSearchParams; + retries?: number; + timeout?: number; +} + +// AMAZON + export type AmazonSearchParams = { add_html?: boolean; category_id?: string; @@ -74,31 +111,160 @@ export type AmazonSearchParams = { [key: string]: any; }; -// WILL BE REMOVED IN A FUTURE VERSION -export interface SpbConfig { - url: string; - headers?: Record; - cookies?: string | Record; - params?: SpbParams; - data?: any; +export interface AmazonSearchConfig { + query: string; + params?: AmazonSearchParams; retries?: number; timeout?: number; } -export interface GoogleSearchConfig { - search: string; - params?: GoogleSearchParams; +export type AmazonProductParams = { + add_html?: boolean; + autoselect_variant?: boolean; + country?: string; + currency?: string; + device?: string; + domain?: string; + language?: string; + light_request?: boolean; + screenshot?: boolean; + zip_code?: string; +} & { + [key: string]: any; +}; + +export interface AmazonProductConfig { + query: string; + params?: AmazonProductParams; retries?: number; timeout?: number; } -export interface AmazonSearchConfig { +// WALMART + +export type WalmartSearchParams = { + add_html?: boolean; + delivery_zip?: string; + device?: string; + domain?: string; + fulfillment_speed?: string; + fulfillment_type?: string; + light_request?: boolean; + max_price?: number; + min_price?: number; + screenshot?: boolean; + sort_by?: string; + store_id?: string; +} & { + [key: string]: any; +}; + +export interface WalmartSearchConfig { query: string; - params?: AmazonSearchParams; + params?: WalmartSearchParams; retries?: number; timeout?: number; } +export type WalmartProductParams = { + add_html?: boolean; + delivery_zip?: string; + device?: string; + domain?: string; + light_request?: boolean; + screenshot?: boolean; + store_id?: string; +} & { + [key: string]: any; +}; + +export interface WalmartProductConfig { + product_id: string; + params?: WalmartProductParams; + retries?: number; + timeout?: number; +} + +// CHATGPT + +export type ChatGPTParams = { + add_html?: boolean; + country_code?: string; + search?: boolean; +} & { + [key: string]: any; +}; + +export interface ChatGPTConfig { + prompt: string; + params?: ChatGPTParams; + retries?: number; + timeout?: number; +} + +// YOUTUBE + +export type YouTubeSearchParams = { + '360'?: boolean; + '3d'?: boolean; + '4k'?: boolean; + creative_commons?: boolean; + duration?: string; + hd?: boolean; + hdr?: boolean; + live?: boolean; + location?: boolean; + purchased?: boolean; + sort_by?: string; + subtitles?: boolean; + type?: string; + upload_date?: string; + vr180?: boolean; +} & { + [key: string]: any; +}; + +export interface YouTubeSearchConfig { + search: string; + params?: YouTubeSearchParams; + retries?: number; + timeout?: number; +} + +export interface YouTubeMetadataConfig { + video_id: string; + retries?: number; + timeout?: number; +} + +export type YouTubeTranscriptParams = { + language?: string; + transcript_origin?: string; +} & { + [key: string]: any; +}; + +export interface YouTubeTranscriptConfig { + video_id: string; + params?: YouTubeTranscriptParams; + retries?: number; + timeout?: number; +} + +export interface YouTubeTrainabilityConfig { + video_id: string; + retries?: number; + timeout?: number; +} + +// USAGE + +export interface UsageConfig { + retries?: number; + timeout?: number; +} + + export class ScrapingBeeClient { readonly api_key: string; @@ -128,7 +294,7 @@ export class ScrapingBeeClient { } // WILL BE REMOVED IN A FUTURE VERSION - public get(config: SpbConfig): AxiosPromise { + public get(config: HtmlApiConfig): AxiosPromise { let params: Record = { ...config.params, url: config.url, @@ -142,7 +308,7 @@ export class ScrapingBeeClient { return this.request({ method: 'GET', - endpoint: API_URL, + endpoint: HTML_API_URL, params: process_params(params), headers: headers, data: config.data, @@ -152,7 +318,7 @@ export class ScrapingBeeClient { } // WILL BE REMOVED IN A FUTURE VERSION - public post(config: SpbConfig): AxiosPromise { + public post(config: HtmlApiConfig): AxiosPromise { let params: Record = { ...config.params, url: config.url, @@ -166,7 +332,7 @@ export class ScrapingBeeClient { return this.request({ method: 'POST', - endpoint: API_URL, + endpoint: HTML_API_URL, params: process_params(params), headers: headers, data: config.data, @@ -205,4 +371,155 @@ export class ScrapingBeeClient { }); } + public amazonProduct(config: AmazonProductConfig): AxiosPromise { + const params: Record = { + query: config.query, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: AMAZON_PRODUCT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public walmartSearch(config: WalmartSearchConfig): AxiosPromise { + const params: Record = { + query: config.query, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: WALMART_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public walmartProduct(config: WalmartProductConfig): AxiosPromise { + const params: Record = { + product_id: config.product_id, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: WALMART_PRODUCT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public chatGPT(config: ChatGPTConfig): AxiosPromise { + const params: Record = { + prompt: config.prompt, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: CHATGPT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public youtubeSearch(config: YouTubeSearchConfig): AxiosPromise { + const params: Record = { + search: config.search, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: YOUTUBE_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public youtubeMetadata(config: YouTubeMetadataConfig): AxiosPromise { + const params: Record = { + video_id: config.video_id, + }; + + return this.request({ + method: 'GET', + endpoint: YOUTUBE_METADATA_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public youtubeTranscript(config: YouTubeTranscriptConfig): AxiosPromise { + const params: Record = { + video_id: config.video_id, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: YOUTUBE_TRANSCRIPT_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public youtubeTrainability(config: YouTubeTrainabilityConfig): AxiosPromise { + const params: Record = { + video_id: config.video_id, + }; + + return this.request({ + method: 'GET', + endpoint: YOUTUBE_TRAINABILITY_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public htmlApi(config: HtmlApiConfig): AxiosPromise { + let params: Record = { + ...config.params, + url: config.url, + cookies: config.cookies + }; + + let headers = process_headers(config.headers); + if (Object.keys(config.headers ?? {}).length > 0) { + params.forward_headers = true; + } + + return this.request({ + method: config.method || 'GET', + endpoint: HTML_API_URL, + params: process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); + } + + public usage(config: UsageConfig = {}): AxiosPromise { + return this.request({ + method: 'GET', + endpoint: USAGE_API_URL, + params: {}, + retries: config.retries, + timeout: config.timeout, + }); + } + } diff --git a/test-manual.js b/test-manual.js index 30f9ee7..45901a8 100644 --- a/test-manual.js +++ b/test-manual.js @@ -105,6 +105,315 @@ async function testAmazonSearch() { } } +async function testAmazonProduct() { + console.log('=== Testing Amazon Product API ==='); + try { + const response = await client.amazonProduct({ + query: 'B0D2Q9397Y', // Example ASIN + params: { + domain: 'com' + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Product title:', data.title?.substring(0, 50) || 'N/A'); + console.log('✅ Amazon Product test passed!\n'); + } catch (error) { + console.log('❌ Amazon Product test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testWalmartSearch() { + console.log('=== Testing Walmart Search API ==='); + try { + const response = await client.walmartSearch({ + query: 'laptop', + params: { + device: 'desktop', + sort_by: 'best_match' + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Results found:', data.products?.length || 0); + console.log('✅ Walmart Search test passed!\n'); + } catch (error) { + console.log('❌ Walmart Search test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testWalmartProduct() { + console.log('=== Testing Walmart Product API ==='); + try { + const response = await client.walmartProduct({ + product_id: '13943258180', + params: { + device: 'desktop' + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Product title:', data.title?.substring(0, 50) || 'N/A'); + console.log('✅ Walmart Product test passed!\n'); + } catch (error) { + console.log('❌ Walmart Product test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testChatGPT() { + console.log('=== Testing ChatGPT API ==='); + try { + const response = await client.chatGPT({ + prompt: 'What is web scraping?', + params: { + search: true + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Response:', data.results_text?.substring(0, 50) || 'N/A'); + console.log('✅ ChatGPT test passed!\n'); + } catch (error) { + console.log('❌ ChatGPT test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testYouTubeSearch() { + console.log('=== Testing YouTube Search API ==='); + try { + const response = await client.youtubeSearch({ + search: 'web scraping tutorial', + params: { + sort_by: 'relevance', + type: 'video' + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Results found:', data.results?.length || 0); + console.log('✅ YouTube Search test passed!\n'); + } catch (error) { + console.log('❌ YouTube Search test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testYouTubeMetadata() { + console.log('=== Testing YouTube Metadata API ==='); + try { + const response = await client.youtubeMetadata({ + video_id: 'dQw4w9WgXcQ' + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Number of Likes:', data.like_count || 'N/A'); + console.log('✅ YouTube Metadata test passed!\n'); + } catch (error) { + console.log('❌ YouTube Metadata test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testYouTubeTranscript() { + console.log('=== Testing YouTube Transcript API ==='); + try { + const response = await client.youtubeTranscript({ + video_id: 'q0aFOxT6TNw', + params: { + language: 'en' + } + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Transcript segments:', data.text?.substring(0, 100) || 'N/A'); + console.log('✅ YouTube Transcript test passed!\n'); + } catch (error) { + console.log('❌ YouTube Transcript test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testYouTubeTrainability() { + console.log('=== Testing YouTube Trainability API ==='); + try { + const response = await client.youtubeTrainability({ + video_id: 'dQw4w9WgXcQ' + }); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Trainability data:', data.permitted || 'N/A'); + console.log('✅ YouTube Trainability test passed!\n'); + } catch (error) { + console.log('❌ YouTube Trainability test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + +async function testHtmlApiGet() { + console.log('=== Testing HTML API - GET ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/get', + method: 'GET', + params: { + render_js: false + } + }); + console.log('Status:', response.status); + console.log('Response:', response.data.toString().substring(0, 200) + '...'); + console.log('✅ HTML API GET test passed!\n'); + } catch (error) { + console.log('❌ HTML API GET test failed:', error.message); + } +} + +async function testHtmlApiPost() { + console.log('=== Testing HTML API - POST ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/post', + method: 'POST', + params: { + render_js: false + }, + data: 'test=data' + }); + console.log('Status:', response.status); + console.log('Response:', response.data.toString().substring(0, 200) + '...'); + console.log('✅ HTML API POST test passed!\n'); + } catch (error) { + console.log('❌ HTML API POST test failed:', error.message); + } +} + +async function testUsage() { + console.log('=== Testing Usage API ==='); + try { + const response = await client.usage(); + console.log('Status:', response.status); + + let data; + if (Buffer.isBuffer(response.data)) { + data = JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + data = JSON.parse(response.data); + } else { + data = response.data; + } + + console.log('Max API credits:', data.max_api_credit); + console.log('Used API credits:', data.used_api_credit); + console.log('Max concurrency:', data.max_concurrency); + console.log('✅ Usage test passed!\n'); + } catch (error) { + console.log('❌ Usage test failed:', error.message); + if (error.response) { + console.log('Response status:', error.response.status); + console.log('Response data:', error.response.data?.toString()); + } + } +} + async function runTests() { console.log('\n🚀 Starting ScrapingBee SDK Tests\n'); @@ -112,6 +421,17 @@ async function runTests() { await testHtmlPost(); await testGoogleSearch(); await testAmazonSearch(); + await testAmazonProduct(); + await testWalmartSearch(); + await testWalmartProduct(); + await testChatGPT(); + await testYouTubeSearch(); + await testYouTubeMetadata(); + await testYouTubeTranscript(); + await testYouTubeTrainability(); + await testHtmlApiGet(); + await testHtmlApiPost(); + await testUsage(); console.log('🏁 All tests completed!\n'); } From 1d97edd429734afa0c02a9666318e8da203df1ce Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Thu, 18 Dec 2025 15:25:28 +0530 Subject: [PATCH 3/7] Added Tests Added more tests to both client and utils test files --- test-manual.js | 2 +- tests/client.test.ts | 565 +++++++++++++++++++++++++++++++++++++++++++ tests/utils.test.ts | 15 ++ 3 files changed, 581 insertions(+), 1 deletion(-) diff --git a/test-manual.js b/test-manual.js index 45901a8..2a6e082 100644 --- a/test-manual.js +++ b/test-manual.js @@ -294,7 +294,7 @@ async function testYouTubeTranscript() { console.log('=== Testing YouTube Transcript API ==='); try { const response = await client.youtubeTranscript({ - video_id: 'q0aFOxT6TNw', + video_id: 'eKVTFXQPAhs', params: { language: 'en' } diff --git a/tests/client.test.ts b/tests/client.test.ts index 3e82983..45b9ab2 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -6,6 +6,10 @@ import { LIB_VERSION } from '../src/version'; var mock = new MockAdapter(axios); +// ============================================ +// HTML API +// ============================================ + describe('test_ScrapingBeeClient.get', function () { var api_key = 'API_KEY'; var target_url = 'https://httpbin-scrapingbee.cleverapps.io/html'; @@ -88,3 +92,564 @@ describe('test_ScrapingBeeClient.post', function () { assert.match(res.config.headers['User-Agent'], /^ScrapingBee-Node\//); }); }); + +describe('test_ScrapingBeeClient.htmlApi', function () { + var api_key = 'API_KEY'; + var target_url = 'https://httpbin.org/get'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + mock.onPost().reply(201); + + it('should make a GET request by default', async function () { + var res = await client.htmlApi({ url: target_url }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['url'], target_url); + // @ts-ignore + assert.match(res.config.headers['User-Agent'], /^ScrapingBee-Node\//); + }); + + it('should make a GET request when explicitly specified', async function () { + var res = await client.htmlApi({ url: target_url, method: 'GET' }); + assert.deepStrictEqual(res.status, 200); + }); + + it('should make a POST request when specified', async function () { + var res = await client.htmlApi({ + url: 'https://httpbin.org/post', + method: 'POST', + data: 'test=data' + }); + assert.deepStrictEqual(res.status, 201); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + // @ts-ignore + assert.match(res.config.headers['User-Agent'], /^ScrapingBee-Node\//); + }); + + it('should add the render_js query param', async function () { + var res = await client.htmlApi({ url: target_url, params: { render_js: true } }); + assert.deepStrictEqual(res.config.params['render_js'], true); + }); + + it('should handle multiple params correctly', async function () { + var res = await client.htmlApi({ + url: target_url, + params: { render_js: true, premium_proxy: true, block_ads: true } + }); + assert.deepStrictEqual(res.config.params['render_js'], true); + assert.deepStrictEqual(res.config.params['premium_proxy'], true); + assert.deepStrictEqual(res.config.params['block_ads'], true); + }); + + it('should prefix header names with Spb- and set forward_headers', async function () { + var res = await client.htmlApi({ + url: target_url, + headers: { 'Content-Type': 'text/html; charset=utf-8' } + }); + // @ts-ignore + assert.deepStrictEqual(res.config.headers['Spb-Content-Type'], 'text/html; charset=utf-8'); + // @ts-ignore + assert.deepStrictEqual(res.config.headers['User-Agent'], `ScrapingBee-Node/${LIB_VERSION}`); + assert.deepStrictEqual(res.config.params['forward_headers'], true); + }); + + it('should format the cookies and add them to the query params', async function () { + var cookies = { name1: 'value1', name2: 'value2' }; + var res = await client.htmlApi({ url: target_url, cookies: cookies }); + assert.deepStrictEqual(res.config.params['cookies'], 'name1=value1;name2=value2'); + }); + + it('should pass string cookies directly', async function () { + var cookies = 'session=abc123;user=john'; + var res = await client.htmlApi({ url: target_url, cookies: cookies }); + assert.deepStrictEqual(res.config.params['cookies'], cookies); + }); + + it('should format the extract_rules and add them to the query params', async function () { + var res = await client.htmlApi({ + url: target_url, + params: { + extract_rules: { + title: 'h1', + subtitle: '#subtitle', + }, + }, + }); + assert.deepStrictEqual( + res.config.params['extract_rules'], + '{"title":"h1","subtitle":"#subtitle"}' + ); + }); + + it('should format the js_scenario and add them to the query params', async function () { + var res = await client.htmlApi({ + url: target_url, + params: { + js_scenario: { + instructions: [{ click: '#buttonId' }], + }, + }, + }); + assert.deepStrictEqual( + res.config.params['js_scenario'], + '{"instructions":[{"click":"#buttonId"}]}' + ); + }); + + it('should format the ai_extract_rules and add them to the query params', async function () { + var res = await client.htmlApi({ + url: target_url, + params: { + ai_extract_rules: { + summary: 'Extract the main content', + }, + }, + }); + assert.deepStrictEqual( + res.config.params['ai_extract_rules'], + '{"summary":"Extract the main content"}' + ); + }); + + it('should handle POST with data and params', async function () { + var res = await client.htmlApi({ + url: 'https://httpbin.org/post', + method: 'POST', + data: 'username=user&password=pass', + params: { render_js: true } + }); + assert.deepStrictEqual(res.status, 201); + assert.deepStrictEqual(res.config.params['render_js'], true); + assert.deepStrictEqual(res.config.data, 'username=user&password=pass'); + }); + + it('should handle POST with headers', async function () { + var res = await client.htmlApi({ + url: 'https://httpbin.org/post', + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + data: 'test=data' + }); + // @ts-ignore + assert.deepStrictEqual(res.config.headers['Spb-Content-Type'], 'application/x-www-form-urlencoded'); + assert.deepStrictEqual(res.config.params['forward_headers'], true); + }); +}); + +// ============================================ +// Google Search API +// ============================================ + +describe('test_ScrapingBeeClient.googleSearch', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.googleSearch({ + search: 'test query', + params: { language: 'en', country_code: 'us' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['search'], 'test query'); + assert.deepStrictEqual(res.config.params['language'], 'en'); + assert.deepStrictEqual(res.config.params['country_code'], 'us'); + }); + + it('should work with only required param', async function () { + var res = await client.googleSearch({ search: 'test' }); + assert.deepStrictEqual(res.config.params['search'], 'test'); + }); + + it('should handle all optional params', async function () { + var res = await client.googleSearch({ + search: 'test', + params: { + add_html: true, + device: 'mobile', + page: 2, + search_type: 'news', + light_request: false, + nfpr: true + } + }); + assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['device'], 'mobile'); + assert.deepStrictEqual(res.config.params['page'], 2); + assert.deepStrictEqual(res.config.params['search_type'], 'news'); + assert.deepStrictEqual(res.config.params['light_request'], false); + assert.deepStrictEqual(res.config.params['nfpr'], true); + }); +}); + +// ============================================ +// Amazon Search API +// ============================================ + +describe('test_ScrapingBeeClient.amazonSearch', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.amazonSearch({ + query: 'laptop', + params: { domain: 'com', language: 'en' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['query'], 'laptop'); + assert.deepStrictEqual(res.config.params['domain'], 'com'); + assert.deepStrictEqual(res.config.params['language'], 'en'); + }); + + it('should work with only required param', async function () { + var res = await client.amazonSearch({ query: 'phone' }); + assert.deepStrictEqual(res.config.params['query'], 'phone'); + }); + + it('should handle all optional params', async function () { + var res = await client.amazonSearch({ + query: 'laptop', + params: { + add_html: true, + country: 'us', + currency: 'USD', + device: 'desktop', + pages: 2, + sort_by: 'price_low_to_high', + start_page: 1 + } + }); + assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['country'], 'us'); + assert.deepStrictEqual(res.config.params['currency'], 'USD'); + assert.deepStrictEqual(res.config.params['pages'], 2); + assert.deepStrictEqual(res.config.params['sort_by'], 'price_low_to_high'); + }); +}); + +// ============================================ +// Amazon Product API +// ============================================ + +describe('test_ScrapingBeeClient.amazonProduct', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.amazonProduct({ + query: 'B0D2Q9397Y', + params: { domain: 'com' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['query'], 'B0D2Q9397Y'); + assert.deepStrictEqual(res.config.params['domain'], 'com'); + }); + + it('should work with only required param', async function () { + var res = await client.amazonProduct({ query: 'B0D2Q9397Y' }); + assert.deepStrictEqual(res.config.params['query'], 'B0D2Q9397Y'); + }); + + it('should handle all optional params', async function () { + var res = await client.amazonProduct({ + query: 'B0D2Q9397Y', + params: { + add_html: true, + autoselect_variant: true, + country: 'us', + currency: 'USD', + device: 'mobile', + language: 'en', + light_request: false, + screenshot: true + } + }); + assert.deepStrictEqual(res.config.params['autoselect_variant'], true); + assert.deepStrictEqual(res.config.params['screenshot'], true); + assert.deepStrictEqual(res.config.params['light_request'], false); + }); +}); + +// ============================================ +// Walmart Search API +// ============================================ + +describe('test_ScrapingBeeClient.walmartSearch', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.walmartSearch({ + query: 'laptop', + params: { sort_by: 'best_match', device: 'desktop' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['query'], 'laptop'); + assert.deepStrictEqual(res.config.params['sort_by'], 'best_match'); + }); + + it('should work with only required param', async function () { + var res = await client.walmartSearch({ query: 'tv' }); + assert.deepStrictEqual(res.config.params['query'], 'tv'); + }); + + it('should handle all optional params', async function () { + var res = await client.walmartSearch({ + query: 'laptop', + params: { + add_html: true, + delivery_zip: '10001', + device: 'mobile', + fulfillment_speed: 'today', + max_price: 1000, + min_price: 100, + screenshot: true, + store_id: '12345' + } + }); + assert.deepStrictEqual(res.config.params['delivery_zip'], '10001'); + assert.deepStrictEqual(res.config.params['max_price'], 1000); + assert.deepStrictEqual(res.config.params['min_price'], 100); + assert.deepStrictEqual(res.config.params['fulfillment_speed'], 'today'); + }); +}); + +// ============================================ +// Walmart Product API +// ============================================ + +describe('test_ScrapingBeeClient.walmartProduct', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.walmartProduct({ + product_id: '123456789', + params: { device: 'desktop' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['product_id'], '123456789'); + assert.deepStrictEqual(res.config.params['device'], 'desktop'); + }); + + it('should work with only required param', async function () { + var res = await client.walmartProduct({ product_id: '123456789' }); + assert.deepStrictEqual(res.config.params['product_id'], '123456789'); + }); + + it('should handle all optional params', async function () { + var res = await client.walmartProduct({ + product_id: '123456789', + params: { + add_html: true, + delivery_zip: '10001', + device: 'tablet', + domain: 'com', + light_request: false, + screenshot: true, + store_id: '12345' + } + }); + assert.deepStrictEqual(res.config.params['delivery_zip'], '10001'); + assert.deepStrictEqual(res.config.params['device'], 'tablet'); + assert.deepStrictEqual(res.config.params['screenshot'], true); + }); +}); + +// ============================================ +// ChatGPT API +// ============================================ + +describe('test_ScrapingBeeClient.chatGPT', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.chatGPT({ + prompt: 'What is web scraping?', + params: { search: true } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['prompt'], 'What is web scraping?'); + assert.deepStrictEqual(res.config.params['search'], true); + }); + + it('should work with only required param', async function () { + var res = await client.chatGPT({ prompt: 'Hello' }); + assert.deepStrictEqual(res.config.params['prompt'], 'Hello'); + }); + + it('should handle all optional params', async function () { + var res = await client.chatGPT({ + prompt: 'Explain AI', + params: { + add_html: true, + country_code: 'us', + search: true + } + }); + assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['country_code'], 'us'); + assert.deepStrictEqual(res.config.params['search'], true); + }); +}); + +// ============================================ +// YouTube Search API +// ============================================ + +describe('test_ScrapingBeeClient.youtubeSearch', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.youtubeSearch({ + search: 'web scraping tutorial', + params: { sort_by: 'relevance', type: 'video' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['search'], 'web scraping tutorial'); + assert.deepStrictEqual(res.config.params['sort_by'], 'relevance'); + }); + + it('should work with only required param', async function () { + var res = await client.youtubeSearch({ search: 'coding' }); + assert.deepStrictEqual(res.config.params['search'], 'coding'); + }); + + it('should handle all optional params', async function () { + var res = await client.youtubeSearch({ + search: 'music', + params: { + '4k': true, + hd: true, + duration: '4-20', + upload_date: 'this_week', + sort_by: 'view_count', + type: 'video', + subtitles: true, + live: false + } + }); + assert.deepStrictEqual(res.config.params['4k'], true); + assert.deepStrictEqual(res.config.params['hd'], true); + assert.deepStrictEqual(res.config.params['duration'], '4-20'); + assert.deepStrictEqual(res.config.params['upload_date'], 'this_week'); + assert.deepStrictEqual(res.config.params['subtitles'], true); + }); +}); + +// ============================================ +// YouTube Metadata API +// ============================================ + +describe('test_ScrapingBeeClient.youtubeMetadata', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.youtubeMetadata({ + video_id: 'dQw4w9WgXcQ' + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); + }); +}); + +// ============================================ +// YouTube Transcript API +// ============================================ + +describe('test_ScrapingBeeClient.youtubeTranscript', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.youtubeTranscript({ + video_id: 'dQw4w9WgXcQ', + params: { language: 'en' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); + assert.deepStrictEqual(res.config.params['language'], 'en'); + }); + + it('should work with only required param', async function () { + var res = await client.youtubeTranscript({ video_id: 'dQw4w9WgXcQ' }); + assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); + }); + + it('should handle all optional params', async function () { + var res = await client.youtubeTranscript({ + video_id: 'dQw4w9WgXcQ', + params: { + language: 'es', + transcript_origin: 'uploader_provided' + } + }); + assert.deepStrictEqual(res.config.params['language'], 'es'); + assert.deepStrictEqual(res.config.params['transcript_origin'], 'uploader_provided'); + }); +}); + +// ============================================ +// YouTube Trainability API +// ============================================ + +describe('test_ScrapingBeeClient.youtubeTrainability', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.youtubeTrainability({ + video_id: 'dQw4w9WgXcQ' + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); + }); +}); + +// ============================================ +// Usage API +// ============================================ + +describe('test_ScrapingBeeClient.usage', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.usage(); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + }); + + it('should work with empty config', async function () { + var res = await client.usage({}); + assert.deepStrictEqual(res.status, 200); + }); + + it('should work with optional config', async function () { + var res = await client.usage({ retries: 3, timeout: 5000 }); + assert.deepStrictEqual(res.status, 200); + }); +}); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 1a82aba..9830468 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -62,6 +62,21 @@ describe('test_process_extract_rules', function () { }); }); +describe('test_process_ai_extract_rules', function () { + it('should stringify JSON', function () { + var ai_extract_rules = { summary: 'Extract main content' }; + var res = process_params({ ai_extract_rules: ai_extract_rules }); + assert.strictEqual(res['ai_extract_rules'], '{"summary":"Extract main content"}'); + }); + + it('should return the same string', function () { + var ai_extract_rules = '{"summary":"Extract main content"}'; + var res = process_params({ ai_extract_rules: ai_extract_rules }); + assert.strictEqual(res['ai_extract_rules'], '{"summary":"Extract main content"}'); + }); +}); + + describe('test_process_js_scenario', function () { it('should stringify JSON', function () { var js_scenario = { From ee51ff8c847f6e58945f07abf38ae7c61d6dde1d Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Thu, 18 Dec 2025 15:25:36 +0530 Subject: [PATCH 4/7] 1.8.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1fb39c5..80d785d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "scrapingbee", - "version": "1.7.6", + "version": "1.8.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "scrapingbee", - "version": "1.7.6", + "version": "1.8.0", "license": "ISC", "dependencies": { "axios": "^1.7", diff --git a/package.json b/package.json index f15e39f..e5375c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scrapingbee", - "version": "1.7.6", + "version": "1.8.0", "description": "ScrapingBee Node SDK", "main": "dist/index.js", "types": "dist/index.d.ts", From 6e90d0bd64fe3d90e971db329044d2bf21308b33 Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Fri, 19 Dec 2025 20:51:02 +0530 Subject: [PATCH 5/7] Final Touches - Updated README.md - Improved manual-test.js - Added information to CHANGELOG.md - Updated version in version.ts --- CHANGELOG.md | 23 ++ README.md | 485 +++++++++++++++++++++++++++------- dist/version.d.ts | 2 +- dist/version.js | 2 +- manual-test.js | 644 ++++++++++++++++++++++++++++++++++++++++++++++ src/version.ts | 2 +- test-manual.js | 439 ------------------------------- 7 files changed, 1066 insertions(+), 531 deletions(-) create mode 100644 manual-test.js delete mode 100644 test-manual.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 64c986f..c56fa01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [1.8.0](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.7.5...v1.8.0) (2024-12-22) + +### Features + +- Added `htmlApi()` method for HTML API +- Added `googleSearch()` method for Google Search API +- Added `amazonSearch()` method for Amazon Search API +- Added `amazonProduct()` method for Amazon Product API +- Added `walmartSearch()` method for Walmart Search API +- Added `walmartProduct()` method for Walmart Product API +- Added `youtubeSearch()` method for YouTube Search API +- Added `youtubeMetadata()` method for YouTube Metadata API +- Added `youtubeTranscript()` method for YouTube Transcript API +- Added `youtubeTrainability()` method for YouTube Trainability API +- Added `chatGPT()` method for ChatGPT API +- Added `usage()` method for Usage endpoint +- Added new HTML API parameters: `ai_query`, `ai_selector`, `return_page_markdown`, `return_page_text`, `scraping_config` +- Refactored internal `request()` method to be API-agnostic + +### Deprecated + +- `get()` and `post()` methods are deprecated in favor of `htmlApi()` method + ## [1.7.5](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.7.4...v1.7.5) (2023-06-06) ### Bugfix diff --git a/README.md b/README.md index 5ee7ca6..7822ce2 100644 --- a/README.md +++ b/README.md @@ -12,140 +12,447 @@ npm install scrapingbee ## Usage -The ScrapingBee Node SDK is a wrapper around the [axios](https://axios-http.com/docs/intro) library. ScrapingBee supports GET and POST requests. +The ScrapingBee Node SDK is a wrapper around the [axios](https://axios-http.com/docs/intro) library. Signup to ScrapingBee to [get your API key](https://app.scrapingbee.com/account/register) and some free credits to get started. -### Making a GET request +## Table of Contents + +- [HTML API](#html-api) +- [Google Search API](#google-search-api) +- [Amazon API](#amazon-api) +- [Walmart API](#walmart-api) +- [YouTube API](#youtube-api) +- [ChatGPT API](#chatgpt-api) +- [Usage API](#usage-api) + +--- + +## HTML API + +The HTML API allows you to scrape any webpage and get the HTML content. + +### GET Request ```javascript -const scrapingbee = require('scrapingbee'); +const { ScrapingBeeClient } = require('scrapingbee'); async function get(url) { - var client = new scrapingbee.ScrapingBeeClient('REPLACE-WITH-YOUR-API-KEY'); - var response = await client.get({ - // The URL you want to scrape + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.htmlApi({ url: url, params: { - // Block ads on the page you want to scrape - block_ads: false, - // Block images and CSS on the page you want to scrape - block_resources: true, - // Premium proxy geolocation - country_code: '', - // Control the device the request will be sent from - device: 'desktop', - // Use some data extraction rules - extract_rules: { title: 'h1' }, - // Use AI data extraction rules - ai_extract_rules: { summary: '5 words summary of the post' }, - // Wrap response in JSON - json_response: false, - // JavaScript scenario to execute (clicking on button, scrolling ...) + render_js: true, + extract_rules: { title: 'h1', links: 'a @href' }, js_scenario: { instructions: [ - { wait_for: '#slow_button' }, - { click: '#slow_button' }, - { scroll_x: 1000 }, - { wait: 1000 }, - { scroll_x: 1000 }, - { wait: 1000 }, - ], + { click: '#button' }, + { wait: 500 }, + { scroll_y: 1000 }, + ] }, - // Use premium proxies to bypass difficult to scrape websites (10-25 credits/request) - premium_proxy: false, - // Execute JavaScript code with a Headless Browser (5 credits/request) - render_js: true, - // Return the original HTML before the JavaScript rendering - return_page_source: false, - // Return page screenshot as a png image - screenshot: false, - // Take a full page screenshot without the window limitation - screenshot_full_page: false, - // Transparently return the same HTTP code of the page requested. - transparent_status_code: false, - // Wait, in miliseconds, before returning the response - wait: 0, - // Wait for CSS selector before returning the response, ex ".title" - wait_for: '', - // Set the browser window width in pixel - window_width: 1920, - // Set the browser window height in pixel - window_height: 1080, + } + }); + + const decoder = new TextDecoder(); + const text = decoder.decode(response.data); + console.log(text); +} + +get('https://example.com'); +``` + +### POST Request + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function post(url) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.htmlApi({ + url: url, + method: 'POST', + 'username=user&password=pass', + params: { + render_js: false, }, headers: { - // Forward custom headers to the target website - key: 'value', + 'Custom-Header': 'value', }, cookies: { - // Forward custom cookies to the target website - name: 'value', + session: 'abc123', }, - // `timeout` specifies the number of milliseconds before the request times out. - // If the request takes longer than `timeout`, the request will be aborted. - timeout: 10000, // here 10sec, default is `0` (no timeout) }); - var decoder = new TextDecoder(); - var text = decoder.decode(response.data); + const decoder = new TextDecoder(); + const text = decoder.decode(response.data); console.log(text); } -get('https://httpbin-scrapingbee.cleverapps.io/html').catch((e) => console.log('A problem occurs : ' + e.message)); - -/* -- output - ... -*/ +post('https://httpbin.org/post'); ``` -ScrapingBee takes various parameters to render JavaScript, execute a custom JavaScript script, use a premium proxy from a specific geolocation and more. - -You can find all the supported parameters on [ScrapingBee's documentation](https://www.scrapingbee.com/documentation/). - -You can send custom cookies and headers like you would normally do with the requests library. - -## Screenshot - -Here a little exemple on how to retrieve and store a screenshot from the ScrapingBee blog in its mobile resolution. +### Screenshot ```javascript const fs = require('fs'); -const scrapingbee = require('scrapingbee'); +const { ScrapingBeeClient } = require('scrapingbee'); async function screenshot(url, path) { - var client = new scrapingbee.ScrapingBeeClient('REPLACE-WITH-YOUR-API-KEY'); - var response = await client.get({ + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.htmlApi({ url: url, params: { - screenshot: true, // Take a screenshot - screenshot_full_page: true, // Specify that we need the full height - window_width: 375, // Specify a mobile width in pixel + screenshot: true, + screenshot_full_page: true, + window_width: 375, }, }); fs.writeFileSync(path, response.data); } -screenshot('https://httpbin-scrapingbee.cleverapps.io/html', './httpbin.png').catch((e) => - console.log('A problem occurs : ' + e.message) -); +screenshot('https://example.com', './screenshot.png'); +``` + +--- + +## Google Search API + +Scrape Google search results in real-time. + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function googleSearch(query) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.googleSearch({ + search: query, + params: { + language: 'en', + country_code: 'us', + page: 1, + search_type: 'classic', + device: 'desktop', + light_request: true, + nfpr: false, + add_html: false, + } + }); + + console.log(response.data); +} + +googleSearch('web scraping tools'); +``` + +--- + +## Amazon API + +Scrape Amazon search results and product details. + +### Amazon Search + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function amazonSearch(query) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.amazonSearch({ + query: query, + params: { + domain: 'com', + language: 'en', + country: 'us', + device: 'desktop', + pages: 1, + start_page: 1, + sort_by: 'featured', + currency: 'USD', + add_html: false, + screenshot: false, + } + }); + + console.log(response.data); +} + +amazonSearch('laptop'); +``` + +### Amazon Product + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function amazonProduct(asin) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.amazonProduct({ + query: asin, + params: { + domain: 'com', + language: 'en', + country: 'us', + device: 'desktop', + autoselect_variant: false, + add_html: false, + screenshot: false, + } + }); + + console.log(response.data); +} + +amazonProduct('B0D2Q9397Y'); +``` + +--- + +## Walmart API + +Scrape Walmart search results and product details. + +### Walmart Search + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function walmartSearch(query) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.walmartSearch({ + query: query, + params: { + device: 'desktop', + sort_by: 'best_match', + min_price: 10, + max_price: 1000, + delivery_zip: '10001', + store_id: '', + fulfillment_speed: '', + add_html: false, + screenshot: false, + } + }); + + console.log(response.data); +} + +walmartSearch('laptop'); +``` + +### Walmart Product + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function walmartProduct(productId) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.walmartProduct({ + product_id: productId, + params: { + device: 'desktop', + delivery_zip: '10001', + store_id: '', + add_html: false, + screenshot: false, + } + }); + + console.log(response.data); +} + +walmartProduct('123456789'); +``` + +--- + +## YouTube API + +Scrape YouTube search results, video metadata, transcripts, and trainability data. + +### YouTube Search + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function youtubeSearch(query) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.youtubeSearch({ + search: query, + params: { + sort_by: 'relevance', + type: 'video', + upload_date: '', + duration: '', + hd: false, + '4k': false, + subtitles: false, + live: false, + } + }); + + console.log(response.data); +} + +youtubeSearch('web scraping tutorial'); +``` + +### YouTube Metadata + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function youtubeMetadata(videoId) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.youtubeMetadata({ + video_id: videoId, + }); + + console.log(response.data); +} + +youtubeMetadata('dQw4w9WgXcQ'); +``` + +### YouTube Transcript + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function youtubeTranscript(videoId) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.youtubeTranscript({ + video_id: videoId, + params: { + language: 'en', + transcript_origin: 'auto_generated', + } + }); + + console.log(response.data); +} + +youtubeTranscript('dQw4w9WgXcQ'); +``` + +### YouTube Trainability + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function youtubeTrainability(videoId) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.youtubeTrainability({ + video_id: videoId, + }); + + console.log(response.data); +} + +youtubeTrainability('dQw4w9WgXcQ'); +``` + +--- + +## ChatGPT API + +Use ChatGPT with optional web search capabilities. + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function askChatGPT(prompt) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.chatGPT({ + prompt: prompt, + params: { + search: true, + country_code: 'us', + add_html: false, + } + }); + + console.log(response.data); +} + +askChatGPT('What are the latest web scraping trends?'); +``` + +--- + +## Usage API + +Check your API credit usage and account limits. + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function checkUsage() { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.usage(); + + console.log(response.data); +} + +checkUsage(); ``` +--- + ## Retries -The client includes a retry mechanism for 5XX responses. +All API methods support automatic retry for 5XX responses. ```javascript -const spb = require('scrapingbee'); +const { ScrapingBeeClient } = require('scrapingbee'); -async function get(url) { - let client = new spb.ScrapingBeeClient('REPLACE-WITH-YOUR-API-KEY'); - let resp = await client.get({ url: url, params: { render_js: false }, retries: 5 }); +async function scrapeWithRetry() { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.htmlApi({ + url: 'https://example.com', + retries: 5, + timeout: 30000, + }); - let decoder = new TextDecoder(); - let text = decoder.decode(resp.data); - console.log(text); + console.log(response.data); } +``` + +--- + +## Promise and Async/Await + +All methods return Promises, so you can use either `.then()` or `async/await`: + +```javascript +// Using async/await +const response = await client.googleSearch({ search: 'test' }); -get('https://httpbin-scrapingbee.cleverapps.io/html').catch((e) => console.log('A problem occured: ' + e.message)); +// Using .then() +client.googleSearch({ search: 'test' }) + .then(response => console.log(response.data)) + .catch(error => console.error(error)); ``` + +--- + +## Legacy Methods (Deprecated) + +The `get()` and `post()` methods are deprecated and will be removed in a future version. Please use `htmlApi()` instead. + +```javascript +// Deprecated +await client.get({ url: '...' }); +await client.post({ url: '...' }); + +// Use instead +await client.htmlApi({ url: '...', method: 'GET' }); +await client.htmlApi({ url: '...', method: 'POST' }); +``` + +--- + +## Documentation + +For more details on all available parameters, visit [ScrapingBee's documentation](https://www.scrapingbee.com/documentation/). \ No newline at end of file diff --git a/dist/version.d.ts b/dist/version.d.ts index 0cee7dd..6b4927c 100644 --- a/dist/version.d.ts +++ b/dist/version.d.ts @@ -1 +1 @@ -export declare const LIB_VERSION = "1.7.6"; +export declare const LIB_VERSION = "1.8.0"; diff --git a/dist/version.js b/dist/version.js index 1dafffc..aae18d2 100644 --- a/dist/version.js +++ b/dist/version.js @@ -1,4 +1,4 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.LIB_VERSION = void 0; -exports.LIB_VERSION = "1.7.6"; +exports.LIB_VERSION = "1.8.0"; diff --git a/manual-test.js b/manual-test.js new file mode 100644 index 0000000..9903b64 --- /dev/null +++ b/manual-test.js @@ -0,0 +1,644 @@ +const { ScrapingBeeClient } = require('./dist/index'); + +const API_KEY = process.env.SCRAPINGBEE_API_KEY; +const client = new ScrapingBeeClient(API_KEY); + +// Helper function to parse response data +function parseResponse(response) { + if (Buffer.isBuffer(response.data)) { + return JSON.parse(response.data.toString()); + } else if (typeof response.data === 'string') { + return JSON.parse(response.data); + } + return response.data; +} + +// Helper function to assert +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +// ============================================ +// Legacy HTML API Tests +// ============================================ + +async function testHtmlGet() { + console.log('=== Testing HTML API - GET ==='); + try { + const response = await client.get({ + url: 'https://httpbin.org/get', + params: { render_js: false } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data, 'Response data is empty'); + assert(response.data.toString().includes('httpbin'), 'Response does not contain expected content'); + + console.log('Status:', response.status); + console.log('✅ HTML GET test passed!\n'); + } catch (error) { + console.log('❌ HTML GET test failed:', error.message); + throw error; + } +} + +async function testHtmlPost() { + console.log('=== Testing HTML API - POST ==='); + try { + const response = await client.post({ + url: 'https://httpbin.org/post', + params: { render_js: false }, + data: 'test=data' + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data, 'Response data is empty'); + assert(response.data.toString().includes('test'), 'Response does not contain posted data'); + + console.log('Status:', response.status); + console.log('✅ HTML POST test passed!\n'); + } catch (error) { + console.log('❌ HTML POST test failed:', error.message); + throw error; + } +} + +// ============================================ +// New HTML API Tests +// ============================================ + +async function testHtmlApiGet() { + console.log('=== Testing HTML API (New) - GET ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/get', + method: 'GET', + params: { render_js: false } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data, 'Response data is empty'); + assert(response.data.toString().includes('httpbin'), 'Response does not contain expected content'); + + console.log('Status:', response.status); + console.log('✅ HTML API GET test passed!\n'); + } catch (error) { + console.log('❌ HTML API GET test failed:', error.message); + throw error; + } +} + +async function testHtmlApiPost() { + console.log('=== Testing HTML API (New) - POST ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/post', + method: 'POST', + params: { render_js: false }, + data: 'test=data' + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data, 'Response data is empty'); + assert(response.data.toString().includes('test'), 'Response does not contain posted data'); + + console.log('Status:', response.status); + console.log('✅ HTML API POST test passed!\n'); + } catch (error) { + console.log('❌ HTML API POST test failed:', error.message); + throw error; + } +} + +async function testHtmlApiExtractRules() { + console.log('=== Testing HTML API - Extract Rules ==='); + try { + const response = await client.htmlApi({ + url: 'https://www.scrapingbee.com/blog/', + params: { + render_js: false, + extract_rules: { + title: 'h1', + posts: { + selector: '.container > div > div > div', + type: 'list', + output: { + title: 'h4', + link: 'a@href' + } + } + } + } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.title, 'Extracted title is missing'); + assert(Array.isArray(data.posts), 'Extracted posts is not an array'); + assert(data.posts.length > 0, 'No posts extracted'); + + console.log('Status:', response.status); + console.log('Extracted title:', data.title); + console.log('Extracted posts count:', data.posts.length); + console.log('✅ HTML API Extract Rules test passed!\n'); + } catch (error) { + console.log('❌ HTML API Extract Rules test failed:', error.message); + throw error; + } +} + +async function testHtmlApiJsScenario() { + console.log('=== Testing HTML API - JS Scenario ==='); + try { + const response = await client.htmlApi({ + url: 'https://www.scrapingbee.com', + params: { + render_js: true, + js_scenario: { + instructions: [ + { wait: 1000 }, + { scroll_y: 500 }, + { wait: 500 } + ] + } + } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data, 'Response data is empty'); + + console.log('Status:', response.status); + console.log('Content:', response.data.toString().substring(0, 300)); + console.log('✅ HTML API JS Scenario test passed!\n'); + } catch (error) { + console.log('❌ HTML API JS Scenario test failed:', error.message); + throw error; + } +} + +async function testHtmlApiScreenshot() { + console.log('=== Testing HTML API - Screenshot ==='); + try { + const response = await client.htmlApi({ + url: 'https://www.scrapingbee.com', + params: { + render_js: true, + screenshot: true, + window_width: 1920, + window_height: 1080 + } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data, 'Response data is empty'); + assert(response.data.length > 10000, 'Screenshot seems too small'); + + // Check PNG signature (first 8 bytes) + const pngSignature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); + assert(response.data.slice(0, 8).equals(pngSignature), 'Response is not a valid PNG'); + + console.log('Status:', response.status); + console.log('Screenshot size:', response.data.length, 'bytes'); + console.log('✅ HTML API Screenshot test passed!\n'); + } catch (error) { + console.log('❌ HTML API Screenshot test failed:', error.message); + throw error; + } +} + +async function testHtmlApiJsonResponse() { + console.log('=== Testing HTML API - JSON Response ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/get', + params: { + render_js: false, + json_response: true + } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.body, 'JSON response missing body field'); + assert(data.xhr, 'JSON response missing xhr field'); + + const bodyPreview = typeof data.body === 'string' + ? data.body.substring(0, 300) + : JSON.stringify(data.body).substring(0, 300); + + console.log('Status:', response.status); + console.log('Content:', bodyPreview); + console.log('✅ HTML API JSON Response test passed!\n'); + } catch (error) { + console.log('❌ HTML API JSON Response test failed:', error.message); + throw error; + } +} + +async function testHtmlApiWithHeaders() { + console.log('=== Testing HTML API - Custom Headers ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/headers', + params: { render_js: false }, + headers: { + 'X-Custom-Header': 'CustomValue123' + } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data.toString().includes('CustomValue123'), 'Custom header not forwarded'); + + console.log('Status:', response.status); + console.log('✅ HTML API Custom Headers test passed!\n'); + } catch (error) { + console.log('❌ HTML API Custom Headers test failed:', error.message); + throw error; + } +} + +async function testHtmlApiWithCookies() { + console.log('=== Testing HTML API - Custom Cookies ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/cookies', + params: { render_js: false }, + cookies: { + session_id: 'abc123', + user_token: 'xyz789' + } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const responseText = response.data.toString(); + assert(responseText.includes('abc123') || responseText.includes('xyz789'), 'Cookies not forwarded'); + + console.log('Status:', response.status); + console.log('✅ HTML API Custom Cookies test passed!\n'); + } catch (error) { + console.log('❌ HTML API Custom Cookies test failed:', error.message); + throw error; + } +} + +async function testHtmlApiPostWithHeadersAndCookies() { + console.log('=== Testing HTML API - POST with Headers + Cookies ==='); + try { + const response = await client.htmlApi({ + url: 'https://httpbin.org/post', + method: 'POST', + params: { render_js: false }, + headers: { 'X-Test-Header': 'TestValue' }, + cookies: { session: 'mysession123' }, + data: JSON.stringify({ action: 'submit' }) + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + assert(response.data.toString().includes('submit'), 'Posted data not in response'); + + console.log('Status:', response.status); + console.log('✅ HTML API POST with Headers + Cookies test passed!\n'); + } catch (error) { + console.log('❌ HTML API POST with Headers + Cookies test failed:', error.message); + throw error; + } +} + +// ============================================ +// Google Search API +// ============================================ + +async function testGoogleSearch() { + console.log('=== Testing Google Search API ==='); + try { + const response = await client.googleSearch({ + search: 'scrapingbee', + params: { language: 'en', country_code: 'us' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.organic_results, 'Missing organic_results in response'); + assert(Array.isArray(data.organic_results), 'organic_results is not an array'); + assert(data.organic_results.length > 0, 'No organic results found'); + + console.log('Status:', response.status); + console.log('Results found:', data.organic_results.length); + console.log('✅ Google Search test passed!\n'); + } catch (error) { + console.log('❌ Google Search test failed:', error.message); + throw error; + } +} + +// ============================================ +// Amazon API +// ============================================ + +async function testAmazonSearch() { + console.log('=== Testing Amazon Search API ==='); + try { + const response = await client.amazonSearch({ + query: 'laptop', + params: { domain: 'com', pages: 1 } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.products, 'Missing products in response'); + assert(Array.isArray(data.products), 'products is not an array'); + assert(data.products.length > 0, 'No products found'); + + console.log('Status:', response.status); + console.log('Results found:', data.products.length); + console.log('✅ Amazon Search test passed!\n'); + } catch (error) { + console.log('❌ Amazon Search test failed:', error.message); + throw error; + } +} + +async function testAmazonProduct() { + console.log('=== Testing Amazon Product API ==='); + try { + const response = await client.amazonProduct({ + query: 'B0D2Q9397Y', + params: { domain: 'com' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.title, 'Missing product title in response'); + + console.log('Status:', response.status); + console.log('Product title:', data.title.substring(0, 50)); + console.log('✅ Amazon Product test passed!\n'); + } catch (error) { + console.log('❌ Amazon Product test failed:', error.message); + throw error; + } +} + +// ============================================ +// Walmart API +// ============================================ + +async function testWalmartSearch() { + console.log('=== Testing Walmart Search API ==='); + try { + const response = await client.walmartSearch({ + query: 'laptop', + params: { device: 'desktop', sort_by: 'best_match' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.products, 'Missing products in response'); + assert(Array.isArray(data.products), 'products is not an array'); + assert(data.products.length > 0, 'No products found'); + + console.log('Status:', response.status); + console.log('Results found:', data.products.length); + console.log('✅ Walmart Search test passed!\n'); + } catch (error) { + console.log('❌ Walmart Search test failed:', error.message); + throw error; + } +} + +async function testWalmartProduct() { + console.log('=== Testing Walmart Product API ==='); + try { + const response = await client.walmartProduct({ + product_id: '454408250', + params: { device: 'desktop' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.title, 'Missing product title in response'); + + console.log('Status:', response.status); + console.log('Product title:', data.title.substring(0, 50)); + console.log('✅ Walmart Product test passed!\n'); + } catch (error) { + console.log('❌ Walmart Product test failed:', error.message); + throw error; + } +} + +// ============================================ +// ChatGPT API +// ============================================ + +async function testChatGPT() { + console.log('=== Testing ChatGPT API ==='); + try { + const response = await client.chatGPT({ + prompt: 'What is web scraping? Answer in one sentence.', + params: { search: true } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.results_text || data.results_markdown, 'Missing response text'); + + console.log('Status:', response.status); + console.log('Response:', (data.results_text || data.results_markdown).substring(0, 100)); + console.log('✅ ChatGPT test passed!\n'); + } catch (error) { + console.log('❌ ChatGPT test failed:', error.message); + throw error; + } +} + +// ============================================ +// YouTube API +// ============================================ + +async function testYouTubeSearch() { + console.log('=== Testing YouTube Search API ==='); + try { + const response = await client.youtubeSearch({ + search: 'web scraping tutorial', + params: { sort_by: 'relevance', type: 'video' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.results, 'Missing results in response'); + assert(Array.isArray(data.results), 'results is not an array'); + assert(data.results.length > 0, 'No results found'); + + console.log('Status:', response.status); + console.log('Results found:', data.results.length); + console.log('✅ YouTube Search test passed!\n'); + } catch (error) { + console.log('❌ YouTube Search test failed:', error.message); + throw error; + } +} + +async function testYouTubeMetadata() { + console.log('=== Testing YouTube Metadata API ==='); + try { + const response = await client.youtubeMetadata({ + video_id: 'dQw4w9WgXcQ' + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.title || data.like_count !== undefined, 'Missing expected metadata fields'); + + console.log('Status:', response.status); + console.log('Like count:', data.like_count); + console.log('✅ YouTube Metadata test passed!\n'); + } catch (error) { + console.log('❌ YouTube Metadata test failed:', error.message); + throw error; + } +} + +async function testYouTubeTranscript() { + console.log('=== Testing YouTube Transcript API ==='); + try { + const response = await client.youtubeTranscript({ + video_id: 'sfyL4BswUeE', + params: { language: 'en' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.text || data.transcript, 'Missing transcript in response'); + + console.log('Status:', response.status); + console.log('Transcript preview:', (data.text || JSON.stringify(data.transcript)).substring(0, 100)); + console.log('✅ YouTube Transcript test passed!\n'); + } catch (error) { + console.log('❌ YouTube Transcript test failed:', error.message); + throw error; + } +} + +async function testYouTubeTrainability() { + console.log('=== Testing YouTube Trainability API ==='); + try { + const response = await client.youtubeTrainability({ + video_id: 'dQw4w9WgXcQ' + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.permitted !== undefined, 'Missing permitted field in response'); + + console.log('Status:', response.status); + console.log('Permitted:', data.permitted); + console.log('✅ YouTube Trainability test passed!\n'); + } catch (error) { + console.log('❌ YouTube Trainability test failed:', error.message); + throw error; + } +} + +// ============================================ +// Usage API +// ============================================ + +async function testUsage() { + console.log('=== Testing Usage API ==='); + try { + const response = await client.usage(); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.max_api_credit !== undefined, 'Missing max_api_credit'); + assert(data.used_api_credit !== undefined, 'Missing used_api_credit'); + assert(data.max_concurrency !== undefined, 'Missing max_concurrency'); + + console.log('Status:', response.status); + console.log('Max API credits:', data.max_api_credit); + console.log('Used API credits:', data.used_api_credit); + console.log('Max concurrency:', data.max_concurrency); + console.log('✅ Usage test passed!\n'); + } catch (error) { + console.log('❌ Usage test failed:', error.message); + throw error; + } +} + +// ============================================ +// Run All Tests +// ============================================ + +async function runTests() { + console.log('\n🚀 Starting ScrapingBee SDK Tests\n'); + + let passed = 0; + let failed = 0; + + const tests = [ + // Legacy HTML API + testHtmlGet, + testHtmlPost, + + // New HTML API + testHtmlApiGet, + testHtmlApiPost, + testHtmlApiExtractRules, + testHtmlApiJsScenario, + testHtmlApiScreenshot, + testHtmlApiJsonResponse, + testHtmlApiWithHeaders, + testHtmlApiWithCookies, + testHtmlApiPostWithHeadersAndCookies, + + // Other APIs + testGoogleSearch, + testAmazonSearch, + testAmazonProduct, + testWalmartSearch, + testWalmartProduct, + testChatGPT, + testYouTubeSearch, + testYouTubeMetadata, + testYouTubeTranscript, + testYouTubeTrainability, + testUsage, + ]; + + for (const test of tests) { + try { + await test(); + passed++; + } catch (error) { + failed++; + } + } + + console.log('🏁 All tests completed!'); + console.log(`✅ Passed: ${passed}`); + console.log(`❌ Failed: ${failed}`); + console.log(`📊 Total: ${tests.length}\n`); + + if (failed > 0) { + process.exit(1); + } +} + +runTests(); diff --git a/src/version.ts b/src/version.ts index 7866235..0121859 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const LIB_VERSION = "1.7.6"; +export const LIB_VERSION = "1.8.0"; diff --git a/test-manual.js b/test-manual.js deleted file mode 100644 index 2a6e082..0000000 --- a/test-manual.js +++ /dev/null @@ -1,439 +0,0 @@ -const { ScrapingBeeClient } = require('./dist/index'); - -const API_KEY = process.env.SCRAPINGBEE_API_KEY; - -const client = new ScrapingBeeClient(API_KEY); - -async function testHtmlGet() { - console.log('=== Testing HTML API - GET ==='); - try { - const response = await client.get({ - url: 'https://httpbin.org/get', - params: { - render_js: false - } - }); - console.log('Status:', response.status); - console.log('Response:', response.data.toString().substring(0, 200) + '...'); - console.log('✅ HTML GET test passed!\n'); - } catch (error) { - console.log('❌ HTML GET test failed:', error.message); - } -} - -async function testHtmlPost() { - console.log('=== Testing HTML API - POST ==='); - try { - const response = await client.post({ - url: 'https://httpbin.org/post', - params: { - render_js: false - }, - data: 'test=data' - }); - console.log('Status:', response.status); - console.log('Response:', response.data.toString().substring(0, 200) + '...'); - console.log('✅ HTML POST test passed!\n'); - } catch (error) { - console.log('❌ HTML POST test failed:', error.message); - } -} - -async function testGoogleSearch() { - console.log('=== Testing Google Search API ==='); - try { - const response = await client.googleSearch({ - search: 'scrapingbee', - params: { - language: 'en', - country_code: 'us' - } - }); - console.log('Status:', response.status); - console.log('Response type:', typeof response.data); - - // Parse JSON response - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Results found:', data.organic_results?.length || 0); - console.log('✅ Google Search test passed!\n'); - } catch (error) { - console.log('❌ Google Search test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testAmazonSearch() { - console.log('=== Testing Amazon Search API ==='); - try { - const response = await client.amazonSearch({ - query: 'laptop', - params: { - domain: 'com', - pages: 1 - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Results found:', data.products?.length || 0); - console.log('✅ Amazon Search test passed!\n'); - } catch (error) { - console.log('❌ Amazon Search test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testAmazonProduct() { - console.log('=== Testing Amazon Product API ==='); - try { - const response = await client.amazonProduct({ - query: 'B0D2Q9397Y', // Example ASIN - params: { - domain: 'com' - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Product title:', data.title?.substring(0, 50) || 'N/A'); - console.log('✅ Amazon Product test passed!\n'); - } catch (error) { - console.log('❌ Amazon Product test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testWalmartSearch() { - console.log('=== Testing Walmart Search API ==='); - try { - const response = await client.walmartSearch({ - query: 'laptop', - params: { - device: 'desktop', - sort_by: 'best_match' - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Results found:', data.products?.length || 0); - console.log('✅ Walmart Search test passed!\n'); - } catch (error) { - console.log('❌ Walmart Search test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testWalmartProduct() { - console.log('=== Testing Walmart Product API ==='); - try { - const response = await client.walmartProduct({ - product_id: '13943258180', - params: { - device: 'desktop' - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Product title:', data.title?.substring(0, 50) || 'N/A'); - console.log('✅ Walmart Product test passed!\n'); - } catch (error) { - console.log('❌ Walmart Product test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testChatGPT() { - console.log('=== Testing ChatGPT API ==='); - try { - const response = await client.chatGPT({ - prompt: 'What is web scraping?', - params: { - search: true - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Response:', data.results_text?.substring(0, 50) || 'N/A'); - console.log('✅ ChatGPT test passed!\n'); - } catch (error) { - console.log('❌ ChatGPT test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testYouTubeSearch() { - console.log('=== Testing YouTube Search API ==='); - try { - const response = await client.youtubeSearch({ - search: 'web scraping tutorial', - params: { - sort_by: 'relevance', - type: 'video' - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Results found:', data.results?.length || 0); - console.log('✅ YouTube Search test passed!\n'); - } catch (error) { - console.log('❌ YouTube Search test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testYouTubeMetadata() { - console.log('=== Testing YouTube Metadata API ==='); - try { - const response = await client.youtubeMetadata({ - video_id: 'dQw4w9WgXcQ' - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Number of Likes:', data.like_count || 'N/A'); - console.log('✅ YouTube Metadata test passed!\n'); - } catch (error) { - console.log('❌ YouTube Metadata test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testYouTubeTranscript() { - console.log('=== Testing YouTube Transcript API ==='); - try { - const response = await client.youtubeTranscript({ - video_id: 'eKVTFXQPAhs', - params: { - language: 'en' - } - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Transcript segments:', data.text?.substring(0, 100) || 'N/A'); - console.log('✅ YouTube Transcript test passed!\n'); - } catch (error) { - console.log('❌ YouTube Transcript test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testYouTubeTrainability() { - console.log('=== Testing YouTube Trainability API ==='); - try { - const response = await client.youtubeTrainability({ - video_id: 'dQw4w9WgXcQ' - }); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Trainability data:', data.permitted || 'N/A'); - console.log('✅ YouTube Trainability test passed!\n'); - } catch (error) { - console.log('❌ YouTube Trainability test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function testHtmlApiGet() { - console.log('=== Testing HTML API - GET ==='); - try { - const response = await client.htmlApi({ - url: 'https://httpbin.org/get', - method: 'GET', - params: { - render_js: false - } - }); - console.log('Status:', response.status); - console.log('Response:', response.data.toString().substring(0, 200) + '...'); - console.log('✅ HTML API GET test passed!\n'); - } catch (error) { - console.log('❌ HTML API GET test failed:', error.message); - } -} - -async function testHtmlApiPost() { - console.log('=== Testing HTML API - POST ==='); - try { - const response = await client.htmlApi({ - url: 'https://httpbin.org/post', - method: 'POST', - params: { - render_js: false - }, - data: 'test=data' - }); - console.log('Status:', response.status); - console.log('Response:', response.data.toString().substring(0, 200) + '...'); - console.log('✅ HTML API POST test passed!\n'); - } catch (error) { - console.log('❌ HTML API POST test failed:', error.message); - } -} - -async function testUsage() { - console.log('=== Testing Usage API ==='); - try { - const response = await client.usage(); - console.log('Status:', response.status); - - let data; - if (Buffer.isBuffer(response.data)) { - data = JSON.parse(response.data.toString()); - } else if (typeof response.data === 'string') { - data = JSON.parse(response.data); - } else { - data = response.data; - } - - console.log('Max API credits:', data.max_api_credit); - console.log('Used API credits:', data.used_api_credit); - console.log('Max concurrency:', data.max_concurrency); - console.log('✅ Usage test passed!\n'); - } catch (error) { - console.log('❌ Usage test failed:', error.message); - if (error.response) { - console.log('Response status:', error.response.status); - console.log('Response data:', error.response.data?.toString()); - } - } -} - -async function runTests() { - console.log('\n🚀 Starting ScrapingBee SDK Tests\n'); - - await testHtmlGet(); - await testHtmlPost(); - await testGoogleSearch(); - await testAmazonSearch(); - await testAmazonProduct(); - await testWalmartSearch(); - await testWalmartProduct(); - await testChatGPT(); - await testYouTubeSearch(); - await testYouTubeMetadata(); - await testYouTubeTranscript(); - await testYouTubeTrainability(); - await testHtmlApiGet(); - await testHtmlApiPost(); - await testUsage(); - - console.log('🏁 All tests completed!\n'); -} - -runTests(); \ No newline at end of file From 2e4129100e132df85ace4d17dfe64824f0bbe92b Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Fri, 16 Jan 2026 16:08:49 +0530 Subject: [PATCH 6/7] Minor Changes - Used util.deprecate for legacy get and post methods and added a warning that it will be removed on version 2.0.0 - Updated changelog to reflect the correct date - Removed DELETE, PATCH, HEAD, and OPTIONS from request methods in HtmlApiConfig as they are not supported by the API. --- .DS_Store | Bin 6148 -> 6148 bytes CHANGELOG.md | 2 +- dist/index.d.ts | 12 ++++++-- dist/index.js | 77 ++++++++++++++++++++++++++---------------------- src/index.ts | 19 +++++++----- 5 files changed, 63 insertions(+), 47 deletions(-) diff --git a/.DS_Store b/.DS_Store index 2803a9f70b43c3c6feb4f2c69970dee45f074349..36ce1767e3f1078dd938d62e36da25f8bbde4ae1 100644 GIT binary patch delta 45 zcmZoMXffE3&mzvpkjs$BkjGF8By$*a8A=#Z8HyQ7CYP|OZq8?U$GCYWTecto2gM9A delta 45 ycmZoMXffE3&$4+n%R5GKK86y8REA<8smqYdkjRk7Pzoe-CbzMuZl1}OEeHTEy$pQ- diff --git a/CHANGELOG.md b/CHANGELOG.md index c56fa01..766e7f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [1.8.0](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.7.5...v1.8.0) (2024-12-22) +## [1.8.0](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.7.5...v1.8.0) (2026-01-16) ### Features diff --git a/dist/index.d.ts b/dist/index.d.ts index 2504b39..e2db34b 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -38,7 +38,7 @@ export declare type HtmlApiParams = { }; export interface HtmlApiConfig { url: string; - method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS'; + method?: 'GET' | 'POST' | 'PUT'; headers?: Record; cookies?: string | Record; params?: HtmlApiParams; @@ -216,8 +216,14 @@ export declare class ScrapingBeeClient { readonly api_key: string; constructor(api_key: string); private request; - get(config: HtmlApiConfig): AxiosPromise; - post(config: HtmlApiConfig): AxiosPromise; + /** + * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. + */ + get: (config: HtmlApiConfig) => AxiosPromise; + /** + * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. + */ + post: (config: HtmlApiConfig) => AxiosPromise; googleSearch(config: GoogleSearchConfig): AxiosPromise; amazonSearch(config: AmazonSearchConfig): AxiosPromise; amazonProduct(config: AmazonProductConfig): AxiosPromise; diff --git a/dist/index.js b/dist/index.js index 7b5a565..d5d122e 100644 --- a/dist/index.js +++ b/dist/index.js @@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { Object.defineProperty(exports, "__esModule", { value: true }); exports.ScrapingBeeClient = void 0; const axios_1 = __importDefault(require("axios")); +const util_1 = require("util"); const axios_retry_1 = __importDefault(require("axios-retry")); const utils_1 = require("./utils"); const HTML_API_URL = 'https://app.scrapingbee.com/api/v1/'; @@ -21,6 +22,46 @@ const YOUTUBE_TRAINABILITY_API_URL = 'https://app.scrapingbee.com/api/v1/youtube const USAGE_API_URL = 'https://app.scrapingbee.com/api/v1/usage'; class ScrapingBeeClient { constructor(api_key) { + /** + * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. + */ + this.get = util_1.deprecate((config) => { + var _a; + let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); + let headers = utils_1.process_headers(config.headers); + if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { + params.forward_headers = true; + } + return this.request({ + method: 'GET', + endpoint: HTML_API_URL, + params: utils_1.process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); + }, 'ScrapingBeeClient.get() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); + /** + * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. + */ + this.post = util_1.deprecate((config) => { + var _a; + let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); + let headers = utils_1.process_headers(config.headers); + if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { + params.forward_headers = true; + } + return this.request({ + method: 'POST', + endpoint: HTML_API_URL, + params: utils_1.process_params(params), + headers: headers, + data: config.data, + retries: config.retries, + timeout: config.timeout, + }); + }, 'ScrapingBeeClient.post() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); this.api_key = api_key; } request(config) { @@ -41,42 +82,6 @@ class ScrapingBeeClient { } return axios_1.default(axiosConfig); } - // WILL BE REMOVED IN A FUTURE VERSION - get(config) { - var _a; - let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); - let headers = utils_1.process_headers(config.headers); - if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { - params.forward_headers = true; - } - return this.request({ - method: 'GET', - endpoint: HTML_API_URL, - params: utils_1.process_params(params), - headers: headers, - data: config.data, - retries: config.retries, - timeout: config.timeout, - }); - } - // WILL BE REMOVED IN A FUTURE VERSION - post(config) { - var _a; - let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); - let headers = utils_1.process_headers(config.headers); - if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { - params.forward_headers = true; - } - return this.request({ - method: 'POST', - endpoint: HTML_API_URL, - params: utils_1.process_params(params), - headers: headers, - data: config.data, - retries: config.retries, - timeout: config.timeout, - }); - } googleSearch(config) { const params = Object.assign({ search: config.search }, config.params); return this.request({ diff --git a/src/index.ts b/src/index.ts index 5d9d95b..eb650f7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ import axios, { AxiosPromise, AxiosRequestConfig, Method } from 'axios'; +import { deprecate } from 'util'; import axiosRetry from 'axios-retry'; import { process_params, process_headers } from './utils'; @@ -58,7 +59,7 @@ export type HtmlApiParams = { export interface HtmlApiConfig { url: string; - method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS'; // Some of these methods are not supported by HTML API + method?: 'GET' | 'POST' | 'PUT'; headers?: Record; cookies?: string | Record; params?: HtmlApiParams; @@ -293,8 +294,10 @@ export class ScrapingBeeClient { return axios(axiosConfig); } - // WILL BE REMOVED IN A FUTURE VERSION - public get(config: HtmlApiConfig): AxiosPromise { + /** + * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. + */ + public get = deprecate((config: HtmlApiConfig): AxiosPromise => { let params: Record = { ...config.params, url: config.url, @@ -315,10 +318,12 @@ export class ScrapingBeeClient { retries: config.retries, timeout: config.timeout, }); - } + }, 'ScrapingBeeClient.get() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); - // WILL BE REMOVED IN A FUTURE VERSION - public post(config: HtmlApiConfig): AxiosPromise { + /** + * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. + */ + public post = deprecate((config: HtmlApiConfig): AxiosPromise => { let params: Record = { ...config.params, url: config.url, @@ -339,7 +344,7 @@ export class ScrapingBeeClient { retries: config.retries, timeout: config.timeout, }); - } + }, 'ScrapingBeeClient.post() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); public googleSearch(config: GoogleSearchConfig): AxiosPromise { const params: Record = { From 9d5c18bd3949fe5693b3e32b4277a81db7c602cc Mon Sep 17 00:00:00 2001 From: Sahil Sunny Date: Fri, 16 Jan 2026 16:42:52 +0530 Subject: [PATCH 7/7] Changing date in changelog based on expected Merging date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 766e7f0..4dbf2f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [1.8.0](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.7.5...v1.8.0) (2026-01-16) +## [1.8.0](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.7.5...v1.8.0) (2026-01-19) ### Features