Merge pull request #6 from ScrapingBee/update-sdk-retry

Add retry option and param interfaces

Author: daolf

README.md

@@ -125,3 +125,22 @@ screenshot('https://httpbin-scrapingbee.cleverapps.io/html', './httpbin.png').ca
     console.log('A problem occurs : ' + e.message)
 );
 ```
+
+## Retries
+
+The client includes a retry mechanism for 5XX responses.
+
+```javascript
+const spb = require('scrapingbee');
+
+async function get(url) {
+    let client = new spb.ScrapingBeeClient('REPLACE-WITH-YOUR-API-KEY');
+    let resp = await client.get({ url: url, params: { render_js: false }, retries: 5 });
+
+    let decoder = new TextDecoder();
+    let text = decoder.decode(resp.data);
+    console.log(text);
+}
+
+get('https://httpbin-scrapingbee.cleverapps.io/html').catch((e) => console.log('A problem occured: ' + e.message));
+```

dist/index.d.ts

@@ -1,8 +1,46 @@
 import { AxiosPromise } from 'axios';
+export declare type SpbParams = {
+    block_ads?: boolean;
+    block_resources?: boolean;
+    cookies?: string | Record<string, any>;
+    country_code?: string;
+    custom_google?: boolean;
+    device?: string;
+    extract_rules?: object | string;
+    forward_headers?: boolean;
+    forward_headers_pure?: boolean;
+    js_scenario?: object | string;
+    json_response?: boolean;
+    own_proxy?: string;
+    premium_proxy?: boolean;
+    render_js?: boolean;
+    return_page_source?: boolean;
+    screenshot?: boolean;
+    screenshot_full_page?: boolean;
+    screenshot_selector?: string;
+    session_id?: number;
+    stealth_proxy?: boolean;
+    timeout?: number;
+    transparent_status_code?: boolean;
+    wait?: number;
+    wait_for?: string;
+    window_height?: number;
+    window_width?: number;
+} & {
+    [key: string]: any;
+};
+export interface SpbConfig {
+    url: string;
+    headers?: Record<string, any>;
+    cookies?: string | Record<string, any>;
+    params?: SpbParams;
+    data?: any;
+    retries?: number;
+}
 export declare class ScrapingBeeClient {
     readonly api_key: string;
     constructor(api_key: string);
     private request;
-    get(config?: Record<string, any>): AxiosPromise<any>;
-    post(config?: Record<string, any>): AxiosPromise<any>;
+    get(config: SpbConfig): AxiosPromise<any>;
+    post(config: SpbConfig): AxiosPromise<any>;
 }

dist/index.js

@@ -5,37 +5,38 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ScrapingBeeClient = void 0;
 const axios_1 = __importDefault(require("axios"));
-const version_1 = require("./version");
+const axios_retry_1 = __importDefault(require("axios-retry"));
 const utils_1 = require("./utils");
 const API_URL = 'https://app.scrapingbee.com/api/v1/';
 class ScrapingBeeClient {
     constructor(api_key) {
         this.api_key = api_key;
     }
-    request(method, config = {}) {
+    request(method, config) {
+        var _a;
         let params = config.params || {};
-        let raw_headers = config.headers || {};
-        let headers = utils_1.process_headers(raw_headers);
-        if (headers != {}) {
-            params['forward_headers'] = true;
+        // Headers
+        let headers = utils_1.process_headers(config.headers);
+        if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) {
+            params.forward_headers = true;
         }
-        headers["User-Agent"] = `ScrapingBee-Node/${version_1.LIB_VERSION}`;
-        let cookies = config.cookies || null;
-        if (cookies != null) {
-            params['cookies'] = cookies;
-        }
-        let url = config.url;
-        let data = config.data || {};
+        // Cookies
+        params.cookies = config.cookies;
+        // Other query params
         params['api_key'] = this.api_key;
-        params['url'] = url;
+        params['url'] = config.url;
         params = utils_1.process_params(params);
         let axios_params = {
             method: method,
             headers: headers,
             params: params,
-            data: data,
+            data: config.data,
             responseType: 'arraybuffer',
         };
+        // Retry policy
+        if (config.retries !== undefined) {
+            axios_retry_1.default(axios_1.default, { retries: config.retries });
+        }
         return axios_1.default(API_URL, axios_params);
     }
     get(config) {

dist/utils.d.ts

@@ -1,2 +1,3 @@
+export declare function is_empty(value: any): boolean;
 export declare function process_params(params: Record<string, any>): Record<string, any>;
-export declare function process_headers(headers: Record<string, any>, prefix?: string): Record<string, any>;
+export declare function process_headers(headers?: Record<string, any>, prefix?: string): Record<string, any>;

dist/utils.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.process_headers = exports.process_params = void 0;
+exports.process_headers = exports.process_params = exports.is_empty = void 0;
 const version_1 = require("./version");
-const DEFAULT_HEADERS = { 'User-Agent': 'ScrapingBee-Node/' + version_1.LIB_VERSION };
+const DEFAULT_HEADERS = { 'User-Agent': `ScrapingBee-Node/${version_1.LIB_VERSION}` };
 function process_js_snippet(js_snippet) {
     return Buffer.from(js_snippet).toString('base64');
 }
@@ -21,15 +21,15 @@ function process_json_stringify_param(param) {
     return JSON.stringify(param);
 }
 function is_empty(value) {
-    switch (typeof value) {
-        case 'string':
-            return value === '';
-        case 'object':
-            return value && Object.keys(value).length === 0 && value.constructor === Object;
-        default:
-            return false;
+    if (typeof value === 'number') {
+        return false;
     }
+    if (typeof value === 'object' && value !== null) {
+        return Object.keys(value).length === 0;
+    }
+    return !value;
 }
+exports.is_empty = is_empty;
 function process_params(params) {
     var clean_params = {};
     for (let key in params) {
@@ -45,7 +45,8 @@ function process_params(params) {
                 break;
             case 'extract_rules':
             case 'js_scenario':
-                clean_params[key] = process_json_stringify_param(params[key]);
+                clean_params[key] =
+                    typeof params[key] === 'string' ? params[key] : process_json_stringify_param(params[key]);
                 break;
             default:
                 clean_params[key] = params[key];
@@ -54,7 +55,7 @@ function process_params(params) {
     return clean_params;
 }
 exports.process_params = process_params;
-function process_headers(headers, prefix = 'Spb-') {
+function process_headers(headers = {}, prefix = 'Spb-') {
     var new_headers = {};
     for (let key in headers) {
         new_headers[`${prefix}${key}`] = headers[key];

dist/version.d.ts

@@ -1 +1 @@
-export declare const LIB_VERSION = "1.6.1";
+export declare const LIB_VERSION = "1.7.0";

dist/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LIB_VERSION = void 0;
-exports.LIB_VERSION = "1.6.1";
+exports.LIB_VERSION = "1.7.0";