Implement retry (#13)

* Implement retry

* Update supported python versions

* Drop support for Python3.6

* Lint max col

* Linter complient

* Update version and README

Author: kirby81

.github/workflows/main.yaml

@@ -6,7 +6,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Install dependencies
       run: |
         make deps
@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

README.md

@@ -122,3 +122,22 @@ Here a little exemple on how to retrieve and store a screenshot from the Scrapin
 ## Using ScrapingBee with Scrapy
 
 Scrapy is the most popular Python web scraping framework. You can easily [integrate ScrapingBee's API with the Scrapy middleware](https://github.com/ScrapingBee/scrapy-scrapingbee).
+
+
+## Retries
+
+The client includes a retry mechanism for 5XX responses.
+
+```python
+>>> from scrapingbee import ScrapingBeeClient
+
+>>> client = ScrapingBeeClient(api_key='REPLACE-WITH-YOUR-API-KEY')
+
+>>> response = client.get(
+    'https://www.scrapingbee.com/blog/', 
+    params={
+        'render_js': True,
+    },
+    retries=5
+)
+```

requirements.txt

@@ -1,4 +1,24 @@
+attrs==22.2.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+distlib==0.3.6
+filelock==3.10.0
 flake8==3.9.2
-pytest==5.4.2
+idna==3.4
+iniconfig==2.0.0
+mccabe==0.6.1
+more-itertools==9.1.0
+packaging==23.0
+platformdirs==3.1.1
+pluggy==0.13.1
+py==1.11.0
+pycodestyle==2.7.0
+pyflakes==2.3.1
+pytest==7.2.2
+requests==2.28.2
+six==1.16.0
+toml==0.10.2
 tox==3.23.1
-wheel==0.36.2
+urllib3==1.26.15
+virtualenv==20.21.0
+wcwidth==0.2.6

scrapingbee/__version__.py

@@ -1 +1 @@
-__version__ = '1.1.8'
+__version__ = '1.2.0'

scrapingbee/client.py

@@ -1,6 +1,9 @@
-from requests import request, Response
+from typing import Optional
+
+from requests import Response, Session
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
-from .default_headers import default_headers
 from .utils import get_scrapingbee_url, process_headers
 
 
@@ -14,23 +17,21 @@ def request(
         self,
         method: str,
         url: str,
-        params: dict = None,
-        data: dict = None,
-        json: dict = None,
-        headers: dict = None,
-        cookies: dict = None,
+        params: Optional[dict] = None,
+        data: Optional[dict] = None,
+        json: Optional[dict] = None,
+        headers: Optional[dict] = None,
+        cookies: Optional[dict] = None,
+        retries: Optional[int] = None,
         **kwargs
     ) -> Response:
         if not params:
             params = {}
 
         # Process headers and set forward_headers
         if headers:
-            headers = process_headers(headers)
             params["forward_headers"] = True
-        else:
-            headers = {}
-        headers.update(default_headers)
+        headers = process_headers(headers)
 
         # Add cookies to params
         if cookies:
@@ -40,12 +41,27 @@ def request(
         # Get ScrapingBee API URL
         spb_url = get_scrapingbee_url(self.api_url, self.api_key, url, params)
 
+        session = Session()
+        if retries:
+            # Retries if it is a network error or a 5xx error on an idempotent request (GET)
+            retries = Retry(total=retries, raise_on_status=False, status_forcelist=frozenset(range(500, 600)))
+            session.mount('https://', HTTPAdapter(max_retries=retries))
+            session.mount('http://', HTTPAdapter(max_retries=retries))
+
         if not data and json is not None:
-            return request(method, spb_url, json=json, headers=headers, **kwargs)
-        return request(method, spb_url, data=data, headers=headers, **kwargs)
+            return session.request(method, spb_url, json=json, headers=headers, **kwargs)
+        return session.request(method, spb_url, data=data, headers=headers, **kwargs)
 
-    def get(self, url: str, params: dict = None, headers: dict = None, cookies: dict = None, **kwargs) -> Response:
-        return self.request("GET", url, params=params, headers=headers, cookies=cookies, **kwargs)
+    def get(
+        self,
+        url: str,
+        params: dict = None,
+        headers: dict = None,
+        cookies: dict = None,
+        retries: Optional[int] = None,
+        **kwargs
+    ) -> Response:
+        return self.request("GET", url, params=params, headers=headers, cookies=cookies, retries=retries, **kwargs)
 
     def post(
         self,

scrapingbee/default_headers.py

@@ -1,6 +1,11 @@
 import base64
 import json
 import urllib
+from typing import Optional
+
+from .__version__ import __version__
+
+DEFAULT_HEADERS = {"User-Agent": f"ScrapingBee-Python/{__version__}"}
 
 
 def process_url(url: str) -> str:
@@ -11,8 +16,10 @@ def process_js_snippet(js_snippet: str) -> str:
     return base64.b64encode(js_snippet.encode()).decode()
 
 
-def process_headers(headers: dict, prefix: str = 'Spb-') -> dict:
-    return {f'{prefix}{k}': v for k, v in headers.items()}
+def process_headers(headers: Optional[dict], prefix: str = 'Spb-') -> dict:
+    headers = headers or {}
+    headers = {f'{prefix}{k}': v for k, v in headers.items()}
+    return {**DEFAULT_HEADERS, **headers}
 
 
 def process_cookies(cookies: dict) -> str:

scrapingbee/utils.py

@@ -24,11 +24,13 @@
         'License :: OSI Approved :: MIT License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     install_requires=['requests'],
 )

setup.py

@@ -3,76 +3,76 @@
 import pytest
 
 from scrapingbee import ScrapingBeeClient
-from scrapingbee.default_headers import default_headers
+from scrapingbee.utils import DEFAULT_HEADERS
 
 
 @pytest.fixture(scope='module')
 def client():
     return ScrapingBeeClient(api_key='API_KEY')
 
 
-@mock.patch('scrapingbee.client.request')
-def test_get(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_get(mock_session, client):
     '''It should make a GET request with the url and API key'''
     client.get('https://httpbin.org')
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'GET',
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org',
         data=None,
-        headers=default_headers
+        headers=DEFAULT_HEADERS
     )
 
 
-@mock.patch('scrapingbee.client.request')
-def test_get_with_params(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_get_with_params(mock_session, client):
     '''It should add parameters to the url'''
     client.get('https://httpbin.org', params={'render_js': True})
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'GET',
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&render_js=True',
         data=None,
-        headers=default_headers,
+        headers=DEFAULT_HEADERS,
     )
 
 
-@mock.patch('scrapingbee.client.request')
-def test_get_with_headers(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_get_with_headers(mock_session, client):
     '''It should prefix header names with Spb- and set forward_headers'''
     client.get('https://httpbin.org', headers={'Content-Type': 'text/html; charset=utf-8'})
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'GET',
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&forward_headers=True',
         data=None,
         headers={'Spb-Content-Type': 'text/html; charset=utf-8',
-                 **default_headers},
+                 **DEFAULT_HEADERS},
     )
 
 
-@mock.patch('scrapingbee.client.request')
-def test_get_with_cookies(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_get_with_cookies(mock_session, client):
     '''It should format the cookies and add them to the url'''
     client.get('https://httpbin.org', cookies={
         'name_1': 'value_1',
         'name_2': 'value_2',
     })
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'GET',
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&cookies=name_1=value_1;name_2=value_2',
         data=None,
-        headers=default_headers,
+        headers=DEFAULT_HEADERS,
     )
 
 
-@mock.patch('scrapingbee.client.request')
-def test_get_with_extract_rules(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_get_with_extract_rules(mock_session, client):
     '''It should format the extract_rules and add them to the url'''
     client.get('https://httpbin.org', params={
         'extract_rules': {
@@ -81,19 +81,19 @@ def test_get_with_extract_rules(mock_request, client):
         }
     })
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'GET',
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&'
         'extract_rules=%7B%22title%22%3A%20%22h1%22%2C%20%22'
         'subtitle%22%3A%20%22%23subtitle%22%7D',
         data=None,
-        headers=default_headers,
+        headers=DEFAULT_HEADERS,
     )
 
 
-@mock.patch('scrapingbee.client.request')
-def test_get_with_js_scenario(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_get_with_js_scenario(mock_session, client):
     '''It should format the extract_rules and add them to the url'''
     client.get('https://httpbin.org', params={
         'js_scenario': {
@@ -103,24 +103,24 @@ def test_get_with_js_scenario(mock_request, client):
         }
     })
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'GET',
         'https://app.scrapingbee.com/api/v1/'
         '?api_key=API_KEY&url=https%3A//httpbin.org&'
         'js_scenario=%7B%22instructions%22%3A%20%5B%7B%22click%22%3A%20%22%23buttonId%22%7D%5D%7D',
         data=None,
-        headers=default_headers,
+        headers=DEFAULT_HEADERS,
     )
 
 
-@mock.patch('scrapingbee.client.request')
-def test_post(mock_request, client):
+@mock.patch('scrapingbee.client.Session')
+def test_post(mock_session, client):
     '''It should make a POST request with some data'''
     client.post('https://httpbin.org', data={'KEY_1': 'VALUE_1'})
 
-    mock_request.assert_called_with(
+    mock_session.return_value.request.assert_called_with(
         'POST',
         'https://app.scrapingbee.com/api/v1/?api_key=API_KEY&url=https%3A//httpbin.org',
         data={'KEY_1': 'VALUE_1'},
-        headers=default_headers
+        headers=DEFAULT_HEADERS
     )

tests/test_client.py

@@ -26,14 +26,17 @@ def test_process_js_snippet():
 def test_process_headers():
     '''It should add a Spb- prefix to header names'''
     output = process_headers({'Accept-Language': 'En-US'})
-    assert output == {'Spb-Accept-Language': 'En-US'}
+    assert output == {
+        'User-Agent': 'ScrapingBee-Python/1.2.0',
+        'Spb-Accept-Language': 'En-US',
+    }
 
 
 def test_process_cookies():
     '''It should format cookies to a string'''
     output = process_cookies({
         'name_1': 'value_1',
-        'name_2': 'value_2'
+        'name_2': 'value_2',
     })
     assert output == 'name_1=value_1;name_2=value_2'