improve proxy & session

Signed-off-by: Zhiyuan Chen <this@zyc.ai>

Author: ZhiyuanChen

icrawler/crawler.py

@@ -5,13 +5,13 @@
 import time
 from importlib import import_module
 
+from . import defaults
 from . import storage as storage_package
 from .downloader import Downloader
 from .feeder import Feeder
 from .parser import Parser
 from .storage import BaseStorage
 from .utils import ProxyPool, Session, Signal
-from . import defaults
 
 
 class Crawler:

icrawler/defaults.py

@@ -1,5 +1,10 @@
+MAX_RETRIES = 3
+BACKOFF_BASE = 1.2
+
 ACCEPT_LANGUAGES = "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"
-USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"
+)
 
 DEFAULT_HEADERS = {
     "Accept-Language": ACCEPT_LANGUAGES,

icrawler/utils/proxy_pool.py

@@ -1,10 +1,10 @@
-import json
 import logging
 import queue
 import random
 import threading
 import time
 
+import chanfig
 import requests
 from bs4 import BeautifulSoup
 
@@ -44,7 +44,12 @@ def to_dict(self):
             dict: A dict with four keys: ``addr``, ``protocol``,
                   ``weight`` and ``last_checked``
         """
-        return dict(addr=self.addr, protocol=self.protocol, weight=self.weight, last_checked=self.last_checked)
+        return {
+            "addr": self.addr,
+            "protocol": self.protocol,
+            "weight": self.weight,
+            "last_checked": self.last_checked,
+        }
 
 
 class ProxyPool:
@@ -146,17 +151,15 @@ def save(self, filename):
             for proxy in self.proxies[protocol]:
                 serializable_proxy = self.proxies[protocol][proxy].to_dict()
                 proxies[protocol].append(serializable_proxy)
-        with open(filename, "w") as fout:
-            json.dump(proxies, fout)
+        chanfig.save(proxies, filename)
 
     def load(self, filename):
         """Load proxies from file"""
-        with open(filename) as fin:
-            proxies = json.load(fin)
-        for protocol in proxies:
-            for proxy in proxies[protocol]:
+        proxies = chanfig.load(filename)
+        for protocol, protocol_proxies in proxies.items():
+            for proxy in protocol_proxies:
                 self.proxies[protocol][proxy["addr"]] = Proxy(
-                    proxy["addr"], proxy["protocol"], proxy["weight"], proxy["last_checked"]
+                    proxy["addr"], protocol, proxy.get("weight", 1.0), proxy.get("last_checked")
                 )
                 self.addr_list[protocol].append(proxy["addr"])
 
@@ -215,7 +218,7 @@ def is_valid(self, addr, protocol="http", timeout=5):
             raise
         except requests.exceptions.Timeout:
             return {"valid": False, "msg": "timeout"}
-        except:
+        except BaseException:  # noqa: B036
             return {"valid": False, "msg": "exception"}
         else:
             if r.status_code == 200:
@@ -278,20 +281,20 @@ def scan(
                 t = threading.Thread(
                     name=f"val-{i + 1:0>2d}",
                     target=self.validate,
-                    kwargs=dict(
-                        proxy_scanner=proxy_scanner,
-                        expected_num=expected_num,
-                        queue_timeout=queue_timeout,
-                        val_timeout=val_timeout,
-                    ),
+                    kwargs={
+                        "proxy_scanner": proxy_scanner,
+                        "expected_num": expected_num,
+                        "queue_timeout": queue_timeout,
+                        "val_timeout": val_timeout,
+                    },
                 )
                 t.daemon = True
                 val_threads.append(t)
                 t.start()
             for t in val_threads:
                 t.join()
             self.logger.info("Proxy scanning done!")
-        except:
+        except BaseException:
             raise
         finally:
             if out_file is not None:
@@ -466,18 +469,14 @@ def scan_free_proxy_list(self):
     def scan_file(self, src_file):
         """Scan candidate proxies from an existing file"""
         self.logger.info(f"start scanning file {src_file} for proxy list...")
-        with open(src_file) as fin:
-            proxies = json.load(fin)
+        proxies = chanfig.load(src_file)
         for protocol in proxies.keys():
             for proxy in proxies[protocol]:
                 self.proxy_queue.put({"addr": proxy["addr"], "protocol": protocol})
 
     def is_scanning(self):
         """Return whether at least one scanning thread is alive"""
-        for t in self.scan_threads:
-            if t.is_alive():
-                return True
-        return False
+        return any(t.is_alive() for t in self.scan_threads)
 
     def scan(self):
         """Start a thread for each registered scan function to scan proxy lists"""

icrawler/utils/session.py

@@ -1,42 +1,59 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
 from urllib.parse import urlsplit
 
 import requests
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential
+
+from .. import defaults
+from .proxy_pool import ProxyPool
 
 
 class Session(requests.Session):
-    def __init__(self, proxy_pool):
+    def __init__(
+        self, proxy_pool: ProxyPool | None = None, headers: Mapping | None = None, cookies: Mapping | None = None
+    ):
         super().__init__()
+        self.logger = logging.getLogger("cscholars.connection")
         self.proxy_pool = proxy_pool
+        if headers is not None:
+            self.headers.update(headers)
+        if cookies is not None:
+            self.cookies.update(cookies)
 
     def _url_scheme(self, url):
         return urlsplit(url).scheme
 
-    def get(self, url, **kwargs):
-        proxy = self.proxy_pool.get_next(protocol=self._url_scheme(url))
-        if proxy is None:
-            return super().get(url, **kwargs)
-        try:
-            response = super().get(url, proxies=proxy.format(), **kwargs)
-        except requests.exceptions.ConnectionError:
-            self.proxy_pool.decrease_weight(proxy)
-            raise
-        except:
-            raise
-        else:
-            self.proxy_pool.increase_weight(proxy)
-            return response
-
-    def post(self, url, data=None, json=None, **kwargs):
-        proxy = self.proxy_pool.get_next(protocol=self._url_scheme(url))
-        if proxy is None:
-            return super().get(url, data, json, **kwargs)
-        try:
-            response = super().post(url, data, json, proxies=proxy.format(), **kwargs)
-        except requests.exceptions.ConnectionError:
-            self.proxy_pool.decrease_weight(proxy)
-            raise
-        except:
-            raise
+    @retry(
+        stop=stop_after_attempt(defaults.MAX_RETRIES),
+        wait=wait_random_exponential(exp_base=defaults.BACKOFF_BASE),
+        retry=retry_if_exception_type((requests.RequestException, requests.HTTPError, requests.ConnectionError)),
+    )
+    def request(self, method, url, *args, **kwargs):
+        message = f"{method}ing {url}"
+        if args and kwargs:
+            message += f" with {args} and {kwargs}"
+        elif args:
+            message += f" with {args}"
+        elif kwargs:
+            message += f" with {kwargs}"
+        self.logger.debug(message)
+
+        if self.proxy_pool is not None:
+            proxy = self.proxy_pool.get_next(protocol=self._url_scheme(url))
+            self.logger.debug(f"Using proxy: {proxy.format()}")
+            try:
+                response = super().request(method, url, *args, proxies=proxy.format(), **kwargs)
+                response.raise_for_status()
+                self.proxy_pool.increase_weight(proxy)
+            except (requests.ConnectionError, requests.HTTPError):
+                self.proxy_pool.decrease_weight(proxy)
+                raise
         else:
-            self.proxy_pool.increase_weight(proxy)
-            return response
+            response = super().request(method, url, *args, **kwargs)
+
+        if "set-cookie" in response.headers:
+            self.cookies.update(response.cookies)
+        return response

pyproject.toml

@@ -46,11 +46,12 @@ dynamic = [
 dependencies = [
   "beautifulsoup4",
   "bs4",
+  "chanfig",
   "lxml",
   "pillow",
-  "pyyaml",
   "requests",
   "six",
+  "tenacity",
 ]
 urls.documentation = "https://icrawler.readthedocs.io/"
 urls.homepage = "https://icrawler.readthedocs.io/"