add defaults for default headers, etc.

Signed-off-by: Zhiyuan Chen <this@zyc.ai>

Author: ZhiyuanChen

icrawler/crawler.py

@@ -11,6 +11,7 @@
 from .parser import Parser
 from .storage import BaseStorage
 from .utils import ProxyPool, Session, Signal
+from . import defaults
 
 
 class Crawler:
@@ -133,14 +134,7 @@ def set_session(self, headers=None):
                      header to init the session)
         """
         if headers is None:
-            headers = {
-                "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
-                "User-Agent": (
-                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
-                    " AppleWebKit/537.36 (KHTML, like Gecko) "
-                    "Chrome/88.0.4324.104 Safari/537.36"
-                ),
-            }
+            headers = defaults.DEFAULT_HEADERS
         elif not isinstance(headers, dict):
             raise TypeError('"headers" must be a dict object')
 

icrawler/defaults.py

@@ -0,0 +1,7 @@
+ACCEPT_LANGUAGES = "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"
+
+DEFAULT_HEADERS = {
+    "Accept-Language": ACCEPT_LANGUAGES,
+    "User-Agent": USER_AGENT,
+}