Merge pull request #3 from drunkpig/dev

feat: init project

Author: drunkpig

.gitignore

@@ -192,3 +192,18 @@ cython_debug/
 #  refer to https://docs.cursor.com/context/ignore-files
 .cursorignore
 .cursorindexingignore
+
+# sphinx docs
+_build/
+
+
+output/
+**/temp.py
+
+# coverage file
+.coverage*
+coverage.xml
+
+
+.env
+*.egg-info/

.pre-commit-config.yaml

@@ -40,11 +40,11 @@ repos:
           - mdformat_frontmatter
           - linkify-it-py
         exclude: '^tests/.*/assets/'
-  - repo: https://github.com/myint/docformatter
-    rev: v1.3.1
-    hooks:
-      - id: docformatter
-        args: [ "--in-place", "--wrap-descriptions", "119" ]
+#  - repo: https://github.com/myint/docformatter
+#    rev: v1.3.1
+#    hooks:
+#      - id: docformatter
+#        args: [ "--in-place", "--wrap-descriptions", "119" ]
   - repo: local
     hooks:
       - id: clear-jupyter-notebook-output

docs/.gitkeep

@@ -0,0 +1,62 @@
+"""统一的配置读取函数
+配置文件位于
+1. 环境变量 REAL_CRAWL_CONFIG_PATH
+2. ~/.realcrawl/.realcrawl.jsonc
+"""
+
+import os
+
+import commentjson as json
+from loguru import logger
+
+from realcrawl.exception.base import ConfigFileNotFoundException
+
+
+def load_config(suppress_error: bool = False) -> dict:
+    """Load the configuration file for the web kit. First try to read the
+    configuration file from the environment variable REAL_CRAWL_CONFIG_PATH. If
+    the environment variable is not set, use the default configuration file
+    path ~/.realcrawl/.realcrawl.jsonc. If the configuration file does not exist, raise
+    an exception.
+
+    Raises:
+        ConfigFileNotFoundException: REAL_CRAWL_CONFIG_PATH points to a non-exist file
+        ConfigFileNotFoundException: cfg_path does not exist
+
+    Returns:
+        config(dict): The configuration dictionary
+    """
+    # 首先从环境变量LLM_WEB_KIT_CFG_PATH 读取配置文件的位置
+    # 如果没有配置，就使用默认的配置文件位置
+    # 如果配置文件不存在，就抛出异常
+    env_cfg_path = os.getenv('REAL_CRAWL_CONFIG_PATH')
+    if env_cfg_path:
+        cfg_path = env_cfg_path
+        if not os.path.exists(cfg_path):
+            if suppress_error:
+                return {}
+
+            logger.warning(
+                f'environment variable REAL_CRAWL_CONFIG_PATH points to a non-exist file: {cfg_path}'
+            )
+            raise ConfigFileNotFoundException(
+                f'environment variable REAL_CRAWL_CONFIG_PATH points to a non-exist file: {cfg_path}'
+            )
+    else:
+        cfg_path = os.path.expanduser('~/.realcrawl/.realcrawl.jsonc')
+        if not os.path.exists(cfg_path):
+            if suppress_error:
+                return {}
+
+            logger.warning(
+                f'{cfg_path} does not exist, please create one or set environment variable REAL_CRAWL_CONFIG_PATH to a valid file path'
+            )
+            raise ConfigFileNotFoundException(
+                f'{cfg_path} does not exist, please create one or set environment variable REAL_CRAWL_CONFIG_PATH to a valid file path'
+            )
+
+    # 读取配置文件
+    with open(cfg_path, 'r', encoding='utf-8') as f:
+        config = json.load(f)
+
+    return config

realcrawl/__init__.py

@@ -0,0 +1,86 @@
+import inspect
+from pathlib import Path
+
+import commentjson as json
+
+
+class ErrorMsg:
+    """Error message manager class."""
+    _errors = {}
+
+    @classmethod
+    def _load_errors(cls):
+        """Load error codes and messages from JSON file."""
+        exception_defs_file_path = Path(__file__).parent / 'exception.jsonc'
+        with open(exception_defs_file_path, 'r', encoding='utf-8') as file:
+            jso = json.load(file)
+            for module, module_defs in jso.items():
+                for err_name, err_info in module_defs.items():
+                    err_code = err_info['code']
+                    cls._errors[str(err_code)] = {
+                        'message': err_info['message'],
+                        'module': module,
+                        'error_name': err_name,
+                    }
+
+    @classmethod
+    def get_error_message(cls, error_code: int):
+        # 根据错误代码获取错误消息
+        if str(error_code) not in cls._errors:
+            return f'unknown error code {error_code}'
+        return cls._errors[str(error_code)]['message']
+
+    @classmethod
+    def get_error_code(cls, module: str, error_name: str) -> int:
+        """根据模块名和错误名获取错误代码."""
+        for code, info in cls._errors.items():
+            if info['module'] == module and info['error_name'] == error_name:
+                return int(code)
+        raise ValueError(f'error code not found: module={module}, error_name={error_name}')
+
+
+ErrorMsg._load_errors()
+
+
+class RealCrawlBaseException(Exception):
+    """Base exception class for realcrawl."""
+
+    def __init__(self, custom_message: str | None = None, error_code: int | None = None):
+        if error_code is None:
+            error_code = ErrorMsg.get_error_code('realcrawlBase', 'realcrawlBaseException')
+
+        self.error_code = error_code
+        self.message = ErrorMsg.get_error_message(self.error_code)
+        self.custom_message = custom_message
+        self.dataset_name = ''
+        super().__init__(self.message)
+        frame = inspect.currentframe().f_back
+        self.__py_filename = frame.f_code.co_filename
+        self.__py_file_line_number = frame.f_lineno
+
+    def __str__(self):
+        return (
+            f'{self.__py_filename}: {self.__py_file_line_number}#{self.error_code}#{self.message}#{self.custom_message}'
+        )
+
+
+##############################################################################
+#
+#  Config Exceptions
+#
+##############################################################################
+
+class ConfigBaseException(RealCrawlBaseException):
+    """Base exception class for Config."""
+    def __init__(self, custom_message: str | None = None, error_code: int | None = None):
+        if error_code is None:
+            error_code = ErrorMsg.get_error_code('Config', 'ConfigBaseException')
+        super().__init__(custom_message, error_code)
+
+
+class ConfigFileNotFoundException(ConfigBaseException):
+    """Config file not found exception."""
+    def __init__(self, custom_message: str | None = None, error_code: int | None = None):
+        if error_code is None:
+            error_code = ErrorMsg.get_error_code('Config', 'ConfigFileNotFoundException')
+        super().__init__(custom_message, error_code)

realcrawl/cfg.py

@@ -0,0 +1,21 @@
+{
+    // Base基础异常 (10000000)
+    "realcrawlBase": {
+      "realcrawlBaseException": {
+        "code": 10000000,
+        "message": "realcrawl base exception"
+      }
+    },
+
+    // 配置相关异常 (20000000)
+    "Config": {
+      "ConfigBaseException": {
+        "code": 20000000,
+        "message": "Config base exception"
+      },
+      "ConfigFileNotFoundException": {
+        "code": 21000000,
+        "message": "Config file not found exception"
+      }
+    }
+  }