重构数据格式从brotli格式改为parquet

.github/workflows/build-wheels.yml

@@ -36,7 +36,6 @@ jobs:
     - name: Install system dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y libhdf5-dev
 
         # 从源码编译安装ta-lib
         wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz

.github/workflows/publish.yml

@@ -31,7 +31,6 @@ jobs:
       - name: Install system dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y libhdf5-dev
 
           # 从源码编译安装ta-lib
           wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz

.github/workflows/test.yml

@@ -43,7 +43,6 @@ jobs:
       - name: Install system dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y libhdf5-dev
 
           # 从源码编译安装ta-lib
           wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
@@ -120,7 +119,6 @@ jobs:
       - name: Install system dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y libhdf5-dev
 
           # 从源码编译安装ta-lib
           wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
@@ -171,7 +169,6 @@ jobs:
       - name: Install system dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y libhdf5-dev
 
           # 从源码编译安装ta-lib
           wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz

.gitignore

@@ -68,10 +68,9 @@ logs/
 *.png
 !docs/sponsor/*.png
 *.log
+*.gz
 CLAUDE.md
 GEMINI.md
 .pre-commit-config.yaml
-data/*.h5
-data/.keys_cache
 strategies/*/optimization/results
-
+data*/

README.md

@@ -5,7 +5,7 @@
 [![Python](https://img.shields.io/badge/Python-3.9+-blue.svg)](https://www.python.org/)
 [![License](https://img.shields.io/badge/License-AGPL--3.0-blue.svg)](LICENSE)
 [![License: Commercial](https://img.shields.io/badge/License-Commercial--Available-red)](licenses/LICENSE-COMMERCIAL.md)
-[![Version](https://img.shields.io/badge/Version-2.0.0-orange.svg)](#)
+[![Version](https://img.shields.io/badge/Version-2.1.0-orange.svg)](#)
 [![PyPI](https://img.shields.io/pypi/v/simtradelab.svg)](https://pypi.org/project/simtradelab/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/simtradelab.svg)](https://pypi.org/project/simtradelab/)
 
@@ -27,7 +27,7 @@ SimTradeLab（深测Lab） 是一个由社区独立开发的开源策略回测
 - 📊 **完整统计报告** - 收益、风险、交易明细、持仓批次、FIFO分红税
 - 🔌 **模块化设计** - 清晰的代码结构，易于扩展和定制
 
-**当前版本：** v2.0.0 | **开发状态：** Beta - 核心功能完善，正在策略实战中持续优化
+**当前版本：** v2.1.0 | **开发状态：** Beta - 核心功能完善，正在策略实战中持续优化
 
 ---
 
@@ -49,18 +49,19 @@ pip install simtradelab[optimizer]
 ```
 
 **系统依赖：**
-- macOS: `brew install hdf5 ta-lib`
-- Linux: `sudo apt-get install libhdf5-dev` + [ta-lib源码编译](docs/INSTALLATION.md)
+- macOS: `brew install ta-lib`
+- Linux: [ta-lib源码编译](docs/INSTALLATION.md)
 
 > 详细安装指南：[docs/INSTALLATION.md](docs/INSTALLATION.md)
 
 ### 📁 准备数据
 
-将PTrade数据文件放到 `data/` 目录：
+将数据文件放到 `data/` 目录：
 ```
 data/
-├── ptrade_data.h5           # 股票价格、除权数据
-└── ptrade_fundamentals.h5   # 基本面数据
+├── price/               # 股票价格数据
+├── fundamentals/        # 基本面数据
+└── exrights/            # 除权除息数据
 ```
 
 **数据获取：** 推荐使用 [SimTradeData](https://github.com/kay-ou/SimTradeData) 项目获取A股历史数据

pyproject.toml

@@ -1,7 +1,7 @@
 # Poetry configuration
 [tool.poetry]
 name = "simtradelab"
-version = "2.0.0"
+version = "2.1.0"
 description = "开源策略回测框架，灵感来自PTrade的事件驱动模型，提供轻量、清晰、可插拔的策略验证环境"
 authors = ["kay <kayou@duck.com>"]
 license = "AGPL-3.0-or-later"
@@ -30,27 +30,14 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-# 核心数据处理
-pandas = "*" # 0.23.4
-numpy = "<2.0" # 1.11.2
-h5py = ">=3.8,<4.0"
-tables = "<3.10"
-
-# 性能优化
-cachetools = "^5.3.0"  # LRU缓存
-joblib = "^1.3.0"  # 并行处理
-
-# 图表与界面
-matplotlib = "^3.7.0"  # 图表绘制
-tqdm = "^4.67.1"  # 进度条
-
-# 配置验证
+pandas = "*"
+numpy = "<2.0"
+cachetools = "^5.3.0"
+joblib = "^1.3.0"
+matplotlib = "^3.7.0"
+tqdm = "^4.67.1"
 pydantic = "^2.12.5"
-
-# 技术指标计算
-ta-lib = "*" #0.4.10
-
-# 优化器（可选）
+ta-lib = "*"
 optuna = {version = "^3.0.0", optional = true}
 
 [tool.poetry.extras]

src/simtradelab/backtest/runner.py

@@ -47,10 +47,10 @@ def __init__(self):
         self.stock_metadata = None
         self.index_constituents = {}
         self.stock_status_history = {}
-        self.stock_data_store = None
-        self.fundamentals_store = None
         self.adj_pre_cache = None
+        self.adj_post_cache = None
         self.dividend_cache = None
+        self.trade_days = None
 
         # 注册信号处理（仅在主线程）
         try:
@@ -193,10 +193,10 @@ def _load_data(self, required_data=None) -> pd.DataFrame:
         self.stock_metadata = data_server.stock_metadata
         self.index_constituents = data_server.index_constituents
         self.stock_status_history = data_server.stock_status_history
-        self.stock_data_store = data_server.stock_data_store
-        self.fundamentals_store = data_server.fundamentals_store
         self.adj_pre_cache = data_server.adj_pre_cache
+        self.adj_post_cache = data_server.adj_post_cache
         self.dividend_cache = data_server.dividend_cache
+        self.trade_days = getattr(data_server, 'trade_days', None)
         self._data_loaded = True
         return data_server.get_benchmark_data()
 
@@ -268,12 +268,12 @@ def _initialize_context(self, config: BacktestConfig, start_date, log) -> tuple:
             exrights_dict=self.exrights_dict,
             benchmark_data=self.benchmark_data,
             stock_metadata=self.stock_metadata,
-            stock_data_store=self.stock_data_store,
-            fundamentals_store=self.fundamentals_store,
             index_constituents=self.index_constituents,
             stock_status_history=self.stock_status_history,
             adj_pre_cache=self.adj_pre_cache,
-            dividend_cache=self.dividend_cache
+            adj_post_cache=self.adj_post_cache,
+            dividend_cache=self.dividend_cache,
+            trade_days=self.trade_days
         )
 
         # 创建API

src/simtradelab/cli/data_tools.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (c) 2025 Kay
+#
+# This file is part of SimTradeLab, dual-licensed under AGPL-3.0 and a
+# commercial license. See LICENSE-COMMERCIAL.md or contact kayou@duck.com
+#
+"""
+数据包解包工具
+"""
+
+from __future__ import annotations
+import json
+import tarfile
+from pathlib import Path
+from tqdm import tqdm
+
+
+class DataUnpacker:
+    """数据解包器"""
+
+    def __init__(self, data_dir):
+        """初始化解包器
+
+        Args:
+            data_dir: 数据目录路径
+        """
+        self.data_dir = Path(data_dir)
+
+    def unpack_all(self, download_dir, verify=True):
+        """解包所有下载的tar.gz文件
+
+        Args:
+            download_dir: 下载目录路径
+            verify: 是否验证校验和
+        """
+        download_dir = Path(download_dir)
+
+        print("=" * 70)
+        print("SimTradeLab 数据解包工具")
+        print("=" * 70)
+
+        # 加载清单
+        manifest_file = download_dir / 'manifest.json'
+        if not manifest_file.exists():
+            print("错误：找不到manifest.json文件")
+            return
+
+        with open(manifest_file, 'r') as f:
+            manifest = json.load(f)
+
+        print("数据版本: {}".format(manifest['version']))
+        print("导出日期: {}".format(manifest['export_date']))
+        print("=" * 70)
+
+        # 确保数据目录存在
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+
+        # 解压所有包
+        for pkg_info in tqdm(manifest['packages'], desc="解包"):
+            pkg_path = download_dir / pkg_info['name']
+
+            if not pkg_path.exists():
+                print("\n警告：文件不存在 {}".format(pkg_info['name']))
+                continue
+
+            # 解压
+            with tarfile.open(pkg_path, 'r:gz') as tar:
+                tar.extractall(path=self.data_dir)
+
+        # 保存版本信息
+        import pandas as pd
+        version_info = {
+            'version': manifest['version'],
+            'export_date': manifest['export_date'],
+            'install_date': str(pd.Timestamp.now())
+        }
+
+        version_file = self.data_dir / 'version.json'
+        with open(version_file, 'w') as f:
+            json.dump(version_info, f, ensure_ascii=False, indent=2)
+
+        print("\n" + "=" * 70)
+        print("解包完成！数据目录: {}".format(self.data_dir))
+        print("=" * 70)
+
+
+def unpack_command(download_dir, data_dir=None):
+    """解包命令
+
+    Args:
+        download_dir: 下载目录路径
+        data_dir: 数据目录路径，默认为./data
+    """
+    if data_dir is None:
+        from simtradelab.utils.paths import DATA_PATH
+        data_dir = DATA_PATH
+
+    unpacker = DataUnpacker(data_dir)
+    unpacker.unpack_all(download_dir)
+
+
+if __name__ == '__main__':
+    import sys
+
+    if len(sys.argv) < 2:
+        print("用法：")
+        print("  python data_tools.py unpack <download_dir>")
+        sys.exit(1)
+
+    command = sys.argv[1]
+
+    if command == 'unpack':
+        if len(sys.argv) < 3:
+            print("错误：需要指定下载目录")
+            sys.exit(1)
+        unpack_command(sys.argv[2])
+    else:
+        print("未知命令: {}".format(command))
+        sys.exit(1)