diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Readme.md b/Readme.md index 7a34740d..0ab62c58 100644 --- a/Readme.md +++ b/Readme.md @@ -1,8 +1,15 @@ -# OpenManus +# OpenManus-RL +🤗 Dataset (OpenManus-RL) -OpenManus is an open-source initiative that aims to build a robust agent ecosystem capable of harnessing various powerful tools—such as the MCP tool base and web browsing—to achieve sophisticated reasoning and execution. We draw inspiration from existing works like **OSWorld**, **open-interpretor**, and more. **Code coming soon**! +OpenManus-RL is an open-source initiative collaboratively led by **Ulab-UIUC** and **MetaGPT**. ---- +This project is an extended version of the original [@OpenManus](https://github.com/mannaandpoem/OpenManus) initiative. Inspired by successful RL tunning for reasoning LLM such as Deepseek-R1, QwQ-32B, we will explore new paradigms for RL-based LLM agent tuning, particularly building upon foundations. + +We are committed to regularly updating our exploration directions and results in a dynamic, live-streaming fashion. All progress, including rigorous testing on agent benchmarks such as GAIA, AgentBench, WebShop, and OSWorld, and tuned models, will be openly shared and continuously updated. + +We warmly welcome contributions from the broader community—join us in pushing the boundaries of agent reasoning and tool integration! + +Code and dataset coming soon! Stay tuned!
@@ -10,48 +17,367 @@ OpenManus is an open-source initiative that aims to build a robust agent ecosyst
+## 📖 Table of Contents + +- [OpenManus-RL](#openmanus-rl) + - [🔔 News](#-news) + - [Current Team Members](#current-team-members) + - [How to Contribute](#how-to-contribute) + - [Roadmap](#roadmap) + - [Method](#method) + - [Reasoning Models Exploration](#reasoning-models-exploration) + - [Alternative Rollout Strategies](#alternative-rollout-strategies) + - [Environment and Benchmark](#environment-and-benchmark) + - [Post-Training Strategies](#post-training-strategies) + - [Training of Agent Reward Model](#training-of-agent-reward-model) + - [Test-time Scaling of Trajectories](#test-time-scaling-of-trajectories) + - [Action Space Awareness and Strategic Exploration](#action-space-awareness-and-strategic-exploration) + - [Integration with RL Tuning Frameworks](#integration-with-rl-tuning-frameworks) + - [Dataset](#dataset) + - [Dataset Overbiew](#dataset-overview) + - [Data Instances](#data-instances) +- [Running](#Running) +- [Related Work](#related-work) + - [Agent tuning](#agent-tuning) + - [Tool using](#tool-using) + - [Agent tuning instruction dataset](#agent-tuning-instruction-dataset) + - [RL tuning](#rl-tuning) + - [Benchmark](#benchmark) + - [Similar Code](#similar-code) +- [Acknowledgement](#acknowledgement) +- [Community Group](#community-group) +- [Citation](#citation) + +--- + + +## 🔔 News +- **[2025-03-09]** 🍺 We collect and opensource our Agent SFT dataset at [Huggingface](https://huggingface.co/datasets/CharlieDreemur/OpenManus-RL), go try it! +- **[2025-03-08]** 🎉 We are collaborating with [@OpenManus](https://github.com/mannaandpoem/OpenManus) from Metagpt to work on this project together! +- **[2025-03-06]** 🥳 We(UIUC-Ulab) are announcing our live-streaming project, OpenManus-RL. -## Team Members -- **@Kunlun Zhu** -- **@Haofei Yu** +## Current Team Members +[@Kunlun Zhu](https://github.com/Kunlun-Zhu)(Ulab-UIUC), [@Jiayi Zhang](https://github.com/didiforgithub)(MetaGPT), [@Xiangxin Zhou](https://github.com/zhouxiangxin1998), [@Yanfei Zhang](https://github.com/yanfei-zhang-95), [@Yingxuan Yang](https://github.com/zoe-yyx), [@Weijia Zhang](https://github.com/CharlieDreemur), [@Muxin Tian](https://github.com/realtmxi), [@Haofei Yu](https://github.com/lwaekfjlk)(Ulab-UIUC) --- # How to Contribute We wholeheartedly welcome suggestions, feedback, and contributions from the community! Feel free to: +We welcome contributions, including fine-tuning codebase, tuning dataset, environment setup, and computing resources. Create issues for feature requests, bug reports, or ideas. -Submit pull requests to help improve OpenManus. +Submit pull requests to help improve OpenManus-RL. Or simply reach out to us for direct collaboration. +Important contributors will be listed as co-authors to our paper. # Roadmap -1. Web Environment Support -Integrate the ability to browse the web and interact with external resources seamlessly. +1. Agent Environment Support +Setting up LLM agent environment for online RL tunning. -2. Advanced Reasoning Models -Connect to specialized reasoning models such as deepseek-r1 for more complex inference tasks. +2. Agent Trajectories Data Collection +Connect to specialized reasoning models such as deepseek-r1, QwQ-32B for more complex inference tasks to collect comprehensive agent trajectories. -3. Test on Agent Benchmarks +3. RL-Tuning Model Paradigm +Provide an RL fine-tuning approach for customizing the agent’s behavior in our agent environment. + +4. Test on Agent Benchmarks Evaluate our framework on agentic benchmark such as Webshop, GAIA, OSWorld, AgentBench -4. RL-Tuning Model Paradigm -Provide an RL fine-tuning approach for customizing the agent’s behavior in various real-world applications. + + +
+
+ marble +
+
+ +## Method + +Our method proposes an advanced reinforcement learning (RL)-based agent tuning framework designed to significantly enhance reasoning and decision-making capabilities of large language models (LLMs). Drawing inspiration from RAGEN's Reasoning-Interaction Chain Optimization (RICO), our approach further explores novel algorithmic structures, diverse reasoning paradigms, sophisticated reward strategies, and extensive benchmark environments. + +### Reasoning Models Exploration +To benchmark the reasoning capabilities effectively, we evaluate multiple state-of-the-art reasoning models: +- **GPT-O1** +- **Deepseek-R1** +- **QwQ-32B** + +Each model provides unique reasoning capabilities that inform downstream optimization and training strategies. + +### Alternative Rollout Strategies +We experiment with a variety of rollout strategies to enhance agent planning efficiency and reasoning robustness, including: + +- **Tree-of-Thoughts (ToT)**: Employs tree-based reasoning paths, enabling agents to explore branching possibilities systematically. +- **Graph-of-Thoughts (GoT)**: Utilizes graph structures to represent complex reasoning dependencies effectively. +- **DFSDT (Depth-First Search Decision Trees)**: Optimizes action selection through depth-first search, enhancing long-horizon planning. +- **Monte Carlo Tree Search (MCTS)**: Explores reasoning and decision paths probabilistically, balancing exploration and exploitation effectively. + +These methods help identify optimal rollout techniques for various reasoning tasks. + +### Diverse Reasoning Formats +We specifically analyze and compare several reasoning output formats, notably: + +- **ReAct**: Integrates reasoning and action explicitly, encouraging structured decision-making. +- **Outcome-based Reasoning**: Optimizes toward explicit outcome predictions, driving focused goal alignment. + +These formats are rigorously compared to derive the most effective reasoning representation for various tasks. + +### Post-Training Strategies +We investigate multiple post-training methodologies to fine-tune agent reasoning effectively: + +- **Supervised Fine-Tuning (SFT)**: Initializes reasoning capabilities using human-annotated instructions. +- **Generalized Reward-based Policy Optimization (GRPO)**: Incorporates: + - **Format-based Rewards**: Rewards adherence to specified reasoning structures. + - **Outcome-based Rewards**: Rewards accurate task completion and goal attainment. +- **Proximal Policy Optimization (PPO)**: Enhances agent stability through proximal updates. +- **Direct Preference Optimization (DPO)**: Leverages explicit human preferences to optimize agent outputs directly. +- **Preference-based Reward Modeling (PRM)**: Uses learned reward functions derived from human preference data. + +### Training of Agent Reward Model +We train specialized agent reward models using annotated data to accurately quantify nuanced reward signals. These models are then leveraged to guide agent trajectory selection during both training and evaluation phases. + +### Test-time Scaling of Trajectories +During the inference phase, trajectory scaling methods are implemented, allowing agents to flexibly adapt to varying task complexities, thus enhancing robustness and performance in real-world scenarios. + +### Action Space Awareness and Strategic Exploration +Agents are equipped with action-space awareness, employing systematic exploration strategies designed to navigate complex action spaces effectively, ultimately maximizing expected rewards. + +### Integration with RL Tuning Frameworks +We integrate insights and methodologies from leading RL tuning frameworks, including: + +- **Verl** +- **TinyZero** +- **OpenR1** +- **Trlx** + +Through these frameworks, agents can effectively balance exploration and exploitation, optimize reasoning processes, and adapt dynamically to novel environments. + +In summary, our method systematically integrates advanced reasoning paradigms, diverse rollout strategies, sophisticated reward modeling, and robust RL frameworks, significantly advancing the capability and adaptability of reasoning-enhanced LLM agents. + +
+
+ marble +
+
+ +# Dataset +[**OpenManusRL-Dataset**](https://huggingface.co/datasets/CharlieDreemur/OpenManus-RL) combines agent trajectories from [AgentInstruct](https://huggingface.co/datasets/THUDM/AgentInstruct) and [Agent-FLAN](https://huggingface.co/datasets/internlm/Agent-FLAN) with features: + +- 🔍 **ReAct Framework** - Reasoning-Acting integration +- 🧠 **Structured Training** - Separate format/reasoning learning +- 🚫 **Anti-Hallucination** - Negative samples + environment grounding +- 🌐 **6 Domains** - OS, DB, Web, KG, Household, E-commerce + +## Dataset Overview + +| Source | Trajectories | Avg Turns | Key Features | +|--------|--------------|-----------|--------------| +| [AgentInstruct](https://huggingface.co/datasets/THUDM/AgentInstruct) | 1,866 | 5.24 | Multi-task QA, CoT reasoning | +| [Agent-FLAN](https://huggingface.co/datasets/internlm/Agent-FLAN) | 34,442 | 3-35 | Error recovery patterns, diverse real-world tasks | +| **Combined** | 36,308 | 4-20 | Enhanced generalization, broader task coverage | + +### Supported Tasks +- **text-generation**: ReAct-style instruction following +- **conversational-ai**: Tool-augmented dialogues + +### Languages +English + +## Data Instances + +**ReAct Pattern Example**: +```json +{ + "id": "os_0", + "conversations": [ + {"role": "user", "content": "Count files in /etc"}, + {"role": "assistant", "content": "Think: Need reliable counting method\nAct: bash\n```bash\nls -1 /etc | wc -l\n```"}, + {"role": "user", "content": "OS Output: 220"}, + {"role": "assistant", "content": "Think: Verified through execution\nAct: answer(220)"} + ] +} +``` + +# Running + +## OpenManus-RL + +A simplified library for Supervised Fine-Tuning (SFT) and GRPO tunning of language models for agentic system. (developed upon [Open-R1](https://github.com/huggingface/open-r1) from huggingface) +We are still laboriously developing this part, welcome feedback. + +## Installation + +First, create a conda environment and activate it: + +```bash +conda create -n openmanus-rl python=3.10 +conda activate openmanus-rl +``` + +Then, install the required dependencies: + + +```bash +pip install -r requirements.txt +``` + +Supervised Fine-Tuning (SFT) + +Basic Usage + +To fine-tune a model on a single GPU: + + +```bash +python -m openmanus_rl.sft \ + --model_name_or_path Qwen/Qwen2.5-1.5B-Instruct \ + --dataset_name CharlieDreemur/OpenManus-RL \ + --learning_rate 2.0e-5 \ + --num_train_epochs 1 \ + --packing \ + --max_seq_length 4096 \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 8 \ + --gradient_checkpointing \ + --bf16 \ + --logging_steps 5 \ + --output_dir data/sft-output +``` + +Distributed Training with Accelerate + +For multi-GPU training using Accelerate: + + +```bash +accelerate launch --config_file=configs/accelerate_configs/zero3.yaml openmanus_rl/sft.py \ + --model_name_or_path Qwen/Qwen2.5-1.5B-Instruct \ + --dataset_name CharlieDreemur/OpenManus-RL \ + --learning_rate 2.0e-5 \ + --num_train_epochs 1 \ + --packing \ + --max_seq_length 4096 \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 8 \ + --gradient_checkpointing \ + --bf16 \ + --logging_steps 5 \ + --output_dir data/sft-output +``` + +## Gradient-based Reinforcement for Policy Optimization (GRPO) for agent tunning +Basic Usage +To fine-tune a model using GRPO on a single GPU: + +```bash +python -m openmanus_rl.grpo \ + --model_name_or_path Qwen/Qwen2.5-1.5B-Instruct \ + --dataset_name CharlieDreemur/OpenManus-RL-GRPO \ + --learning_rate 2.0e-5 \ + --num_train_epochs 1 \ + --max_seq_length 4096 \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 8 \ + --gradient_checkpointing \ + --bf16 \ + --reward_funcs accuracy format tag_count \ + --logging_steps 5 \ + --output_dir data/grpo-output +``` +Distributed Training with Accelerate +For multi-GPU training using Accelerate: + +```bash +accelerate launch --config_file=configs/accelerate_configs/zero3.yaml openmanus_rl/grpo.py \ + --model_name_or_path Qwen/Qwen2.5-1.5B-Instruct \ + --dataset_name CharlieDreemur/OpenManus-RL-GRPO \ + --learning_rate 2.0e-5 \ + --num_train_epochs 1 \ + --max_seq_length 4096 \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 8 \ + --gradient_checkpointing \ + --bf16 \ + --reward_funcs accuracy format tag_count \ + --logging_steps 5 \ + --output_dir data/grpo-output +``` + + +# Related Work + +## Agent tuning + +1. **Offline Training of Language Model Agents with Functions as Learnable Weights**. [[paper](https://arxiv.org/pdf/2402.11359)] +2. **FIREACT : TOWARD LANGUAGE AGENT FINE-TUNING**. [[paper](https://arxiv.org/pdf/2310.05915)] +3. **AgentTuning: Enabling Generalized Agent Abilities for LLMs**. [[paper](https://arxiv.org/pdf/2310.12823)] +4. **ReAct Meets ActRe: When Language Agents Enjoy Training Data Autonomy**. [[paper](https://arxiv.org/pdf/2403.14589)] +5. **UI-TARS: Pioneering Automated GUI Interaction with Native Agents**. [[paper](https://arxiv.org/pdf/2501.12326#page=16.83)] +6. **ATLAS: Agent Tuning via Learning Critical Steps**. [[paper](https://arxiv.org/pdf/2503.02197)] + +## Tool using + +1. **Toolformer: Language Models Can Teach Themselves to Use Tools**. [[paper](https://arxiv.org/pdf/2302.04761)] +2. **ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs**. [[paper](https://arxiv.org/abs/2307.16789)] + +## Agent tuning instruction dataset + +1. **Agent-FLAN: Designing Data and Methods of Effective Agent Tuning for Large Language Models**. [[paper](https://arxiv.org/pdf/2403.12881)] +2. **AgentOhana: Design Unified Data and Training Pipeline for Effective Agent Learning**. [[paper](https://arxiv.org/pdf/2402.15506)] + +## RL tuning + +1. **Training Language Models to Follow Instructions with Human Feedback**. [[paper](https://arxiv.org/pdf/2305.18438)] +2. **Deepseekmath: Pushing the Limits of Mathematical Reasoning in Open Language Models**. [[paper](https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf)] +3. **DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning**. [[paper](https://arxiv.org/pdf/2501.12948)] + +## **Benchmark:** + +1. **AgentBench: Evaluating LLMs as Agents**. [paper](https://arxiv.org/abs/2308.03688) +2. **OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments**. [paper](https://arxiv.org/abs/2404.07972) +3. **AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents**. [paper](https://openreview.net/forum?id=il5yUQsrjC) +4. **WebShop: Towards Scalable Real-World Web Interaction with Autonomous Agents**. [paper](https://arxiv.org/pdf/2207.01206) +5. **GAIA: a benchmark for General AI Assistants**. [paper](https://arxiv.org/abs/2311.12983) +6. **TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks**. [paper](https://arxiv.org/abs/2412.14161) + + +## Similar Code + +1. **RAGEN: Training Agents by Reinforcing Reasoning**. [[code](https://github.com/ZihanWang314/RAGEN)] # Acknowledgement -We extend our thanks to ulab-uiuc (https://ulab-uiuc.github.io/) for their support and shared knowledge. Their mission and community contributions help drive innovations like OpenManus forward. +We extend our thanks to ulab-uiuc (https://ulab-uiuc.github.io/) and Openmanus (https://github.com/mannaandpoem/OpenManus)) team from MetaGPT for their support and shared knowledge. Their mission and community contributions help drive innovations like OpenManus forward. We welcome all developers who are interested in this project can reach out to (kunlunz2@illinois.edu) Stay tuned for updates and the official release of our repository. Together, let's build a thriving open-source agent ecosystem! -## Citation +# Community Group + +Join our networking group on Wecgat and share your experience with other developers! + +
+ OpenManus-RL 交流群 +
+ +# Citation Please cite the following paper if you find OpenManus helpful! ```bibtex @misc{OpenManus, - author = {Kunlun Zhu, Haofei Yu, Jiaxuan You}, - title = {OpenManus: Open Platform for Generalist Reasoning Agents}, + author = {OpenManus-RL Team}, + title = {OpenManus-RL: Open Platform for Generalist LLM Reasoning Agents with RL optimization}, year = {2025}, organization = {GitHub}, - url = {https://github.com/OpenManus/OpenManus}, + url = {https://github.com/OpenManus/OpenManus-RL}, } -``` \ No newline at end of file +``` + +

+ + + + + Star History Chart + + +

+ diff --git a/assets/method_overview.png b/assets/method_overview.png new file mode 100644 index 00000000..61a01480 Binary files /dev/null and b/assets/method_overview.png differ diff --git a/assets/openmanus-roadmap.png b/assets/openmanus-roadmap.png new file mode 100644 index 00000000..e23bfbe5 Binary files /dev/null and b/assets/openmanus-roadmap.png differ diff --git a/assets/wechat-link.jpg b/assets/wechat-link.jpg new file mode 100644 index 00000000..8752e8f8 Binary files /dev/null and b/assets/wechat-link.jpg differ diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/openmanus-rl/configs.py b/openmanus-rl/configs.py new file mode 100644 index 00000000..98cd0d10 --- /dev/null +++ b/openmanus-rl/configs.py @@ -0,0 +1,85 @@ +# coding=utf-8 +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Optional + +import trl + + +# TODO: add the shared options with a mixin to reduce code duplication +@dataclass +class GRPOConfig(trl.GRPOConfig): + """ + args for callbacks, benchmarks etc + """ + + benchmarks: list[str] = field( + default_factory=lambda: [], metadata={"help": "The benchmarks to run after training."} + ) + callbacks: list[str] = field( + default_factory=lambda: [], metadata={"help": "The callbacks to run during training."} + ) + chat_template: Optional[str] = field(default=None, metadata={"help": "The chat template to use."}) + system_prompt: Optional[str] = field( + default=None, + metadata={"help": "The optional system prompt to use."}, + ) + hub_model_revision: Optional[str] = field( + default="main", metadata={"help": "The Hub model branch to push the model to."} + ) + overwrite_hub_revision: bool = field(default=False, metadata={"help": "Whether to overwrite the Hub revision."}) + push_to_hub_revision: bool = field(default=False, metadata={"help": "Whether to push to a Hub revision/branch."}) + wandb_entity: Optional[str] = field( + default=None, + metadata={"help": ("The entity to store runs under.")}, + ) + wandb_project: Optional[str] = field( + default=None, + metadata={"help": ("The project to store runs under.")}, + ) + + +@dataclass +class SFTConfig(trl.SFTConfig): + """ + args for callbacks, benchmarks etc + """ + + benchmarks: list[str] = field( + default_factory=lambda: [], metadata={"help": "The benchmarks to run after training."} + ) + callbacks: list[str] = field( + default_factory=lambda: [], metadata={"help": "The callbacks to run during training."} + ) + chat_template: Optional[str] = field(default=None, metadata={"help": "The chat template to use."}) + system_prompt: Optional[str] = field( + default=None, + metadata={"help": "The optional system prompt to use for benchmarking."}, + ) + hub_model_revision: Optional[str] = field( + default="main", + metadata={"help": "The Hub model branch to push the model to."}, + ) + overwrite_hub_revision: bool = field(default=False, metadata={"help": "Whether to overwrite the Hub revision."}) + push_to_hub_revision: bool = field(default=False, metadata={"help": "Whether to push to a Hub revision/branch."}) + wandb_entity: Optional[str] = field( + default=None, + metadata={"help": ("The entity to store runs under.")}, + ) + wandb_project: Optional[str] = field( + default=None, + metadata={"help": ("The project to store runs under.")}, + ) diff --git a/openmanus-rl/grpo.py b/openmanus-rl/grpo.py new file mode 100644 index 00000000..91cf51c2 --- /dev/null +++ b/openmanus-rl/grpo.py @@ -0,0 +1,272 @@ +# Copyright 2025 The OpenManus-RL Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Simplified Gradient-based Reinforcement for Policy Optimization (GRPO) script for language models. + +Usage: +# Fine-tune a model on a single GPU +python -m openmanus_rl.grpo \ + --model_name_or_path Qwen/Qwen2.5-1.5B-Instruct \ + --dataset_name HuggingFaceH4/Bespoke-Stratos-17k \ + --learning_rate 2.0e-5 \ + --num_train_epochs 1 \ + --max_seq_length 4096 \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 8 \ + --gradient_checkpointing \ + --bf16 \ + --logging_steps 5 \ + --output_dir data/grpo-output +""" + +import logging +import os +import sys +from dataclasses import dataclass, field + +import datasets +import torch +import transformers +from datasets import load_dataset +from transformers import set_seed +from transformers.trainer_utils import get_last_checkpoint + +from .configs import GRPOConfig +from .rewards import ( + accuracy_reward, + format_reward, + get_cosine_scaled_reward, + get_repetition_penalty_reward, + get_trajectories_format_reward, + len_reward, + reasoning_steps_reward, + tag_count_reward, +) +from .utils import get_tokenizer +from trl import GRPOTrainer, ModelConfig, ScriptArguments, TrlParser, get_peft_config + + +logger = logging.getLogger(__name__) + + +def init_wandb_training(training_args): + """ + Helper function for setting up Weights & Biases logging tools. + """ + if training_args.wandb_entity is not None: + os.environ["WANDB_ENTITY"] = training_args.wandb_entity + if training_args.wandb_project is not None: + os.environ["WANDB_PROJECT"] = training_args.wandb_project + + +@dataclass +class GRPOScriptArguments(ScriptArguments): + """ + Script arguments for the GRPO training script. + + Args: + reward_funcs (`list[str]`): + List of reward functions. Possible values: 'accuracy', 'format', 'reasoning_steps', 'cosine', 'repetition_penalty', 'length', 'tag_count', 'trajectories_format'. + cosine_min_value_wrong (`float`): + Minimum reward for cosine scaling for wrong answers. + cosine_max_value_wrong (`float`): + Maximum reward for cosine scaling for wrong answers. + cosine_min_value_correct (`float`): + Minimum reward for cosine scaling for correct answers. + cosine_max_value_correct (`float`): + Maximum reward for cosine scaling for correct answers. + cosine_max_len (`int`): + Maximum length for cosine scaling. + """ + + reward_funcs: list[str] = field( + default_factory=lambda: ["accuracy", "format", "tag_count"], + metadata={ + "help": "List of reward functions. Possible values: 'accuracy', 'format', 'reasoning_steps', 'cosine', 'repetition_penalty', 'length', tag_count', 'trajectories_format'" + }, + ) + cosine_min_value_wrong: float = field( + default=0.0, + metadata={"help": "Minimum reward for wrong answers"}, + ) + cosine_max_value_wrong: float = field( + default=-0.5, + metadata={"help": "Maximum reward for wrong answers"}, + ) + cosine_min_value_correct: float = field( + default=0.5, + metadata={"help": "Minimum reward for correct answers"}, + ) + cosine_max_value_correct: float = field( + default=1.0, + metadata={"help": "Maximum reward for correct answers"}, + ) + cosine_max_len: int = field( + default=1000, + metadata={"help": "Maximum length for scaling"}, + ) + repetition_n_grams: int = field( + default=3, + metadata={"help": "Number of n-grams for repetition penalty reward"}, + ) + repetition_max_penalty: float = field( + default=-1.0, + metadata={"help": "Maximum (negative) penalty for for repetition penalty reward"}, + ) + + +def main(script_args, training_args, model_args): + # Set seed for reproducibility + set_seed(training_args.seed) + + ############### + # Setup logging + ############### + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process a small summary + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f" distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + logger.info(f"Model parameters {model_args}") + logger.info(f"Script parameters {script_args}") + logger.info(f"Training parameters {training_args}") + + # Check for last checkpoint + last_checkpoint = None + if os.path.isdir(training_args.output_dir): + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info(f"Checkpoint detected, resuming training at {last_checkpoint=}.") + + if "wandb" in training_args.report_to: + init_wandb_training(training_args) + + # Load the dataset + dataset = load_dataset(script_args.dataset_name, name=script_args.dataset_config) + + ################ + # Load tokenizer + ################ + tokenizer = get_tokenizer(model_args, training_args) + + # Get reward functions + REWARD_FUNCS_REGISTRY = { + "accuracy": accuracy_reward, + "format": format_reward, + "reasoning_steps": reasoning_steps_reward, + "cosine": get_cosine_scaled_reward( + min_value_wrong=script_args.cosine_min_value_wrong, + max_value_wrong=script_args.cosine_max_value_wrong, + min_value_correct=script_args.cosine_min_value_correct, + max_value_correct=script_args.cosine_max_value_correct, + max_len=script_args.cosine_max_len, + ), + "repetition_penalty": get_repetition_penalty_reward( + ngram_size=script_args.repetition_n_grams, + max_penalty=script_args.repetition_max_penalty, + ), + "length": len_reward, + "trajectories_format": get_trajectories_format_reward(), + "tag_count": tag_count_reward, + } + reward_funcs = [REWARD_FUNCS_REGISTRY[func] for func in script_args.reward_funcs] + + # Format into conversation + def make_conversation(example): + prompt = [] + + if training_args.system_prompt is not None: + prompt.append({"role": "system", "content": training_args.system_prompt}) + + prompt.append({"role": "user", "content": example["problem"]}) + return {"prompt": prompt} + + dataset = dataset.map(make_conversation) + + for split in dataset: + if "messages" in dataset[split].column_names: + dataset[split] = dataset[split].remove_columns("messages") + + logger.info("*** Initializing model kwargs ***") + torch_dtype = ( + model_args.torch_dtype if model_args.torch_dtype in ["auto", None] else getattr(torch, model_args.torch_dtype) + ) + model_kwargs = dict( + revision=model_args.model_revision, + trust_remote_code=model_args.trust_remote_code, + attn_implementation=model_args.attn_implementation, + torch_dtype=torch_dtype, + use_cache=False if training_args.gradient_checkpointing else True, + ) + training_args.model_init_kwargs = model_kwargs + + ############################# + # Initialize the GRPO trainer + ############################# + trainer = GRPOTrainer( + model=model_args.model_name_or_path, + reward_funcs=reward_funcs, + args=training_args, + train_dataset=dataset[script_args.dataset_train_split], + eval_dataset=dataset[script_args.dataset_test_split] if training_args.eval_strategy != "no" else None, + peft_config=get_peft_config(model_args), + processing_class=tokenizer, + ) + + ############### + # Training loop + ############### + logger.info("*** Train ***") + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + metrics = train_result.metrics + metrics["train_samples"] = len(dataset[script_args.dataset_train_split]) + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + ################################## + # Save model + ################################## + logger.info("*** Save model ***") + trainer.save_model(training_args.output_dir) + logger.info(f"Model saved to {training_args.output_dir}") + + # Restore k,v cache for fast inference + if trainer.accelerator.is_main_process: + trainer.model.config.use_cache = True + trainer.model.config.save_pretrained(training_args.output_dir) + + +if __name__ == "__main__": + parser = TrlParser((GRPOScriptArguments, GRPOConfig, ModelConfig)) + script_args, training_args, model_args = parser.parse_args_and_config() + main(script_args, training_args, model_args) \ No newline at end of file diff --git a/openmanus-rl/rewards.py b/openmanus-rl/rewards.py new file mode 100644 index 00000000..f8fdfe76 --- /dev/null +++ b/openmanus-rl/rewards.py @@ -0,0 +1,515 @@ +# coding=utf-8 +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reward functions for GRPO training.""" + +import asyncio +import json +import math +import re +from typing import Dict + +from latex2sympy2_extended import NormalizationConfig +from math_verify import LatexExtractionConfig, parse, verify + +from .utils import is_e2b_available + + +if is_e2b_available(): + from dotenv import load_dotenv + from e2b_code_interpreter import AsyncSandbox + + load_dotenv() + + +def accuracy_reward(completions, solution, **kwargs): + """Reward function that checks if the completion is the same as the ground truth.""" + contents = [completion[0]["content"] for completion in completions] + rewards = [] + for content, sol in zip(contents, solution): + gold_parsed = parse( + sol, + extraction_mode="first_match", + extraction_config=[LatexExtractionConfig()], + ) + if len(gold_parsed) != 0: + # We require the answer to be provided in correct latex (no malformed operators) + answer_parsed = parse( + content, + extraction_config=[ + LatexExtractionConfig( + normalization_config=NormalizationConfig( + nits=False, + malformed_operators=False, + basic_latex=True, + equations=True, + boxed="all", + units=True, + ), + # Ensures that boxed is tried first + boxed_match_priority=0, + try_extract_without_anchor=False, + ) + ], + extraction_mode="first_match", + ) + # Reward 1 if the content is the same as the ground truth, 0 otherwise + try: + reward = float(verify(answer_parsed, gold_parsed)) + except Exception as e: + print(f"verify failed: {e}, answer: {answer_parsed}, gold: {gold_parsed}") + reward = 0.0 + else: + # If the gold solution is not parseable, we reward 1 to skip this example + reward = 1.0 + print("Failed to parse gold solution: ", sol) + rewards.append(reward) + + return rewards + + +def format_reward(completions, **kwargs): + """Reward function that checks if the reasoning process is enclosed within and tags, while the final answer is enclosed within and tags.""" + pattern = r"^\n.*?\n\n\n.*?\n$" + completion_contents = [completion[0]["content"] for completion in completions] + matches = [re.match(pattern, content, re.DOTALL | re.MULTILINE) for content in completion_contents] + return [1.0 if match else 0.0 for match in matches] + + +def tag_count_reward(completions, **kwargs) -> list[float]: + """Reward function that checks if we produce the desired number of think and answer tags associated with `format_reward()`. + + Adapted from: https://gist.github.com/willccbb/4676755236bb08cab5f4e54a0475d6fb#file-grpo_demo-py-L90 + """ + + def count_tags(text: str) -> float: + count = 0.0 + if text.count("\n") == 1: + count += 0.25 + if text.count("\n\n") == 1: + count += 0.25 + if text.count("\n\n") == 1: + count += 0.25 + if text.count("\n") == 1: + count += 0.25 + return count + + contents = [completion[0]["content"] for completion in completions] + return [count_tags(c) for c in contents] + + +def reasoning_steps_reward(completions, **kwargs): + r"""Reward function that checks for clear step-by-step reasoning. + Regex pattern: + Step \d+: - matches "Step 1:", "Step 2:", etc. + ^\d+\. - matches numbered lists like "1.", "2.", etc. at start of line + \n- - matches bullet points with hyphens + \n\* - matches bullet points with asterisks + First,|Second,|Next,|Finally, - matches transition words + """ + pattern = r"(Step \d+:|^\d+\.|\n-|\n\*|First,|Second,|Next,|Finally,)" + completion_contents = [completion[0]["content"] for completion in completions] + matches = [len(re.findall(pattern, content)) for content in completion_contents] + + # Magic number 3 to encourage 3 steps and more, otherwise partial reward + return [min(1.0, count / 3) for count in matches] + + +def len_reward(completions: list[Dict[str, str]], solution: list[str], **kwargs) -> float: + """Compute length-based rewards to discourage overthinking and promote token efficiency. + + Taken from the Kimi 1.5 tech report: https://arxiv.org/abs/2501.12599 + + Args: + completions: List of model completions + solution: List of ground truth solutions + + Returns: + List of rewards where: + - For correct answers: reward = 0.5 - (len - min_len)/(max_len - min_len) + - For incorrect answers: reward = min(0, 0.5 - (len - min_len)/(max_len - min_len)) + """ + contents = [completion[0]["content"] for completion in completions] + + # First check correctness of answers + correctness = [] + for content, sol in zip(contents, solution): + gold_parsed = parse( + sol, + extraction_mode="first_match", + extraction_config=[LatexExtractionConfig()], + ) + if len(gold_parsed) == 0: + # Skip unparseable examples + correctness.append(True) # Treat as correct to avoid penalizing + print("Failed to parse gold solution: ", sol) + continue + + answer_parsed = parse( + content, + extraction_config=[ + LatexExtractionConfig( + normalization_config=NormalizationConfig( + nits=False, + malformed_operators=False, + basic_latex=True, + equations=True, + boxed=True, + units=True, + ), + boxed_match_priority=0, + try_extract_without_anchor=False, + ) + ], + extraction_mode="first_match", + ) + correctness.append(verify(answer_parsed, gold_parsed)) + + # Calculate lengths + lengths = [len(content) for content in contents] + min_len = min(lengths) + max_len = max(lengths) + + # If all responses have the same length, return zero rewards + if max_len == min_len: + return [0.0] * len(completions) + + rewards = [] + for length, is_correct in zip(lengths, correctness): + lambda_val = 0.5 - (length - min_len) / (max_len - min_len) + + if is_correct: + reward = lambda_val + else: + reward = min(0, lambda_val) + + rewards.append(float(reward)) + + return rewards + + +def get_cosine_scaled_reward( + min_value_wrong: float = -1.0, + max_value_wrong: float = -0.5, + min_value_correct: float = 0.5, + max_value_correct: float = 1.0, + max_len: int = 1000, +): + def cosine_scaled_reward(completions, solution, **kwargs): + """Reward function that scales based on completion length using a cosine schedule. + + Shorter correct solutions are rewarded more than longer ones. + Longer incorrect solutions are penalized less than shorter ones. + + Args: + completions: List of model completions + solution: List of ground truth solutions + + This function is parameterized by the following arguments: + min_value_wrong: Minimum reward for wrong answers + max_value_wrong: Maximum reward for wrong answers + min_value_correct: Minimum reward for correct answers + max_value_correct: Maximum reward for correct answers + max_len: Maximum length for scaling + """ + contents = [completion[0]["content"] for completion in completions] + rewards = [] + + for content, sol in zip(contents, solution): + gold_parsed = parse(sol, extraction_mode="first_match", extraction_config=[LatexExtractionConfig()]) + if len(gold_parsed) == 0: + rewards.append(1.0) # Skip unparseable examples + print("Failed to parse gold solution: ", sol) + continue + + answer_parsed = parse( + content, + extraction_config=[ + LatexExtractionConfig( + normalization_config=NormalizationConfig( + nits=False, + malformed_operators=False, + basic_latex=True, + equations=True, + boxed=True, + units=True, + ), + boxed_match_priority=0, + try_extract_without_anchor=False, + ) + ], + extraction_mode="first_match", + ) + + is_correct = verify(answer_parsed, gold_parsed) + gen_len = len(content) + + # Apply cosine scaling based on length + progress = gen_len / max_len + cosine = math.cos(progress * math.pi) + + if is_correct: + min_value = min_value_correct + max_value = max_value_correct + else: + # Swap min/max for incorrect answers + min_value = max_value_wrong + max_value = min_value_wrong + + reward = min_value + 0.5 * (max_value - min_value) * (1.0 + cosine) + rewards.append(float(reward)) + + return rewards + + return cosine_scaled_reward + + +def get_repetition_penalty_reward(ngram_size: int, max_penalty: float): + """ + Computes N-gram repetition penalty as described in Appendix C.2 of https://arxiv.org/abs/2502.03373. + Reference implementation from: https://github.com/eddycmu/demystify-long-cot/blob/release/openrlhf/openrlhf/reward/repetition.py + + Args: + ngram_size: size of the n-grams + max_penalty: Maximum (negative) penalty for wrong answers + """ + if max_penalty > 0: + raise ValueError(f"max_penalty {max_penalty} should not be positive") + + def zipngram(text: str, ngram_size: int): + words = text.lower().split() + return zip(*[words[i:] for i in range(ngram_size)]) + + def repetition_penalty_reward(completions, **kwargs) -> float: + """ + reward function the penalizes repetitions + ref implementation: https://github.com/eddycmu/demystify-long-cot/blob/release/openrlhf/openrlhf/reward/repetition.py + + Args: + completions: List of model completions + """ + + contents = [completion[0]["content"] for completion in completions] + rewards = [] + for completion in contents: + if completion == "": + rewards.append(0.0) + continue + if len(completion.split()) < ngram_size: + rewards.append(0.0) + continue + + ngrams = set() + total = 0 + for ng in zipngram(completion, ngram_size): + ngrams.add(ng) + total += 1 + + scaling = 1 - len(ngrams) / total + reward = scaling * max_penalty + rewards.append(reward) + return rewards + + return repetition_penalty_reward + + +def extract_code(completion: str) -> str: + pattern = re.compile(r"```python\n(.*?)```", re.DOTALL) + matches = pattern.findall(completion) + extracted_answer = matches[-1] if len(matches) >= 1 else "" + return extracted_answer + + +def code_reward(completions, **kwargs) -> list[float]: + """Reward function that evaluates code snippets using the E2B code interpreter. + + Assumes the dataset contains a `verification_info` column with test cases. + """ + if not is_e2b_available(): + raise ImportError( + "E2B is not available and required for this reward function. Please install E2B with " + "`pip install e2b-code-interpreter` and add an API key to a `.env` file." + ) + + # TODO: add support for other languages in E2B: https://e2b.dev/docs/code-interpreting/supported-languages + """Returns a reward function that evaluates code snippets in a sandbox.""" + evaluation_script_template = """ + import subprocess + import json + + def evaluate_code(code, test_cases): + passed = 0 + total = len(test_cases) + exec_timeout = 5 + + for case in test_cases: + process = subprocess.run( + ["python3", "-c", code], + input=case["input"], + text=True, + capture_output=True, + timeout=exec_timeout + ) + + if process.returncode != 0: # Error in execution + continue + + output = process.stdout.strip() + if output.strip() == case["output"].strip(): + passed += 1 + + success_rate = (passed / total) + return success_rate + + code_snippet = {code} + test_cases = json.loads({test_cases}) + + evaluate_code(code_snippet, test_cases) + """ + code_snippets = [extract_code(completion[-1]["content"]) for completion in completions] + verification_info = kwargs["verification_info"] + scripts = [ + evaluation_script_template.format(code=json.dumps(code), test_cases=json.dumps(json.dumps(info["test_cases"]))) + for code, info in zip(code_snippets, verification_info) + ] + try: + rewards = run_async_from_sync(scripts, verification_info["language"]) + + except Exception as e: + print(f"Error from E2B executor: {e}") + rewards = [0.0] * len(completions) + + return rewards + + +def get_trajectories_format_reward(min_steps: int = 3, partial_reward: bool = True): + """Reward function that checks if the reasoning process follows the ReAct (Reasoning and Acting) format. + + Args: + min_steps: Minimum number of reasoning steps required for full reward. + partial_reward: If True, provide partial rewards for partially correct formats. + + Returns: + A reward function that evaluates compliance with the ReAct format. + """ + # Pattern looks for ReAct format sequences: Thought, Action, Observation + thought_pattern = r"Thought:[\s\S]+?" + action_pattern = r"Action:[\s\S]+?" + observation_pattern = r"Observation:[\s\S]+?" + + # Full ReAct step pattern (all three components) + react_step_pattern = rf"({thought_pattern}{action_pattern}{observation_pattern})" + + # Final answer pattern + final_answer_pattern = r"Answer:[\s\S]+" + + def trajectories_format_reward(completions, **kwargs): + """Evaluates if completions follow the ReAct format with proper Thought/Action/Observation sequences. + + Args: + completions: List of model completions + + Returns: + List of rewards between 0.0 and 1.0 based on format compliance + """ + contents = [completion[0]["content"] for completion in completions] + rewards = [] + + for content in contents: + # Count full ReAct steps (Thought/Action/Observation sequences) + react_steps = re.findall(react_step_pattern, content) + num_steps = len(react_steps) + + # Check for final answer + has_final_answer = bool(re.search(final_answer_pattern, content)) + + if num_steps >= min_steps and has_final_answer: + # Full reward for meeting minimum steps and having final answer + rewards.append(1.0) + elif not partial_reward: + # No partial reward if disabled + rewards.append(0.0) + else: + # Partial rewards based on components present + reward = 0.0 + + # Reward for steps (up to 0.7) + step_reward = min(0.7, (num_steps / min_steps) * 0.7) + reward += step_reward + + # Reward for final answer (0.3) + if has_final_answer: + reward += 0.3 + + rewards.append(reward) + + return rewards + + return trajectories_format_reward + +def get_code_format_reward(language: str = "python"): + """Format reward function specifically for code responses. + + Args: + language: Programming language supported by E2B https://e2b.dev/docs/code-interpreting/supported-languages + """ + pattern = rf"^\n.*?\n\n\n.*?```{language}.*?```.*?\n$" + + def code_format_reward(completions, **kwargs): + completion_contents = [completion[0]["content"] for completion in completions] + matches = [re.match(pattern, content, re.DOTALL | re.MULTILINE) for content in completion_contents] + return [1.0 if match else 0.0 for match in matches] + + return code_format_reward + + +def run_async_from_sync(scripts: list[str], language: str) -> list[float]: + """Function wrapping the `run_async` function.""" + # Create a new event loop and set it + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + # Run the async function and get the result + rewards = loop.run_until_complete(run_async(scripts, language)) + finally: + loop.close() + + return rewards + + +async def run_async(scripts: list[str], language: str) -> list[float]: + # Create the sandbox by hand, currently there's no context manager for this version + sbx = await AsyncSandbox.create(timeout=30, request_timeout=3) + + # Create a list of tasks for running scripts concurrently + tasks = [run_script(sbx, script) for script in scripts] + + # Wait for all tasks to complete and gather their results as they finish + results = await asyncio.gather(*tasks) + rewards = list(results) # collect results + + # Kill the sandbox after all the tasks are complete + await sbx.kill() + + return rewards + + +async def run_script(sbx, script: str, language: str) -> float: + execution = await sbx.run_code(script, language=language) + try: + return float(execution.text) + except (TypeError, ValueError): + return 0.0 \ No newline at end of file diff --git a/openmanus-rl/sft.py b/openmanus-rl/sft.py new file mode 100644 index 00000000..0c6f025f --- /dev/null +++ b/openmanus-rl/sft.py @@ -0,0 +1,178 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Simplified supervised fine-tuning script for decoder language models. + +Usage: +# Fine-tune a model on a single GPU +python -m openmanus_rl.sft \ + --model_name_or_path Qwen/Qwen2.5-1.5B-Instruct \ + --dataset_name HuggingFaceH4/Bespoke-Stratos-17k \ + --learning_rate 2.0e-5 \ + --num_train_epochs 1 \ + --packing \ + --max_seq_length 4096 \ + --per_device_train_batch_size 2 \ + --gradient_accumulation_steps 8 \ + --gradient_checkpointing \ + --bf16 \ + --logging_steps 5 \ + --output_dir data/sft-output +""" + +import logging +import os +import sys + +import datasets +import torch +import transformers +from datasets import load_dataset +from transformers import set_seed +from transformers.trainer_utils import get_last_checkpoint + +from .utils import get_tokenizer +from .configs import SFTConfig +from trl import ( + ModelConfig, + ScriptArguments, + SFTTrainer, + TrlParser, + get_kbit_device_map, + get_peft_config, + get_quantization_config, +) + + +logger = logging.getLogger(__name__) + + +def init_wandb_training(training_args): + """ + Helper function for setting up Weights & Biases logging tools. + """ + if training_args.wandb_entity is not None: + os.environ["WANDB_ENTITY"] = training_args.wandb_entity + if training_args.wandb_project is not None: + os.environ["WANDB_PROJECT"] = training_args.wandb_project + + +def main(script_args, training_args, model_args): + # Set seed for reproducibility + set_seed(training_args.seed) + + ############### + # Setup logging + ############### + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + logger.info(f"Model parameters {model_args}") + logger.info(f"Script parameters {script_args}") + logger.info(f"Training parameters {training_args}") + + # Check for last checkpoint + last_checkpoint = None + if os.path.isdir(training_args.output_dir): + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info(f"Checkpoint detected, resuming training at {last_checkpoint=}.") + + if "wandb" in training_args.report_to: + init_wandb_training(training_args) + + ################ + # Load datasets + ################ + dataset = load_dataset(script_args.dataset_name, name=script_args.dataset_config) + + ################ + # Load tokenizer + ################ + tokenizer = get_tokenizer(model_args, training_args) + tokenizer.pad_token = tokenizer.eos_token + + ################### + # Model init kwargs + ################### + logger.info("*** Initializing model kwargs ***") + torch_dtype = ( + model_args.torch_dtype if model_args.torch_dtype in ["auto", None] else getattr(torch, model_args.torch_dtype) + ) + quantization_config = get_quantization_config(model_args) + model_kwargs = dict( + revision=model_args.model_revision, + trust_remote_code=model_args.trust_remote_code, + attn_implementation=model_args.attn_implementation, + torch_dtype=torch_dtype, + use_cache=False if training_args.gradient_checkpointing else True, + device_map=get_kbit_device_map() if quantization_config is not None else None, + quantization_config=quantization_config, + ) + training_args.model_init_kwargs = model_kwargs + + ############################ + # Initialize the SFT Trainer + ############################ + trainer = SFTTrainer( + model=model_args.model_name_or_path, + args=training_args, + train_dataset=dataset[script_args.dataset_train_split], + processing_class=tokenizer, + peft_config=get_peft_config(model_args), + ) + + ############### + # Training loop + ############### + logger.info("*** Train ***") + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + metrics = train_result.metrics + metrics["train_samples"] = len(dataset[script_args.dataset_train_split]) + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + ################################## + # Save model + ################################## + logger.info("*** Save model ***") + trainer.save_model(training_args.output_dir) + logger.info(f"Model saved to {training_args.output_dir}") + + # Restore k,v cache for fast inference + if trainer.accelerator.is_main_process: + trainer.model.config.use_cache = True + trainer.model.config.save_pretrained(training_args.output_dir) + + +if __name__ == "__main__": + parser = TrlParser((ScriptArguments, SFTConfig, ModelConfig)) + script_args, training_args, model_args = parser.parse_args_and_config() + main(script_args, training_args, model_args) \ No newline at end of file diff --git a/openmanus-rl/utils.py b/openmanus-rl/utils.py new file mode 100644 index 00000000..955f88ad --- /dev/null +++ b/openmanus-rl/utils.py @@ -0,0 +1,38 @@ +from transformers import AutoTokenizer, PreTrainedTokenizer + +from trl import ModelConfig + +from ..configs import GRPOConfig, SFTConfig + + +DEFAULT_CHAT_TEMPLATE = "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}" + + +def get_tokenizer( + model_args: ModelConfig, training_args: SFTConfig | GRPOConfig, auto_set_chat_template: bool = True +) -> PreTrainedTokenizer: + """Get the tokenizer for the model.""" + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, + revision=model_args.model_revision, + trust_remote_code=model_args.trust_remote_code, + ) + + if training_args.chat_template is not None: + tokenizer.chat_template = training_args.chat_template + elif auto_set_chat_template and tokenizer.get_chat_template() is None: + tokenizer.chat_template = DEFAULT_CHAT_TEMPLATE + + return tokenizer + +import os + + +def init_wandb_training(training_args): + """ + Helper function for setting up Weights & Biases logging tools. + """ + if training_args.wandb_entity is not None: + os.environ["WANDB_ENTITY"] = training_args.wandb_entity + if training_args.wandb_project is not None: + os.environ["WANDB_PROJECT"] = training_args.wandb_project \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7de44c25 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +accelerate==1.4.0 +bitsandbytes>=0.43.0 +datasets>=3.2.0 +deepspeed==0.15.4 +distilabel[vllm,ray,openai]>=1.5.2 +e2b-code-interpreter>=1.0.5 +einops>=0.8.0 +flake8>=6.0.0 +hf_transfer>=0.1.4 +huggingface-hub[cli]>=0.19.2,<1.0 +isort>=5.12.0 +langdetect +latex2sympy2_extended>=1.0.6 +liger_kernel==0.5.3 +git+https://github.com/huggingface/lighteval.git@ed084813e0bd12d82a06d9f913291fdbee774905 +math-verify==0.5.2 +packaging>=23.0 +parameterized>=0.9.0 +peft>=0.14.0 +pytest +python-dotenv +ruff>=0.9.0 +safetensors>=0.3.3 +sentencepiece>=0.1.99 +torch==2.5.1 +transformers==4.49.0 +git+https://github.com/huggingface/trl.git@69ad852e5654a77f1695eb4c608906fe0c7e8624 +vllm==0.7.2 +wandb>=0.19.1