auto_deploy_all.py

#!/usr/bin/env python3
"""
完全自动化部署脚本
支持批量处理多个GitHub项目的部署、文档生成和RAG上传

目录结构：
- projects/{project_name}/ - 存放克隆的项目代码
- deployment/{project_name}/ - 存放部署相关的JSON和MD文件
"""

import os
import sys
import subprocess
import json
import time
import re
from datetime import datetime

class AutoDeployManager:
    def __init__(self):
        self.base_dir = "/Users/agenthub/code/testCC"
        self.projects_dir = os.path.join(self.base_dir, "projects")
        self.wechat_dir = os.path.join(self.base_dir, "wechat")  # 改为wechat目录
        self.autodeploy_file = os.path.join(self.base_dir, "autodeploy.js")
        self.rag_file = os.path.join(self.base_dir, "rag.py")
        
        # 确保目录存在
        os.makedirs(self.projects_dir, exist_ok=True)
        os.makedirs(self.wechat_dir, exist_ok=True)
        
    def extract_project_name(self, github_url):
        """从GitHub URL提取项目名称"""
        # 支持多种URL格式
        patterns = [
            r'github\.com/[^/]+/([^/]+?)(?:\.git)?/?$',
            r'github\.com/[^/]+/([^/]+)',
        ]
        
        for pattern in patterns:
            match = re.search(pattern, github_url)
            if match:
                return match.group(1)  # 保持原始大小写
        
        # 如果无法提取，使用URL的最后部分
        return github_url.split('/')[-1].replace('.git', '')  # 保持原始大小写
    
    def modify_autodeploy_url(self, github_url):
        """修改autodeploy.js中的GitHub URL"""
        try:
            with open(self.autodeploy_file, 'r', encoding='utf-8') as f:
                content = f.read()
            
            # 查找第60行附近的GitHub URL
            lines = content.split('\n')
            for i, line in enumerate(lines):
                if 'github.com' in line and 'prompt:' in line:
                    # 替换GitHub URL
                    new_line = re.sub(
                        r'https://github\.com/[^,\s]+',
                        github_url,
                        line
                    )
                    lines[i] = new_line
                    print(f"✅ 已修改第{i+1}行的GitHub URL为: {github_url}")
                    break
            
            # 写回文件
            with open(self.autodeploy_file, 'w', encoding='utf-8') as f:
                f.write('\n'.join(lines))
            
            return True
            
        except Exception as e:
            print(f"❌ 修改autodeploy.js失败: {e}")
            return False
    
    def run_node_deployment(self, project_name):
        """运行node部署命令并捕获输出"""
        print(f"🚀 开始部署项目: {project_name}")
        
        try:
            # 运行node命令并实时显示输出
            process = subprocess.Popen(
                ['node', 'autodeploy.js'],
                cwd=self.base_dir,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                universal_newlines=True,
                bufsize=1
            )
            
            output_lines = []
            
            # 实时显示输出并收集
            while True:
                line = process.stdout.readline()
                if not line and process.poll() is not None:
                    break
                if line:
                    print(line.strip())
                    output_lines.append(line.strip())
            
            # 等待进程完成
            return_code = process.wait()
            
            if return_code == 0:
                print(f"✅ {project_name} 部署完成")
                return '\n'.join(output_lines)
            else:
                print(f"❌ {project_name} 部署失败，返回码: {return_code}")
                return '\n'.join(output_lines)
                
        except Exception as e:
            error_msg = f"❌ 运行node命令失败: {e}"
            print(error_msg)
            return error_msg
    
    def extract_summary(self, full_output):
        """从完整输出中提取最后的总结部分"""
        lines = full_output.split('\n')
        
        # 寻找总结标志
        summary_keywords = [
            '=== 处理完成 ===',
            'Final Result:',
            '部署完成',
            'Deployment completed',
            '总结',
            'Summary'
        ]
        
        summary_start = -1
        for i in range(len(lines) - 1, -1, -1):
            line = lines[i].strip()
            for keyword in summary_keywords:
                if keyword in line:
                    summary_start = max(0, i - 5)  # 包含前5行作为上下文
                    break
            if summary_start != -1:
                break
        
        if summary_start != -1:
            return '\n'.join(lines[summary_start:])
        else:
            # 如果没找到特定标志，返回最后20行
            return '\n'.join(lines[-20:]) if len(lines) > 20 else full_output
    
    def create_deployment_doc(self, project_name, summary, github_url):
        """创建部署文档"""
        # 创建项目专用的wechat子目录
        project_wechat_dir = os.path.join(self.wechat_dir, project_name)
        os.makedirs(project_wechat_dir, exist_ok=True)
        
        doc_filename = f"{project_name}_deployment.md"
        doc_path = os.path.join(project_wechat_dir, doc_filename)
        
        doc_content = f"""# {project_name.title()} 部署文档

## 项目信息
- **GitHub URL**: {github_url}
- **部署时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
- **项目名称**: {project_name}

## 部署总结

```
{summary}
```

## 注意事项
- 此文档由自动化脚本生成
- 包含了部署过程的最终输出和总结
- 如有问题请检查完整的部署日志

---
*Generated by auto_deploy_all.py*
"""
        
        try:
            with open(doc_path, 'w', encoding='utf-8') as f:
                f.write(doc_content)
            print(f"✅ 已创建部署文档: {doc_filename}")
            return True
        except Exception as e:
            print(f"❌ 创建部署文档失败: {e}")
            return False
    
    def modify_rag_json_path(self, project_name):
        """修改rag.py中的JSON文件路径"""
        # JSON文件应该在wechat/project_name/目录下
        project_wechat_dir = os.path.join(self.wechat_dir, project_name)
        json_filename = f"{project_name}_deployment_issues_solution.json"
        json_path = os.path.join(project_wechat_dir, json_filename)
        
        try:
            with open(self.rag_file, 'r', encoding='utf-8') as f:
                content = f.read()
            
            # 修改第106行附近的json_file_path
            lines = content.split('\n')
            for i, line in enumerate(lines):
                if 'json_file_path = ' in line and '.json' in line:
                    lines[i] = f'    json_file_path = "{json_path}"'
                    print(f"✅ 已修改第{i+1}行的JSON文件路径为: {json_path}")
                    break
            
            # 写回文件
            with open(self.rag_file, 'w', encoding='utf-8') as f:
                f.write('\n'.join(lines))
            
            return json_path
            
        except Exception as e:
            print(f"❌ 修改rag.py失败: {e}")
            return None
    
    def run_rag_script(self, json_path):
        """运行RAG脚本"""
        # 检查JSON文件是否存在
        if not os.path.exists(json_path):
            print(f"⚠️  警告: JSON文件 {json_path} 不存在，跳过RAG上传")
            return False
        
        try:
            print(f"🔄 开始运行RAG脚本，处理文件: {json_path}")
            
            result = subprocess.run(
                ['python', 'rag.py'],
                cwd=self.base_dir,
                capture_output=True,
                text=True,
                timeout=300  # 5分钟超时
            )
            
            print("RAG脚本输出:")
            print(result.stdout)
            
            if result.stderr:
                print("RAG脚本错误:")
                print(result.stderr)
            
            if result.returncode == 0:
                print("✅ RAG脚本执行成功")
                return True
            else:
                print(f"❌ RAG脚本执行失败，返回码: {result.returncode}")
                return False
                
        except subprocess.TimeoutExpired:
            print("❌ RAG脚本执行超时")
            return False
        except Exception as e:
            print(f"❌ 运行RAG脚本失败: {e}")
            return False
    
    def deploy_project(self, github_url):
        """部署单个项目的完整流程"""
        print(f"\n{'='*60}")
        print(f"🎯 开始处理项目: {github_url}")
        print(f"{'='*60}")
        
        # 1. 提取项目名称
        project_name = self.extract_project_name(github_url)
        print(f"📝 项目名称: {project_name}")
        
        # 2. 修改autodeploy.js中的GitHub URL
        if not self.modify_autodeploy_url(github_url):
            print(f"❌ 跳过项目 {project_name}：无法修改autodeploy.js")
            return False
        
        # 3. 运行node部署
        full_output = self.run_node_deployment(project_name)
        
        # 4. 提取总结并创建文档
        summary = self.extract_summary(full_output)
        self.create_deployment_doc(project_name, summary, github_url)
        
        # 5. 修改rag.py中的JSON路径
        json_path = self.modify_rag_json_path(project_name)
        
        # 6. 运行RAG脚本
        if json_path:
            self.run_rag_script(json_path)
        
        print(f"✅ 项目 {project_name} 处理完成")
        return True
    
    def deploy_multiple_projects(self, github_urls):
        """批量部署多个项目"""
        print(f"🚀 开始批量部署 {len(github_urls)} 个项目")
        print(f"开始时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        
        results = []
        
        for i, url in enumerate(github_urls, 1):
            print(f"\n🔄 进度: {i}/{len(github_urls)}")
            
            try:
                success = self.deploy_project(url)
                results.append({
                    'url': url,
                    'project_name': self.extract_project_name(url),
                    'success': success
                })
                
                # 项目间添加间隔
                if i < len(github_urls):
                    print("⏳ 等待30秒后处理下一个项目...")
                    time.sleep(30)
                    
            except Exception as e:
                print(f"❌ 处理项目 {url} 时发生错误: {e}")
                results.append({
                    'url': url,
                    'project_name': self.extract_project_name(url),
                    'success': False,
                    'error': str(e)
                })
        
        # 生成最终报告
        self.generate_final_report(results)
        
        return results
    
    def generate_final_report(self, results):
        """生成最终部署报告"""
        report_file = os.path.join(self.base_dir, "deployment_report.md")
        
        successful = [r for r in results if r['success']]
        failed = [r for r in results if not r['success']]
        
        report_content = f"""# 批量部署报告

## 部署概况
- **部署时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
- **总项目数**: {len(results)}
- **成功数量**: {len(successful)}
- **失败数量**: {len(failed)}

## 成功项目
"""
        
        for result in successful:
            report_content += f"- ✅ **{result['project_name']}**: {result['url']}\n"
        
        if failed:
            report_content += "\n## 失败项目\n"
            for result in failed:
                error_msg = result.get('error', '未知错误')
                report_content += f"- ❌ **{result['project_name']}**: {result['url']}\n"
                report_content += f"  - 错误: {error_msg}\n"
        
        report_content += f"""
## 生成文件
每个成功部署的项目都生成了以下文件：
- `wechat/{{project_name}}/{{project_name}}_deployment.md` - 部署文档
- `wechat/{{project_name}}/{{project_name}}_deployment_guide.json` - 部署指南
- `wechat/{{project_name}}/{{project_name}}_deployment_issues_solution.json` - 问题解决方案
- `projects/{{project_name}}/` - 项目源代码
- RAG数据已上传到服务器（如果对应的JSON文件存在）

---
*Generated by auto_deploy_all.py*
"""
        
        try:
            with open(report_file, 'w', encoding='utf-8') as f:
                f.write(report_content)
            print(f"\n📊 最终报告已生成: deployment_report.md")
        except Exception as e:
            print(f"❌ 生成报告失败: {e}")

def main():
    if len(sys.argv) < 2:
        print("使用方法:")
        print("  python auto_deploy_all.py <github_url1> [github_url2] [github_url3] ...")
        print("\n示例:")
        print("  python auto_deploy_all.py https://github.com/black-forest-labs/flux")
        print("  python auto_deploy_all.py https://github.com/black-forest-labs/flux https://github.com/Stability-AI/generative-models")
        sys.exit(1)
    
    github_urls = sys.argv[1:]
    
    # 验证URL格式
    valid_urls = []
    for url in github_urls:
        if 'github.com' in url:
            valid_urls.append(url)
        else:
            print(f"⚠️  跳过无效URL: {url}")
    
    if not valid_urls:
        print("❌ 没有有效的GitHub URL")
        sys.exit(1)
    
    # 创建部署管理器并开始部署
    manager = AutoDeployManager()
    results = manager.deploy_multiple_projects(valid_urls)
    
    # 显示最终统计
    successful = sum(1 for r in results if r['success'])
    print(f"\n{'='*60}")
    print(f"🎉 批量部署完成！")
    print(f"成功: {successful}/{len(results)} 个项目")
    print(f"结束时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'='*60}")

if __name__ == "__main__":
    main()