From b9c4aebe671de80107e1bba15cd6f16db05535f1 Mon Sep 17 00:00:00 2001 From: LING71671 <1739677116@qq.com> Date: Tue, 5 May 2026 23:56:11 +0800 Subject: [PATCH] feat: add proxy discovery foundation --- .github/workflows/ci.yml | 33 +++ .gitignore | 12 + AGENTS.md | 58 +++++ README.md | 120 +++++++++- cmd/plugproxy/main.go | 248 ++++++++++++++++++++ docs/ci-cd.md | 39 ++++ docs/project-conventions.md | 42 ++++ docs/proxy-sources.md | 363 ++++++++++++++++++++++++++++++ docs/roadmap.md | 39 ++++ docs/source-discovery.md | 151 +++++++++++++ go.mod | 3 + internal/app/app.go | 124 ++++++++++ internal/checker/checker.go | 19 ++ internal/checker/http.go | 55 +++++ internal/discover/analyzer.go | 170 ++++++++++++++ internal/discover/extract.go | 170 ++++++++++++++ internal/discover/extract_test.go | 83 +++++++ internal/discover/github.go | 235 +++++++++++++++++++ internal/discover/http.go | 54 +++++ internal/discover/openai.go | 239 ++++++++++++++++++++ internal/discover/openai_test.go | 41 ++++ internal/discover/types.go | 71 ++++++ internal/discover/validate.go | 99 ++++++++ internal/fetcher/fetcher.go | 32 +++ internal/pool/memory.go | 69 ++++++ internal/pool/pool.go | 21 ++ internal/server/server.go | 67 ++++++ internal/source/source.go | 12 + internal/source/static.go | 38 ++++ pkg/model/proxy.go | 36 +++ 30 files changed, 2741 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 cmd/plugproxy/main.go create mode 100644 docs/ci-cd.md create mode 100644 docs/project-conventions.md create mode 100644 docs/proxy-sources.md create mode 100644 docs/roadmap.md create mode 100644 docs/source-discovery.md create mode 100644 go.mod create mode 100644 internal/app/app.go create mode 100644 internal/checker/checker.go create mode 100644 internal/checker/http.go create mode 100644 internal/discover/analyzer.go create mode 100644 internal/discover/extract.go create mode 100644 internal/discover/extract_test.go create mode 100644 internal/discover/github.go create mode 100644 internal/discover/http.go create mode 100644 internal/discover/openai.go create mode 100644 internal/discover/openai_test.go create mode 100644 internal/discover/types.go create mode 100644 internal/discover/validate.go create mode 100644 internal/fetcher/fetcher.go create mode 100644 internal/pool/memory.go create mode 100644 internal/pool/pool.go create mode 100644 internal/server/server.go create mode 100644 internal/source/source.go create mode 100644 internal/source/static.go create mode 100644 pkg/model/proxy.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a82a9c0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + go: + name: Go + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Check formatting + run: test -z "$(gofmt -l .)" + + - name: Test + run: go test ./... + + - name: Build + run: go build ./cmd/plugproxy diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d0421d --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# Binaries +bin/ +dist/ +*.exe + +# Local/editor state +.idea/ +.vscode/ + +# Logs +*.log + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6f9ef12 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,58 @@ +# AGENTS.md + +## 项目说明 + +plugproxy 是一个 Go 语言轻量代理采集、检测、代理池管理和接入工具。项目目标是先提供稳定 CLI 和 HTTP API，再演进到轻量前端管理面板。 + +## 文档规则 + +- 默认使用中文。 +- `README.md` 保持仓库入口性质，只放项目简介、快速命令和文档索引。 +- 详细文档默认放到 `docs/` 目录。 +- 新增设计文档、路线图、规范和方案时，优先创建或更新 `docs/*.md`。 + +## Go 规则 + +- 遵循 Google Go 风格。 +- 提交前运行 `gofmt`。 +- 提交前运行 `go test ./...`。 +- 标准库优先，谨慎增加第三方依赖。 +- 并发代码必须支持 `context.Context` 取消，避免 goroutine 泄漏。 +- 公共模型和 SDK 相关代码放在 `pkg/`，内部实现放在 `internal/`。 + +## GitHub 规则 + +- 默认使用 GitHub CLI：`gh`。 +- 创建、查看、推送仓库和 PR 时优先使用 `gh`。 +- 当前仓库使用 GitHub Actions 作为 CI/CD。 +- 所有非临时改动默认先创建 issue，再基于 issue 创建分支和 PR。 +- PR 描述需要关联 issue，例如 `Closes #123`。 +- PR 检查没有问题后启用自动合并。 +- 合并后删除已合并分支。 +- 不要在没有明确要求时改写 Git 历史。 +- 不要回滚用户未明确要求回滚的改动。 + +## 常用命令 + +```bash +go test ./... +go build -o bin/plugproxy ./cmd/plugproxy +go run ./cmd/plugproxy version +go run ./cmd/plugproxy fetch +go run ./cmd/plugproxy list +go run ./cmd/plugproxy run -addr 127.0.0.1:8899 +``` + +## 当前架构 + +```text +cmd/plugproxy/ CLI 入口 +internal/app/ 应用编排 +internal/checker/ 代理检测 +internal/fetcher/ 并发代理源采集 +internal/pool/ 代理池接口与内存实现 +internal/server/ 轻量 HTTP API +internal/source/ 代理源接口与实现 +pkg/model/ 公开代理数据模型 +docs/ 项目文档 +``` diff --git a/README.md b/README.md index caffca5..bc8d749 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,121 @@ # plugproxy -Lightweight proxy crawling, checking, pooling, and integration toolkit written in Go. +plugproxy 是一个使用 Go 编写的轻量级代理采集、检测、代理池管理和接入工具。 -> Status: early design and initialization. +> 当前状态：早期设计与项目初始化阶段。 + +## 目标 + +- 从多种免费代理源采集代理。 +- 并发检测代理，并保留有价值的健康状态与元数据。 +- 通过 CLI、HTTP API 和 Go SDK 让任何项目都能轻松接入代理池。 +- 支持 GitHub、Web、Raw URL 和可选 AI 搜索发现候选代理源。 +- 以 worker pool 支持高并发抓取、验证和检测。 +- 保持轻量：单个 Go 二进制，无强制外部服务依赖。 + +## 架构 + +```mermaid +flowchart LR + subgraph Discovery["代理源发现"] + GH["GitHub API\nREADME / sources / fetcher"] + WEB["Web Search\n可选 AI Provider"] + RAW["Raw URL\nTXT / JSON / HTML"] + AI["AI Analyst\n搜索规划 / 结果理解 / 规则草案"] + end + + subgraph Candidates["候选源层"] + CS["CandidateSource\n候选待审"] + VR["Validate Workers\n抽样验证 / 去重 / 打分"] + SR["SourceRecipe\n解析规则草案"] + end + + subgraph Fetch["采集与检测"] + SRC["Source Adapter\nRaw / JSON / HTML / API"] + FW["Fetch Workers\n高并发采集"] + CK["Check Workers\n高并发检测"] + POOL["Proxy Pool\n健康评分 / 策略选择"] + end + + subgraph Access["接入层"] + CLI["CLI"] + API["HTTP API"] + SDK["Go SDK"] + UI["轻量管理面板\n后续"] + end + + GH --> CS + WEB --> AI --> CS + RAW --> CS + CS --> VR --> SR + SR --> SRC + SRC --> FW --> CK --> POOL + POOL --> CLI + POOL --> API + POOL --> SDK + POOL --> UI +``` + +## 逻辑链 + +```text +discover -> candidates -> validate -> review -> source config -> fetch -> check -> pool -> CLI / HTTP API / SDK +``` + +plugproxy 的核心原则是“发现候选源”和“使用可用代理”分离。AI、GitHub 搜索和页面分析只进入候选源队列；代理必须经过抽样验证、采集、检测和健康评分后，才会进入代理池。 + +## 扩展点 + +- `AIProvider`：适配 OpenAI、Responses-compatible 服务，以及后续 Anthropic、Gemini、OpenRouter、Ollama 等。 +- `Source`：适配 Raw TXT、JSON、HTML 表格、公开 API 和项目源码引用的页面型源。 +- `Checker`：扩展 HTTP、HTTPS、SOCKS4、SOCKS5 和多目标检测。 +- `Pool`：扩展内存池、持久化池、健康评分和选择策略。 +- `Access`：扩展 CLI、HTTP API、Go SDK 和后续嵌入式前端管理面板。 + +## 当前命令 + +```bash +go run ./cmd/plugproxy version +go run ./cmd/plugproxy fetch +go run ./cmd/plugproxy list +go run ./cmd/plugproxy get +go run ./cmd/plugproxy check -workers 32 -target https://httpbin.org/ip -timeout 8s +go run ./cmd/plugproxy run -addr 127.0.0.1:8899 +go run ./cmd/plugproxy discover repo jhao104/proxy_pool -workers 32 +go run ./cmd/plugproxy discover url https://raw.githubusercontent.com/gfpcom/free-proxy-list/main/sources/http.txt +go run ./cmd/plugproxy discover search -query "free proxy list socks5" -limit 10 -workers 32 +go run ./cmd/plugproxy discover validate candidates.json -workers 128 +``` + +运行后可用的 HTTP API： + +```text +GET /health +GET /proxies +GET /proxy +GET /proxy?protocol=http +GET /proxy?strategy=fastest +``` + +## 项目结构 + +```text +cmd/plugproxy/ CLI 入口 +internal/app/ 应用编排 +internal/checker/ 代理检测 +internal/fetcher/ 并发代理源采集 +internal/pool/ 代理池接口与内存实现 +internal/server/ 轻量 HTTP API +internal/source/ 代理源接口与实现 +internal/discover/ 代理源发现、验证和 AI Provider +pkg/model/ 公开代理数据模型 +docs/ 项目文档 +``` + +## 文档 + +- [项目约定](docs/project-conventions.md) +- [GitHub Actions CI/CD](docs/ci-cd.md) +- [代理源清单](docs/proxy-sources.md) +- [代理源发现爬虫设计](docs/source-discovery.md) +- [开发路线图](docs/roadmap.md) diff --git a/cmd/plugproxy/main.go b/cmd/plugproxy/main.go new file mode 100644 index 0000000..494b221 --- /dev/null +++ b/cmd/plugproxy/main.go @@ -0,0 +1,248 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/LING71671/plugproxy/internal/app" + "github.com/LING71671/plugproxy/internal/discover" + "github.com/LING71671/plugproxy/internal/pool" +) + +const version = "0.1.0-dev" + +func main() { + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + application := app.New(log) + + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + ctx := context.Background() + + switch os.Args[1] { + case "version": + fmt.Println(version) + case "fetch": + count := application.Fetch(ctx) + fmt.Printf("fetched %d proxies\n", count) + case "check": + fs := flag.NewFlagSet("check", flag.ExitOnError) + workers := fs.Int("workers", 32, "number of concurrent proxy checks") + target := fs.String("target", "https://httpbin.org/ip", "target URL used to check proxies") + timeout := fs.Duration("timeout", 8*time.Second, "per-proxy check timeout") + _ = fs.Parse(os.Args[2:]) + application.Fetch(ctx) + healthy := application.Check(ctx, *workers, *target, *timeout) + fmt.Printf("healthy %d proxies\n", healthy) + case "list": + application.Fetch(ctx) + writeJSON(application.Pool().List(pool.Filter{})) + case "get": + application.Fetch(ctx) + proxy, ok := application.Pool().Get(pool.StrategyAny, pool.Filter{}) + if !ok { + fmt.Fprintln(os.Stderr, "no proxy available") + os.Exit(1) + } + writeJSON(proxy) + case "run": + fs := flag.NewFlagSet("run", flag.ExitOnError) + addr := fs.String("addr", "127.0.0.1:8899", "HTTP API listen address") + workers := fs.Int("workers", 32, "number of concurrent proxy checks") + target := fs.String("target", "https://httpbin.org/ip", "target URL used to check proxies") + timeout := fs.Duration("timeout", 8*time.Second, "per-proxy check timeout") + skipCheck := fs.Bool("skip-check", true, "skip proxy checking on startup") + _ = fs.Parse(os.Args[2:]) + + application.Fetch(ctx) + if !*skipCheck { + application.Check(ctx, *workers, *target, *timeout) + } + if err := application.Serve(*addr); err != nil { + log.Error("server stopped", "error", err) + os.Exit(1) + } + case "discover": + if err := runDiscover(ctx, os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + default: + usage() + os.Exit(2) + } +} + +func usage() { + fmt.Println(`plugproxy is a lightweight proxy crawling, checking, pooling, and integration toolkit. + +Usage: + plugproxy version + plugproxy fetch + plugproxy check [-workers 32] [-target URL] [-timeout 8s] + plugproxy list + plugproxy get + plugproxy run [-addr 127.0.0.1:8899] [-skip-check=true] + plugproxy discover repo owner/name + plugproxy discover url URL + plugproxy discover validate FILE + plugproxy discover search -query QUERY [-ai]`) +} + +func writeJSON(value any) { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + _ = encoder.Encode(value) +} + +func runDiscover(ctx context.Context, args []string) error { + if len(args) < 1 { + discoverUsage() + return fmt.Errorf("missing discover command") + } + + switch args[0] { + case "repo": + fs := flag.NewFlagSet("discover repo", flag.ExitOnError) + timeout := fs.Duration("timeout", 15*time.Second, "GitHub request timeout") + workers := fs.Int("workers", 16, "concurrent GitHub file scans") + _ = fs.Parse(reorderFlagArgs(args[1:], map[string]bool{"timeout": false, "workers": false})) + if fs.NArg() != 1 { + return fmt.Errorf("usage: plugproxy discover repo owner/name") + } + client := discover.NewGitHubClient(*timeout, *workers) + writeJSON(client.DiscoverRepo(ctx, fs.Arg(0))) + case "url": + fs := flag.NewFlagSet("discover url", flag.ExitOnError) + timeout := fs.Duration("timeout", 12*time.Second, "URL request timeout") + _ = fs.Parse(reorderFlagArgs(args[1:], map[string]bool{"timeout": false})) + if fs.NArg() != 1 { + return fmt.Errorf("usage: plugproxy discover url URL") + } + httpClient := discover.NewHTTPClient(*timeout, 0) + content, err := httpClient.FetchSample(ctx, fs.Arg(0)) + if err != nil { + return err + } + report := discover.NewReport(fs.Arg(0), "url") + report.Candidates = discover.NewAnalyzer().AnalyzeURLContent(fs.Arg(0), content, "url:"+fs.Arg(0)) + writeJSON(report) + case "validate": + fs := flag.NewFlagSet("discover validate", flag.ExitOnError) + timeout := fs.Duration("timeout", 12*time.Second, "per-source validation timeout") + workers := fs.Int("workers", 128, "concurrent source validations") + _ = fs.Parse(reorderFlagArgs(args[1:], map[string]bool{"timeout": false, "workers": false})) + if fs.NArg() != 1 { + return fmt.Errorf("usage: plugproxy discover validate FILE") + } + report, err := readDiscoveryInput(fs.Arg(0)) + if err != nil { + return err + } + report.Source = "validate" + report.Generated = time.Now() + report.Candidates = discover.NewValidator(*timeout, *workers).Validate(ctx, report.Candidates) + writeJSON(report) + case "search": + fs := flag.NewFlagSet("discover search", flag.ExitOnError) + query := fs.String("query", "", "search query") + limit := fs.Int("limit", 10, "maximum results") + useAI := fs.Bool("ai", false, "enable AI web search") + aiProviderName := fs.String("ai-provider", "openai", "AI provider: openai or responses-compatible") + aiModel := fs.String("ai-model", "gpt-5", "AI model") + aiBaseURL := fs.String("ai-base-url", os.Getenv("PLUGPROXY_AI_BASE_URL"), "Responses-compatible API base URL") + timeout := fs.Duration("timeout", 45*time.Second, "search request timeout") + workers := fs.Int("workers", 16, "concurrent GitHub search enrichment workers") + _ = fs.Parse(reorderFlagArgs(args[1:], map[string]bool{ + "query": false, "limit": false, "ai": true, "ai-provider": false, + "ai-model": false, "ai-base-url": false, "timeout": false, "workers": false, + })) + if *query == "" { + return fmt.Errorf("discover search requires -query") + } + report := discover.NewGitHubClient(*timeout, *workers).SearchRepos(ctx, *query, *limit) + if *useAI { + provider, err := discover.NewAIProvider(*aiProviderName, *aiModel, *aiBaseURL, *timeout) + if err != nil { + return err + } + aiReport, err := provider.Search(ctx, *query, *limit) + if err != nil { + report.Failures = append(report.Failures, err.Error()) + } else { + report.Candidates = append(report.Candidates, aiReport.Candidates...) + } + report.Source = "github_search+ai" + report.Candidates = discover.Deduplicate(report.Candidates) + } + writeJSON(report) + default: + discoverUsage() + return fmt.Errorf("unknown discover command %q", args[0]) + } + return nil +} + +func discoverUsage() { + fmt.Println(`Discover commands: + plugproxy discover repo owner/name + plugproxy discover url URL + plugproxy discover validate FILE + plugproxy discover search -query QUERY [-limit 10] [-workers 16] [-ai] [-ai-provider openai] [-ai-model gpt-5]`) +} + +func readDiscoveryInput(path string) (discover.DiscoveryReport, error) { + data, err := os.ReadFile(path) + if err != nil { + return discover.DiscoveryReport{}, err + } + + var report discover.DiscoveryReport + if err := json.Unmarshal(data, &report); err == nil && len(report.Candidates) > 0 { + return report, nil + } + + var candidates []discover.CandidateSource + if err := json.Unmarshal(data, &candidates); err != nil { + return discover.DiscoveryReport{}, err + } + report = discover.NewReport(path, "file") + report.Candidates = candidates + return report, nil +} + +func reorderFlagArgs(args []string, boolFlags map[string]bool) []string { + flags := make([]string, 0, len(args)) + positionals := make([]string, 0, len(args)) + for i := 0; i < len(args); i++ { + arg := args[i] + if !strings.HasPrefix(arg, "-") || arg == "-" { + positionals = append(positionals, arg) + continue + } + + nameValue := strings.TrimLeft(arg, "-") + name := nameValue + if index := strings.Index(nameValue, "="); index >= 0 { + name = nameValue[:index] + } + flags = append(flags, arg) + if strings.Contains(nameValue, "=") || boolFlags[name] { + continue + } + if i+1 < len(args) { + flags = append(flags, args[i+1]) + i++ + } + } + return append(flags, positionals...) +} diff --git a/docs/ci-cd.md b/docs/ci-cd.md new file mode 100644 index 0000000..b480fb8 --- /dev/null +++ b/docs/ci-cd.md @@ -0,0 +1,39 @@ +# GitHub Actions CI/CD + +本项目使用 GitHub Actions 作为默认 CI/CD。 + +## 目标 + +- PR 自动运行 Go 检查。 +- 检查通过后可以启用自动合并。 +- 合并后自动删除功能分支。 +- 所有 GitHub 操作默认使用 GitHub CLI，也就是 `gh`。 + +## 推荐流程 + +```text +issue -> branch -> PR -> GitHub Actions 检查 -> 自动合并 -> 删除分支 +``` + +## 当前检查项 + +- `gofmt` 检查。 +- `go test ./...`。 +- `go build ./cmd/plugproxy`。 + +## 常用命令 + +```bash +gh auth status +gh issue create +gh pr create +gh pr checks +gh pr merge --auto --squash --delete-branch +``` + +## 约定 + +- 非临时改动默认先创建 issue。 +- PR 描述需要关联 issue，例如 `Closes #123`。 +- CI 通过后再自动合并。 +- 不在本地绕过 CI 直接合并重要改动。 diff --git a/docs/project-conventions.md b/docs/project-conventions.md new file mode 100644 index 0000000..ae79a0f --- /dev/null +++ b/docs/project-conventions.md @@ -0,0 +1,42 @@ +# 项目约定 + +## 文档语言 + +- 默认使用中文编写项目文档。 +- `README.md` 作为仓库入口，保持简洁。 +- 详细设计、规范、路线图和方案文档默认放在 `docs/` 目录。 +- 面向外部生态必须使用英文的内容可以保留英文，例如包名、命令、API 字段、错误码和协议名。 + +## Go 编码规范 + +本项目默认遵循 Google Go 风格： + +- 代码必须通过 `gofmt`。 +- 包名使用简短、清晰的小写名称，不使用下划线。 +- 导出的类型、函数、常量和变量需要有清晰命名；公共 API 稳定性优先于内部实现便利。 +- 错误处理显式返回，不隐藏关键错误。 +- `context.Context` 作为需要取消、超时或请求生命周期控制的函数首个参数。 +- 并发逻辑必须考虑退出路径，避免 goroutine 泄漏。 +- 保持接口小而稳定，只在真实调用方需要时抽象。 +- 标准库优先；只有当收益明显时才引入第三方依赖。 + +## GitHub 操作 + +- 默认使用 GitHub CLI，也就是 `gh`。 +- 查看认证状态使用 `gh auth status`。 +- 创建仓库、查看仓库、创建 issue、创建 PR 和查看 Actions 等优先使用 `gh`。 +- 当前仓库使用 GitHub Actions 作为 CI/CD。 +- 非临时改动默认先创建 issue，确认任务边界后再创建 PR。 +- PR 应关联对应 issue，优先使用 `Closes #issue_number` 让合并后自动关闭 issue。 +- PR 检查通过且没有阻塞问题时启用自动合并。 +- PR 合并后删除功能分支。 +- 提交信息使用简洁英文，例如 `chore: initialize project`。 +- 未经明确要求，不改写已有 Git 历史。 + +## 项目设计偏好 + +- CLI 优先，HTTP API 次之，最后再做轻量前端管理面板。 +- 默认无外部服务依赖。 +- 可作为独立二进制运行，也要能作为 Go SDK 被其他项目接入。 +- 核心能力保持小模块化：代理源、检测器、代理池、HTTP API 彼此解耦。 +- 免费代理质量不稳定，检测与健康评分是核心能力，不只是附加功能。 diff --git a/docs/proxy-sources.md b/docs/proxy-sources.md new file mode 100644 index 0000000..4017848 --- /dev/null +++ b/docs/proxy-sources.md @@ -0,0 +1,363 @@ +# 代理源清单 + +> 更新时间：2026-05-05。免费代理源变化很快，所有源都必须经过 plugproxy 自己的检测、去重、评分和隔离流程。 + +## 收集原则 + +- 优先选择可程序化接入的源：Raw TXT、JSON、CSV、公开 API。 +- 优先选择协议明确的源：HTTP、HTTPS、SOCKS4、SOCKS5 分文件或带字段。 +- 默认低频抓取，尊重源站限制，避免对公共源造成压力。 +- 免费代理不可信，不能直接进入可用池，必须先进入候选池并检测。 +- 页面型源先降级处理，优先做稳定 Raw/API 源。 + +## 第一批优先接入 + +### ProxyScrape + +- 类型：公开 API +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：TXT、JSON、CSV +- 优先级：高 +- 入口： + - `https://api.proxyscrape.com/v4/free-proxy-list/get?request=display_proxies&proxy_format=protocolipport&format=text` +- 备注：官方页面提供 API URL，并说明免费列表持续更新和检测。 + +### Proxifly Free Proxy List + +- 类型：GitHub/CDN Raw +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT、JSON、CSV +- 优先级：高 +- 入口： + - `https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.txt` + - `https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/protocols/http/data.txt` + - `https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/protocols/https/data.txt` + - `https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/protocols/socks4/data.txt` + - `https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/protocols/socks5/data.txt` +- 备注：README 标注每 5 分钟更新，支持 `.json`、`.txt`、`.csv`。 + +### dpangestuw/Free-Proxy + +- 类型：GitHub Raw +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：高 +- 入口： + - `https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/All_proxies.txt` + - `https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt` + - `https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/socks4_proxies.txt` + - `https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/socks5_proxies.txt` +- 备注：README 标注每 5 分钟更新，已按协议拆分。 + +### TheSpeedX/PROXY-List + +- 类型：GitHub Raw +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：高 +- 入口： + - `https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt` + - `https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt` + - `https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt` +- 备注：项目历史较久，适合作为基础候选源。 + +### ProxyScraper/ProxyScraper + +- 类型：GitHub Raw +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：高 +- 入口： + - `https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/http.txt` + - `https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/socks4.txt` + - `https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/socks5.txt` +- 备注：项目页面标注每 30 分钟更新。 + +### OpenProxyList + +- 类型：公开 Raw API +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：高 +- 入口： + - `https://api.openproxylist.xyz/http.txt` + - `https://api.openproxylist.xyz/https.txt` + - `https://api.openproxylist.xyz/socks4.txt` + - `https://api.openproxylist.xyz/socks5.txt` +- 备注：页面标注每 10 分钟更新。 + +### proxy-list.download + +- 类型：公开 API +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：中 +- 入口： + - `https://www.proxy-list.download/api/v1/get?type=http` + - `https://www.proxy-list.download/api/v1/get?type=https` + - `https://www.proxy-list.download/api/v1/get?type=socks4` + - `https://www.proxy-list.download/api/v1/get?type=socks5` +- 备注：API 文档明确 `type` 参数。 + +## 第二批增强源 + +### Firmfox/Proxify + +- 类型：GitHub Raw +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：中 +- 入口： + - `https://raw.githubusercontent.com/Firmfox/Proxify/main/proxies/http.txt` + - `https://raw.githubusercontent.com/Firmfox/Proxify/main/proxies/https.txt` + - `https://raw.githubusercontent.com/Firmfox/Proxify/main/proxies/socks4.txt` + - `https://raw.githubusercontent.com/Firmfox/Proxify/main/proxies/socks5.txt` +- 备注：项目 README 说明会从公开源收集并维护多协议代理。 + +### LoneKingCode/free-proxy-db + +- 类型：GitHub Raw +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：TXT、JSON +- 优先级：中 +- 入口： + - `https://raw.githubusercontent.com/LoneKingCode/free-proxy-db/main/proxies/all.txt` + - `https://raw.githubusercontent.com/LoneKingCode/free-proxy-db/main/proxies/http.txt` + - `https://raw.githubusercontent.com/LoneKingCode/free-proxy-db/main/proxies/socks4.txt` + - `https://raw.githubusercontent.com/LoneKingCode/free-proxy-db/main/proxies/socks5.txt` + - `https://raw.githubusercontent.com/LoneKingCode/free-proxy-db/main/proxies/all.json` +- 备注：包含 TXT 和 JSON 两种格式，适合验证通用 adapter。 + +### monosans/proxy-list + +- 类型：GitHub Raw JSON +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：JSON +- 优先级：中 +- 入口： + - `https://raw.githubusercontent.com/monosans/proxy-list/main/proxies.json` + - `https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_pretty.json` +- 备注：包含地理信息，适合做 JSON adapter 和元数据映射。 + +### joy-deploy/free-proxy-list + +- 类型：GitHub Raw +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：TXT、JSON +- 优先级：中 +- 入口： + - `https://raw.githubusercontent.com/thenasty1337/free-proxy-list/main/data/latest/proxies.txt` + - `https://raw.githubusercontent.com/thenasty1337/free-proxy-list/main/data/latest/types/http/proxies.txt` + - `https://raw.githubusercontent.com/thenasty1337/free-proxy-list/main/data/latest/types/socks4/proxies.txt` + - `https://raw.githubusercontent.com/thenasty1337/free-proxy-list/main/data/latest/types/socks5/proxies.txt` +- 备注：GitHub 页面发生重定向，接入前需要确认仓库归属和稳定性。 + +### gfpcom/free-proxy-list + +- 类型：GitHub Wiki Raw +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT +- 优先级：中 +- 入口： + - `https://raw.githubusercontent.com/wiki/gfpcom/free-proxy-list/lists/http.txt` + - `https://raw.githubusercontent.com/wiki/gfpcom/free-proxy-list/lists/https.txt` + - `https://raw.githubusercontent.com/wiki/gfpcom/free-proxy-list/lists/socks4.txt` + - `https://raw.githubusercontent.com/wiki/gfpcom/free-proxy-list/lists/socks5.txt` +- 备注：规模很大，需要限量读取、流式解析和严格检测，避免候选池膨胀。 + +### gfpcom/free-proxy-list sources 目录 + +- 类型：上游源清单 +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 等 +- 格式：TXT，内容是代理源 URL +- 优先级：高，用于“发现代理源”，不直接作为代理列表 +- 入口： + - `https://raw.githubusercontent.com/gfpcom/free-proxy-list/main/sources/http.txt` + - `https://raw.githubusercontent.com/gfpcom/free-proxy-list/main/sources/https.txt` + - `https://raw.githubusercontent.com/gfpcom/free-proxy-list/main/sources/socks4.txt` + - `https://raw.githubusercontent.com/gfpcom/free-proxy-list/main/sources/socks5.txt` +- 代表性新增源： + - `https://raw.githubusercontent.com/ALIILAPRO/Proxy/main/http.txt` + - `https://raw.githubusercontent.com/ALIILAPRO/Proxy/main/socks4.txt` + - `https://raw.githubusercontent.com/ALIILAPRO/Proxy/main/socks5.txt` + - `https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt` + - `https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt` + - `https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt` + - `https://raw.githubusercontent.com/SevenworksDev/proxy-list/main/proxies/http.txt` + - `https://raw.githubusercontent.com/SevenworksDev/proxy-list/main/proxies/socks4.txt` + - `https://raw.githubusercontent.com/SevenworksDev/proxy-list/main/proxies/socks5.txt` + - `https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt` + - `https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt` + - `https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt` + - `https://raw.githubusercontent.com/Tsprnay/Proxy-lists/master/proxies/http.txt` + - `https://raw.githubusercontent.com/Tsprnay/Proxy-lists/master/proxies/socks4.txt` + - `https://raw.githubusercontent.com/Tsprnay/Proxy-lists/master/proxies/socks5.txt` + - `https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt` + - `https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/socks4.txt` + - `https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/socks5.txt` + - `https://raw.githubusercontent.com/fyvri/fresh-proxy-list/archive/storage/classic/http.txt` + - `https://raw.githubusercontent.com/fyvri/fresh-proxy-list/archive/storage/classic/socks5.txt` +- 备注：这些 URL 需要由辅助发现流程去重、抽样访问、打分后再进入候选源配置。 + +### free-proxy Python 生态提到的页面源 + +- 类型：HTML 页面 +- 协议：HTTP、HTTPS +- 格式：HTML 表格 +- 优先级：低 +- 入口： + - `https://www.sslproxies.org/` + - `https://www.us-proxy.org/` + - `https://free-proxy-list.net/uk-proxy.html` + - `https://free-proxy-list.net/` +- 备注：`jundymek/free-proxy` 和同类项目使用这些页面作为代理来源；页面型源容易变动，先不作为 MVP 默认源。 + +## 辅助代理源发现 + +很多 GitHub 项目会把代理源写在 README、源码、配置文件、`sources/` 目录或 workflow 里。plugproxy 可以做一个辅助爬虫，但它的职责应该是“发现代理源”，不是直接把发现到的代理投入可用池。 + +建议能力： + +- GitHub 仓库发现：搜索 `free proxy`、`proxy scraper`、`socks5.txt`、`sources/http.txt` 等关键词。 +- README/源码扫描：抽取 `raw.githubusercontent.com`、`cdn.jsdelivr.net/gh`、`.txt`、`.json`、`.csv`、API URL。 +- 源 URL 分类：按协议、格式、宿主、更新频率、是否需要认证分类。 +- 抽样验证：只读取少量字节，判断是否像代理列表或源列表。 +- 源评分：根据可访问性、格式稳定性、协议明确性、重复率和最近更新时间打分。 +- 人工审核队列：新发现源先进入候选源清单，不自动启用。 + +建议命令： + +```bash +plugproxy discover github --query "free proxy sources" --limit 50 +plugproxy discover url https://github.com/gfpcom/free-proxy-list +plugproxy discover validate docs/proxy-sources.candidates.json +``` + +输出建议： + +```json +{ + "name": "example-source", + "url": "https://raw.githubusercontent.com/example/proxy/main/http.txt", + "format": "text", + "protocol_hint": "http", + "host": "raw.githubusercontent.com", + "confidence": 0.82, + "discovered_from": "github:gfpcom/free-proxy-list:sources/http.txt" +} +``` + +边界： + +- 默认只抓 GitHub API、Raw 文件、公开 API 文档和明确导出链接。 +- 不绕过登录、验证码、付费墙或 robots 限制。 +- 不高频请求公共源。 +- 不把未检测代理暴露给用户项目。 + +## 页面/API 待验证源 + +### ProxyRadar + +- 类型：页面/API +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT、CSV、JSON +- 优先级：中 +- 入口： + - `https://proxyradar.net/` +- 备注：页面说明支持过滤、导出和 API，但需要进一步确认具体接口。 + +### FreeProxy24 + +- 类型：页面/API +- 协议：HTTP、HTTPS、SOCKS4、SOCKS5 +- 格式：TXT、CSV、JSON +- 优先级：中 +- 入口： + - `https://freeproxy24.com/free-proxy-list` +- 备注：页面说明提供 API 和导出，具体下载接口需要抓包或查看页面脚本。 + +### litport.net + +- 类型：页面/API +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：JSON、CSV、TXT +- 优先级：低 +- 入口： + - `https://litport.net/free-proxy` +- 备注：页面说明 API/导出需要登录或注册，暂不作为默认内置源。 + +### GimmeProxy + +- 类型：公开 API +- 协议：HTTP、SOCKS4、SOCKS5 +- 格式：JSON +- 优先级：低 +- 入口： + - `https://gimmeproxy.com/` +- 备注：搜索结果显示 JSON API，但当前访问不稳定，需要后续复核。 + +### Geonode + +- 类型：商业/账号 API +- 协议：HTTP、SOCKS5 等 +- 格式：JSON +- 优先级：低 +- 入口： + - `https://docs.geonode.com/api-reference/introduction` +- 备注：需要认证和服务配置，不适合作为默认免费源，但可作为未来付费/账号源适配样例。 + +## Adapter 设计建议 + +第一阶段只实现两个通用 adapter： + +1. `raw_text_url` + - 输入：URL、默认协议、source name。 + - 支持格式： + - `ip:port` + - `protocol://ip:port` + - 适配大多数 GitHub Raw、CDN Raw 和 TXT API。 + +2. `json_url` + - 输入：URL、字段映射。 + - 支持数组对象和数组字符串。 + - 先适配 `monosans/proxy-list`，后续扩展到 ProxyScrape JSON、Proxifly JSON。 + +第二阶段再做页面型 adapter。页面型源需要独立限速、缓存和失败降级，不能影响主流程。 + +## 默认内置源建议 + +MVP 默认只启用这些相对稳定、易解析的源： + +- ProxyScrape TXT API +- Proxifly TXT +- dpangestuw TXT +- TheSpeedX TXT +- ProxyScraper TXT +- OpenProxyList TXT + +其他源放进示例配置，由用户手动启用。 + +## 风险 + +- 免费代理高失效率是常态，不能用采集数量衡量质量。 +- 大型源可能重复率高，需要全局去重。 +- 同一代理可能被不同源标记为不同协议，需要检测器确认。 +- 部分代理可能返回污染内容，需要检测目标做响应指纹校验。 +- 公共代理存在隐私和安全风险，默认不应用于敏感流量。 + +## 参考来源 + +- Proxifly Free Proxy List: https://github.com/proxifly/free-proxy-list +- dpangestuw/Free-Proxy: https://github.com/dpangestuw/Free-Proxy +- TheSpeedX/PROXY-List: https://github.com/TheSpeedX/PROXY-List +- ProxyScraper: https://proxyscraper.github.io/ProxyScraper/ +- ProxyScrape: https://proxyscrape.com/free-proxy-list +- OpenProxyList: https://api.openproxylist.xyz/ +- proxy-list.download: https://www.proxy-list.download/api/v1 +- monosans/proxy-list: https://github.com/monosans/proxy-list +- joy-deploy/free-proxy-list: https://github.com/joy-deploy/free-proxy-list +- gfpcom/free-proxy-list: https://github.com/gfpcom/free-proxy-list +- ProxyRadar: https://proxyradar.net/ +- FreeProxy24: https://freeproxy24.com/free-proxy-list +- litport.net: https://litport.net/free-proxy +- Geonode API Reference: https://docs.geonode.com/api-reference/introduction diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..cc85000 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,39 @@ +# 开发路线图 + +## 第一阶段：可运行骨架 + +- 初始化 Go module。 +- 提供 CLI 入口。 +- 定义代理模型、代理源、检测器、代理池基础接口。 +- 实现内存代理池。 +- 实现轻量 HTTP API。 + +## 第二阶段：真实代理源 + +- 支持纯文本代理列表。 +- 支持 GitHub Raw 代理源。 +- 支持 JSON 代理源。 +- 支持从配置文件加载代理源。 +- 增加代理去重和基础格式归一化。 +- 增加 `discover` 体系，支持 GitHub、Raw URL 和可选 AI 搜索发现候选代理源。 + +## 第三阶段：检测与评分 + +- 支持 HTTP 和 HTTPS 代理检测。 +- 支持 SOCKS4 和 SOCKS5 代理检测。 +- 支持可配置检测目标。 +- 增加延迟、失败次数、成功率、最后可用时间等评分字段。 +- 增加 `any`、`fastest`、`healthy` 等选择策略。 + +## 第四阶段：接入能力 + +- 稳定 HTTP API。 +- 提供 Go SDK。 +- 提供 CLI JSON 输出，方便脚本接入。 +- 提供本地代理获取命令，例如 `plugproxy get`。 + +## 第五阶段：管理与可视化 + +- 增加可选轻量持久化。 +- 增加嵌入式管理面板。 +- 支持代理源启停、代理状态查看、手动刷新和检测。 diff --git a/docs/source-discovery.md b/docs/source-discovery.md new file mode 100644 index 0000000..99e6a89 --- /dev/null +++ b/docs/source-discovery.md @@ -0,0 +1,151 @@ +# 代理源发现爬虫设计 + +这个辅助爬虫用于发现“代理源”，不是直接发现“可用代理”。它像一个侦察员：去 GitHub 和公开页面里找可能的代理列表 URL，然后交给 plugproxy 的采集、检测、评分流程处理。 + +## 为什么需要 + +免费代理源变化非常快。很多项目会维护自己的 `sources/` 目录、README 下载链接、Raw 文件或 API 示例。手工整理可以启动项目，但长期维护需要自动发现和周期性复核。 + +## 输入 + +- GitHub 搜索关键词。 +- GitHub 仓库地址。 +- 已知代理源 URL。 +- 已知源清单文件，例如 `sources/http.txt`。 + +## 输出 + +输出候选代理源，不直接写入默认启用配置。 + +```json +{ + "name": "vakhov-http", + "url": "https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt", + "format": "text", + "protocol_hint": "http", + "confidence": 0.8, + "status": "candidate", + "discovered_from": "github:gfpcom/free-proxy-list:sources/http.txt" +} +``` + +## 发现策略 + +### GitHub 仓库搜索 + +关键词： + +- `free proxy list` +- `proxy scraper` +- `socks5.txt` +- `sources/http.txt` +- `raw.githubusercontent.com proxy socks4` + +优先检查： + +- README。 +- `sources/`。 +- `proxies/`。 +- `.github/workflows/`。 +- 常见配置文件，例如 `.json`、`.yaml`、`.txt`。 + +### URL 抽取 + +识别这些 URL： + +- `https://raw.githubusercontent.com/...` +- `https://cdn.jsdelivr.net/gh/...` +- `https://github.com/.../raw/...` +- `https://api.*` +- 以 `.txt`、`.json`、`.csv` 结尾的公开下载链接。 + +### 抽样验证 + +- 使用 `HEAD` 或小范围 `GET`。 +- 限制响应体大小。 +- 判断内容是否符合以下模式： + - `ip:port` + - `protocol://ip:port` + - JSON 数组。 + - 每行一个 URL 的源清单。 + +## 评分 + +候选源评分维度： + +- 可访问性。 +- 是否协议明确。 +- 是否机器可读。 +- 是否无需认证。 +- 是否来自稳定宿主。 +- 是否最近更新。 +- 与已有源的重复率。 + +## 命令 + +```bash +plugproxy discover search -query "free proxy list socks5" -limit 10 +plugproxy discover search -query "free proxy list socks5" -limit 10 -ai +plugproxy discover repo jhao104/proxy_pool -workers 16 +plugproxy discover url https://raw.githubusercontent.com/gfpcom/free-proxy-list/main/sources/http.txt +plugproxy discover validate candidates.json -workers 128 +``` + +AI 默认关闭。开启 AI 搜索时需要配置： + +```bash +OPENAI_API_KEY=... +``` + +也可以使用 Responses-compatible Provider： + +```bash +PLUGPROXY_AI_API_KEY=... +PLUGPROXY_AI_BASE_URL=https://example.com/v1 +plugproxy discover search -query "proxy sources" -ai -ai-provider responses-compatible -ai-model gpt-5 +``` + +## 数据流 + +```text +discover -> candidates -> validate source -> human review -> source config -> fetch -> check -> pool +``` + +## 安全边界 + +- 不绕过登录、验证码、付费墙。 +- 不扫描无关网站。 +- 不高频请求公共服务。 +- 不默认启用新发现源。 +- 不把未经检测的代理暴露给用户项目。 + +## AI Provider + +发现模块只依赖 `AIProvider` 抽象，不和具体模型厂商强绑定。 + +第一版内置： + +- `openai`：使用 OpenAI Responses API + `web_search`。 +- `responses-compatible`：使用兼容 Responses API 的服务，通过 `-ai-base-url` 或 `PLUGPROXY_AI_BASE_URL` 配置。 + +后续可以新增： + +- Anthropic。 +- Gemini。 +- OpenRouter。 +- Ollama 或其他本地模型。 + +AI 的职责是搜索规划、结果理解和候选规则草案生成。网络请求、限速、抽样验证和候选源状态仍由 Go 代码控制。 + +## 第一版实现建议 + +第一版只做 GitHub 和 Raw URL： + +- 使用 GitHub API 搜索仓库。 +- 读取 README 和 `sources/`、`proxies/` 目录。 +- 从文本中提取 URL。 +- 对 URL 做抽样验证。 +- 输出 `docs/proxy-sources.candidates.json` 或本地缓存文件。 +- repo 文件扫描和候选源验证使用 worker pool；默认值保守，但可以通过 `-workers` 提高并发。 + +页面型源发现放到第二版。 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c159486 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/LING71671/plugproxy + +go 1.25.0 diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..14c2f1c --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,124 @@ +package app + +import ( + "context" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/LING71671/plugproxy/internal/checker" + "github.com/LING71671/plugproxy/internal/fetcher" + "github.com/LING71671/plugproxy/internal/pool" + "github.com/LING71671/plugproxy/internal/server" + "github.com/LING71671/plugproxy/internal/source" + "github.com/LING71671/plugproxy/pkg/model" +) + +type App struct { + pool *pool.MemoryPool + sources []source.Source + log *slog.Logger +} + +func New(log *slog.Logger) *App { + if log == nil { + log = slog.Default() + } + + return &App{ + pool: pool.NewMemory(), + sources: []source.Source{ + source.NewStatic("example", []model.Proxy{ + {Address: "127.0.0.1:8080", Protocol: model.ProtocolHTTP}, + }), + }, + log: log, + } +} + +func (a *App) Fetch(ctx context.Context) int { + count := 0 + for _, result := range fetcher.FetchAll(ctx, a.sources) { + if result.Error != nil { + a.log.Warn("source fetch failed", "source", result.Source, "error", result.Error) + continue + } + + for _, proxy := range result.Proxies { + a.pool.Add(proxy) + count++ + } + } + + return count +} + +func (a *App) Check(ctx context.Context, workers int, targetURL string, timeout time.Duration) int { + if workers <= 0 { + workers = 32 + } + + items := a.pool.List(pool.Filter{}) + jobs := make(chan model.Proxy) + results := make(chan checker.Result) + httpChecker := checker.NewHTTP(targetURL, timeout) + + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for proxy := range jobs { + result := httpChecker.Check(ctx, proxy) + select { + case <-ctx.Done(): + return + case results <- result: + } + } + }() + } + + go func() { + defer close(jobs) + for _, proxy := range items { + select { + case <-ctx.Done(): + return + case jobs <- proxy: + } + } + }() + + go func() { + wg.Wait() + close(results) + }() + + healthy := 0 + for result := range results { + proxy := result.Proxy + proxy.LastCheckedAt = time.Now() + proxy.Latency = result.Latency + if result.OK { + proxy.SuccessCount++ + healthy++ + } else { + proxy.FailureCount++ + } + a.pool.Add(proxy) + } + + return healthy +} + +func (a *App) Pool() pool.Pool { + return a.pool +} + +func (a *App) Serve(addr string) error { + srv := server.New(a.pool, a.log) + a.log.Info("api server listening", "addr", addr) + return http.ListenAndServe(addr, srv.Handler()) +} diff --git a/internal/checker/checker.go b/internal/checker/checker.go new file mode 100644 index 0000000..4f8026c --- /dev/null +++ b/internal/checker/checker.go @@ -0,0 +1,19 @@ +package checker + +import ( + "context" + "time" + + "github.com/LING71671/plugproxy/pkg/model" +) + +type Result struct { + Proxy model.Proxy + OK bool + Latency time.Duration + Error error +} + +type Checker interface { + Check(ctx context.Context, proxy model.Proxy) Result +} diff --git a/internal/checker/http.go b/internal/checker/http.go new file mode 100644 index 0000000..e10f1cf --- /dev/null +++ b/internal/checker/http.go @@ -0,0 +1,55 @@ +package checker + +import ( + "context" + "net/http" + "time" + + "github.com/LING71671/plugproxy/pkg/model" +) + +type HTTPChecker struct { + TargetURL string + Timeout time.Duration +} + +func NewHTTP(targetURL string, timeout time.Duration) HTTPChecker { + if targetURL == "" { + targetURL = "https://httpbin.org/ip" + } + if timeout <= 0 { + timeout = 8 * time.Second + } + + return HTTPChecker{TargetURL: targetURL, Timeout: timeout} +} + +func (c HTTPChecker) Check(ctx context.Context, proxy model.Proxy) Result { + proxyURL, err := proxy.URL() + if err != nil { + return Result{Proxy: proxy, Error: err} + } + + client := &http.Client{ + Timeout: c.Timeout, + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.TargetURL, nil) + if err != nil { + return Result{Proxy: proxy, Error: err} + } + + start := time.Now() + resp, err := client.Do(req) + latency := time.Since(start) + if err != nil { + return Result{Proxy: proxy, Latency: latency, Error: err} + } + defer resp.Body.Close() + + ok := resp.StatusCode >= 200 && resp.StatusCode < 400 + return Result{Proxy: proxy, OK: ok, Latency: latency} +} diff --git a/internal/discover/analyzer.go b/internal/discover/analyzer.go new file mode 100644 index 0000000..8ff7c03 --- /dev/null +++ b/internal/discover/analyzer.go @@ -0,0 +1,170 @@ +package discover + +import ( + "sort" + "strings" +) + +type Analyzer struct{} + +func NewAnalyzer() Analyzer { + return Analyzer{} +} + +func (a Analyzer) AnalyzeURLContent(rawURL, content, discoveredFrom string) []CandidateSource { + if LooksLikeSourceList(content) { + urls := ExtractURLs(content) + candidates := make([]CandidateSource, 0, len(urls)) + for _, discoveredURL := range urls { + evidence := evidenceForURL(content, discoveredURL) + if !IsLikelyProxySourceURL(discoveredURL, evidence) { + continue + } + candidates = append(candidates, NewCandidate(discoveredURL, "", KindSourceList, discoveredFrom, evidence)) + } + return Deduplicate(candidates) + } + + if LooksLikeProxyList(content) || InferFormat(rawURL, content) != FormatUnknown { + candidate := NewCandidate(rawURL, content, InferKind(rawURL, content), discoveredFrom, sampleEvidence(content)) + if candidate.Format == FormatHTML { + candidate.AdapterRequired = true + } + return []CandidateSource{candidate} + } + + return a.AnalyzeText(content, discoveredFrom) +} + +func (a Analyzer) AnalyzeText(content, discoveredFrom string) []CandidateSource { + urls := ExtractURLs(content) + candidates := make([]CandidateSource, 0, len(urls)) + codeReference := LooksLikeCrawlerCode(content) + for _, rawURL := range urls { + evidence := evidenceForURL(content, rawURL) + if !IsLikelyProxySourceURL(rawURL, evidence) { + continue + } + kind := InferKind(rawURL, "") + if codeReference { + kind = KindCrawlerCodeReference + } + candidates = append(candidates, NewCandidate(rawURL, "", kind, discoveredFrom, evidence)) + } + return Deduplicate(candidates) +} + +func NewCandidate(rawURL, content string, kind SourceKind, discoveredFrom, evidence string) CandidateSource { + format := InferFormat(rawURL, content) + if format == FormatUnknown { + switch kind { + case KindJSON: + format = FormatJSON + case KindHTMLTable, KindCrawlerCodeReference: + format = FormatHTML + default: + format = FormatText + } + } + + adapterRequired := kind == KindHTMLTable || kind == KindCrawlerCodeReference + confidence := 0.55 + switch kind { + case KindSourceList: + confidence = 0.70 + case KindRawText, KindJSON, KindAPI: + confidence = 0.75 + case KindHTMLTable: + confidence = 0.60 + case KindCrawlerCodeReference: + confidence = 0.65 + } + if LooksLikeProxyList(content) { + confidence += 0.15 + } + if confidence > 0.95 { + confidence = 0.95 + } + + return CandidateSource{ + Name: CandidateName(rawURL), + URL: rawURL, + Format: format, + ProtocolHint: InferProtocolHint(rawURL, content), + SourceKind: kind, + Confidence: confidence, + Status: StatusCandidate, + AdapterRequired: adapterRequired, + DiscoveredFrom: discoveredFrom, + Evidence: evidence, + Recipe: &SourceRecipe{ + Kind: kind, + Format: format, + Parser: parserFor(kind, format), + URL: rawURL, + ProtocolHint: InferProtocolHint(rawURL, content), + }, + } +} + +func Deduplicate(candidates []CandidateSource) []CandidateSource { + byURL := make(map[string]CandidateSource, len(candidates)) + for _, candidate := range candidates { + if candidate.URL == "" { + continue + } + existing, ok := byURL[candidate.URL] + if !ok || candidate.Confidence > existing.Confidence { + byURL[candidate.URL] = candidate + } + } + + result := make([]CandidateSource, 0, len(byURL)) + for _, candidate := range byURL { + result = append(result, candidate) + } + sort.SliceStable(result, func(i, j int) bool { + if result[i].Confidence == result[j].Confidence { + return result[i].URL < result[j].URL + } + return result[i].Confidence > result[j].Confidence + }) + return result +} + +func parserFor(kind SourceKind, format SourceFormat) string { + switch { + case kind == KindSourceList: + return "source_list_urls" + case format == FormatJSON: + return "json_auto" + case format == FormatHTML: + return "html_table_auto" + default: + return "raw_text_lines" + } +} + +func evidenceForURL(content, rawURL string) string { + index := strings.Index(content, rawURL) + if index < 0 { + return "" + } + start := index - 80 + if start < 0 { + start = 0 + } + end := index + len(rawURL) + 80 + if end > len(content) { + end = len(content) + } + return strings.TrimSpace(content[start:end]) +} + +func sampleEvidence(content string) string { + content = strings.TrimSpace(content) + if len(content) > 240 { + return content[:240] + } + return content +} diff --git a/internal/discover/extract.go b/internal/discover/extract.go new file mode 100644 index 0000000..4e33973 --- /dev/null +++ b/internal/discover/extract.go @@ -0,0 +1,170 @@ +package discover + +import ( + "encoding/json" + "net/url" + "path" + "regexp" + "sort" + "strings" +) + +var ( + urlPattern = regexp.MustCompile(`https?://[A-Za-z0-9\-._~:/?#\[\]@!$&()*+,;=%]+`) + proxyLinePattern = regexp.MustCompile(`(?m)^\s*(?:(https?|socks4|socks5)://)?((?:\d{1,3}\.){3}\d{1,3}):(\d{2,5})\s*$`) + htmlTablePattern = regexp.MustCompile(`(?is)]*>.*?`) + methodNamePattern = regexp.MustCompile(`(?m)def\s+(freeProxy\d+|[A-Za-z_][A-Za-z0-9_]*Proxy[A-Za-z0-9_]*)\s*\(`) +) + +func ExtractURLs(text string) []string { + matches := urlPattern.FindAllString(text, -1) + seen := make(map[string]struct{}, len(matches)) + urls := make([]string, 0, len(matches)) + for _, match := range matches { + normalized := normalizeURL(match) + if normalized == "" { + continue + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + urls = append(urls, normalized) + } + sort.Strings(urls) + return urls +} + +func normalizeURL(value string) string { + value = strings.TrimSpace(value) + value = strings.Trim(value, "`\"'()[]{}<>") + value = strings.Split(value, "](")[0] + value = strings.Split(value, ",,")[0] + value = strings.TrimRight(value, ".,;:)]}，。）、") + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "" + } + return parsed.String() +} + +func CandidateName(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Host == "" { + return "unknown-source" + } + base := strings.TrimSuffix(path.Base(parsed.Path), path.Ext(parsed.Path)) + if base == "." || base == "/" || base == "" { + base = strings.Split(parsed.Host, ".")[0] + } + base = strings.ToLower(base) + base = strings.NewReplacer("_", "-", " ", "-").Replace(base) + if hint := InferProtocolHint(rawURL, ""); hint != "" && !strings.Contains(base, hint) { + base = base + "-" + hint + } + return base +} + +func InferProtocolHint(rawURL, content string) string { + lower := strings.ToLower(rawURL + "\n" + content) + for _, protocol := range []string{"socks5", "socks4", "https", "http"} { + if strings.Contains(lower, protocol) { + return protocol + } + } + return "" +} + +func InferFormat(rawURL, content string) SourceFormat { + lowerURL := strings.ToLower(rawURL) + trimmed := strings.TrimSpace(content) + switch { + case strings.HasSuffix(lowerURL, ".json") || json.Valid([]byte(trimmed)): + return FormatJSON + case strings.Contains(strings.ToLower(trimmed), " 0 +} + +func LooksLikeSourceList(content string) bool { + lines := strings.Split(content, "\n") + urlLines := 0 + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "http://") || strings.HasPrefix(line, "https://") { + urlLines++ + } + } + return urlLines >= 2 +} + +func LooksLikeCrawlerCode(content string) bool { + return methodNamePattern.MatchString(content) || strings.Contains(content, "yield") && strings.Contains(content, "proxy") +} + +func IsLikelyProxySourceURL(rawURL, evidence string) bool { + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Host == "" { + return false + } + host := strings.ToLower(parsed.Host) + path := strings.ToLower(parsed.Path) + joined := strings.ToLower(host + " " + path + " " + evidence) + + for _, blocked := range []string{ + "127.0.0.1", "localhost", "example.com", "shields.io", "travis-ci.org", + "readthedocs.org", "readthedocs.io", "hellogithub.com", + } { + if strings.Contains(host, blocked) { + return false + } + } + for _, suffix := range []string{".svg", ".png", ".jpg", ".jpeg", ".gif", ".webp"} { + if strings.HasSuffix(path, suffix) { + return false + } + } + if host == "github.com" { + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) < 2 { + return false + } + } + + for _, keyword := range []string{ + "proxy", "proxies", "socks", "free-proxy", "freeproxy", "openproxy", + "daili", "kuaidaili", "kxdaili", "zdaye", "66ip", "89ip", "docip", + "ip3366", "jiangxianli", "binglx", "ihuan", "proxylist", + } { + if strings.Contains(joined, keyword) { + return true + } + } + return false +} diff --git a/internal/discover/extract_test.go b/internal/discover/extract_test.go new file mode 100644 index 0000000..ac39192 --- /dev/null +++ b/internal/discover/extract_test.go @@ -0,0 +1,83 @@ +package discover + +import "testing" + +func TestExtractURLsNormalizesAndDeduplicates(t *testing.T) { + text := ` + https://raw.githubusercontent.com/a/proxy/main/http.txt + "https://raw.githubusercontent.com/a/proxy/main/http.txt", + https://github.com/zloi-user/hideip.me/raw/refs/heads/master/http.txt,,ColonURL + ` + + got := ExtractURLs(text) + if len(got) != 2 { + t.Fatalf("expected 2 URLs, got %d: %#v", len(got), got) + } + if got[0] == "" || got[1] == "" { + t.Fatalf("expected normalized URLs, got %#v", got) + } +} + +func TestLooksLikeProxyList(t *testing.T) { + content := "1.1.1.1:8080\nhttp://2.2.2.2:3128\n" + if !LooksLikeProxyList(content) { + t.Fatal("expected proxy list") + } +} + +func TestLooksLikeSourceList(t *testing.T) { + content := "https://raw.githubusercontent.com/a/b/main/http.txt\nhttps://api.example.com/proxies.txt\n" + if !LooksLikeSourceList(content) { + t.Fatal("expected source list") + } +} + +func TestAnalyzeCrawlerCodeReference(t *testing.T) { + content := ` + def freeProxy02(): + url = "http://www.66ip.cn/" + yield "1.1.1.1:8080" + ` + + candidates := NewAnalyzer().AnalyzeText(content, "github:test/repo:fetcher/proxyFetcher.py") + if len(candidates) != 1 { + t.Fatalf("expected 1 candidate, got %d", len(candidates)) + } + if candidates[0].SourceKind != KindCrawlerCodeReference { + t.Fatalf("expected crawler code reference, got %s", candidates[0].SourceKind) + } + if !candidates[0].AdapterRequired { + t.Fatal("expected adapter required") + } +} + +func TestAnalyzeTextSkipsNonSourceLinks(t *testing.T) { + content := ` + [badge](https://img.shields.io/badge/test.svg) + localhost http://127.0.0.1:5010/get/ + 代理源 https://www.66ip.cn/ + ` + + candidates := NewAnalyzer().AnalyzeText(content, "readme") + if len(candidates) != 1 { + t.Fatalf("expected 1 candidate, got %d: %#v", len(candidates), candidates) + } + if candidates[0].URL != "https://www.66ip.cn/" { + t.Fatalf("expected 66ip candidate, got %s", candidates[0].URL) + } +} + +func TestDeduplicateKeepsHigherConfidence(t *testing.T) { + low := NewCandidate("https://example.com/http.txt", "", KindRawText, "a", "") + high := low + high.Confidence = 0.9 + high.DiscoveredFrom = "b" + + got := Deduplicate([]CandidateSource{low, high}) + if len(got) != 1 { + t.Fatalf("expected 1 candidate, got %d", len(got)) + } + if got[0].DiscoveredFrom != "b" { + t.Fatalf("expected higher confidence candidate, got %#v", got[0]) + } +} diff --git a/internal/discover/github.go b/internal/discover/github.go new file mode 100644 index 0000000..2aa58cc --- /dev/null +++ b/internal/discover/github.go @@ -0,0 +1,235 @@ +package discover + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" +) + +type GitHubClient struct { + client *http.Client + token string + analyzer Analyzer + workers int +} + +func NewGitHubClient(timeout time.Duration, workers ...int) GitHubClient { + if timeout <= 0 { + timeout = 15 * time.Second + } + workerCount := 8 + if len(workers) > 0 && workers[0] > 0 { + workerCount = workers[0] + } + token := os.Getenv("GITHUB_TOKEN") + if token == "" { + token = os.Getenv("GH_TOKEN") + } + return GitHubClient{ + client: &http.Client{Timeout: timeout}, + token: token, + analyzer: NewAnalyzer(), + workers: workerCount, + } +} + +func (g GitHubClient) DiscoverRepo(ctx context.Context, repo string) DiscoveryReport { + report := NewReport(repo, "github_repo") + files := []string{"README.md"} + + for _, dir := range []string{"sources", "proxies", "fetcher", "spider", "crawler", "collector", ".github/workflows"} { + items, err := g.listContents(ctx, repo, dir) + if err != nil { + continue + } + for _, item := range items { + if item.Type != "file" || !isInterestingRepoFile(item.Name) { + continue + } + files = append(files, item.Path) + } + } + + report.Candidates = append(report.Candidates, g.discoverFiles(ctx, repo, files)...) + report.Candidates = Deduplicate(report.Candidates) + if len(report.Candidates) == 0 { + report.Failures = append(report.Failures, "no candidates found") + } + return report +} + +func (g GitHubClient) discoverFiles(ctx context.Context, repo string, files []string) []CandidateSource { + if len(files) == 0 { + return nil + } + workerCount := g.workers + if workerCount > len(files) { + workerCount = len(files) + } + + jobs := make(chan string) + results := make(chan []CandidateSource) + var wg sync.WaitGroup + for range workerCount { + wg.Add(1) + go func() { + defer wg.Done() + for filePath := range jobs { + select { + case <-ctx.Done(): + return + default: + } + results <- g.discoverFile(ctx, repo, filePath, "github:"+repo+":"+filePath) + } + }() + } + + go func() { + defer close(jobs) + for _, filePath := range files { + select { + case <-ctx.Done(): + return + case jobs <- filePath: + } + } + }() + go func() { + wg.Wait() + close(results) + }() + + var candidates []CandidateSource + for result := range results { + candidates = append(candidates, result...) + } + return candidates +} + +func (g GitHubClient) SearchRepos(ctx context.Context, query string, limit int) DiscoveryReport { + if limit <= 0 { + limit = 10 + } + if limit > 50 { + limit = 50 + } + report := NewReport(query, "github_search") + + endpoint := "https://api.github.com/search/repositories?q=" + url.QueryEscape(query) + "&sort=updated&per_page=" + fmt.Sprint(limit) + var response struct { + Items []struct { + FullName string `json:"full_name"` + Description string `json:"description"` + HTMLURL string `json:"html_url"` + } `json:"items"` + } + if err := g.getJSON(ctx, endpoint, &response); err != nil { + report.Failures = append(report.Failures, err.Error()) + return report + } + + for _, item := range response.Items { + candidate := NewCandidate(item.HTMLURL, item.Description, KindCrawlerCodeReference, "github_search:"+query, item.Description) + candidate.Name = strings.ReplaceAll(item.FullName, "/", "-") + candidate.AdapterRequired = true + report.Candidates = append(report.Candidates, candidate) + } + report.Candidates = Deduplicate(report.Candidates) + return report +} + +func (g GitHubClient) discoverFile(ctx context.Context, repo, filePath, discoveredFrom string) []CandidateSource { + content, err := g.fetchRawFile(ctx, repo, filePath) + if err != nil { + return nil + } + + if strings.Contains(filePath, "sources/") || strings.Contains(filePath, "proxies/") { + return g.analyzer.AnalyzeURLContent(rawGitHubURL(repo, filePath), content, discoveredFrom) + } + return g.analyzer.AnalyzeText(content, discoveredFrom) +} + +func (g GitHubClient) fetchRawFile(ctx context.Context, repo, filePath string) (string, error) { + endpoint := "https://api.github.com/repos/" + repo + "/contents/" + strings.TrimPrefix(filePath, "/") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return "", err + } + g.setHeaders(req) + req.Header.Set("Accept", "application/vnd.github.raw") + resp, err := g.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return "", fmt.Errorf("github raw %s: %s", filePath, resp.Status) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, defaultSampleLimit)) + if err != nil { + return "", err + } + return string(data), nil +} + +func (g GitHubClient) listContents(ctx context.Context, repo, dir string) ([]githubContent, error) { + endpoint := "https://api.github.com/repos/" + repo + "/contents/" + strings.TrimPrefix(dir, "/") + var items []githubContent + err := g.getJSON(ctx, endpoint, &items) + return items, err +} + +func (g GitHubClient) getJSON(ctx context.Context, endpoint string, value any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return err + } + g.setHeaders(req) + resp, err := g.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("github api %s: %s", endpoint, resp.Status) + } + return json.NewDecoder(resp.Body).Decode(value) +} + +func (g GitHubClient) setHeaders(req *http.Request) { + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("User-Agent", "plugproxy-discover/0.1") + if g.token != "" { + req.Header.Set("Authorization", "Bearer "+g.token) + } +} + +type githubContent struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + DownloadURL string `json:"download_url"` +} + +func isInterestingRepoFile(name string) bool { + lower := strings.ToLower(name) + for _, suffix := range []string{".md", ".txt", ".json", ".yaml", ".yml", ".py", ".go", ".js", ".ts"} { + if strings.HasSuffix(lower, suffix) { + return true + } + } + return false +} + +func rawGitHubURL(repo, filePath string) string { + return "https://raw.githubusercontent.com/" + repo + "/HEAD/" + strings.TrimPrefix(filePath, "/") +} diff --git a/internal/discover/http.go b/internal/discover/http.go new file mode 100644 index 0000000..4b1f751 --- /dev/null +++ b/internal/discover/http.go @@ -0,0 +1,54 @@ +package discover + +import ( + "context" + "fmt" + "io" + "net/http" + "time" +) + +const defaultSampleLimit = 128 * 1024 + +type HTTPClient struct { + client *http.Client + limit int64 +} + +func NewHTTPClient(timeout time.Duration, limit int64) HTTPClient { + if timeout <= 0 { + timeout = 12 * time.Second + } + if limit <= 0 { + limit = defaultSampleLimit + } + return HTTPClient{ + client: &http.Client{Timeout: timeout}, + limit: limit, + } +} + +func (c HTTPClient) FetchSample(ctx context.Context, rawURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", err + } + req.Header.Set("User-Agent", "plugproxy-discover/0.1") + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", c.limit-1)) + + resp, err := c.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return "", fmt.Errorf("unexpected status: %s", resp.Status) + } + + data, err := io.ReadAll(io.LimitReader(resp.Body, c.limit)) + if err != nil { + return "", err + } + return string(data), nil +} diff --git a/internal/discover/openai.go b/internal/discover/openai.go new file mode 100644 index 0000000..e8b1efa --- /dev/null +++ b/internal/discover/openai.go @@ -0,0 +1,239 @@ +package discover + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +type OpenAIProvider struct { + apiKey string + model string + baseURL string + client *http.Client +} + +type AIProvider interface { + Search(ctx context.Context, query string, limit int) (DiscoveryReport, error) +} + +func NewAIProvider(name, model, baseURL string, timeout time.Duration) (AIProvider, error) { + switch strings.ToLower(strings.TrimSpace(name)) { + case "", "openai": + return NewOpenAIProvider(model, baseURL, timeout), nil + case "responses-compatible": + return NewOpenAIProvider(model, baseURL, timeout), nil + default: + return nil, fmt.Errorf("unsupported AI provider %q", name) + } +} + +func NewOpenAIProvider(model, baseURL string, timeout time.Duration) OpenAIProvider { + if model == "" { + model = "gpt-5" + } + if baseURL == "" { + baseURL = "https://api.openai.com/v1" + } + if timeout <= 0 { + timeout = 45 * time.Second + } + return OpenAIProvider{ + apiKey: firstNonEmpty(os.Getenv("PLUGPROXY_AI_API_KEY"), os.Getenv("OPENAI_API_KEY")), + model: model, + baseURL: strings.TrimRight(baseURL, "/"), + client: &http.Client{Timeout: timeout}, + } +} + +func (p OpenAIProvider) Search(ctx context.Context, query string, limit int) (DiscoveryReport, error) { + report := NewReport(query, "openai_web_search") + if p.apiKey == "" { + return report, errors.New("OPENAI_API_KEY is required for AI web search") + } + if limit <= 0 { + limit = 10 + } + + input := fmt.Sprintf(`你是 plugproxy 的代理源发现助手。请使用 web search 搜索免费代理源、代理池项目、Raw 代理列表和 API 文档。 + +要求： +- 只输出候选“代理源”，不要输出单个可用代理。 +- 候选源必须是公开 URL。 +- 标记 format: text/json/html/unknown。 +- 标记 source_kind: raw_text/json/html_table/api/crawler_code_reference/source_list。 +- AI 结果默认 status=candidate，不要自动启用。 +- 最多输出 %d 个候选。 + +查询：%s + +只返回 JSON，格式： +{"candidates":[{"name":"","url":"","format":"text","protocol_hint":"http","source_kind":"raw_text","confidence":0.8,"status":"candidate","adapter_required":false,"discovered_from":"openai_web_search","evidence":""}]}`, limit, query) + + body := map[string]any{ + "model": p.model, + "tools": []map[string]any{ + {"type": "web_search"}, + }, + "input": input, + } + payload, err := json.Marshal(body) + if err != nil { + return report, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.baseURL+"/responses", bytes.NewReader(payload)) + if err != nil { + return report, err + } + req.Header.Set("Authorization", "Bearer "+p.apiKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := p.client.Do(req) + if err != nil { + return report, err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) + if err != nil { + return report, err + } + if resp.StatusCode >= 400 { + return report, fmt.Errorf("openai responses api: %s: %s", resp.Status, string(data)) + } + + text := extractResponseText(data) + candidates, err := parseAICandidates(text) + if err != nil { + return report, err + } + for i := range candidates { + if candidates[i].Status == "" { + candidates[i].Status = StatusCandidate + } + if candidates[i].DiscoveredFrom == "" { + candidates[i].DiscoveredFrom = "openai_web_search:" + query + } + if candidates[i].Name == "" { + candidates[i].Name = CandidateName(candidates[i].URL) + } + } + report.Candidates = Deduplicate(candidates) + return report, nil +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func extractResponseText(data []byte) string { + var response struct { + OutputText string `json:"output_text"` + Output []struct { + Type string `json:"type"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } `json:"output"` + } + if err := json.Unmarshal(data, &response); err != nil { + return string(data) + } + if response.OutputText != "" { + return response.OutputText + } + var builder strings.Builder + for _, output := range response.Output { + for _, content := range output.Content { + if content.Text != "" { + builder.WriteString(content.Text) + builder.WriteByte('\n') + } + } + } + return builder.String() +} + +func parseAICandidates(text string) ([]CandidateSource, error) { + jsonText := extractJSONObject(text) + if jsonText == "" { + return nil, errors.New("AI response did not contain JSON") + } + var wrapper struct { + Candidates []CandidateSource `json:"candidates"` + } + if err := json.Unmarshal([]byte(jsonText), &wrapper); err != nil { + var candidates []CandidateSource + if arrayErr := json.Unmarshal([]byte(jsonText), &candidates); arrayErr == nil { + return sanitizeAICandidates(candidates), nil + } + return nil, err + } + return sanitizeAICandidates(wrapper.Candidates), nil +} + +func sanitizeAICandidates(candidates []CandidateSource) []CandidateSource { + result := make([]CandidateSource, 0, len(candidates)) + for _, candidate := range candidates { + if normalizeURL(candidate.URL) == "" { + continue + } + candidate.URL = normalizeURL(candidate.URL) + if candidate.Status == "" { + candidate.Status = StatusCandidate + } + if candidate.Format == "" { + candidate.Format = InferFormat(candidate.URL, "") + } + if candidate.SourceKind == "" { + candidate.SourceKind = InferKind(candidate.URL, "") + } + if candidate.Confidence <= 0 || candidate.Confidence > 1 { + candidate.Confidence = 0.5 + } + if candidate.AdapterRequired || candidate.SourceKind == KindHTMLTable || candidate.SourceKind == KindCrawlerCodeReference { + candidate.AdapterRequired = true + } + result = append(result, candidate) + } + return result +} + +func extractJSONObject(text string) string { + text = strings.TrimSpace(text) + if strings.HasPrefix(text, "```") { + text = strings.TrimPrefix(text, "```json") + text = strings.TrimPrefix(text, "```") + text = strings.TrimSuffix(text, "```") + text = strings.TrimSpace(text) + } + startObj := strings.Index(text, "{") + startArray := strings.Index(text, "[") + start := startObj + endChar := byte('}') + if startArray >= 0 && (startObj < 0 || startArray < startObj) { + start = startArray + endChar = ']' + } + if start < 0 { + return "" + } + end := strings.LastIndexByte(text, endChar) + if end <= start { + return "" + } + return text[start : end+1] +} diff --git a/internal/discover/openai_test.go b/internal/discover/openai_test.go new file mode 100644 index 0000000..6702e47 --- /dev/null +++ b/internal/discover/openai_test.go @@ -0,0 +1,41 @@ +package discover + +import "testing" + +func TestParseAICandidatesRejectsInvalidJSON(t *testing.T) { + if _, err := parseAICandidates("not json"); err == nil { + t.Fatal("expected invalid JSON error") + } +} + +func TestParseAICandidatesSanitizesCandidates(t *testing.T) { + text := `{ + "candidates": [ + { + "name": "Example", + "url": "https://raw.githubusercontent.com/example/proxy/main/http.txt", + "format": "text", + "source_kind": "raw_text", + "confidence": 1.5 + }, + { + "name": "Bad", + "url": "not a url" + } + ] + }` + + candidates, err := parseAICandidates(text) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 1 { + t.Fatalf("expected 1 candidate, got %d", len(candidates)) + } + if candidates[0].Confidence != 0.5 { + t.Fatalf("expected confidence fallback, got %v", candidates[0].Confidence) + } + if candidates[0].Status != StatusCandidate { + t.Fatalf("expected candidate status, got %s", candidates[0].Status) + } +} diff --git a/internal/discover/types.go b/internal/discover/types.go new file mode 100644 index 0000000..8008335 --- /dev/null +++ b/internal/discover/types.go @@ -0,0 +1,71 @@ +package discover + +import "time" + +type SourceFormat string + +const ( + FormatUnknown SourceFormat = "unknown" + FormatText SourceFormat = "text" + FormatJSON SourceFormat = "json" + FormatHTML SourceFormat = "html" +) + +type SourceKind string + +const ( + KindRawText SourceKind = "raw_text" + KindJSON SourceKind = "json" + KindHTMLTable SourceKind = "html_table" + KindAPI SourceKind = "api" + KindCrawlerCodeReference SourceKind = "crawler_code_reference" + KindSourceList SourceKind = "source_list" +) + +type CandidateStatus string + +const ( + StatusCandidate CandidateStatus = "candidate" + StatusValid CandidateStatus = "valid" + StatusInvalid CandidateStatus = "invalid" +) + +type CandidateSource struct { + Name string `json:"name"` + URL string `json:"url"` + Format SourceFormat `json:"format"` + ProtocolHint string `json:"protocol_hint,omitempty"` + SourceKind SourceKind `json:"source_kind"` + Confidence float64 `json:"confidence"` + Status CandidateStatus `json:"status"` + AdapterRequired bool `json:"adapter_required"` + DiscoveredFrom string `json:"discovered_from"` + Evidence string `json:"evidence,omitempty"` + Error string `json:"error,omitempty"` + Recipe *SourceRecipe `json:"recipe,omitempty"` +} + +type SourceRecipe struct { + Kind SourceKind `json:"kind"` + Format SourceFormat `json:"format"` + Parser string `json:"parser"` + URL string `json:"url,omitempty"` + ProtocolHint string `json:"protocol_hint,omitempty"` + Notes string `json:"notes,omitempty"` +} + +type DiscoveryReport struct { + Query string `json:"query,omitempty"` + Source string `json:"source,omitempty"` + Generated time.Time `json:"generated"` + Candidates []CandidateSource `json:"candidates"` + Failures []string `json:"failures,omitempty"` +} + +func NewReport(query, source string) DiscoveryReport { + return DiscoveryReport{ + Query: query, + Source: source, + Generated: time.Now(), + } +} diff --git a/internal/discover/validate.go b/internal/discover/validate.go new file mode 100644 index 0000000..37fd8d3 --- /dev/null +++ b/internal/discover/validate.go @@ -0,0 +1,99 @@ +package discover + +import ( + "context" + "sync" + "time" +) + +type Validator struct { + http HTTPClient + analyzer Analyzer + workers int +} + +func NewValidator(timeout time.Duration, workers ...int) Validator { + workerCount := 32 + if len(workers) > 0 && workers[0] > 0 { + workerCount = workers[0] + } + return Validator{ + http: NewHTTPClient(timeout, defaultSampleLimit), + analyzer: NewAnalyzer(), + workers: workerCount, + } +} + +func (v Validator) Validate(ctx context.Context, candidates []CandidateSource) []CandidateSource { + if len(candidates) == 0 { + return nil + } + + workerCount := v.workers + if workerCount > len(candidates) { + workerCount = len(candidates) + } + + jobs := make(chan CandidateSource) + results := make(chan CandidateSource) + var wg sync.WaitGroup + for range workerCount { + wg.Add(1) + go func() { + defer wg.Done() + for candidate := range jobs { + results <- v.validateOne(ctx, candidate) + } + }() + } + + go func() { + defer close(jobs) + for _, candidate := range candidates { + if candidate.URL == "" { + continue + } + select { + case <-ctx.Done(): + return + case jobs <- candidate: + } + } + }() + go func() { + wg.Wait() + close(results) + }() + + result := make([]CandidateSource, 0, len(candidates)) + for candidate := range results { + result = append(result, candidate) + } + return Deduplicate(result) +} + +func (v Validator) validateOne(ctx context.Context, candidate CandidateSource) CandidateSource { + content, err := v.http.FetchSample(ctx, candidate.URL) + if err != nil { + candidate.Status = StatusInvalid + candidate.Error = err.Error() + return candidate + } + + analyzed := v.analyzer.AnalyzeURLContent(candidate.URL, content, candidate.DiscoveredFrom) + if len(analyzed) == 0 { + candidate.Status = StatusInvalid + candidate.Error = "content does not look like a proxy source" + return candidate + } + candidate.Status = StatusValid + candidate.Error = "" + candidate.Evidence = analyzed[0].Evidence + candidate.Format = analyzed[0].Format + candidate.SourceKind = analyzed[0].SourceKind + candidate.ProtocolHint = analyzed[0].ProtocolHint + if analyzed[0].Confidence > candidate.Confidence { + candidate.Confidence = analyzed[0].Confidence + } + return candidate +} diff --git a/internal/fetcher/fetcher.go b/internal/fetcher/fetcher.go new file mode 100644 index 0000000..248ccca --- /dev/null +++ b/internal/fetcher/fetcher.go @@ -0,0 +1,32 @@ +package fetcher + +import ( + "context" + "sync" + + "github.com/LING71671/plugproxy/internal/source" + "github.com/LING71671/plugproxy/pkg/model" +) + +type Result struct { + Source string + Proxies []model.Proxy + Error error +} + +func FetchAll(ctx context.Context, sources []source.Source) []Result { + results := make([]Result, len(sources)) + var wg sync.WaitGroup + + for i, src := range sources { + wg.Add(1) + go func(i int, src source.Source) { + defer wg.Done() + proxies, err := src.Fetch(ctx) + results[i] = Result{Source: src.Name(), Proxies: proxies, Error: err} + }(i, src) + } + + wg.Wait() + return results +} diff --git a/internal/pool/memory.go b/internal/pool/memory.go new file mode 100644 index 0000000..08f4d23 --- /dev/null +++ b/internal/pool/memory.go @@ -0,0 +1,69 @@ +package pool + +import ( + "sort" + "sync" + "time" + + "github.com/LING71671/plugproxy/pkg/model" +) + +type MemoryPool struct { + mu sync.RWMutex + proxies map[string]model.Proxy +} + +func NewMemory() *MemoryPool { + return &MemoryPool{proxies: make(map[string]model.Proxy)} +} + +func (p *MemoryPool) Add(proxy model.Proxy) { + p.mu.Lock() + defer p.mu.Unlock() + + if proxy.ID == "" { + proxy.ID = string(proxy.Protocol) + "://" + proxy.Address + } + if proxy.CreatedAt.IsZero() { + proxy.CreatedAt = time.Now() + } + + p.proxies[proxy.ID] = proxy +} + +func (p *MemoryPool) Get(strategy Strategy, filter Filter) (model.Proxy, bool) { + items := p.List(filter) + if len(items) == 0 { + return model.Proxy{}, false + } + + if strategy == StrategyFastest { + sort.SliceStable(items, func(i, j int) bool { + return items[i].Latency < items[j].Latency + }) + } + + return items[0], true +} + +func (p *MemoryPool) List(filter Filter) []model.Proxy { + p.mu.RLock() + defer p.mu.RUnlock() + + items := make([]model.Proxy, 0, len(p.proxies)) + for _, proxy := range p.proxies { + if filter.Protocol != "" && proxy.Protocol != filter.Protocol { + continue + } + if filter.Healthy && !proxy.Healthy() { + continue + } + items = append(items, proxy) + } + + sort.SliceStable(items, func(i, j int) bool { + return items[i].CreatedAt.Before(items[j].CreatedAt) + }) + + return items +} diff --git a/internal/pool/pool.go b/internal/pool/pool.go new file mode 100644 index 0000000..399edf2 --- /dev/null +++ b/internal/pool/pool.go @@ -0,0 +1,21 @@ +package pool + +import "github.com/LING71671/plugproxy/pkg/model" + +type Strategy string + +const ( + StrategyAny Strategy = "any" + StrategyFastest Strategy = "fastest" +) + +type Filter struct { + Protocol model.Protocol + Healthy bool +} + +type Pool interface { + Add(proxy model.Proxy) + Get(strategy Strategy, filter Filter) (model.Proxy, bool) + List(filter Filter) []model.Proxy +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..26514e0 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,67 @@ +package server + +import ( + "encoding/json" + "log/slog" + "net/http" + + "github.com/LING71671/plugproxy/internal/pool" + "github.com/LING71671/plugproxy/pkg/model" +) + +type Server struct { + pool pool.Pool + log *slog.Logger +} + +func New(proxyPool pool.Pool, log *slog.Logger) Server { + if log == nil { + log = slog.Default() + } + + return Server{pool: proxyPool, log: log} +} + +func (s Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health", s.health) + mux.HandleFunc("GET /proxies", s.listProxies) + mux.HandleFunc("GET /proxy", s.getProxy) + return mux +} + +func (s Server) health(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (s Server) listProxies(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, s.pool.List(parseFilter(r))) +} + +func (s Server) getProxy(w http.ResponseWriter, r *http.Request) { + strategy := pool.Strategy(r.URL.Query().Get("strategy")) + if strategy == "" { + strategy = pool.StrategyAny + } + + proxy, ok := s.pool.Get(strategy, parseFilter(r)) + if !ok { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "no proxy available"}) + return + } + + writeJSON(w, http.StatusOK, proxy) +} + +func parseFilter(r *http.Request) pool.Filter { + return pool.Filter{ + Protocol: model.Protocol(r.URL.Query().Get("protocol")), + Healthy: r.URL.Query().Get("healthy") == "true", + } +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 0000000..1024915 --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,12 @@ +package source + +import ( + "context" + + "github.com/LING71671/plugproxy/pkg/model" +) + +type Source interface { + Name() string + Fetch(ctx context.Context) ([]model.Proxy, error) +} diff --git a/internal/source/static.go b/internal/source/static.go new file mode 100644 index 0000000..2ca093e --- /dev/null +++ b/internal/source/static.go @@ -0,0 +1,38 @@ +package source + +import ( + "context" + "time" + + "github.com/LING71671/plugproxy/pkg/model" +) + +type StaticSource struct { + name string + proxies []model.Proxy +} + +func NewStatic(name string, proxies []model.Proxy) StaticSource { + now := time.Now() + for i := range proxies { + proxies[i].Source = name + if proxies[i].CreatedAt.IsZero() { + proxies[i].CreatedAt = now + } + } + + return StaticSource{name: name, proxies: proxies} +} + +func (s StaticSource) Name() string { + return s.name +} + +func (s StaticSource) Fetch(ctx context.Context) ([]model.Proxy, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + return append([]model.Proxy(nil), s.proxies...), nil + } +} diff --git a/pkg/model/proxy.go b/pkg/model/proxy.go new file mode 100644 index 0000000..40d6baf --- /dev/null +++ b/pkg/model/proxy.go @@ -0,0 +1,36 @@ +package model + +import ( + "net/url" + "time" +) + +type Protocol string + +const ( + ProtocolHTTP Protocol = "http" + ProtocolHTTPS Protocol = "https" + ProtocolSOCKS4 Protocol = "socks4" + ProtocolSOCKS5 Protocol = "socks5" +) + +type Proxy struct { + ID string `json:"id"` + Address string `json:"address"` + Protocol Protocol `json:"protocol"` + Source string `json:"source,omitempty"` + Country string `json:"country,omitempty"` + Latency time.Duration `json:"latency"` + SuccessCount int `json:"success_count"` + FailureCount int `json:"failure_count"` + LastCheckedAt time.Time `json:"last_checked_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +func (p Proxy) URL() (*url.URL, error) { + return url.Parse(string(p.Protocol) + "://" + p.Address) +} + +func (p Proxy) Healthy() bool { + return p.SuccessCount > 0 && p.FailureCount == 0 +}