From 020f553a2a30381d2f3ff086e8514e6c4181be2a Mon Sep 17 00:00:00 2001 From: chaz6chez Date: Mon, 31 Mar 2025 11:42:23 +0800 Subject: [PATCH 01/12] fixed #85 --- composer.json | 1 + tools/src/Phpy/Application.php | 2 + .../src/Phpy/Commands/PushMetadataCommand.php | 85 ++++++++ tools/src/Phpy/Commands/ScanCommand.php | 2 +- tools/src/Phpy/Config.php | 2 +- tools/src/Phpy/Helpers/Process.php | 123 +++++++++++ tools/src/Phpy/Helpers/PythonMetadata.php | 202 +++++++++--------- tools/src/Phpy/Helpers/Version.php | 119 ++++++++++- tools/src/Phpy/Installers/ModuleInstaller.php | 169 +++++++++------ tools/src/Phpy/Installers/PhpyInstaller.php | 2 +- tools/src/Phpy/Installers/PythonInstaller.php | 4 +- 11 files changed, 534 insertions(+), 177 deletions(-) create mode 100644 tools/src/Phpy/Commands/PushMetadataCommand.php diff --git a/composer.json b/composer.json index fdfdf56..8e11ce3 100644 --- a/composer.json +++ b/composer.json @@ -11,6 +11,7 @@ "require": { "php": ">=8.1", "symfony/console": "^6.0 | ^7.0", + "symfony/cache": "^6.0 | ^7.0", "nikic/php-parser": "^5.4" }, "require-dev": { diff --git a/tools/src/Phpy/Application.php b/tools/src/Phpy/Application.php index 5c8f876..1156100 100644 --- a/tools/src/Phpy/Application.php +++ b/tools/src/Phpy/Application.php @@ -11,6 +11,7 @@ use PhpyTool\Phpy\Commands\PhpyInstall; use PhpyTool\Phpy\Commands\PipMirrorConfig; use PhpyTool\Phpy\Commands\PipModuleInstall; +use PhpyTool\Phpy\Commands\PushMetadataCommand; use PhpyTool\Phpy\Commands\PythonInstall; use PhpyTool\Phpy\Commands\ScanCommand; use PhpyTool\Phpy\Commands\ShowCommand; @@ -36,6 +37,7 @@ public function __construct() System::setcwd(getcwd()); parent::__construct('PHPy', static::VERSION); $this->addCommands([ + new PushMetadataCommand(), new ScanCommand(), new InitConfigCommand(), new InstallCommand(), diff --git a/tools/src/Phpy/Commands/PushMetadataCommand.php b/tools/src/Phpy/Commands/PushMetadataCommand.php new file mode 100644 index 0000000..cb6930e --- /dev/null +++ b/tools/src/Phpy/Commands/PushMetadataCommand.php @@ -0,0 +1,85 @@ +setName('metadata:push') + ->setDescription('Push metadata'); + } + + /** @inheritdoc */ + protected function handler(): int + { + $pushed = []; + while (1) { + $metadata = $this->consoleIO?->ask( + "Please enter the metadata information in the format: [top_level]:[module_name]" + ); + $metadataArray = explode(':', $metadata); + if (count($metadataArray) !== 2) { + $this->consoleIO?->error("Invalid metadata information. $metadata"); + continue; + } + list($topLevel, $moduleName) = $metadataArray; + $pushed[$topLevel] = $moduleName; + $continue = $this->consoleIO?->ask( + "Is there anything else? [Y,n]", + true, + questionClass: ConfirmationQuestion::class + ); + if (!$continue) { + break; + } + } + if (!$pushed) { + return $this->consoleIO?->success('No metadata information entered.'); + } + $count = count($pushed); + // 输出 $pushed 的内容供用户确认 + $this->consoleIO?->output("Metadata to be pushing ($count):"); + foreach ($pushed as $topLevel => $moduleName) { + $this->consoleIO?->subOutput( + "top_level: $topLevel module_name: $moduleName" + ); + } + // 确认是否继续 + $confirm = $this->consoleIO?->ask( + 'Do you want to proceed with pushing the metadata? [Y,n]', + true, + questionClass: ConfirmationQuestion::class + ); + if (!$confirm) { + return $this->consoleIO?->comment('Cancelled.'); + } + $bar = new ProgressBar($this->consoleIO?->getOutput(), count($pushed)); + $this->consoleIO?->output('Start pushing metadata.'); + foreach ($pushed as $topLevel => $moduleName) { + PythonMetadata::pushMetadata($topLevel, $moduleName); + $bar->advance(); + } + $bar->finish(); + $bar->clear(); + return $this->consoleIO?->success('completed.'); + } +} diff --git a/tools/src/Phpy/Commands/ScanCommand.php b/tools/src/Phpy/Commands/ScanCommand.php index 6ae8b76..46ed8c5 100644 --- a/tools/src/Phpy/Commands/ScanCommand.php +++ b/tools/src/Phpy/Commands/ScanCommand.php @@ -65,7 +65,7 @@ protected function handler(): int // 解析 import依赖 $moduleInstaller->scan(); // 安装 - $moduleInstaller->install(); + $moduleInstaller->upgrade(); } catch (CommandStopException $exception) { return $this->consoleIO?->success($exception->getMessage() ?: 'Scan stop.'); } catch (CommandSuccessedException $exception) { diff --git a/tools/src/Phpy/Config.php b/tools/src/Phpy/Config.php index 160b7ae..723bd7d 100644 --- a/tools/src/Phpy/Config.php +++ b/tools/src/Phpy/Config.php @@ -93,7 +93,7 @@ public function get(?string $key, mixed $default = null): mixed $config = $config[$index]; } - return $found ? $config : $default; + return $found ? (($config instanceof \stdClass) ? [] : $config) : $default; } return $config; diff --git a/tools/src/Phpy/Helpers/Process.php b/tools/src/Phpy/Helpers/Process.php index 44cb7ad..b6874f1 100644 --- a/tools/src/Phpy/Helpers/Process.php +++ b/tools/src/Phpy/Helpers/Process.php @@ -5,6 +5,7 @@ namespace PhpyTool\Phpy\Helpers; use PhpyTool\Phpy\ConsoleIO; +use PhpyTool\Phpy\Exceptions\PhpyException; class Process { @@ -128,4 +129,126 @@ public function executePythonConfig(string $command, ?array &$output = null, boo $python = System::pythonConfig(); return $this->execute("$python $command", $output, subOutput: $subOutput); } + + /** + * 请求 + * + * @param string $method + * @param string $url + * @param array $data + * @param array $headers + * @return array{httpCode:int, responseBody:string} + */ + public function request(string $method, string $url, array $data = [], array $headers = []): array + { + $method = strtoupper($method); + $headers['Accept'] ??= 'application/json'; + // 统一处理 GET 参数 + if ($method === 'GET') { + $url .= $data + ? (str_contains($url, '?') ? '&' : '?') . http_build_query($data) + : ''; + } else { + $jsonData = json_encode($data, JSON_UNESCAPED_UNICODE); + $headers['Content-Type'] = 'application/json'; + $headers['Content-Length'] = strlen($jsonData); + } + // 没有安装curl，则使用curl命令 + if (!extension_loaded('curl')) { + return $this->requestUseCurlShell($method, $url, $data, $headers); + } + $ch = curl_init(); + try { + $options = [ + CURLOPT_URL => $url, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_CUSTOMREQUEST => $method, + CURLOPT_HTTPHEADER => self::formatHeaders($headers), + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_CONNECTTIMEOUT => 30, + CURLOPT_TIMEOUT => 60, + ]; + if (isset($jsonData)) { + $options[CURLOPT_POSTFIELDS] = $jsonData; + } + curl_setopt_array($ch, $options); + $responseBody = curl_exec($ch); + if ($errorNo = curl_errno($ch)) { + throw new PhpyException("cURL error ($errorNo): " . curl_error($ch), $errorNo); + } + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + return [ + 'httpCode' => (int)$httpCode, + 'responseBody' => (string)$responseBody + ]; + } finally { + curl_close($ch); + } + } + + /** + * 使用curl命令请求 + * + * @param string $method + * @param string $url + * @param array $data + * @param array $headers + * @return array{httpCode:int, responseBody:string} + */ + public function requestUseCurlShell(string $method, string $url, array $data = [], array $headers = []): array + { + $method = strtoupper($method); + $cmd = [ + 'curl', + '-sS', + '-X', $method, + '-o', '-', + '-w', "HTTP_CODE=%{http_code}", + '--connect-timeout', '30', + '--max-time', '60' + ]; + // 添加请求头 + foreach (self::formatHeaders($headers) as $header) { + array_push($cmd, '-H', escapeshellarg($header)); + } + + // 处理请求体 + if ($method !== 'GET' and $data) { + $jsonData = json_encode($data, JSON_UNESCAPED_UNICODE); + array_push($cmd, '--data', escapeshellarg($jsonData)); + } + + $cmd[] = escapeshellarg($url); + $output = []; + $resultCode = $this->execute(implode(' ', $cmd), $output); + // 处理命令行执行错误 + if ($resultCode !== 0) { + throw new PhpyException("cURL command failed (exit $resultCode): " . implode("\n", $output), $resultCode); + } + $rawResponse = implode('', array_reverse($output)); + if (preg_match('/^(?.*)HTTP_CODE=(?\d+)$/s', $rawResponse, $matches)) { + return [ + 'httpCode' => (int)$matches['http_code'], + 'responseBody' => (string)$matches['response_body'] + ]; + } else { + throw new PhpyException("Invalid response format: $rawResponse"); + } + } + + /** + * headers map to headers + * + * @param array $headers + * @return array + */ + private static function formatHeaders(array $headers): array + { + return array_map( + fn($k, $v) => "$k: $v", + array_keys($headers), + array_values($headers) + ); + } + } diff --git a/tools/src/Phpy/Helpers/PythonMetadata.php b/tools/src/Phpy/Helpers/PythonMetadata.php index 8c673b6..0683854 100644 --- a/tools/src/Phpy/Helpers/PythonMetadata.php +++ b/tools/src/Phpy/Helpers/PythonMetadata.php @@ -2,116 +2,126 @@ namespace PhpyTool\Phpy\Helpers; +use PhpyTool\Phpy\Exceptions\PhpyException; + class PythonMetadata { - public static function isStdLibrary(string $module) + /** @var string */ + protected static string $apiKey = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRmYmpnbnBtdGx6eXRiZXp3dGR4Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDI5ODQ5MjQsImV4cCI6MjA1ODU2MDkyNH0.trmgSPSI55IRnU4ko1E6gJQNi-kyXbobsl61bkRZVBE'; + + /** + * 判断是否是标准库 + * + * @param string $module + * @return bool + */ + public static function isStdLibrary(string $module): bool { static $stdLibrary = null; if (!$stdLibrary) { - $stdLibrary = array_flip(self::getStdLibrary()); + $stdLibrary = array_flip(self::getPythonStdModules()); } return isset($stdLibrary[$module]); } - private static function getStdLibrary(): array + /** + * 通过系统函数获取python标准库 + * + * @return array + */ + public static function getPythonStdModules(): array + { + $pythonCode = <<<'PYTHON' +import sys, json +print(json.dumps(sorted(sys.stdlib_module_names))) +PYTHON; + // 执行命令并捕获输出 + (new Process())->executePython("-c '$pythonCode'", $output); + return json_decode($output[0], true) ?? []; + } + + /** + * 上报metadata + * + * @param string $topLevel + * @param string $moduleName + * @param string $version + * @return array + */ + public static function pushMetadata(string $topLevel, string $moduleName, string $version = 'default'): array { - return [ - "os", - "sys", - "re", - "math", - "random", - "dataclasses", - "json", - "datetime", - "time", - "calendar", - "collections", - "heapq", - "bisect", - "array", - "weakref", - "types", - "copy", - "pprint", - "reprlib", - "enum", - "numbers", - "textwrap", - "this", - "cmath", - "decimal", - "fractions", - "statistics", - "itertools", - "functools", - "operator", - "pathlib", - "fileinput", - "stat", - "filecmp", - "tempfile", - "glob", - "fnmatch", - "linecache", - "shutil", - "macpath", - "pickle", - "copyreg", - "shelve", - "marshal", - "dbm", - "sqlite3", - "zlib", - "gzip", - "bz2", - "lzma", - "zipfile", - "tarfile", - "csv", - "configparser", - "netrc", - "xdrlib", - "plistlib", - "hashlib", - "hmac", - "secrets", - "builtins", - "platform", - "asyncio", - "socket", - "turtle", - "typing", - "tkinter", - ]; + $res = (new Process())->request( + 'POST', + "https://tfbjgnpmtlzytbezwtdx.supabase.co/rest/v1/rpc/push_metadata", + [ + 'p_module_name' => $moduleName, + 'p_top_level' => $topLevel, + 'p_version' => $version + ], + [ + 'Content-Type' => 'application/json', + 'apikey' => static::$apiKey, + 'Authorization' => 'Bearer ' . static::$apiKey + ] + ); + if ($res['httpCode'] !== 200) { + throw new PhpyException("push metadata failed: {$res['responseBody']}"); + } + return json_decode($res['responseBody'], true) ?? []; } - static function getPipPackage(string $module): ?string + /** + * 根据topLevel查询对应的模块信息 + * + * @param string $topLevel + * @param int $limit + * @return array + */ + public static function queryByTopLevel(string $topLevel, int $limit = 0): array { - static $map = array( - "torch" => "torch", - "transformers" => "transformers", - "accelerate" => "accelerate", - "TermTk" => "pytermtk", - "paddlenlp" => "paddlenlp", - "cefpython3" => "cefpython3", - "dearpygui" => "dearpygui", - "pygame" => "pygame", - "gi" => "PyGObject", - "gradio_client" => "gradio_client", - "gradio" => "gradio", - "openai" => "openai", - "webview" => "pywebview", - "numpy" => "numpy", - "magicgui" => "magicgui", - "modelscope" => "modelscope", - "tqdm" => "tqdm", - "llama_index" => "llama-index", - "wx" => "wxPython", - "pyqt5" => "PyQt5", + $res = (new Process())->request( + 'POST', + "https://tfbjgnpmtlzytbezwtdx.supabase.co/rest/v1/rpc/query_by_top_level", + [ + 'p_top_level' => $topLevel, + 'p_limit' => $limit + ], + [ + 'Content-Type' => 'application/json', + 'apikey' => static::$apiKey, + 'Authorization' => 'Bearer ' . static::$apiKey + ] ); + if ($res['httpCode'] !== 200) { + throw new PhpyException("push metadata failed: {$res['responseBody']}"); + } + return json_decode($res['responseBody'], true) ?? []; + } - [$root] = explode('.', $module); - return $map[$root] ?? null; + + /** + * 根据topLevel查询对应的模块信息 + * + * @param string $nameString + * @param string $version + * @return string|null + */ + static function getPipPackage(string $nameString, string $version = 'default'): ?string + { + static $metadataMap = []; + [$topLevel] = explode('.', $nameString); + if (!isset($metadataMap[$topLevel])) { + $moduleName = null; + if ($query = static::queryByTopLevel($topLevel)) { + foreach ($query as $item) { + if ($version === $item['version']) { + $moduleName = $item['module_name']; + break; + } + } + } + $metadataMap[$topLevel] = $moduleName; + } + return $metadataMap[$topLevel] ?? null; } } \ No newline at end of file diff --git a/tools/src/Phpy/Helpers/Version.php b/tools/src/Phpy/Helpers/Version.php index 0835d40..dbf1215 100644 --- a/tools/src/Phpy/Helpers/Version.php +++ b/tools/src/Phpy/Helpers/Version.php @@ -5,18 +5,121 @@ namespace PhpyTool\Phpy\Helpers; use Composer\Semver\VersionParser; +use PhpyTool\Phpy\Exceptions\PhpyException; class Version { /** - * @param string $version + * 获取模块的PEP可用版本列表 + * + * @param string $module + * @return array + */ + public static function getPepVersions(string $module): array + { + static $modulePepVersions = []; + if (isset($modulePepVersions[$module])) { + $res = (new Process())->request( + 'GET', + "https://pypi.org/pypi/$module/json", + [], + [ + 'Content-Type' => 'application/json' + ] + ); + $httpCode = $res['httpCode'] ?? 500; + + $res = json_decode($responseBody = $res['responseBody'], true); + if ($httpCode === 404) { + $modulePepVersions[$module] = []; + } else if ($httpCode === 200) { + $modulePepVersions[$module] = array_keys($res['releases'] ?? []); + } else { + throw new PhpyException("Request failed: $responseBody"); + } + } + return $modulePepVersions[$module] ?? []; + } + + /** + * 将符合 PEP 440 稳定版格式（含有限预发行标识）的版本号转换为 SemVer 格式。 + * + * 规则说明： + * 1. 如果版本中包含 epoch（例如 "1!1.2.3"）或 post-release（例如 "1.2.3.post1"），返回 null。 + * 2. 仅支持纯数字部分和预发行部分（预发行部分仅允许 a/alpha、b/beta、rc），格式必须完全符合要求。 + * 3. 数字部分：若不足三段，则补 0；若超过三段，则将第三段及以后部分合并为 patch， + * 如 "2.3.4.5" 转换为 "2.3.45"。 + * 4. 预发行部分转换为 SemVer 格式，形式为 "-