From 76cbcc19a6ca76f4d5aeb903195c0927ac19a7da Mon Sep 17 00:00:00 2001 From: Steve Dignam Date: Tue, 14 Apr 2026 22:28:54 -0400 Subject: [PATCH] internal: combine sync-kwlist and sync-regression-suite into sync-pg --- Cargo.lock | 1 - crates/xtask/Cargo.toml | 1 - crates/xtask/src/main.rs | 15 +- crates/xtask/src/sync_kwlist.rs | 68 ------- .../{sync_regression_suite.rs => sync_pg.rs} | 168 ++++++++---------- postgres/kwlist.h | 10 +- 6 files changed, 80 insertions(+), 183 deletions(-) delete mode 100644 crates/xtask/src/sync_kwlist.rs rename crates/xtask/src/{sync_regression_suite.rs => sync_pg.rs} (84%) diff --git a/Cargo.lock b/Cargo.lock index d5e6915ba..6ea546805 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3431,7 +3431,6 @@ dependencies = [ "proc-macro2", "quote", "regex", - "reqwest", "rustc-hash 2.1.1", "serde", "serde_json", diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml index 2a7a32e05..ff3c915e5 100644 --- a/crates/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -13,7 +13,6 @@ anyhow.workspace = true csv.workspace = true clap.workspace = true enum-iterator.workspace = true -reqwest = { workspace = true, features = ["blocking", "json"] } serde.workspace = true serde_json.workspace = true convert_case.workspace = true diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index fe172ed85..d6b74bf7c 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -4,27 +4,23 @@ use clap::{Args, Parser, Subcommand}; use codegen::codegen; use new_rule::new_lint; use sync_builtins::sync_builtins; -use sync_kwlist::sync_kwlist; -use sync_regression_suite::sync_regression_suite; +use sync_pg::sync_pg; mod codegen; mod keywords; mod new_rule; mod path; mod sync_builtins; -mod sync_kwlist; -mod sync_regression_suite; +mod sync_pg; #[derive(Subcommand, Debug)] enum TaskName { #[command(long_about = "Generate code for AST, SyntaxKind, and TokenSets")] Codegen, - #[command(long_about = "Fetch the latest version of kwlist.h from Postgres")] - SyncKwlist, #[command(long_about = "Create a new linter rule")] NewRule(NewRuleArgs), - #[command(long_about = "Fetch the latest regression suite from Postgres")] - SyncRegressionSuite, + #[command(long_about = "Fetch the latest kwlist.h and regression suite from Postgres")] + SyncPg, #[command(long_about = "Generate builtins.sql from PostgreSQL pg_type catalog")] SyncBuiltins, } @@ -46,8 +42,7 @@ struct Arguments { fn main() -> Result<()> { let args = Arguments::parse(); match args.task { - TaskName::SyncKwlist => sync_kwlist(), - TaskName::SyncRegressionSuite => sync_regression_suite(), + TaskName::SyncPg => sync_pg(), TaskName::NewRule(args) => new_lint(args), TaskName::Codegen => codegen(), TaskName::SyncBuiltins => sync_builtins(), diff --git a/crates/xtask/src/sync_kwlist.rs b/crates/xtask/src/sync_kwlist.rs deleted file mode 100644 index 4e19e36ab..000000000 --- a/crates/xtask/src/sync_kwlist.rs +++ /dev/null @@ -1,68 +0,0 @@ -use anyhow::Result; -use reqwest::header::USER_AGENT; -use serde::Deserialize; - -pub(crate) fn sync_kwlist() -> Result<()> { - latest_pg_git_sha()?; - Ok(()) -} - -use std::fs; -use std::io::Write; - -use crate::path::project_root; - -#[derive(Deserialize, Debug)] -struct CommitResponse { - sha: String, - commit: Commit, -} - -#[derive(Deserialize, Debug)] -struct Commit { - committer: Commiter, -} - -#[derive(Deserialize, Debug)] -struct Commiter { - date: String, -} - -fn latest_pg_git_sha() -> Result<()> { - let client = reqwest::blocking::Client::new(); - let response = client - .get("https://api.github.com/repos/postgres/postgres/commits/master") - .header(USER_AGENT, "squawk xtask sync-kwlist") - .send()?; - - let res: CommitResponse = response.json()?; - - let file_response = client - .get(format!( - "https://raw.githubusercontent.com/postgres/postgres/{}/src/include/parser/kwlist.h", - res.sha - )) - .send()?; - - let file_content = file_response.text()?; - - let preamble = format!( - r"// synced from: -// commit: {} -// committed at: {} -// file: https://github.com/postgres/postgres/blob/{}/src/include/parser/kwlist.h -// -// update via: -// cargo xtask sync-kwlist - -", - res.sha, res.commit.committer.date, res.sha - ) - .to_owned(); - - let kwlist_file = project_root().join("postgres/kwlist.h"); - let mut file = fs::File::create(kwlist_file)?; - file.write_all((preamble + &file_content).as_bytes())?; - - Ok(()) -} diff --git a/crates/xtask/src/sync_regression_suite.rs b/crates/xtask/src/sync_pg.rs similarity index 84% rename from crates/xtask/src/sync_regression_suite.rs rename to crates/xtask/src/sync_pg.rs index 47ec39c0f..b5421ee57 100644 --- a/crates/xtask/src/sync_regression_suite.rs +++ b/crates/xtask/src/sync_pg.rs @@ -1,13 +1,14 @@ use crate::path::project_root; -use anyhow::{Result, bail}; -use camino::Utf8PathBuf; +use anyhow::Result; +use camino::{Utf8Path, Utf8PathBuf}; use regex::Regex; use std::fs::{File, create_dir_all, remove_dir_all}; use std::io::{BufRead, Write}; -use std::process::Command; +use xshell::{Shell, cmd}; const SQL_REGRESSION_SUITE_DIR: &str = "postgres/regression_suite"; const PLPGSQL_REGRESSION_SUITE_DIR: &str = "postgres/plpgsql"; +const KWLIST_PATH: &str = "postgres/kwlist.h"; const START_END_MARKERS: &[(&str, &str)] = &[ ( @@ -109,33 +110,21 @@ const GSET_REPLACEMENTS: &[(&str, &str)] = &[ ), ]; -pub(crate) fn sync_regression_suite() -> Result<()> { - let sql_target_dir = Utf8PathBuf::try_from(std::env::temp_dir()) - .map_err(|_| anyhow::anyhow!("temp dir path is not valid UTF-8"))? - .join("squawk_raw_regression_suite"); +pub(crate) fn sync_pg() -> Result<()> { + let clone_dir = clone_postgres()?; + let (sha, date) = git_head_info(&clone_dir)?; - let plpgsql_target_dir = Utf8PathBuf::try_from(std::env::temp_dir()) - .map_err(|_| anyhow::anyhow!("temp dir path is not valid UTF-8"))? - .join("squawk_raw_plpgsql_suite"); + sync_kwlist(&clone_dir, &sha, &date)?; + sync_regression_suite(&clone_dir)?; + sync_plpgsql_suite(&clone_dir)?; + + println!("Cleaning up clone directory..."); + remove_dir_all(&clone_dir)?; - download_regression_suite(&sql_target_dir, &plpgsql_target_dir)?; - transform_regression_suite(&sql_target_dir)?; - copy_plpgsql_suite(&plpgsql_target_dir)?; Ok(()) } -fn download_regression_suite( - sql_target_dir: &Utf8PathBuf, - plpgsql_target_dir: &Utf8PathBuf, -) -> Result<()> { - for dir in [sql_target_dir, plpgsql_target_dir] { - if dir.exists() { - println!("Cleaning temp directory: {dir:?}"); - remove_dir_all(dir)?; - } - create_dir_all(dir)?; - } - +fn clone_postgres() -> Result { let clone_dir = Utf8PathBuf::try_from(std::env::temp_dir()) .map_err(|_| anyhow::anyhow!("temp dir path is not valid UTF-8"))? .join("postgres_sparse_clone"); @@ -144,94 +133,78 @@ fn download_regression_suite( remove_dir_all(&clone_dir)?; } - println!("Cloning postgres repository with sparse checkout..."); + let sh = Shell::new()?; + let url = "https://github.com/postgres/postgres.git"; + let clone_dir_str = clone_dir.as_str(); - let status = Command::new("git") - .args([ - "clone", - "--filter=blob:none", - "--depth=1", - "--sparse", - "https://github.com/postgres/postgres.git", - ]) - .arg(clone_dir.as_str()) - .status()?; - - if !status.success() { - bail!("Failed to clone postgres repository"); - } + println!("Cloning postgres repository with sparse checkout..."); + cmd!( + sh, + "git clone --filter=blob:none --depth=1 --sparse {url} {clone_dir_str}" + ) + .run()?; println!("Setting up sparse checkout..."); + sh.change_dir(&clone_dir); + cmd!( + sh, + "git sparse-checkout set --no-cone /src/test/regress/sql /src/pl/plpgsql/src/sql /src/include/parser/kwlist.h" + ) + .run()?; + + Ok(clone_dir) +} - let status = Command::new("git") - .args([ - "sparse-checkout", - "set", - "src/test/regress/sql", - "src/pl/plpgsql/src/sql", - ]) - .current_dir(&clone_dir) - .status()?; - - if !status.success() { - bail!("Failed to set sparse checkout"); - } - - println!("Copying regression SQL files..."); - let source_dir = clone_dir.join("src/test/regress/sql"); - - let mut file_count = 0; - for entry in std::fs::read_dir(&source_dir)? { - let entry = entry?; - let path = Utf8PathBuf::try_from(entry.path())?; - if path.extension() == Some("sql") { - let filename = path.file_name().unwrap(); - if !filename.contains("psql") || filename == "plpgsql.sql" { - std::fs::copy(&path, sql_target_dir.join(filename))?; - file_count += 1; - } - } - } - - println!("Copied {file_count} regression SQL files"); - - println!("Copying PL/pgSQL SQL files..."); - let plpgsql_source_dir = clone_dir.join("src/pl/plpgsql/src/sql"); - let mut plpgsql_count = 0; - for entry in std::fs::read_dir(&plpgsql_source_dir)? { - let entry = entry?; - let path = Utf8PathBuf::try_from(entry.path())?; - if path.extension() == Some("sql") { - let filename = path.file_name().unwrap(); - std::fs::copy(&path, plpgsql_target_dir.join(filename))?; - plpgsql_count += 1; - } - } - - println!("Copied {plpgsql_count} PL/pgSQL SQL files"); - - println!("Cleaning up clone directory..."); - remove_dir_all(&clone_dir)?; +fn git_head_info(clone_dir: &Utf8Path) -> Result<(String, String)> { + let sh = Shell::new()?; + sh.change_dir(clone_dir); + let sha = cmd!(sh, "git rev-parse HEAD").read()?; + let date = cmd!(sh, "git log -1 --format=%cI").read()?; + Ok((sha, date)) +} +fn sync_kwlist(clone_dir: &Utf8Path, sha: &str, date: &str) -> Result<()> { + println!("Syncing kwlist.h..."); + let source = clone_dir.join("src/include/parser/kwlist.h"); + let file_content = std::fs::read_to_string(&source)?; + + let preamble = format!( + r"// synced from: +// commit: {sha} +// committed at: {date} +// file: https://github.com/postgres/postgres/blob/{sha}/src/include/parser/kwlist.h +// +// update via: +// cargo xtask sync-pg + +" + ); + + let kwlist_file = project_root().join(KWLIST_PATH); + let mut file = File::create(kwlist_file)?; + file.write_all((preamble + &file_content).as_bytes())?; Ok(()) } -fn transform_regression_suite(input_dir: &Utf8PathBuf) -> Result<()> { +fn sync_regression_suite(clone_dir: &Utf8Path) -> Result<()> { + let source_dir = clone_dir.join("src/test/regress/sql"); let output_dir = project_root().join(SQL_REGRESSION_SUITE_DIR); if output_dir.exists() { println!("Cleaning target directory: {output_dir:?}"); remove_dir_all(&output_dir)?; } - create_dir_all(&output_dir)?; let mut files: Vec = vec![]; - for entry in std::fs::read_dir(input_dir)? { + for entry in std::fs::read_dir(&source_dir)? { let entry = entry?; let path = Utf8PathBuf::try_from(entry.path())?; if path.extension() == Some("sql") { - files.push(path); + let filename = path.file_name().unwrap(); + if !filename.contains("psql") || filename == "plpgsql.sql" { + files.push(path); + } } } @@ -260,18 +233,18 @@ fn transform_regression_suite(input_dir: &Utf8PathBuf) -> Result<()> { Ok(()) } -fn copy_plpgsql_suite(input_dir: &Utf8PathBuf) -> Result<()> { +fn sync_plpgsql_suite(clone_dir: &Utf8Path) -> Result<()> { + let source_dir = clone_dir.join("src/pl/plpgsql/src/sql"); let output_dir = project_root().join(PLPGSQL_REGRESSION_SUITE_DIR); if output_dir.exists() { println!("Cleaning target directory: {output_dir:?}"); remove_dir_all(&output_dir)?; } - create_dir_all(&output_dir)?; let mut file_count = 0; - for entry in std::fs::read_dir(input_dir)? { + for entry in std::fs::read_dir(&source_dir)? { let entry = entry?; let path = Utf8PathBuf::try_from(entry.path())?; if path.extension() == Some("sql") { @@ -282,7 +255,6 @@ fn copy_plpgsql_suite(input_dir: &Utf8PathBuf) -> Result<()> { } println!("Copied {file_count} PL/pgSQL files to {output_dir}"); - Ok(()) } @@ -488,7 +460,7 @@ mod tests { } #[test] - fn test_replacement() { + fn replacement() { let cases = [ ( "SELECT * FROM foo WHERE bar = :'foo' AND baz = :baz;", diff --git a/postgres/kwlist.h b/postgres/kwlist.h index ab6ce56a2..688ae2b27 100644 --- a/postgres/kwlist.h +++ b/postgres/kwlist.h @@ -1,10 +1,10 @@ -// synced from: -// commit: effaa464afd355e8927bf430cfe6a0ddd2ee5695 -// committed at: 2026-04-02T12:39:57Z -// file: https://github.com/postgres/postgres/blob/effaa464afd355e8927bf430cfe6a0ddd2ee5695/src/include/parser/kwlist.h +// synced from: +// commit: 972c14fb9134fdfd76ea6ebcf98a55a945bbc988 +// committed at: 2026-04-14T21:06:27-04:00 +// file: https://github.com/postgres/postgres/blob/972c14fb9134fdfd76ea6ebcf98a55a945bbc988/src/include/parser/kwlist.h // // update via: -// cargo xtask sync-kwlist +// cargo xtask sync-pg /*------------------------------------------------------------------------- *