diff --git a/doc/changes/fixed/10123.md b/doc/changes/fixed/10123.md new file mode 100644 index 00000000000..8650275122a --- /dev/null +++ b/doc/changes/fixed/10123.md @@ -0,0 +1,5 @@ +- Fix package extraction on systems with tar implementations that don't + auto-detect compression (e.g., OpenBSD). Dune now passes explicit + decompression flags (-z for gzip, -j for bzip2) when needed, and provides + clear error messages for unsupported formats like XZ and LZMA. (#13688, + fixes #10123, @Alizter) diff --git a/src/dune_pkg/archive_driver.ml b/src/dune_pkg/archive_driver.ml index 81bd87e984e..837bd3e057f 100644 --- a/src/dune_pkg/archive_driver.ml +++ b/src/dune_pkg/archive_driver.ml @@ -2,6 +2,15 @@ open Stdune module Process = Dune_engine.Process open Fiber.O +(** Tar implementation type, detected via --version output: + - Libarchive: libarchive/bsdtar - auto-detects compression, can extract zip + - Gnu: GNU tar - auto-detects compression, no zip support + - Other: Unknown (e.g. OpenBSD) - needs explicit -z/-j flags *) +type tar_impl = + | Libarchive + | Gnu + | Other + module Command = struct type t = { bin : Path.t @@ -16,8 +25,55 @@ type t = let which bin_name = Bin.which ~path:(Env_path.path Env.initial) bin_name -let make_tar_args ~archive ~target_in_temp = - [ "xf"; Path.to_string archive; "-C"; Path.to_string target_in_temp ] +(* Regexes for detecting tar implementation from --version output *) +let libarchive_re = Re.compile (Re.alt [ Re.str "bsdtar"; Re.str "libarchive" ]) +let gnu_tar_re = Re.compile (Re.str "GNU tar") + +(** Detect tar implementation by running --version *) +let detect_tar_impl bin = + let+ output, _ = Process.run_capture ~display:Quiet Return bin [ "--version" ] in + if Re.execp libarchive_re output + then Libarchive + else if Re.execp gnu_tar_re output + then Gnu + else Other +;; + +(** Generate tar arguments, adding -z/-j for non-auto-detecting tar *) +let make_tar_args ~tar_impl ~archive ~target_in_temp = + let decompress_flag = + match tar_impl with + | Libarchive | Gnu -> [] (* auto-detect compression *) + | Other -> + let has_suffix = Filename.check_suffix (Path.to_string archive) in + (* Need explicit flags for tar implementations that don't auto-detect *) + if has_suffix ".tar.gz" || has_suffix ".tgz" + then [ "-z" ] + else if has_suffix ".tar.bz2" || has_suffix ".tbz" + then [ "-j" ] + else if has_suffix ".tar.xz" || has_suffix ".txz" + then + User_error.raise + ~hints:[ Pp.text "Install GNU tar or bsdtar (libarchive) for XZ support." ] + [ Pp.textf "Cannot extract '%s'" (Path.basename archive) + ; Pp.text + "The detected tar does not support XZ decompression. XZ archives require \ + GNU tar or libarchive." + ] + else if has_suffix ".tar.lzma" || has_suffix ".tlz" + then + User_error.raise + ~hints:[ Pp.text "Install GNU tar or bsdtar (libarchive) for LZMA support." ] + [ Pp.textf "Cannot extract '%s'" (Path.basename archive) + ; Pp.text + "The detected tar does not support LZMA decompression. LZMA archives \ + require GNU tar or libarchive." + ] + else [] + in + [ "-x" ] + @ decompress_flag + @ [ "-f"; Path.to_string archive; "-C"; Path.to_string target_in_temp ] ;; let make_zip_args ~archive ~target_in_temp = @@ -27,12 +83,10 @@ let make_zip_args ~archive ~target_in_temp = let tar = let command = Fiber.Lazy.create (fun () -> - match - (* Test for tar before bsdtar as tar is more likely to be installed - and both work equally well for tarballs. *) - List.find_map [ "tar"; "bsdtar" ] ~f:which - with - | Some bin -> Fiber.return { Command.bin; make_args = make_tar_args } + match List.find_map [ "tar"; "bsdtar" ] ~f:which with + | Some bin -> + let+ tar_impl = detect_tar_impl bin in + { Command.bin; make_args = make_tar_args ~tar_impl } | None -> Fiber.return @@ User_error.raise @@ -40,16 +94,31 @@ let tar = ; Pp.enumerate [ "tar"; "bsdtar" ] ~f:Pp.verbatim ]) in - { command; suffixes = [ ".tar"; ".tar.gz"; ".tgz"; ".tar.bz2"; ".tbz" ] } + { command + ; suffixes = + [ ".tar" + ; ".tar.gz" + ; ".tgz" + ; ".tar.bz2" + ; ".tbz" + ; ".tar.xz" + ; ".txz" + ; ".tar.lzma" + ; ".tlz" + ] + } ;; -let which_bsdtar (bin_name : string) = - match which bin_name with - | None -> Fiber.return None - | Some bin -> - let+ output, _error = Process.run_capture ~display:Quiet Return bin [ "--version" ] in - let re = Re.compile (Re.str "bsdtar") in - if Re.execp re output then Some bin else None +let rec find_libarchive_tar = function + | [] -> Fiber.return None + | name :: rest -> + (match which name with + | None -> find_libarchive_tar rest + | Some bin -> + detect_tar_impl bin + >>= (function + | Libarchive -> Fiber.return (Some bin) + | Gnu | Other -> find_libarchive_tar rest)) ;; let zip = @@ -58,18 +127,11 @@ let zip = match which "unzip" with | Some bin -> Fiber.return { Command.bin; make_args = make_zip_args } | None -> - let rec find_tar programs = - match programs with - | [] -> Fiber.return None - | x :: xs -> - let* res = which_bsdtar x in - (match res with - | Some _ -> Fiber.return res - | None -> find_tar xs) - in - let* program = find_tar [ "bsdtar"; "tar" ] in + (* Only libarchive can extract zip, so find a tar that is libarchive *) + let* program = find_libarchive_tar [ "bsdtar"; "tar" ] in (match program with - | Some bin -> Fiber.return { Command.bin; make_args = make_tar_args } + | Some bin -> + Fiber.return { Command.bin; make_args = make_tar_args ~tar_impl:Libarchive } | None -> Fiber.return @@ User_error.raise diff --git a/test/blackbox-tests/test-cases/pkg/openbsd-tar.t b/test/blackbox-tests/test-cases/pkg/openbsd-tar.t index 5cce42b0171..cd516c82bd2 100644 --- a/test/blackbox-tests/test-cases/pkg/openbsd-tar.t +++ b/test/blackbox-tests/test-cases/pkg/openbsd-tar.t @@ -1,4 +1,4 @@ -Test that demonstrates the issue with OpenBSD-style tar (#10123). +Test that OpenBSD-style tar works when dune passes explicit flags (#10123). Unlike GNU tar and BSD tar (libarchive), OpenBSD tar requires explicit flags for compressed archives: -z for gzip, -j for bzip2, etc. @@ -17,17 +17,17 @@ Set up a fake tar that behaves like OpenBSD tar: > echo "tar (OpenBSD)" > ;; > *) - > # Dune passes: xf archive -C target (or xzf archive -C target with fix) - > flags="$1" - > archive="$2" - > target="$4" + > # Dune passes: -x -z -f archive -C target (or -x -j -f ... for bzip2) + > # $4 = archive, $6 = target + > archive="$4" + > target="$6" > # Require correct flag for compressed archives > case "$archive" in > *.tar.gz|*.tgz) - > echo "$flags" | grep -q 'z' || { echo "tar: Cannot open: compressed archive requires -z flag" >&2; exit 1; } + > echo "$@" | grep -q '\-z' || { echo "tar: Cannot open: compressed archive requires -z flag" >&2; exit 1; } > ;; > *.tar.bz2|*.tbz) - > echo "$flags" | grep -q 'j' || { echo "tar: Cannot open: compressed archive requires -j flag" >&2; exit 1; } + > echo "$@" | grep -q '\-j' || { echo "tar: Cannot open: compressed archive requires -j flag" >&2; exit 1; } > ;; > esac > # Success - create fake extracted content @@ -48,7 +48,17 @@ Set up fake PATH with only our OpenBSD-style tar: $ ln -s $(which grep) .fakebin/grep $ ln -s ../.binaries/openbsd-tar .fakebin/tar -Test with a .tar.gz file - fails because dune doesn't pass -z flag: +Helper to show tar extract args from trace (filters to -x calls only): + + $ tar_extract_args() { + > dune trace cat | jq -c 'include "dune"; processes + > | select(.args.prog | contains("tar")) + > | select(.args.process_args | index("-x")) + > | .args.process_args + > | map(if (startswith("/") or startswith("_build")) then "PATH" else . end)' + > } + +Test with a .tar.gz file: $ echo "fake tarball" > test.tar.gz @@ -59,13 +69,14 @@ Test with a .tar.gz file - fails because dune doesn't pass -z flag: > (version dev) > EOF - $ PATH=.fakebin build_pkg foo 2>&1 | grep -A2 '^Error' - Error: failed to extract 'test.tar.gz' - Reason: 'tar' failed with non-zero exit code '1' and output: - - tar: Cannot open: compressed archive requires -z flag - [1] + $ PATH=.fakebin build_pkg foo -Test with a .tbz file - fails because dune doesn't pass -j flag: +Verify that -z flag was passed: + + $ tar_extract_args + ["-x","-z","-f","PATH","-C","PATH"] + +Test with a .tbz file (bzip2 compressed): $ echo "fake tarball" > test.tbz @@ -76,8 +87,47 @@ Test with a .tbz file - fails because dune doesn't pass -j flag: > (version dev) > EOF - $ PATH=.fakebin build_pkg bar 2>&1 | grep -A2 '^Error' - Error: failed to extract 'test.tbz' - Reason: 'tar' failed with non-zero exit code '1' and output: - - tar: Cannot open: compressed archive requires -j flag + $ PATH=.fakebin build_pkg bar + +Verify that -j flag was passed: + + $ tar_extract_args + ["-x","-j","-f","PATH","-C","PATH"] + +Test with a .tar.xz file - should error with helpful message since OpenBSD tar +doesn't support XZ: + + $ echo "fake tarball" > test.tar.xz + + $ make_lockpkg xzpkg < (source + > (fetch + > (url "file://$PWD/test.tar.xz"))) + > (version dev) + > EOF + + $ PATH=.fakebin build_pkg xzpkg 2>&1 | dune_cmd delete '^File |^ *[0-9]+ \||^ +\^' + Error: Cannot extract 'test.tar.xz' + The detected tar does not support XZ decompression. XZ archives require GNU + tar or libarchive. + Hint: Install GNU tar or bsdtar (libarchive) for XZ support. + [1] + +Test with a .tar.lzma file - should error with helpful message since OpenBSD tar +doesn't support LZMA: + + $ echo "fake tarball" > test.tar.lzma + + $ make_lockpkg lzmapkg < (source + > (fetch + > (url "file://$PWD/test.tar.lzma"))) + > (version dev) + > EOF + + $ PATH=.fakebin build_pkg lzmapkg 2>&1 | dune_cmd delete '^File |^ *[0-9]+ \||^ +\^' + Error: Cannot extract 'test.tar.lzma' + The detected tar does not support LZMA decompression. LZMA archives require + GNU tar or libarchive. + Hint: Install GNU tar or bsdtar (libarchive) for LZMA support. [1] diff --git a/test/blackbox-tests/test-cases/pkg/zip-extract-fail.t b/test/blackbox-tests/test-cases/pkg/zip-extract-fail.t index 0d0c842055e..994b20e429a 100644 --- a/test/blackbox-tests/test-cases/pkg/zip-extract-fail.t +++ b/test/blackbox-tests/test-cases/pkg/zip-extract-fail.t @@ -10,10 +10,19 @@ expected location: > #!/usr/bin/env sh > case "$1" in > --version) - > echo "tar" + > echo "GNU tar" > ;; > *) - > cp "$2" "$4/$(basename "${2%.zip}")" + > # Parse: -x -f archive -C target + > archive=""; target="" + > while [ $# -gt 0 ]; do + > case "$1" in + > -f) archive="$2"; shift 2 ;; + > -C) target="$2"; shift 2 ;; + > *) shift ;; + > esac + > done + > cp "$archive" "$target/$(basename "${archive%.zip}")" > esac > EOF $ chmod +x .binaries/gnutar @@ -24,13 +33,23 @@ expected location: > echo "bsdtar" > ;; > *) - > cp "$2" "$4/$(basename "${2%.zip}")" + > # Parse: -x -f archive -C target + > archive=""; target="" + > while [ $# -gt 0 ]; do + > case "$1" in + > -f) archive="$2"; shift 2 ;; + > -C) target="$2"; shift 2 ;; + > *) shift ;; + > esac + > done + > cp "$archive" "$target/$(basename "${archive%.zip}")" > esac > EOF $ chmod +x .binaries/bsdtar $ cat > .binaries/unzip << 'EOF' > #!/usr/bin/env sh - > cp "$2" "$4/$(basename "${2%.zip}")" + > # unzip archive -d target + > cp "$1" "$3/$(basename "${1%.zip}")" > EOF $ chmod +x .binaries/unzip @@ -40,6 +59,7 @@ Set up a folder that we will inject as fake PATH: $ ln -s $(which dune) .fakebin/dune $ ln -s $(which sh) .fakebin/sh $ ln -s $(which cp) .fakebin/cp + $ ln -s $(which basename) .fakebin/basename $ show_path() { > ls .fakebin | sort | xargs > } @@ -66,7 +86,7 @@ Build the package in an environment without unzip, or tar, or bsdtar. variable can escape to subseqent shell invocations on MacOS.) $ show_path - cp dune sh + basename cp dune sh $ (PATH=.fakebin build_pkg foo 2>&1 | grep '^Error:' -A 3) Error: No program found to extract zip file. Tried: - unzip @@ -78,7 +98,7 @@ Build with only GNU tar that can't extract ZIP archives: $ ln -s ../.binaries/gnutar .fakebin/tar $ show_path - cp dune sh tar + basename cp dune sh tar $ (PATH=.fakebin build_pkg foo 2>&1 | grep '^Error:' -A 3) Error: No program found to extract zip file. Tried: - unzip @@ -91,7 +111,7 @@ Build with bsdtar that can extract ZIP archives, without unzip. It should work: $ rm .fakebin/tar $ ln -s ../.binaries/bsdtar .fakebin/tar $ show_path - cp dune sh tar + basename cp dune sh tar $ (PATH=.fakebin build_pkg foo) Build the package with bsdtar and tar. Now our fake bsdtar will get picked up @@ -101,7 +121,7 @@ and used to extract: $ ln -s ../.binaries/gnutar .fakebin/tar $ ln -s ../.binaries/bsdtar .fakebin/bsdtar $ show_path - bsdtar cp dune sh tar + basename bsdtar cp dune sh tar $ (PATH=.fakebin build_pkg foo) Build with unzip only: @@ -109,5 +129,5 @@ Build with unzip only: $ ln -s ../.binaries/unzip .fakebin/unzip $ rm .fakebin/bsdtar .fakebin/tar $ show_path - cp dune sh unzip + basename cp dune sh unzip $ (PATH=.fakebin build_pkg foo)