diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..700707c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..7f07941 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,54 @@ +name: CI +on: + push: + branches: + - master + tags: ['*'] + pull_request: + branches: + - master +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - 'lts' + - '1' + - 'nightly' + os: + - ubuntu-latest + - windows-latest + - macos-latest + arch: + - default + - x86 + exclude: + - os: macos-latest + arch: x86 + include: + - os: ubuntu-latest + version: 'min' + arch: x64 + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + show-versioninfo: true + - uses: julia-actions/cache@v3 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..90dc100 --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,33 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + inputs: + lookback: + default: 3 +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + # Edit the following line to reflect the actual name of the GitHub Secret containing your private key + ssh: ${{ secrets.DOCUMENTER_KEY }} + # ssh: ${{ secrets.NAME_OF_MY_SSH_PRIVATE_KEY_SECRET }} diff --git a/.gitignore b/.gitignore index f6d60ae..dd85182 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,14 @@ +#Mac OS X +*.DS_Store + +#VS Code +/.vscode/ + *.jl.cov *.jl.*.cov *.jl.mem /Manifest.toml +/test/Manifest.toml /.envrc /deps/deps.jl /deps/build.log diff --git a/Project.toml b/Project.toml index 4003857..6d7eece 100644 --- a/Project.toml +++ b/Project.toml @@ -2,18 +2,13 @@ name = "CodecZlibNG" uuid = "642d12eb-acb5-4437-bcfc-a25e07ad685c" license = "MIT" authors = ["Tomas Drvostep ", "Kenta Sato "] -version = "0.1.0" +version = "0.2.0-dev" [deps] TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" ZlibNG_jll = "c62bbaca-5768-5b75-85e2-9a0ea54e1624" [compat] -TranscodingStreams = "0.9" +TranscodingStreams = "0.9, 0.10, 0.11" +ZlibNG_jll = "2" julia = "1.6" - -[extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[targets] -test = ["Test"] diff --git a/README.md b/README.md index 7a7c4f1..b2a12da 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ CodecZlibNG.jl ============ + +[![CI](https://github.com/JuliaIO/CodecZlibNG.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaIO/CodecZlibNG.jl/actions/workflows/CI.yml) +[![codecov](https://codecov.io/gh/JuliaIO/CodecZlibNG.jl/graph/badge.svg?token=6V3Z847Ywr)](https://codecov.io/gh/JuliaIO/CodecZlibNG.jl) + +CodecZlibNG.jl is an experimental alternative to [CodecZlib.jl](https://github.com/JuliaIO/CodecZlib.jl) that wraps the [zlib-ng](https://github.com/zlib-ng/zlib-ng) C library. + ## Installation ```julia @@ -45,4 +51,4 @@ This package exports following codecs and streams: | `DeflateCompressor` | `DeflateCompressorStream` | | `DeflateDecompressor` | `DeflateDecompressorStream` | -See docstrings and [TranscodingStreams.jl](https://github.com/bicycle1885/TranscodingStreams.jl) for details. +See docstrings and [TranscodingStreams.jl](https://github.com/JuliaIO/TranscodingStreams.jl) for details. diff --git a/src/CodecZlibNG.jl b/src/CodecZlibNG.jl index 6a02806..eae580f 100644 --- a/src/CodecZlibNG.jl +++ b/src/CodecZlibNG.jl @@ -27,7 +27,7 @@ import TranscodingStreams: initialize, finalize, splitkwargs -using ZlibNG_jll +using ZlibNG_jll: libzng include("libzng.jl") include("compression.jl") diff --git a/src/compression.jl b/src/compression.jl index 07ea914..37548e8 100644 --- a/src/compression.jl +++ b/src/compression.jl @@ -24,15 +24,18 @@ Create a gzip compression codec. Arguments --------- -- `level`: compression level (-1..9) -- `windowbits`: size of history buffer (8..15) +- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6). +- `windowbits` (9..15): size of history buffer is `2^windowbits`. + +!!! warning + `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function GzipCompressor(;level::Integer=Z_DEFAULT_COMPRESSION, windowbits::Integer=Z_DEFAULT_WINDOWBITS) if !(-1 ≤ level ≤ 9) throw(ArgumentError("compression level must be within -1..9")) - elseif !(8 ≤ windowbits ≤ 15) - throw(ArgumentError("windowbits must be within 8..15")) + elseif !(9 ≤ windowbits ≤ 15) + throw(ArgumentError("windowbits must be within 9..15")) end # Add 16 to windowBits to write a simple gzip header and trailer around the # compressed data instead of a zlib wrapper. @@ -45,6 +48,9 @@ const GzipCompressorStream{S} = TranscodingStream{GzipCompressor,S} where S<:IO GzipCompressorStream(stream::IO; kwargs...) Create a gzip compression stream (see `GzipCompressor` for `kwargs`). + +!!! warning + `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function GzipCompressorStream(stream::IO; kwargs...) x, y = splitkwargs(kwargs, (:level, :windowbits)) @@ -68,15 +74,18 @@ Create a zlib compression codec. Arguments --------- -- `level`: compression level (-1..9) -- `windowbits`: size of history buffer (8..15) +- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6). +- `windowbits` (9..15): size of history buffer is `2^windowbits`. + +!!! warning + `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function ZlibCompressor(;level::Integer=Z_DEFAULT_COMPRESSION, windowbits::Integer=Z_DEFAULT_WINDOWBITS) if !(-1 ≤ level ≤ 9) throw(ArgumentError("compression level must be within -1..9")) - elseif !(8 ≤ windowbits ≤ 15) - throw(ArgumentError("windowbits must be within 8..15")) + elseif !(9 ≤ windowbits ≤ 15) + throw(ArgumentError("windowbits must be within 9..15")) end return ZlibCompressor(ZNGStream(), level, windowbits) end @@ -87,6 +96,9 @@ const ZlibCompressorStream{S} = TranscodingStream{ZlibCompressor,S} where S<:IO ZlibCompressorStream(stream::IO) Create a zlib compression stream (see `ZlibCompressor` for `kwargs`). + +!!! warning + `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function ZlibCompressorStream(stream::IO; kwargs...) x, y = splitkwargs(kwargs, (:level, :windowbits)) @@ -106,14 +118,14 @@ struct DeflateCompressor <: CompressorCodec end """ - DeflateCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_COMPRESSION)) + DeflateCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS)) Create a deflate compression codec. Arguments --------- -- `level`: compression level (-1..9) -- `windowbits`: size of history buffer (8..15) +- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6). +- `windowbits` (9..15): size of history buffer is `2^windowbits`. - `memlevel`: memory size used for internal compression state (1..9) - `strategy`: compression strategy * 0 <-> Z_DEFAULT_STRATEGY @@ -121,6 +133,9 @@ Arguments * 2 <-> Z_HUFFMAN_ONLY * 3 <-> Z_RLE * 4 <-> Z_FIXED + +!!! warning + `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function DeflateCompressor(; level::Integer=Z_DEFAULT_COMPRESSION, @@ -130,8 +145,8 @@ function DeflateCompressor(; ) if !(-1 ≤ level ≤ 9) throw(ArgumentError("compression level must be within -1..9")) - elseif !(8 ≤ windowbits ≤ 15) - throw(ArgumentError("windowbits must be within 8..15")) + elseif !(9 ≤ windowbits ≤ 15) + throw(ArgumentError("windowbits must be within 9..15")) elseif !(1 ≤ memlevel ≤ 9) throw(ArgumentError("memlevel must be within 1..9")) elseif !(0 ≤ strategy ≤ 4) @@ -156,14 +171,6 @@ end # Methods # ------- -function TranscodingStreams.initialize(codec::CompressorCodec) - code = deflate_init!(codec.zstream, codec.level, codec.windowbits) - if code != Z_OK - zerror(codec.zstream, code) - end - return -end - function TranscodingStreams.finalize(codec::CompressorCodec) zstream = codec.zstream if zstream.state != C_NULL @@ -175,31 +182,55 @@ function TranscodingStreams.finalize(codec::CompressorCodec) return end -function TranscodingStreams.startproc(codec::CompressorCodec, state::Symbol, error::Error) - code = deflate_reset!(codec.zstream) - if code == Z_OK - return :ok +function TranscodingStreams.startproc(codec::CompressorCodec, state::Symbol, error_ref::Error) + if codec.zstream.state == C_NULL + code = deflate_init!(codec.zstream, codec.level, codec.windowbits) + # errors in deflate_init! do not require clean up, so just throw + if code == Z_OK + return :ok + elseif code == Z_MEM_ERROR + throw(OutOfMemoryError()) + elseif code == Z_STREAM_ERROR + error("Z_STREAM_ERROR: invalid parameter, this should be caught in the codec constructor") + elseif code == Z_VERSION_ERROR + error("Z_VERSION_ERROR: zlib library version is incompatible") + else + error("unexpected libzng error code: $(code)") + end else - error[] = ErrorException(zlib_error_message(codec.zstream, code)) - return :error + code = deflate_reset!(codec.zstream) + # errors in deflate_reset! do not require clean up, so just throw + if code == Z_OK + return :ok + elseif code == Z_STREAM_ERROR + error("Z_STREAM_ERROR: the source stream state was inconsistent") + else + error("unexpected libzng error code: $(code)") + end end end -function TranscodingStreams.process(codec::CompressorCodec, input::Memory, output::Memory, error::Error) +function TranscodingStreams.process(codec::CompressorCodec, input::Memory, output::Memory, error_ref::Error) zstream = codec.zstream + if zstream.state == C_NULL + error("startproc must be called before process") + end zstream.next_in = input.ptr - zstream.avail_in = input.size + avail_in = min(input.size, typemax(UInt32)) + zstream.avail_in = avail_in zstream.next_out = output.ptr - zstream.avail_out = output.size - code = deflate!(zstream, input.size > 0 ? Z_NO_FLUSH : Z_FINISH) - Δin = Int(input.size - zstream.avail_in) - Δout = Int(output.size - zstream.avail_out) + avail_out = min(output.size, typemax(UInt32)) + zstream.avail_out = avail_out + code = deflate!(zstream, zstream.avail_in > 0 ? Z_NO_FLUSH : Z_FINISH) + @assert code != Z_STREAM_ERROR # state not clobbered + Δin = Int(avail_in - zstream.avail_in) + Δout = Int(avail_out - zstream.avail_out) if code == Z_OK return Δin, Δout, :ok elseif code == Z_STREAM_END return Δin, Δout, :end else - error[] = ErrorException(zlib_error_message(zstream, code)) + error_ref[] = ErrorException(zlib_error_message(zstream, code)) return Δin, Δout, :error end end diff --git a/src/decompression.jl b/src/decompression.jl index c4e2bc0..22ad9a6 100644 --- a/src/decompression.jl +++ b/src/decompression.jl @@ -25,8 +25,11 @@ If `gziponly` is `false`, this codec can decompress the zlib format as well. Arguments --------- -- `windowbits`: size of history buffer (8..15) +- `windowbits` (8..15): Changing `windowbits` from its default of 15 will prevent decoding data using a history buffer larger than `2^windowbits`. - `gziponly`: flag to inactivate data format detection + +!!! warning + `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function GzipDecompressor(;windowbits::Integer=Z_DEFAULT_WINDOWBITS, gziponly::Bool=false) if !(8 ≤ windowbits ≤ 15) @@ -41,6 +44,9 @@ const GzipDecompressorStream{S} = TranscodingStream{GzipDecompressor,S} where S< GzipDecompressorStream(stream::IO; kwargs...) Create a gzip decompression stream (see `GzipDecompressor` for `kwargs`). + +!!! warning + `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function GzipDecompressorStream(stream::IO; kwargs...) x, y = splitkwargs(kwargs, (:windowbits, :gziponly)) @@ -63,7 +69,10 @@ Create a zlib decompression codec. Arguments --------- -- `windowbits`: size of history buffer (8..15) +- `windowbits` (8..15): Changing `windowbits` from its default of 15 will prevent decoding data using a history buffer larger than `2^windowbits`. + +!!! warning + `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function ZlibDecompressor(;windowbits::Integer=Z_DEFAULT_WINDOWBITS) if !(8 ≤ windowbits ≤ 15) @@ -78,6 +87,9 @@ const ZlibDecompressorStream{S} = TranscodingStream{ZlibDecompressor,S} where S< ZlibDecompressorStream(stream::IO; kwargs...) Create a deflate decompression stream (see `ZlibDecompressor` for `kwargs`). + +!!! warning + `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function ZlibDecompressorStream(stream::IO; kwargs...) x, y = splitkwargs(kwargs, (:windowbits,)) @@ -100,7 +112,10 @@ Create a deflate decompression codec. Arguments --------- -- `windowbits`: size of history buffer (8..15) +- `windowbits` (8..15): Changing `windowbits` from its default of 15 will prevent decoding data using a history buffer larger than `2^windowbits`. + +!!! warning + `serialize` and `deepcopy` will not work with this codec due to stored raw pointers. """ function DeflateDecompressor(;windowbits::Integer=Z_DEFAULT_WINDOWBITS) if !(8 ≤ windowbits ≤ 15) @@ -115,6 +130,9 @@ const DeflateDecompressorStream{S} = TranscodingStream{DeflateDecompressor,S} wh DeflateDecompressorStream(stream::IO; kwargs...) Create a deflate decompression stream (see `DeflateDecompressor` for `kwargs`). + +!!! warning + `serialize` and `deepcopy` will not work with this stream due to stored raw pointers. """ function DeflateDecompressorStream(stream::IO; kwargs...) x, y = splitkwargs(kwargs, (:windowbits,)) @@ -125,14 +143,6 @@ end # Methods # ------- -function TranscodingStreams.initialize(codec::DecompressorCodec) - code = inflate_init!(codec.zstream, codec.windowbits) - if code != Z_OK - zerror(codec.zstream, code) - end - return -end - function TranscodingStreams.finalize(codec::DecompressorCodec) zstream = codec.zstream if zstream.state != C_NULL @@ -144,31 +154,64 @@ function TranscodingStreams.finalize(codec::DecompressorCodec) return end -function TranscodingStreams.startproc(codec::DecompressorCodec, ::Symbol, error::Error) - code = inflate_reset!(codec.zstream) - if code == Z_OK - return :ok +function TranscodingStreams.startproc(codec::DecompressorCodec, ::Symbol, error_ref::Error) + # indicate that no input data is being provided for future zlib compat + codec.zstream.next_in = C_NULL + codec.zstream.avail_in = 0 + if codec.zstream.state == C_NULL + code = inflate_init!(codec.zstream, codec.windowbits) + # errors in inflate_init! do not require clean up, so just throw + if code == Z_OK + return :ok + elseif code == Z_MEM_ERROR + throw(OutOfMemoryError()) + elseif code == Z_STREAM_ERROR + error("Z_STREAM_ERROR: invalid parameter, this should be caught in the codec constructor") + elseif code == Z_VERSION_ERROR + error("Z_VERSION_ERROR: zlib library version is incompatible") + else + error("unexpected libzng error code: $(code)") + end else - error[] = ErrorException(zlib_error_message(codec.zstream, code)) - return :error + code = inflate_reset!(codec.zstream) + # errors in inflate_reset! do not require clean up, so just throw + if code == Z_OK + return :ok + elseif code == Z_STREAM_ERROR + error("Z_STREAM_ERROR: the source stream state was inconsistent") + else + error("unexpected libzng error code: $(code)") + end end end -function TranscodingStreams.process(codec::DecompressorCodec, input::Memory, output::Memory, error::Error) +function TranscodingStreams.process(codec::DecompressorCodec, input::Memory, output::Memory, error_ref::Error) zstream = codec.zstream + if zstream.state == C_NULL + error("startproc must be called before process") + end zstream.next_in = input.ptr - zstream.avail_in = input.size + + avail_in = min(input.size, typemax(UInt32)) + zstream.avail_in = avail_in zstream.next_out = output.ptr - zstream.avail_out = output.size + avail_out = min(output.size, typemax(UInt32)) + zstream.avail_out = avail_out code = inflate!(zstream, Z_NO_FLUSH) - Δin = Int(input.size - zstream.avail_in) - Δout = Int(output.size - zstream.avail_out) + @assert code != Z_STREAM_ERROR # state not clobbered + Δin = Int(avail_in - zstream.avail_in) + Δout = Int(avail_out - zstream.avail_out) if code == Z_OK return Δin, Δout, :ok elseif code == Z_STREAM_END return Δin, Δout, :end + elseif code == Z_MEM_ERROR + throw(OutOfMemoryError()) + elseif code == Z_BUF_ERROR && iszero(input.size) + error_ref[] = ZlibError("the compressed stream may be truncated") + return Δin, Δout, :error else - error[] = ErrorException(zlib_error_message(zstream, code)) + error_ref[] = ZlibError(zlib_error_message(zstream, code)) return Δin, Δout, :error end end diff --git a/src/libzng.jl b/src/libzng.jl index 6c5da16..159d330 100644 --- a/src/libzng.jl +++ b/src/libzng.jl @@ -39,9 +39,17 @@ end const Z_DEFAULT_COMPRESSION = Cint(-1) -const Z_OK = Cint(0) -const Z_STREAM_END = Cint(1) -const Z_BUF_ERROR = Cint(-5) +const Z_OK = Cint(0) +const Z_STREAM_END = Cint(1) +const Z_NEED_DICT = Cint(2) +const Z_ERRNO = Cint(-1) +const Z_STREAM_ERROR = Cint(-2) +const Z_DATA_ERROR = Cint(-3) +const Z_MEM_ERROR = Cint(-4) +const Z_BUF_ERROR = Cint(-5) +const Z_VERSION_ERROR = Cint(-6) +# Return codes for the compression/decompression functions. Negative values +# are errors, positive values are used for special but normal events. const Z_NO_FLUSH = Cint(0) const Z_SYNC_FLUSH = Cint(2) @@ -63,7 +71,8 @@ function version() return unsafe_string(ccall((:zlibng_version, libzng), Ptr{UInt8}, ())) end -const zlibng_version = version() +# This is the version of the zlibng header used to make this wrapper. +const zlibng_version = "2.3.2" function deflate_init!(zstream::ZNGStream, level::Integer=Z_DEFAULT_COMPRESSION, windowbits::Integer=Z_DEFAULT_WINDOWBITS, memlevel::Integer=Z_DEFAULT_MEMLEVEL, strategy::Integer=Z_DEFAULT_STRATEGY) return ccall((:zng_deflateInit2, libzng), Cint, (Ref{ZNGStream}, Cint, Cint, Cint, Cint, Cint), zstream, level, Z_DEFLATED, windowbits, memlevel, strategy) @@ -97,14 +106,28 @@ function inflate!(zstream::ZNGStream, flush::Integer) return ccall((:zng_inflate, libzng), Cint, (Ref{ZNGStream}, Cint), zstream, flush) end +# Error +# ----- + +struct ZlibError <: Exception + msg::String +end + +function Base.showerror(io::IO, err::ZlibError) + print(io, "ZlibError: ") + print(io, err.msg) + nothing +end + + function zerror(zstream::ZNGStream, code::Integer) - return throw(ErrorException(zlib_error_message(zstream, code))) + throw(ZlibError(zlib_error_message(zstream, code))) end function zlib_error_message(zstream::ZNGStream, code::Integer) if zstream.msg == C_NULL - return "zlib error: (code: $(code))" + return " (code: $(code))" else - return "zlib error: $(unsafe_string(zstream.msg)) (code: $(code))" + return "$(unsafe_string(zstream.msg)) (code: $(code))" end end diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 0000000..5f4e128 --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,9 @@ +[deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CodecZlibNG = "642d12eb-acb5-4437-bcfc-a25e07ad685c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestsForCodecPackages = "c2e61002-3542-480d-8b3c-5f05cc4f8554" +TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" + +[sources] +CodecZlibNG = {path = ".."} diff --git a/test/big-mem-tests.jl b/test/big-mem-tests.jl new file mode 100644 index 0000000..7fac572 --- /dev/null +++ b/test/big-mem-tests.jl @@ -0,0 +1,77 @@ +# This file contains tests that require a large amount of memory (at least 25 GB) +# and take a long time to run. The tests are designed to check the +# compression and decompression functionality of the package +# with very large inputs. These tests are not run with CI + +using Test +using CodecZlibNG +using TranscodingStreams + +@testset "memory leak" begin + function foo() + for (encode, decode) in [ + (GzipCompressor, GzipDecompressor), + (ZlibCompressor, ZlibDecompressor), + (DeflateCompressor, DeflateDecompressor), + ] + for i in 1:1000000 + encoder = encode() + try + decoder = decode() + try + c = transcode(encoder, zeros(UInt8,16)) + u = transcode(decoder, c) + finally + TranscodingStreams.finalize(decoder) + end + finally + TranscodingStreams.finalize(encoder) + end + end + end + end + foo() +end + +@testset "Big Memory Tests" begin + Sys.WORD_SIZE == 64 || error("tests require 64 bit word size") + @info "compressing zeros" + for n in (2^32 - 1, 2^32, 2^32 +1) + @info "compressing" + local c = transcode(GzipCompressor, zeros(UInt8, n)) + @info "decompressing" + local u = transcode(GzipDecompressor, c) + c = nothing + all_zero = all(iszero, u) + len_n = length(u) == n + @test all_zero && len_n + end + + @info "compressing random" + for n in (2^32 - 1, 2^32, 2^32 +1) + local u = rand(UInt8, n) + @info "compressing" + local c = transcode(GzipCompressor, u) + @info "decompressing" + local u2 = transcode(GzipDecompressor, c) + c = nothing + are_equal = u == u2 + @test are_equal + end + + @info "decompressing huge concatenation" + uncompressed = rand(UInt8, 2^20) + @info "compressing" + compressed = transcode(GzipCompressor, uncompressed) + total_compressed = UInt8[] + sizehint!(total_compressed, length(compressed)*2^12) + total_uncompressed = UInt8[] + sizehint!(total_uncompressed, length(uncompressed)*2^12) + for i in 1:2^12 + append!(total_uncompressed, uncompressed) + append!(total_compressed, compressed) + end + @test length(total_compressed) > 2^32 + @info "decompressing" + @test total_uncompressed == transcode(GzipDecompressor, total_compressed) +end diff --git a/test/runtests.jl b/test/runtests.jl index cf21d40..a2fca9a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,15 +1,49 @@ using CodecZlibNG +using CodecZlibNG: ZlibError using Test -import TranscodingStreams: +using Aqua: Aqua +using TranscodingStreams: TranscodingStreams, - TranscodingStream, + TranscodingStream +using TestsForCodecPackages: test_roundtrip_read, test_roundtrip_write, + test_roundtrip_transcode, test_roundtrip_lines, - test_roundtrip_transcode + test_roundtrip_seekstart, + test_roundtrip_fileio, + test_chunked_read, + test_chunked_write, + test_reuse_encoder + +Aqua.test_all(CodecZlibNG) const testdir = @__DIR__ +# decompress one byte at a time +function decompress_bytes(decoder, data::Vector{UInt8})::Vector{UInt8} + io = IOBuffer() + s = decoder(io; bufsize=1) + for i in eachindex(data) + write(s, data[i]) + flush(s) + end + write(s, TranscodingStreams.TOKEN_END) + flush(s) + take!(io) +end + +# generate random data to test compression +function generate_data() + thing = rand(UInt8, 100) + d = UInt8[] + for dist in [0:258; 400:200:2000; 2000:1000:33000;] + append!(d, thing) + append!(d, rand(0x00:0x0f, dist)) + end + d +end + @testset "Gzip Codec" begin codec = GzipCompressor() @test codec isa GzipCompressor @@ -47,7 +81,7 @@ const testdir = @__DIR__ gzip_data_corrupted[1] = 0x00 # corrupt header file = IOBuffer(gzip_data_corrupted) stream = GzipDecompressorStream(file) - @test_throws ErrorException read(stream) + @test_throws ZlibError read(stream) @test_throws ArgumentError read(stream) @test !isopen(stream) @test isopen(file) @@ -105,11 +139,11 @@ const testdir = @__DIR__ test_roundtrip_read(GzipCompressorStream, GzipDecompressorStream) test_roundtrip_write(GzipCompressorStream, GzipDecompressorStream) test_roundtrip_lines(GzipCompressorStream, GzipDecompressorStream) + test_roundtrip_seekstart(GzipCompressorStream, GzipDecompressorStream) test_roundtrip_transcode(GzipCompressor, GzipDecompressor) + test_reuse_encoder(GzipCompressor, GzipDecompressor) @test_throws ArgumentError GzipCompressor(level=10) - @test_throws ArgumentError GzipCompressor(windowbits=16) - @test_throws ArgumentError GzipDecompressor(windowbits=16) end @testset "Zlib Codec" begin @@ -149,7 +183,7 @@ end close(stream) stream = TranscodingStream(GzipDecompressor(gziponly=true), IOBuffer(zlib_data)) - @test_throws Exception read(stream) + @test_throws ZlibError read(stream) close(stream) file = IOBuffer(b"foo") @@ -185,11 +219,11 @@ end test_roundtrip_read(ZlibCompressorStream, ZlibDecompressorStream) test_roundtrip_write(ZlibCompressorStream, ZlibDecompressorStream) test_roundtrip_lines(ZlibCompressorStream, ZlibDecompressorStream) + test_roundtrip_seekstart(ZlibCompressorStream, ZlibDecompressorStream) test_roundtrip_transcode(ZlibCompressor, ZlibDecompressor) + test_reuse_encoder(ZlibCompressor, ZlibDecompressor) @test_throws ArgumentError ZlibCompressor(level=10) - @test_throws ArgumentError ZlibCompressor(windowbits=16) - @test_throws ArgumentError ZlibDecompressor(windowbits=16) end @testset "Deflate Codec" begin @@ -197,8 +231,7 @@ end @test codec isa DeflateCompressor @test occursin(r"^(CodecZlibNG\.)?DeflateCompressor\(level=-1, windowbits=-\d+\)$", sprint(show, codec)) @test CodecZlibNG.initialize(codec) === nothing - # FIXME: This test fails. - #@test CodecZlibNG.finalize(codec) === nothing + @test CodecZlibNG.finalize(codec) === nothing codec = DeflateDecompressor() @test codec isa DeflateDecompressor @@ -209,21 +242,56 @@ end test_roundtrip_read(DeflateCompressorStream, DeflateDecompressorStream) test_roundtrip_write(DeflateCompressorStream, DeflateDecompressorStream) test_roundtrip_lines(DeflateCompressorStream, DeflateDecompressorStream) + test_roundtrip_seekstart(DeflateCompressorStream, DeflateDecompressorStream) test_roundtrip_transcode(DeflateCompressor, DeflateDecompressor) + test_reuse_encoder(DeflateCompressor, DeflateDecompressor) @test DeflateCompressorStream <: TranscodingStream @test DeflateDecompressorStream <: TranscodingStream @test_throws ArgumentError DeflateCompressor(level=10) - @test_throws ArgumentError DeflateCompressor(windowbits=16) - @test_throws ArgumentError DeflateDecompressor(windowbits=16) + + # Test decoding byte by byte + d = generate_data() + c = transcode(DeflateCompressor, d) + @test transcode(DeflateDecompressor, c) == d + @test decompress_bytes(DeflateDecompressorStream, c) == d +end + +@testset "roundtrip windowbits" begin + d = generate_data() + for (encoder, decoder) in [ + (GzipCompressorStream, GzipDecompressorStream), + (ZlibCompressorStream, ZlibDecompressorStream), + (DeflateCompressorStream, DeflateDecompressorStream), + ] + for compression_windowbits in 9:15 + for decompression_windowbits in 8:15 + c = read(encoder(IOBuffer(d); windowbits=compression_windowbits, level=9)) + if compression_windowbits ≤ decompression_windowbits + @test d == read(decoder(IOBuffer(c); windowbits=decompression_windowbits)) + else + try + u = read(decoder(IOBuffer(c); windowbits=decompression_windowbits)) + @test u == d + catch e + @test e isa ZlibError + end + end + end + end + @test_throws ArgumentError encoder(IOBuffer(d); windowbits=8) + @test_throws ArgumentError decoder(IOBuffer(d); windowbits=7) + @test_throws ArgumentError encoder(IOBuffer(d); windowbits=16) + @test_throws ArgumentError decoder(IOBuffer(d); windowbits=16) + end end # Test APIs of TranscodingStreams.jl using the gzip compressor/decompressor. @testset "TranscodingStreams" begin - TranscodingStreams.test_chunked_read(GzipCompressor, GzipDecompressor) - TranscodingStreams.test_chunked_write(GzipCompressor, GzipDecompressor) - TranscodingStreams.test_roundtrip_fileio(GzipCompressor, GzipDecompressor) + test_chunked_read(GzipCompressor, GzipDecompressor) + test_chunked_write(GzipCompressor, GzipDecompressor) + test_roundtrip_fileio(GzipCompressor, GzipDecompressor) @testset "seek" begin data = transcode(GzipCompressor, Vector(b"abracadabra")) @@ -232,13 +300,11 @@ end @test read(stream, 3) == b"abr" seekstart(stream) @test read(stream, 3) == b"abr" - seekend(stream) - #@test eof(stream) end @testset "panic" begin stream = TranscodingStream(GzipDecompressor(), IOBuffer("some invalid data")) - @test_throws ErrorException read(stream) + @test_throws ZlibError read(stream) @test_throws ArgumentError eof(stream) end @@ -291,3 +357,43 @@ end @test_throws ArgumentError TranscodingStreams.stats(stream) end end + +@testset "unexpected end of stream errors" begin + tests = [ + (ZlibCompressor, ZlibDecompressor), + (DeflateCompressor, DeflateDecompressor), + (GzipCompressor, GzipDecompressor), + ] + @testset "$(encoder)" for (encoder, decoder) in tests + local uncompressed = rand(UInt8, 1000) + local compressed = transcode(encoder, uncompressed) + for i in 0:length(compressed)-1 + @test_throws ZlibError("the compressed stream may be truncated") transcode(decoder, compressed[1:i]) + end + @test transcode(decoder, compressed) == uncompressed + # compressing empty vector should still work + @test transcode(decoder, transcode(encoder, UInt8[])) == UInt8[] + end +end +@testset "data errors" begin + @test_throws ZlibError transcode(ZlibDecompressor, zeros(UInt8, 10)) + local uncompressed = rand(UInt8, 1000) + local compressed = transcode(ZlibCompressor, uncompressed) + compressed[70] ⊻= 0x01 + @test_throws ZlibError transcode(ZlibDecompressor, compressed) + # Z_NEED_DICT error + try + transcode( + ZlibDecompressor, + UInt8[0x78, 0xbb, 0x00, 0x00, 0x00, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01], + ) + @test false + catch e + @test e isa ZlibError + @test endswith(e.msg, "(code: $(CodecZlibNG.Z_NEED_DICT))") + end +end +@testset "error printing" begin + @test sprint(Base.showerror, ZlibError("test error message")) == + "ZlibError: test error message" +end