Port changes from CodecZlib.jl

.github/dependabot.yml

@@ -0,0 +1,7 @@
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/CI.yml

@@ -0,0 +1,54 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+    tags: ['*']
+  pull_request:
+    branches:
+      - master
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - 'lts'
+          - '1'
+          - 'nightly'
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+        arch:
+          - default
+          - x86
+        exclude:
+          - os: macos-latest
+            arch: x86
+        include:
+          - os: ubuntu-latest
+            version: 'min'
+            arch: x64
+    steps:
+      - uses: actions/checkout@v6
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+          show-versioninfo: true
+      - uses: julia-actions/cache@v3
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v6
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: lcov.info

.github/workflows/TagBot.yml

@@ -0,0 +1,33 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+    inputs:
+      lookback:
+        default: 3
+permissions:
+  actions: read
+  checks: read
+  contents: write
+  deployments: read
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          # Edit the following line to reflect the actual name of the GitHub Secret containing your private key
+          ssh: ${{ secrets.DOCUMENTER_KEY }}
+          # ssh: ${{ secrets.NAME_OF_MY_SSH_PRIVATE_KEY_SECRET }}

.gitignore

@@ -1,7 +1,14 @@
+#Mac OS X
+*.DS_Store
+
+#VS Code
+/.vscode/
+
 *.jl.cov
 *.jl.*.cov
 *.jl.mem
 /Manifest.toml
+/test/Manifest.toml
 /.envrc
 /deps/deps.jl
 /deps/build.log

Project.toml

@@ -2,18 +2,13 @@ name = "CodecZlibNG"
 uuid = "642d12eb-acb5-4437-bcfc-a25e07ad685c"
 license = "MIT"
 authors = ["Tomas Drvostep <tomas.drvostep@gmail.com>", "Kenta Sato <bicycle1885@gmail.com>"]
-version = "0.1.0"
+version = "0.2.0-dev"
 
 [deps]
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
 ZlibNG_jll = "c62bbaca-5768-5b75-85e2-9a0ea54e1624"
 
 [compat]
-TranscodingStreams = "0.9"
+TranscodingStreams = "0.9, 0.10, 0.11"
+ZlibNG_jll = "2"
 julia = "1.6"
-
-[extras]
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Test"]

README.md

@@ -1,5 +1,11 @@
 CodecZlibNG.jl
 ============
+
+[![CI](https://github.com/JuliaIO/CodecZlibNG.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaIO/CodecZlibNG.jl/actions/workflows/CI.yml)
+[![codecov](https://codecov.io/gh/JuliaIO/CodecZlibNG.jl/graph/badge.svg?token=6V3Z847Ywr)](https://codecov.io/gh/JuliaIO/CodecZlibNG.jl)
+
+CodecZlibNG.jl is an experimental alternative to [CodecZlib.jl](https://github.com/JuliaIO/CodecZlib.jl) that wraps the [zlib-ng](https://github.com/zlib-ng/zlib-ng) C library.
+
 ## Installation
 
 ```julia
@@ -45,4 +51,4 @@ This package exports following codecs and streams:
 | `DeflateCompressor`    | `DeflateCompressorStream`    |
 | `DeflateDecompressor`  | `DeflateDecompressorStream`  |
 
-See docstrings and [TranscodingStreams.jl](https://github.com/bicycle1885/TranscodingStreams.jl) for details.
+See docstrings and [TranscodingStreams.jl](https://github.com/JuliaIO/TranscodingStreams.jl) for details.

src/CodecZlibNG.jl

@@ -27,7 +27,7 @@ import TranscodingStreams:
     initialize,
     finalize,
     splitkwargs
-using ZlibNG_jll
+using ZlibNG_jll: libzng
 
 include("libzng.jl")
 include("compression.jl")

src/compression.jl

@@ -24,15 +24,18 @@ Create a gzip compression codec.
 
 Arguments
 ---------
-- `level`: compression level (-1..9)
-- `windowbits`: size of history buffer (8..15)
+- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6).
+- `windowbits` (9..15): size of history buffer is `2^windowbits`.
+
+!!! warning
+    `serialize` and `deepcopy` will not work with this codec due to stored raw pointers.
 """
 function GzipCompressor(;level::Integer=Z_DEFAULT_COMPRESSION,
                          windowbits::Integer=Z_DEFAULT_WINDOWBITS)
     if !(-1 ≤ level ≤ 9)
         throw(ArgumentError("compression level must be within -1..9"))
-    elseif !(8 ≤ windowbits ≤ 15)
-        throw(ArgumentError("windowbits must be within 8..15"))
+    elseif !(9 ≤ windowbits ≤ 15)
+        throw(ArgumentError("windowbits must be within 9..15"))
     end
     # Add 16 to windowBits to write a simple gzip header and trailer around the
     # compressed data instead of a zlib wrapper.
@@ -45,6 +48,9 @@ const GzipCompressorStream{S} = TranscodingStream{GzipCompressor,S} where S<:IO
     GzipCompressorStream(stream::IO; kwargs...)
 
 Create a gzip compression stream (see `GzipCompressor` for `kwargs`).
+
+!!! warning
+    `serialize` and `deepcopy` will not work with this stream due to stored raw pointers.
 """
 function GzipCompressorStream(stream::IO; kwargs...)
     x, y = splitkwargs(kwargs, (:level, :windowbits))
@@ -68,15 +74,18 @@ Create a zlib compression codec.
 
 Arguments
 ---------
-- `level`: compression level (-1..9)
-- `windowbits`: size of history buffer (8..15)
+- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6).
+- `windowbits` (9..15): size of history buffer is `2^windowbits`.
+
+!!! warning
+    `serialize` and `deepcopy` will not work with this codec due to stored raw pointers.
 """
 function ZlibCompressor(;level::Integer=Z_DEFAULT_COMPRESSION,
                          windowbits::Integer=Z_DEFAULT_WINDOWBITS)
     if !(-1 ≤ level ≤ 9)
         throw(ArgumentError("compression level must be within -1..9"))
-    elseif !(8 ≤ windowbits ≤ 15)
-        throw(ArgumentError("windowbits must be within 8..15"))
+    elseif !(9 ≤ windowbits ≤ 15)
+        throw(ArgumentError("windowbits must be within 9..15"))
     end
     return ZlibCompressor(ZNGStream(), level, windowbits)
 end
@@ -87,6 +96,9 @@ const ZlibCompressorStream{S} = TranscodingStream{ZlibCompressor,S} where S<:IO
     ZlibCompressorStream(stream::IO)
 
 Create a zlib compression stream (see `ZlibCompressor` for `kwargs`).
+
+!!! warning
+    `serialize` and `deepcopy` will not work with this stream due to stored raw pointers.
 """
 function ZlibCompressorStream(stream::IO; kwargs...)
     x, y = splitkwargs(kwargs, (:level, :windowbits))
@@ -106,21 +118,24 @@ struct DeflateCompressor <: CompressorCodec
 end
 
 """
-    DeflateCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_COMPRESSION))
+    DeflateCompressor(;level=$(Z_DEFAULT_COMPRESSION), windowbits=$(Z_DEFAULT_WINDOWBITS))
 
 Create a deflate compression codec.
 
 Arguments
 ---------
-- `level`: compression level (-1..9)
-- `windowbits`: size of history buffer (8..15)
+- `level` (-1..9): compression level. 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). -1 requests a default compromise between speed and compression (currently equivalent to level 6).
+- `windowbits` (9..15): size of history buffer is `2^windowbits`.
 - `memlevel`: memory size used for internal compression state (1..9)
 - `strategy`: compression strategy
     * 0 <-> Z_DEFAULT_STRATEGY
     * 1 <-> Z_FILTERED
     * 2 <-> Z_HUFFMAN_ONLY
     * 3 <-> Z_RLE
     * 4 <-> Z_FIXED
+
+!!! warning
+    `serialize` and `deepcopy` will not work with this codec due to stored raw pointers.
 """
 function DeflateCompressor(;
     level::Integer=Z_DEFAULT_COMPRESSION,
@@ -130,8 +145,8 @@ function DeflateCompressor(;
 )
     if !(-1 ≤ level ≤ 9)
         throw(ArgumentError("compression level must be within -1..9"))
-    elseif !(8 ≤ windowbits ≤ 15)
-        throw(ArgumentError("windowbits must be within 8..15"))
+    elseif !(9 ≤ windowbits ≤ 15)
+        throw(ArgumentError("windowbits must be within 9..15"))
     elseif !(1 ≤ memlevel ≤ 9)
         throw(ArgumentError("memlevel must be within 1..9"))
     elseif !(0 ≤ strategy ≤ 4)
@@ -156,14 +171,6 @@ end
 # Methods
 # -------
 
-function TranscodingStreams.initialize(codec::CompressorCodec)
-    code = deflate_init!(codec.zstream, codec.level, codec.windowbits)
-    if code != Z_OK
-        zerror(codec.zstream, code)
-    end
-    return
-end
-
 function TranscodingStreams.finalize(codec::CompressorCodec)
     zstream = codec.zstream
     if zstream.state != C_NULL
@@ -175,31 +182,55 @@ function TranscodingStreams.finalize(codec::CompressorCodec)
     return
 end
 
-function TranscodingStreams.startproc(codec::CompressorCodec, state::Symbol, error::Error)
-    code = deflate_reset!(codec.zstream)
-    if code == Z_OK
-        return :ok
+function TranscodingStreams.startproc(codec::CompressorCodec, state::Symbol, error_ref::Error)
+    if codec.zstream.state == C_NULL
+        code = deflate_init!(codec.zstream, codec.level, codec.windowbits)
+        # errors in deflate_init! do not require clean up, so just throw
+        if code == Z_OK
+            return :ok
+        elseif code == Z_MEM_ERROR
+            throw(OutOfMemoryError())
+        elseif code == Z_STREAM_ERROR
+            error("Z_STREAM_ERROR: invalid parameter, this should be caught in the codec constructor")
+        elseif code == Z_VERSION_ERROR
+            error("Z_VERSION_ERROR: zlib library version is incompatible")
+        else
+            error("unexpected libzng error code: $(code)")
+        end
     else
-        error[] = ErrorException(zlib_error_message(codec.zstream, code))
-        return :error
+        code = deflate_reset!(codec.zstream)
+        # errors in deflate_reset! do not require clean up, so just throw
+        if code == Z_OK
+            return :ok
+        elseif code == Z_STREAM_ERROR
+            error("Z_STREAM_ERROR: the source stream state was inconsistent")
+        else
+            error("unexpected libzng error code: $(code)")
+        end
     end
 end
 
-function TranscodingStreams.process(codec::CompressorCodec, input::Memory, output::Memory, error::Error)
+function TranscodingStreams.process(codec::CompressorCodec, input::Memory, output::Memory, error_ref::Error)
     zstream = codec.zstream
+    if zstream.state == C_NULL
+        error("startproc must be called before process")
+    end
     zstream.next_in = input.ptr
-    zstream.avail_in = input.size
+    avail_in = min(input.size, typemax(UInt32))
+    zstream.avail_in = avail_in
     zstream.next_out = output.ptr
-    zstream.avail_out = output.size
-    code = deflate!(zstream, input.size > 0 ? Z_NO_FLUSH : Z_FINISH)
-    Δin = Int(input.size - zstream.avail_in)
-    Δout = Int(output.size - zstream.avail_out)
+    avail_out = min(output.size, typemax(UInt32))
+    zstream.avail_out = avail_out
+    code = deflate!(zstream, zstream.avail_in > 0 ? Z_NO_FLUSH : Z_FINISH)
+    @assert code != Z_STREAM_ERROR # state not clobbered
+    Δin = Int(avail_in - zstream.avail_in)
+    Δout = Int(avail_out - zstream.avail_out)
     if code == Z_OK
         return Δin, Δout, :ok
     elseif code == Z_STREAM_END
         return Δin, Δout, :end
     else
-        error[] = ErrorException(zlib_error_message(zstream, code))
+        error_ref[] = ErrorException(zlib_error_message(zstream, code))
         return Δin, Δout, :error
     end
 end