From 9353b1cd154a36f6de221145f86e8ae233cc1ed7 Mon Sep 17 00:00:00 2001 From: Luis Ferreira Date: Thu, 17 Oct 2024 16:57:07 -0300 Subject: [PATCH 1/3] chore: Upgrade brod dependency to v4 --- mix.exs | 3 ++- mix.lock | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index d5aca90..44c8e17 100644 --- a/mix.exs +++ b/mix.exs @@ -31,7 +31,8 @@ defmodule Kaffe.Mixfile do defp deps do [ - {:brod, "~> 3.0"}, + {:brod, "~> 4.0"}, + {:crc32cer, "< 1.0.0"}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, {:retry, ">= 0.15.0 and < 0.19.0"} ] diff --git a/mix.lock b/mix.lock index ae38140..27c13e7 100644 --- a/mix.lock +++ b/mix.lock @@ -1,10 +1,10 @@ %{ - "brod": {:hex, :brod, "3.16.4", "8d941d489b9a0de26acf8b2bb800fa22751606221ed3ef52534a582c6c265336", [:rebar3], [{:kafka_protocol, "4.1.0", [hex: :kafka_protocol, repo: "hexpm", optional: false]}, {:snappyer, "1.2.8", [hex: :snappyer, repo: "hexpm", optional: false]}, {:supervisor3, "1.1.11", [hex: :supervisor3, repo: "hexpm", optional: false]}], "hexpm", "872ceaff5c2f99234e744e57580ed5f5dfbea29fe0b1387505a8dcb6cf2812b1"}, - "crc32cer": {:hex, :crc32cer, "0.1.8", "c6c2275c5fb60a95f4935d414f30b50ee9cfed494081c9b36ebb02edfc2f48db", [:rebar3], [], "hexpm", "251499085482920deb6c9b7aadabf9fb4c432f96add97ab42aee4501e5b6f591"}, + "brod": {:hex, :brod, "4.4.0", "9e09abd24c0df25eee87f23004eb8b895d69e82f21d088276e1bb6f422c0e5fb", [:rebar3], [{:kafka_protocol, "4.2.3", [hex: :kafka_protocol, repo: "hexpm", optional: false]}], "hexpm", "51bd4f70ad63f9e92b412db5fc55a30a877f36ac386184caa29d729154f2d923"}, + "crc32cer": {:hex, :crc32cer, "0.1.12", "b018bd5dcbba9c35972822f53ad40b6b483d453204ef67daf92af3a314bbfbf6", [:rebar3], [], "hexpm", "56ad9380651c2c4cb21d7741c91cbcc4709e032fd31a98a33f007ee30e526972"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm", "e3be2bc3ae67781db529b80aa7e7c49904a988596e2dbff897425b48b3581161"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, - "kafka_protocol": {:hex, :kafka_protocol, "4.1.0", "53fac8866969484f783bff204bd4e41e62a97ce9753c83f802a08d5bfc0e0c4c", [:rebar3], [{:crc32cer, "0.1.8", [hex: :crc32cer, repo: "hexpm", optional: false]}], "hexpm", "61cb8b80199bf95122cf8073e0f4c0ad62f82515b4d44c54f946a5972c3f5fa5"}, + "kafka_protocol": {:hex, :kafka_protocol, "4.2.3", "83b11eb1ec90d51999a5dbb03ce7ed1363e34491ccf6b0f13c269c29294d59ba", [:rebar3], [{:crc32cer, "0.1.12", [hex: :crc32cer, repo: "hexpm", optional: false]}], "hexpm", "4db4bdd80f26574a0653a2784df00b0e8454c29bc3f65138ae8bc6497482934d"}, "logfmt": {:hex, :logfmt, "3.2.0", "887a091adad28acc6e4d8b3d3bce177b934e7c61e7655c86946410f44aca6d84", [:mix], []}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, From 97bfb54c9c28ca3dd0bb3062cd32b478d3cb4475 Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Tue, 29 Jul 2025 14:34:07 -0700 Subject: [PATCH 2/3] Add changelog entry + documentation Also removes unused deps from mix.lock --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 16 ++++++++++++++++ mix.lock | 5 ----- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e239ce..0366f26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ * Allow keyword configuration for subscribers. Note, keywords require atom keys, so if your current version of `kaffe` is 1.27.0 or higher, adopting to the keyword subscribers is a breaking (and highly encouraged) change. +* Compression support by default was removed from Brod. While this allows less dependencies out of the box, it also means that topics with compression now need additional config for `snappy` or `lz4` compression. + + To support compression as `brod` did by default before this change, add `snappyer` and `lz4b` to your `deps`, add the following to `config` + + ```elixir + config :kafka_protocol, provide_compression: [ + snappy: :snappyer, + lz4: :lz4b + ] + ``` + + For more information and to see other supported compression types, see [kafka_protocol's README](https://github.com/kafka4beam/kafka_protocol/blob/master/README.md#compression-support). + +### Enhancements + +* Bumps `:brod` to v4, which drops `snappyer` dependency requirement + # 1.28.0 ### Enhancements diff --git a/README.md b/README.md index ab6fa5c..6d6b8e6 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ An opinionated, highly specific, Elixir wrapper around [Brod](https://github.com - [Kaffe Producer Usage](#kaffe-producer-usage) - [Heroku Configuration](#heroku-configuration) - [Producing to Kafka](#producing-to-kafka) +- [Compression Config](#compression-config) - [Testing](#testing) - [Setup](#setup) - [Running](#running) @@ -379,6 +380,21 @@ There are several ways to produce: **NOTE**: With this approach Kaffe will not calculate the next partition since it assumes you're taking over that job by giving it a specific partition. +## Compression Config + +Compression support by default was removed from Brod. While this allows less dependencies out of the box, it also means that topics with compression now need additional config for `snappy` or `lz4` compression. + +To support compression as `brod` did by default before this change, add `snappyer` and `lz4b` to your `deps`, add the following to `config` + +```elixir +config :kafka_protocol, provide_compression: [ + snappy: :snappyer, + lz4: :lz4b +] +``` + +For more information and to see other supported compression types, see [kafka_protocol's README](https://github.com/kafka4beam/kafka_protocol/blob/master/README.md#compression-support). + ## Testing ### Setup diff --git a/mix.lock b/mix.lock index 27c13e7..d551e8d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,17 +1,12 @@ %{ "brod": {:hex, :brod, "4.4.0", "9e09abd24c0df25eee87f23004eb8b895d69e82f21d088276e1bb6f422c0e5fb", [:rebar3], [{:kafka_protocol, "4.2.3", [hex: :kafka_protocol, repo: "hexpm", optional: false]}], "hexpm", "51bd4f70ad63f9e92b412db5fc55a30a877f36ac386184caa29d729154f2d923"}, "crc32cer": {:hex, :crc32cer, "0.1.12", "b018bd5dcbba9c35972822f53ad40b6b483d453204ef67daf92af3a314bbfbf6", [:rebar3], [], "hexpm", "56ad9380651c2c4cb21d7741c91cbcc4709e032fd31a98a33f007ee30e526972"}, - "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm", "e3be2bc3ae67781db529b80aa7e7c49904a988596e2dbff897425b48b3581161"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, "kafka_protocol": {:hex, :kafka_protocol, "4.2.3", "83b11eb1ec90d51999a5dbb03ce7ed1363e34491ccf6b0f13c269c29294d59ba", [:rebar3], [{:crc32cer, "0.1.12", [hex: :crc32cer, repo: "hexpm", optional: false]}], "hexpm", "4db4bdd80f26574a0653a2784df00b0e8454c29bc3f65138ae8bc6497482934d"}, - "logfmt": {:hex, :logfmt, "3.2.0", "887a091adad28acc6e4d8b3d3bce177b934e7c61e7655c86946410f44aca6d84", [:mix], []}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, - "metrix": {:git, "https://github.com/rwdaigle/metrix.git", "a6738df9346da0412ca68f82a24a67d2a32b066e", [branch: "master"]}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, - "snappyer": {:hex, :snappyer, "1.2.8", "201ce9067a33c71a6a5087c0c3a49a010b17112d461e6df696c722dcb6d0934a", [:rebar3], [], "hexpm", "35518e79a28548b56d8fd6aee2f565f12f51c2d3d053f9cfa817c83be88c4f3d"}, - "supervisor3": {:hex, :supervisor3, "1.1.11", "d81cdec31d102fde407423e1d05b569572850deebed86b951d5233c387cba80b", [:rebar3], [], "hexpm", "e6c2dedbcabcba24995a218aca12db5e208b80d3252692b22ef0f1a266104b50"}, } From 09b7b7ba7784dfd0a4b4679570841430c095a0c8 Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Wed, 14 Jan 2026 12:15:22 -0800 Subject: [PATCH 3/3] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0366f26..794476d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ ] ``` - For more information and to see other supported compression types, see [kafka_protocol's README](https://github.com/kafka4beam/kafka_protocol/blob/master/README.md#compression-support). + For more information and to see other supported compression types, see [kafka_protocol's README](https://github.com/kafka4beam/kafka_protocol/blob/master/README.md#compression). ### Enhancements