diff --git a/.gitignore b/.gitignore index 2886ad76..e630c22e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /bazel-* target Cargo.lock +# Generated protobuf code +examples/envoy_tcp_routing/src/generated/ diff --git a/README.md b/README.md index fa53c221..4acad661 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ - [HTTP Configuration](./examples/http_config/) - [gRPC Auth (random)](./examples/grpc_auth_random/) - [Envoy filter metadata](./examples/envoy_filter_metadata/) +- [Envoy TCP Routing](./examples/envoy_tcp_routing/) ## Articles & blog posts from the community diff --git a/examples/envoy_tcp_routing/Cargo.toml b/examples/envoy_tcp_routing/Cargo.toml new file mode 100644 index 00000000..ab7c38b5 --- /dev/null +++ b/examples/envoy_tcp_routing/Cargo.toml @@ -0,0 +1,21 @@ +[package] +publish = false +name = "proxy-wasm-example-envoy-tcp-routing" +version = "0.0.1" +authors = ["Proxy-Wasm contributors"] +description = "Proxy-Wasm plugin example: Envoy TCP Routing based on source IP" +license = "Apache-2.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +log = "0.4" +proxy-wasm = { path = "../../" } +prost = "0.14" + +[build-dependencies] +prost-build = "0.14" +proc-macro2 = "1.0" + diff --git a/examples/envoy_tcp_routing/README.md b/examples/envoy_tcp_routing/README.md new file mode 100644 index 00000000..a128c454 --- /dev/null +++ b/examples/envoy_tcp_routing/README.md @@ -0,0 +1,89 @@ +## Proxy-Wasm plugin example: Envoy TCP Routing + +Proxy-Wasm TCP filter that dynamically routes connections to different upstream clusters based on the source IP address. + +This example operates at the [TCP stream context](https://github.com/proxy-wasm/spec/tree/main/abi-versions/v0.2.1#tcp-streams) (L4) rather than the [HTTP application layer](https://github.com/proxy-wasm/spec/tree/main/abi-versions/v0.2.1#http-streams), making it useful for use cases where application-layer processing should be avoided for performance or protocol-agnostic routing decisions. + +This example is inspired by the [wasmerang](https://github.com/SiiiTschiii/wasmerang) project, which demonstrates advanced TCP routing patterns in Envoy/Istio/K8s using WASM filters. + +### Overview + +This example demonstrates how to build a TCP filter that: + +- Intercepts incoming TCP connections +- Extracts the source IP address +- Routes traffic to different upstream clusters based on whether the last octet is even or odd + - **Even last octet** → routes to `egress-router1` + - **Odd last octet** → routes to `egress-router2` + +The filter uses Envoy's `set_envoy_filter_state` foreign function to dynamically override the TCP proxy cluster at runtime, requiring proper protobuf encoding via the included `set_envoy_filter_state.proto` file. + +### Building + +Build the WASM plugin from the example directory: + +```sh +$ cargo build --target wasm32-wasip1 --release +``` + +### Running with Docker Compose + +This example can be run with [`docker compose`](https://docs.docker.com/compose/install/) and has a matching Envoy configuration. + +From the example directory: + +```sh +$ docker compose up +``` + +### Test the Routing + +In a separate terminal test the routing behavior with different source IP addresses: + +```bash +# Even IP (last octet 10) → routes to egress-router1 +docker run --rm -it --network envoy_tcp_routing_envoymesh --ip 172.22.0.10 curlimages/curl curl http://proxy:10000/ip -H "Host: httpbin.org" + +# Odd IP (last octet 11) → routes to egress-router2 +docker run --rm -it --network envoy_tcp_routing_envoymesh --ip 172.22.0.11 curlimages/curl curl http://proxy:10000/ip -H "Host: httpbin.org" +``` + +### Expected Output + +Check the Docker Compose logs to see the WASM filter in action: + +```console +$ docker compose logs -f +``` + +**For even IP (last octet 10) → routes via egress-router1:** + +``` +proxy-1 | [TCP WASM] Source address: 172.22.0.10:39484 +proxy-1 | [TCP WASM] Source IP last octet: 10, intercepting ALL traffic +proxy-1 | [TCP WASM] Routing to egress-router1 +proxy-1 | [TCP WASM] set_envoy_filter_state status (envoy.tcp_proxy.cluster): Ok(None) +proxy-1 | [TCP WASM] Rerouting to egress-router1 via filter state +proxy-1 | [2025-11-20T03:08:18.423Z] cluster=egress-router1 src=172.22.0.10:39484 dst=172.22.0.2:10000 -> 35.170.145.70:80 +``` + +**For odd IP (last octet 11) → routes via egress-router2:** + +``` +proxy-1 | [TCP WASM] Source address: 172.22.0.11:55320 +proxy-1 | [TCP WASM] Source IP last octet: 11, intercepting ALL traffic +proxy-1 | [TCP WASM] Routing to egress-router2 +proxy-1 | [TCP WASM] set_envoy_filter_state status (envoy.tcp_proxy.cluster): Ok(None) +proxy-1 | [TCP WASM] Rerouting to egress-router2 via filter state +proxy-1 | [2025-11-20T03:08:39.974Z] cluster=egress-router2 src=172.22.0.11:55320 dst=172.22.0.2:10000 -> 52.44.182.178:80 +``` + +The `Ok(None)` status confirms that the filter state was successfully set, and you can see in the access logs that traffic is being routed to the correct clusters (`egress-router1` for even IPs, `egress-router2` for odd IPs). + +### Note on Destination Information + +In this example, both Envoy clusters (`egress-router1` and `egress-router2`) have `httpbin.org` hardcoded as their load balancer endpoints. + +In real world scenarios where destination information needs to be preserved across TCP hops, the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) could be used to forward metadata like the original source and destination addresses between proxies. This could be implemented using advanced WASM stream context capabilities combined with Envoy's PROXY protocol configuration. + +**Future Enhancement:** With Istio (Envoy-based) in a Kubernetes setup and PROXY protocol support, this example could be extended to serve as a source-based egress router. One practical use case would be routing user's web traffic via different external IP addresses based on source-based routing decisions (e.g., even vs. odd source IP like in the example). In conventional networks, this would be achieved with routing tables and IP rules to select egress interfaces based on source addresses. However, this approach is close to impossible with managed Kubernetes services (without additional network plugins like [Multus CNI](https://github.com/k8snetworkplumbingwg/multus-cni), as Kubernetes largely abstracts away the network layer where such layer 3 routing (different outgoing IPs) would normally be configured. diff --git a/examples/envoy_tcp_routing/build.rs b/examples/envoy_tcp_routing/build.rs new file mode 100644 index 00000000..5e904520 --- /dev/null +++ b/examples/envoy_tcp_routing/build.rs @@ -0,0 +1,25 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fs; + +fn main() { + let out_dir = "src/generated"; + fs::create_dir_all(out_dir).unwrap(); + prost_build::Config::new() + .out_dir(out_dir) + .compile_protos(&["src/set_envoy_filter_state.proto"], &["src/"]) + .unwrap(); + println!("cargo:rerun-if-changed=src/set_envoy_filter_state.proto"); +} diff --git a/examples/envoy_tcp_routing/docker-compose.yaml b/examples/envoy_tcp_routing/docker-compose.yaml new file mode 100644 index 00000000..fc294f33 --- /dev/null +++ b/examples/envoy_tcp_routing/docker-compose.yaml @@ -0,0 +1,32 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + proxy: + image: envoyproxy/envoy:v1.34.1 + entrypoint: /usr/local/bin/envoy -c /etc/envoy.yaml -l info --service-cluster proxy + volumes: + - ./envoy/envoy.yaml:/etc/envoy.yaml + - ./target/wasm32-wasip1/release/proxy_wasm_example_envoy_tcp_routing.wasm:/etc/envoy_tcp_routing.wasm + networks: + - envoymesh + ports: + - "10000:10000" + - "8001:8001" + +networks: + envoymesh: + ipam: + config: + - subnet: 172.22.0.0/16 diff --git a/examples/envoy_tcp_routing/envoy/envoy.yaml b/examples/envoy_tcp_routing/envoy/envoy.yaml new file mode 100644 index 00000000..cffebd7a --- /dev/null +++ b/examples/envoy_tcp_routing/envoy/envoy.yaml @@ -0,0 +1,90 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Envoy configuration for TCP rerouting example +static_resources: + listeners: + - name: main + address: + socket_address: + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + # WASM filter for TCP rerouting + - name: envoy.filters.network.wasm + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm + config: + name: envoy_tcp_routing_filter + root_id: envoy_tcp_routing_filter + configuration: + "@type": type.googleapis.com/google.protobuf.StringValue + value: "standalone" + vm_config: + vm_id: vm.envoy_tcp_routing + runtime: envoy.wasm.runtime.v8 + code: + local: + filename: /etc/envoy_tcp_routing.wasm + allow_precompiled: true + # TCP proxy filter + - name: envoy.filters.network.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: destination + cluster: egress-router1 # Default cluster, overridden by WASM filter + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + log_format: + text_format: "[%START_TIME%] cluster=%UPSTREAM_CLUSTER% src=%DOWNSTREAM_REMOTE_ADDRESS% dst=%DOWNSTREAM_LOCAL_ADDRESS% -> %UPSTREAM_HOST%\n" + + clusters: + - name: egress-router1 + connect_timeout: 30s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + load_assignment: + cluster_name: egress-router1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: httpbin.org + port_value: 80 + + - name: egress-router2 + connect_timeout: 30s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + load_assignment: + cluster_name: egress-router2 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: httpbin.org + port_value: 80 + +admin: + access_log_path: "/dev/null" + address: + socket_address: + address: 0.0.0.0 + port_value: 8001 diff --git a/examples/envoy_tcp_routing/src/lib.rs b/examples/envoy_tcp_routing/src/lib.rs new file mode 100644 index 00000000..ca5b2e61 --- /dev/null +++ b/examples/envoy_tcp_routing/src/lib.rs @@ -0,0 +1,174 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! TCP Rerouting Example +//! +//! This example demonstrates dynamic TCP routing based on source IP address. +//! Inspired by https://github.com/SiiiTschiii/wasmerang +//! +//! The filter intercepts TCP connections and routes them to different upstream +//! clusters based on whether the last octet of the source IP is even or odd: +//! - Even last octet → egress-router1 +//! - Odd last octet → egress-router2 + +use log::{info, warn}; +use proxy_wasm::traits::*; +use proxy_wasm::types::*; + +// Include the generated protobuf code +pub mod set_envoy_filter_state { + include!("generated/envoy.source.extensions.common.wasm.rs"); +} + +proxy_wasm::main! {{ + proxy_wasm::set_log_level(LogLevel::Info); + proxy_wasm::set_root_context(|_| -> Box { + Box::new(TcpReroutingRoot) + }); +}} + +struct TcpReroutingRoot; + +impl Context for TcpReroutingRoot {} + +impl RootContext for TcpReroutingRoot { + fn on_configure(&mut self, _plugin_configuration_size: usize) -> bool { + if let Some(config_bytes) = self.get_plugin_configuration() { + info!( + "[TCP WASM] Configuration: {:?}", + std::str::from_utf8(&config_bytes).unwrap_or("invalid UTF-8") + ); + } + true + } + + fn create_stream_context(&self, _context_id: u32) -> Option> { + Some(Box::new(TcpReroutingStream)) + } + + fn get_type(&self) -> Option { + Some(ContextType::StreamContext) + } +} + +struct TcpReroutingStream; + +impl Context for TcpReroutingStream {} + +impl StreamContext for TcpReroutingStream { + fn on_new_connection(&mut self) -> Action { + if let Some(source_addr_bytes) = self.get_property(vec!["source", "address"]) { + if let Ok(source_addr) = std::str::from_utf8(&source_addr_bytes) { + info!("[TCP WASM] Source address: {}", source_addr); + + // Extract the last octet from the source IP address + if let Some(last_octet) = extract_last_octet(source_addr) { + info!( + "[TCP WASM] Source IP last octet: {}, intercepting ALL traffic", + last_octet + ); + + // Determine target cluster based on even/odd last octet + let cluster = if last_octet % 2 == 0 { + "egress-router1" + } else { + "egress-router2" + }; + + info!("[TCP WASM] Routing to {}", cluster); + + // Set the cluster via Envoy's filter state mechanism using proper protobuf encoding + use set_envoy_filter_state::{LifeSpan, SetEnvoyFilterStateArguments}; + + let args = SetEnvoyFilterStateArguments { + path: "envoy.tcp_proxy.cluster".to_string(), + value: cluster.to_string(), + span: LifeSpan::FilterChain as i32, + }; + + let mut buf = Vec::new(); + if let Err(e) = prost::Message::encode(&args, &mut buf) { + warn!("[TCP WASM] Failed to encode filter state: {}", e); + return Action::Continue; + } + + // Use the Envoy-specific filter state mechanism + // https://github.com/envoyproxy/envoy/issues/28128 + let status = self.call_foreign_function("set_envoy_filter_state", Some(&buf)); + + info!( + "[TCP WASM] set_envoy_filter_state status (envoy.tcp_proxy.cluster): {:?}", + status + ); + info!("[TCP WASM] Rerouting to {} via filter state", cluster); + } + } + } + Action::Continue + } +} + +/// Extracts the last octet from an IP address string. +/// +/// Handles both IPv4 addresses with and without port numbers. +/// Examples: +/// - "192.168.1.10" → Some(10) +/// - "192.168.1.10:8080" → Some(10) +/// - "172.21.0.11:58762" → Some(11) +fn extract_last_octet(addr: &str) -> Option { + // Remove port if present (format: "ip:port") + let ip_part = addr.split(':').next()?; + + // Split by '.' and get the last segment + let last_segment = ip_part.split('.').next_back()?; + + // Parse as u8 + last_segment.parse::().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_last_octet() { + assert_eq!(extract_last_octet("192.168.1.10"), Some(10)); + assert_eq!(extract_last_octet("192.168.1.10:8080"), Some(10)); + assert_eq!(extract_last_octet("172.21.0.11:58762"), Some(11)); + assert_eq!(extract_last_octet("10.0.0.2"), Some(2)); + assert_eq!(extract_last_octet("invalid"), None); + assert_eq!(extract_last_octet(""), None); + } + + #[test] + fn test_routing_logic() { + // Even last octet should route to egress-router1 + let last_octet = 10; + let cluster = if last_octet % 2 == 0 { + "egress-router1" + } else { + "egress-router2" + }; + assert_eq!(cluster, "egress-router1"); + + // Odd last octet should route to egress-router2 + let last_octet = 11; + let cluster = if last_octet % 2 == 0 { + "egress-router1" + } else { + "egress-router2" + }; + assert_eq!(cluster, "egress-router2"); + } +} diff --git a/examples/envoy_tcp_routing/src/set_envoy_filter_state.proto b/examples/envoy_tcp_routing/src/set_envoy_filter_state.proto new file mode 100644 index 00000000..8d709c92 --- /dev/null +++ b/examples/envoy_tcp_routing/src/set_envoy_filter_state.proto @@ -0,0 +1,31 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package envoy.source.extensions.common.wasm; + +enum LifeSpan { + FilterChain = 0; + DownstreamRequest = 1; + DownstreamConnection = 2; +} + +// Argument expected by set_envoy_filter_state in envoy +// https://github.com/envoyproxy/envoy/blob/d741713c376d1e024236519fb59406c05702ad77/source/extensions/common/wasm/foreign.cc#L116 +message SetEnvoyFilterStateArguments { + string path = 1; + string value = 2; + LifeSpan span = 3; +}