Skip to content

sqm: Replace iptables marking with nftables #190

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dhrm1k wants to merge 1 commit into
base: main
Choose a base branch
from
Open

sqm: Replace iptables marking with nftables #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions platform/openwrt/sqm.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,3 @@ SQM_STATE_DIR=/var/run/sqm
SQM_QDISC_STATE_DIR=${SQM_STATE_DIR}/available_qdiscs
SQM_CHECK_QDISCS="fq_codel codel pie sfq cake"
SQM_SYSLOG=1
IP6TABLES_BINARY=$(command -v ip6tables-nft)
IPTABLES_BINARY=$(command -v iptables-nft)
16 changes: 6 additions & 10 deletions src/defaults.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,9 @@
[ -z "$TC_BINARY" ] && TC_BINARY=$(command -v tc)
[ -z "$IP" ] && IP=ip_wrapper
[ -z "$IP_BINARY" ] && IP_BINARY=$(command -v ip)
[ -z "$IPTABLES" ] && IPTABLES=iptables_wrapper
[ -z "$IPTABLES_BINARY" ] && IPTABLES_BINARY=$(command -v iptables)
[ -z "$IPTABLES_BINARY" ] && IPTABLES_BINARY=$(command -v iptables-nft)
[ -z "$IP6TABLES" ] && IP6TABLES=ip6tables_wrapper
[ -z "$IP6TABLES_BINARY" ] && IP6TABLES_BINARY=$(command -v ip6tables)
[ -z "$IP6TABLES_BINARY" ] && IP6TABLES_BINARY=$(command -v ip6tables-nft)
[ -z "$IPTABLES_ARGS" ] && IPTABLES_ARGS="-w 1"
[ -z "$NFT" ] && NFT=nft_wrapper
[ -z "$NFT_BINARY" ] && NFT_BINARY=$(command -v nft)
[ -z "$NFT_TABLE" ] && NFT_TABLE="sqm_$(printf '%s' "$IFACE" | sed 's/[^A-Za-z0-9_]/_/g')"


# Try modprobe first, fall back to insmod
Expand All @@ -52,7 +48,7 @@ if [ -z "$INSMOD" ]; then
fi

[ -z "$TARGET" ] && TARGET="5ms"
[ -z "$IPT_MASK" ] && IPT_MASK="0xff" # to disable: set mask to 0xffffffff
[ -z "$MARK_MASK" ] && MARK_MASK="0xff" # to disable: set mask to 0xffffffff
#sm: we need the functions above before trying to set the ingress IFB device
#sm: *_CAKE_OPTS should contain the diffserv keyword for cake
[ -z "$INGRESS_CAKE_OPTS" ] && INGRESS_CAKE_OPTS="diffserv3 nat"
Expand Down Expand Up @@ -108,8 +104,8 @@ SILENT=0
# stop operation
[ -z "$CLEANUP" ] && CLEANUP=0

# Transaction log for unwinding ipt rules
IPT_TRANS_LOG="${SQM_STATE_DIR}/${IFACE}.iptables.log"
# Firewall state used for cleanup
NFT_RULESET_FILE="${SQM_STATE_DIR}/${IFACE}.nft"

# These are the modules that do_modules() will attempt to load
ALL_MODULES="sch_$QDISC sch_ingress act_mirred cls_fw cls_flow cls_u32 sch_htb"
108 changes: 17 additions & 91 deletions src/functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,102 +95,29 @@ fn_exists() {
}


# Transaction logging for ipt rules to allow for gracefull final teardown
ipt_log_restart() {
[ -f "$IPT_TRANS_LOG" ] && rm -f "$IPT_TRANS_LOG"
nft_log_restart() {
[ -f "$NFT_RULESET_FILE" ] && rm -f "$NFT_RULESET_FILE"
}


# Function to negate iptables commands. Turns addition and insertion into
# deletion, and creation of new chains into deletion
# Its output has quotes around all parameters so we can preserve arguments
# containing whitespace across log file write & re-read
ipt_negate()
{
for var in "$@"; do
case "$var" in
"-A"|"-I") echo -n '"-D" ' ;;
"-N") echo -n '"-X" ' ;;
*) echo -n "\"$var\" " ;;
esac
done
echo ""
}

ipt_log()
{
echo "$@" >> $IPT_TRANS_LOG
nft_cleanup() {
[ -n "$NFT_BINARY" ] || return 0
SILENT=1 ${NFT} destroy table inet ${NFT_TABLE} >/dev/null 2>&1
nft_log_restart
}

# Split a string containing an iptables command line parameter invocation, then
# run it through ipt(). This is used to turn lines read from the log file, or
# output from ipt_negate() back into proper parameters contained in $@
ipt_run_split()
{
eval "set -- $1"
ipt "$@"
}

# Read the transaction log in reverse and execute using ipt to undo changes.
# Since we logged only ipt '-D' commands, ipt won't add them again to the
# transaction log, but will include them in the syslog/debug log.
ipt_log_rewind() {
[ -f "$IPT_TRANS_LOG" ] || return 0
sed -n '1!G;h;$p' "$IPT_TRANS_LOG" |
while read line; do
[ -n "$line" ] || continue
ipt_run_split "$line"
done

# We just rewound the log, make sure to restart it
ipt_log_restart
# wrapper to call nft to allow debug logging
nft_wrapper(){
cmd_wrapper nft ${NFT_BINARY} "$@"
}

ipt() {
local neg

for var in "$@"; do
case "$var" in
"-A"|"-I"|"-N")
# If the rule is an addition rule, we first run its negation,
# then log that negation to be used by ipt_log_rewind() on
# shutdown
neg="$(ipt_negate "$@")"
ipt_run_split "$neg"
ipt_log "$neg"
;;
esac
done

SILENT=1 ${IPTABLES} $IPTABLES_ARGS "$@"
SILENT=1 ${IP6TABLES} $IPTABLES_ARGS "$@"
}


# wrapper to call iptables to allow debug logging
iptables_wrapper(){
cmd_wrapper iptables ${IPTABLES_BINARY} "$@"
}

# wrapper to call ip6tables to allow debug logging
ip6tables_wrapper(){
cmd_wrapper ip6tables ${IP6TABLES_BINARY} "$@"
}

verify_iptables()
verify_nft()
{
local ret
ret=0

if [ -z "$IPTABLES_BINARY" ]; then
sqm_error "No iptables binary found, please install 'iptables' or 'iptables-nft' to use this script"
ret=1
if [ -z "$NFT_BINARY" ]; then
sqm_error "No nft binary found, please install 'nftables' to use this script"
return 1
fi
if [ -z "$IP6TABLES_BINARY" ]; then
sqm_error "No ip6tables binary found, please install 'ip6tables' or 'ip6tables-nft' to use this script"
ret=1
fi
return $ret
return 0
}

# wrapper to call tc to allow debug logging
Expand Down Expand Up @@ -505,8 +432,8 @@ sqm_start_default() {
#sqm_error "sqm_start_default"
[ -n "$IFACE" ] || return 1

# reset the iptables trace log
ipt_log_restart
# reset the firewall rule file
nft_log_restart

if fn_exists sqm_prepare_script ; then
sqm_debug "sqm_start_default: starting sqm_prepare_script"
Expand Down Expand Up @@ -553,8 +480,7 @@ sqm_cleanup()
local silent
silent=${1:-0}

# undo accumulated ipt commands during shutdown
ipt_log_rewind
nft_cleanup

[ -n "$CUR_IFB" ] || return 0

Expand Down
111 changes: 66 additions & 45 deletions src/simple.qos
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# This is a three band fq_codel and ipv6 enabled shaping script for Ethernet
# gateways. Compared to the complexity that debloat had become this cleanly
# shows a means of going from diffserv marking to prioritization using the
# current tools ip(6)tables and tc. We should note that the complexity of
# current tools nftables and tc. We should note that the complexity of
# debloat exists for a reason, and it is expected that script is run first to
# setup various other parameters such as BQL and ethtool.
#
Expand Down Expand Up @@ -36,57 +36,78 @@

################################################################################

ipt_setup() {
nft_rule() {
printf '%s\n' "$*" >> "$NFT_RULESET_FILE"
}

nft_mark_set() {
local mark

mark=$1
printf 'meta mark set (meta mark & %s) | %s' "$NFT_CLEAR_MASK" "$mark"
}

nft_setup() {
local mark_be
local mark_bulk
local mark_prio

NFT_CLEAR_MASK=$(printf '0x%x' $((0xffffffff & ~MARK_MASK)))
mark_be="$(nft_mark_set 0x2)"
mark_bulk="$(nft_mark_set 0x3)"
mark_prio="$(nft_mark_set 0x1)"

nft_cleanup

ipt -t mangle -N QOS_MARK_${IFACE}
nft_rule "table inet ${NFT_TABLE} {"
nft_rule " chain qos_mark {"

case $QDISC in
cake*)
sqm_debug "cake does all the diffserv work - no need for iptables rules"
sqm_debug "cake does all the diffserv work - no need for nftables classification rules"
;;
*)
ipt -t mangle -A QOS_MARK_${IFACE} -j MARK --set-mark 0x2/${IPT_MASK}
nft_rule " ${mark_be}"
# You can go further with classification but...
ipt -t mangle -A QOS_MARK_${IFACE} -m dscp --dscp-class CS1 -j MARK --set-mark 0x3/${IPT_MASK}
ipt -t mangle -A QOS_MARK_${IFACE} -m dscp --dscp-class CS6 -j MARK --set-mark 0x1/${IPT_MASK}
ipt -t mangle -A QOS_MARK_${IFACE} -m dscp --dscp-class EF -j MARK --set-mark 0x1/${IPT_MASK}
ipt -t mangle -A QOS_MARK_${IFACE} -m dscp --dscp-class AF42 -j MARK --set-mark 0x1/${IPT_MASK}
ipt -t mangle -A QOS_MARK_${IFACE} -m tos --tos Minimize-Delay -j MARK --set-mark 0x1/${IPT_MASK}
nft_rule " meta nfproto ipv4 ip dscp cs1 ${mark_bulk}"
nft_rule " meta nfproto ipv6 ip6 dscp cs1 ${mark_bulk}"
nft_rule " meta nfproto ipv4 ip dscp cs6 ${mark_prio}"
nft_rule " meta nfproto ipv6 ip6 dscp cs6 ${mark_prio}"
nft_rule " meta nfproto ipv4 ip dscp ef ${mark_prio}"
nft_rule " meta nfproto ipv6 ip6 dscp ef ${mark_prio}"
nft_rule " meta nfproto ipv4 ip dscp af42 ${mark_prio}"
nft_rule " meta nfproto ipv6 ip6 dscp af42 ${mark_prio}"
nft_rule " meta nfproto ipv4 ip dscp 0x04 ${mark_prio}"
;;
esac

# Turn it on. Preserve classification if already performed
#
#sm: is it correct to do this in $IFACE? Should ingress not be on $DEV? since HTB acts on $DEV?
#
# ZERO also does not work on $DEV (that is the IFB will still see the
# incoming ToS bits whether we squash or not)
#
# ZERO is still useful to protect internal machines...
nft_rule " }"
nft_rule " chain prerouting {"
nft_rule " type filter hook prerouting priority mangle + 10; policy accept;"
nft_rule " iifname \"vtun*\" meta l4proto tcp ${mark_be}"

if [ "$ZERO_DSCP_INGRESS" = "1" ]; then
sqm_debug "Squashing differentiated services code points (DSCP) from ingress."
ipt -t mangle -I PREROUTING -i $IFACE -m dscp ! --dscp 0 -j DSCP --set-dscp-class be
nft_rule " iif \"${IFACE}\" meta nfproto ipv4 ip dscp != cs0 ip dscp set cs0"
nft_rule " iif \"${IFACE}\" meta nfproto ipv6 ip6 dscp != cs0 ip6 dscp set cs0"
else
sqm_debug "Keeping differentiated services code points (DSCP) from ingress."
ipt -t mangle -A PREROUTING -i $IFACE -m mark --mark 0x00/${IPT_MASK} -g QOS_MARK_${IFACE}
nft_rule " iif \"${IFACE}\" meta mark & ${MARK_MASK} == 0x0 goto qos_mark"
fi

ipt -t mangle -A POSTROUTING -o $IFACE -m mark --mark 0x00/${IPT_MASK} -g QOS_MARK_${IFACE}

# The Syn optimization was nice but fq_codel does it for us
# ipt -t mangle -A PREROUTING -i s+ -p tcp -m tcp --tcp-flags SYN,RST,ACK SYN -j MARK --set-mark 0x01
# Not sure if this will work. Encapsulation is a problem period

ipt -t mangle -I PREROUTING -i vtun+ -p tcp -j MARK --set-mark 0x2/${IPT_MASK} # tcp tunnels need ordering

# Emanating from router, do a little more optimization
# but don't bother with it too much.

ipt -t mangle -A OUTPUT -p udp -m multiport --ports 123,53 -j DSCP --set-dscp-class AF42

#Not clear if the second line is needed
#ipt -t mangle -A OUTPUT -o $IFACE -g QOS_MARK_${IFACE}

nft_rule " }"
nft_rule " chain postrouting {"
nft_rule " type filter hook postrouting priority mangle + 10; policy accept;"
nft_rule " oif \"${IFACE}\" meta mark & ${MARK_MASK} == 0x0 goto qos_mark"
nft_rule " }"
nft_rule " chain output {"
nft_rule " type filter hook output priority mangle + 10; policy accept;"
nft_rule " meta nfproto ipv4 meta l4proto udp ct original proto-dst { 123, 53 } ip dscp set af42"
nft_rule " meta nfproto ipv6 meta l4proto udp ct original proto-dst { 123, 53 } ip6 dscp set af42"
nft_rule " }"
nft_rule "}"

${NFT} -f "$NFT_RULESET_FILE"
}


Expand Down Expand Up @@ -133,19 +154,19 @@ egress() {

# FIXME should probably change the filter here to do pre-nat

$TC filter add dev $IFACE parent 1:0 protocol ip prio 1 handle 1/${IPT_MASK} fw classid 1:11
$TC filter add dev $IFACE parent 1:0 protocol ip prio 2 handle 2/${IPT_MASK} fw classid 1:12
$TC filter add dev $IFACE parent 1:0 protocol ip prio 3 handle 3/${IPT_MASK} fw classid 1:13
$TC filter add dev $IFACE parent 1:0 protocol ip prio 1 handle 1/${MARK_MASK} fw classid 1:11
$TC filter add dev $IFACE parent 1:0 protocol ip prio 2 handle 2/${MARK_MASK} fw classid 1:12
$TC filter add dev $IFACE parent 1:0 protocol ip prio 3 handle 3/${MARK_MASK} fw classid 1:13

# ipv6 support. Note that the handle indicates the fw mark bucket that is looked for

$TC filter add dev $IFACE parent 1:0 protocol ipv6 prio 4 handle 1/${IPT_MASK} fw classid 1:11
$TC filter add dev $IFACE parent 1:0 protocol ipv6 prio 5 handle 2/${IPT_MASK} fw classid 1:12
$TC filter add dev $IFACE parent 1:0 protocol ipv6 prio 6 handle 3/${IPT_MASK} fw classid 1:13
$TC filter add dev $IFACE parent 1:0 protocol ipv6 prio 4 handle 1/${MARK_MASK} fw classid 1:11
$TC filter add dev $IFACE parent 1:0 protocol ipv6 prio 5 handle 2/${MARK_MASK} fw classid 1:12
$TC filter add dev $IFACE parent 1:0 protocol ipv6 prio 6 handle 3/${MARK_MASK} fw classid 1:13

# Arp traffic

$TC filter add dev $IFACE parent 1:0 protocol arp prio 7 handle 1/${IPT_MASK} fw classid 1:11
$TC filter add dev $IFACE parent 1:0 protocol arp prio 7 handle 1/${MARK_MASK} fw classid 1:11

# ICMP traffic - Don't impress your friends. Deoptimize to manage ping floods
# better instead
Expand Down Expand Up @@ -230,6 +251,6 @@ ingress() {
sqm_prepare_script() {
do_modules
verify_qdisc "htb" || return 1
verify_iptables || return 1
ipt_setup
verify_nft || return 1
nft_setup
}