From cc638f93c7ef4a0d2e235052a55d2b30bb19191b Mon Sep 17 00:00:00 2001 From: Wei-Y <83482178+w-yue@users.noreply.github.com> Date: Wed, 18 Jan 2023 19:48:20 +0000 Subject: [PATCH 1/4] Add security group and conntrack support DP support code. This is currently only WIP code, processing logic are not checked in yet. This is mainly for the conviniency of arion_agent SG related development. --- src/dmn/trn_transit_xdp_usr.c | 250 ++++++++++++++++++++++++++++++++- src/dmn/trn_transit_xdp_usr.h | 28 ++-- src/include/trn_datamodel.h | 82 +++++++++++ src/xdp/trn_kern.h | 2 +- src/xdp/trn_transit_xdp_maps.h | 50 ++++++- 5 files changed, 399 insertions(+), 13 deletions(-) diff --git a/src/dmn/trn_transit_xdp_usr.c b/src/dmn/trn_transit_xdp_usr.c index 1f9e12e..87df8cf 100644 --- a/src/dmn/trn_transit_xdp_usr.c +++ b/src/dmn/trn_transit_xdp_usr.c @@ -34,6 +34,7 @@ #include #include #include "extern/linux/err.h" + #include "trn_transitd.h" static trn_xdp_prog_t trn_prog_tbl[TRAN_MAX_PROG] = { @@ -63,13 +64,19 @@ static char * pin_path = "/sys/fs/bpf"; static trn_xdp_map_t trn_xdp_bpfmaps[] = { {"jmp_table", true, -1, NULL}, {"endpoints_map", true, -1, NULL}, -#if turnOn - {"hosted_eps_if", true, -1, NULL}, -#endif {"if_config_map", true, -1, NULL}, {"interfaces_map", true, -1, NULL}, {"xsks_map", true, -1,NULL}, +#if connTrack + {"contrack_map", true, -1, NULL}, +#endif +#if sgSupport + {"sg_cidr_map", true, -1, NULL}, + {"security_group_map", true, -1, NULL}, + {"port_range_map", true, -1, NULL}, +#endif #if turnOn + {"hosted_eps_if", true, -1, NULL}, {"oam_queue_map", true, -1, NULL}, {"fwd_flow_cache", true, -1, NULL}, {"rev_flow_cache", true, -1, NULL}, @@ -374,6 +381,243 @@ static int trn_prog_load(trn_prog_t *prog, int prog_idx) return 1; } +#if sgSupport +int trn_update_sg_cidr_get_ctx(void) +{ + int fd; + + fd = trn_transit_map_get_fd("sg_cidr_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get sg_cidr_map fd"); + } + + return fd; +} + +int trn_update_sg_cidr(int fd, sg_cidr_key_t *sgkey, sg_cidr_t *sg) +{ + int err; + + if (fd < 0) { + TRN_LOG_ERROR("Invalid sg_cidr_map fd"); + return 1; + } + + err = bpf_map_update_elem(fd, sgkey, sg, 0); + if (err) { + TRN_LOG_ERROR("Store sg_cidr mapping failed (err:%d).", err); + return 1; + } + + return 0; +} + +int trn_get_sg_cidr(sg_cidr_key_t *epkey, sg_cidr_t *ep) +{ + int fd, err; + + fd = trn_transit_map_get_fd("sg_cidr_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get sg_cidr_map fd"); + return 1; + } + + err = bpf_map_lookup_elem(fd, epkey, ep); + if (err) { + TRN_LOG_ERROR("Querying sg_cidr mapping failed (err:%d).", + err); + return 1; + } + return 0; +} + +int trn_delete_sg_cidr(sg_cidr_key_t *epkey) +{ + sg_cidr_t ep; + int fd, err; + + fd = trn_transit_map_get_fd("sg_cidr_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get sg_cidr_map fd"); + return 1; + } + + err = bpf_map_lookup_elem(fd, epkey, &ep); + + if (err) { + TRN_LOG_ERROR("Querying sg_cidr for delete failed (err:%d).", + err); + return 1; + } + + err = bpf_map_delete_elem(fd, epkey); + if (err) { + TRN_LOG_ERROR("Deleting sg_cidr mapping failed (err:%d).", + err); + return 1; + } + + return 0; +} +// security_group +int trn_update_sg_get_ctx(void) +{ + int fd; + + fd = trn_transit_map_get_fd("security_group_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get security_group_map fd"); + } + + return fd; +} + +int trn_update_sg(int fd, security_group_key_t *sgkey, security_group_t *sg) +{ + int err; + + if (fd < 0) { + TRN_LOG_ERROR("Invalid security_group_map fd"); + return 1; + } + + err = bpf_map_update_elem(fd, sgkey, sg, 0); + if (err) { + TRN_LOG_ERROR("Store sg_cidr mapping failed (err:%d).", err); + return 1; + } + + return 0; +} + +int trn_get_sg(security_group_key_t *sgkey, security_group_t *sg) +{ + int fd, err; + + fd = trn_transit_map_get_fd("security_group_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get security_group_map fd"); + return 1; + } + + err = bpf_map_lookup_elem(fd, sgkey, sg); + if (err) { + TRN_LOG_ERROR("Querying sg_cidr mapping failed (err:%d).", + err); + return 1; + } + return 0; +} + +int trn_delete_sg(security_group_key_t *sgkey) +{ + security_group_t sg; + int fd, err; + + fd = trn_transit_map_get_fd("security_group_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get security_group_map fd"); + return 1; + } + + err = bpf_map_lookup_elem(fd, sgkey, &sg); + + if (err) { + TRN_LOG_ERROR("Querying security_group for delete failed (err:%d).", + err); + return 1; + } + + err = bpf_map_delete_elem(fd, sgkey); + if (err) { + TRN_LOG_ERROR("Deleting security_group mapping failed (err:%d).", + err); + return 1; + } + + return 0; +} + +//port_range_map +int trn_update_port_range_get_ctx(void) +{ + int fd; + + fd = trn_transit_map_get_fd("port_range_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get port_range_map fd"); + } + + return fd; +} + +int trn_update_port_range(int fd, port_range_key_t *prkey, port_range_t *pr) +{ + int err; + + if (fd < 0) { + TRN_LOG_ERROR("Invalid port_range_map fd"); + return 1; + } + + err = bpf_map_update_elem(fd, prkey, pr, 0); + if (err) { + TRN_LOG_ERROR("Store port_range mapping failed (err:%d).", err); + return 1; + } + + return 0; +} + +int trn_get_port_range(port_range_key_t *prkey, port_range_t *pr) +{ + int fd, err; + + fd = trn_transit_map_get_fd("port_range_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get port_range_map fd"); + return 1; + } + + err = bpf_map_lookup_elem(fd, prkey, pr); + if (err) { + TRN_LOG_ERROR("Querying port_range mapping failed (err:%d).", + err); + return 1; + } + return 0; +} + +int trn_delete_port_range(port_range_key_t *prkey) +{ + port_range_t pr; + int fd, err; + + fd = trn_transit_map_get_fd("port_range_map"); + if (fd < 0) { + TRN_LOG_ERROR("Failed to get port_range_map fd"); + return 1; + } + + err = bpf_map_lookup_elem(fd, prkey, &pr); + + if (err) { + TRN_LOG_ERROR("Querying port_range for delete failed (err:%d).", + err); + return 1; + } + + err = bpf_map_delete_elem(fd, prkey); + if (err) { + TRN_LOG_ERROR("Deleting port_range mapping failed (err:%d).", + err); + return 1; + } + + return 0; +} +#endif + int trn_update_endpoints_get_ctx(void) { int fd; diff --git a/src/dmn/trn_transit_xdp_usr.h b/src/dmn/trn_transit_xdp_usr.h index 35973b9..9880ecf 100644 --- a/src/dmn/trn_transit_xdp_usr.h +++ b/src/dmn/trn_transit_xdp_usr.h @@ -9,7 +9,7 @@ * @brief User space APIs and data structures to program transit * xdp program. * - * @copyright Copyright (c) 2019-2022 The Authors. + * @copyright Copyright (c) 2019-2023 The Authors. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -51,6 +51,8 @@ #include "extern/cJSON.h" #define turnOn 0 +#define sgSupport 1 + typedef struct { int prog_id; // definition in trn_xdp_prog_id_t char *prog_path; // full path of xdp program @@ -99,23 +101,33 @@ typedef struct { } user_metadata_t; trn_iface_t *trn_get_itf_context(char *interface); - int trn_update_itf_config(struct tunnel_iface_t *itf); int trn_update_endpoints_get_ctx(void); - int trn_update_endpoint(int fd, endpoint_key_t *epkey, endpoint_t *ep); - int trn_get_endpoint(endpoint_key_t *epkey, endpoint_t *ep); - int trn_delete_endpoint(endpoint_key_t *epkey); -int trn_transit_xdp_load(char **interfaces, unsigned short ibo_port, bool debug); +#if sgSupport +int trn_update_sg_cidr_get_ctx(void); +int trn_update_sg_cidr(int fd, sg_cidr_key_t *sgkey, sg_cidr_t *sg); +int trn_get_sg_cidr(sg_cidr_key_t *sgkey, sg_cidr_t *sg); +int trn_delete_sg_cidr(sg_cidr_key_t *sgkey); + +int trn_update_sg_get_ctx(void); +int trn_update_sg(int fd, security_group_key_t *sgkey, security_group_t *sg); +int trn_get_sg(security_group_key_t *sgkey, security_group_t *sg); +int trn_delete_sg(security_group_key_t *sgkey); + +int trn_update_port_range_get_ctx(void); +int trn_update_port_range(int fd, port_range_key_t *prkey, port_range_t *pr); +int trn_get_port_range(port_range_key_t *prkey, port_range_t *pr); +int trn_delete_port_range(port_range_key_t *prkey); +#endif +int trn_transit_xdp_load(char **interfaces, unsigned short ibo_port, bool debug); int trn_transit_xdp_unload(char **interfaces); - int trn_transit_ebpf_load(int prog_idx); - int trn_transit_ebpf_unload(int prog_idx); #if turnOn diff --git a/src/include/trn_datamodel.h b/src/include/trn_datamodel.h index 74f909b..9106bac 100644 --- a/src/include/trn_datamodel.h +++ b/src/include/trn_datamodel.h @@ -65,6 +65,21 @@ /* Size for OAM message queue bpfmap */ #define TRAN_OAM_QUEUE_LEN 1024 +#define turnOn 0 +#define cnOn 0 +#define sgSupport 1 +#define connTrack 0 + +#if sgSupport +#define TRAN_MAX_CIDRS 1024*1024 +#define TRAN_SG_STOP 1 +#define TRAN_SG_PASS 0 + +#define SG_STATIC_PREFIX (sizeof(__be32) * 8) +#define SG_PREFIX_LEN(PREFIX) (SG_STATIC_PREFIX + (PREFIX)) +#define SG_IPV4_PREFIX SG_PREFIX_LEN(32) +#endif + /* XDP interface_map keys for packet redirect */ enum trn_itf_ma_key_t { TRAN_ITF_MAP_TENANT = 0, // id map to ifindex connected to tenant network @@ -148,6 +163,73 @@ typedef struct { unsigned char hmac[6]; } __attribute__((packed, aligned(4))) endpoint_t; +#if connTrack +struct ipv4_tuple_t { + __u32 saddr; + __u32 daddr; + + /* ports */ + __u16 sport; + __u16 dport; + + /* Addresses */ + __u8 protocol; + + /*TODO: include TCP flags, no use case for the moment! */ +} __attribute__((packed)); + +typedef struct { + __u32 vni; + struct ipv4_tuple_t tuple; +} __attribute__((packed)) contrack_key_t; + +typedef struct { + __u32 hip; + unsigned char mac[6]; + unsigned char hmac[6]; +} __attribute__((packed, aligned(4))) contrack_t; +#endif + +#if sgSupport +typedef struct { + __u32 prefixlen; /* up to 32 for AF_INET, 128 for AF_INET6*/ + __u32 vni; + __u32 ip; + __u16 port; + __u8 direction; + __u8 protocol; +} __attribute__((packed, aligned(4))) sg_cidr_key_t; + +typedef struct { + __u32 sg_id; + __u8 action; +} __attribute__((packed, aligned(4))) sg_cidr_t; + +typedef struct { + __u32 vni; + __u32 ip; + __u8 direction; +} __attribute__((packed, aligned(4))) security_group_key_t; + +typedef struct { + __u32 sg_id; + __u8 action; +} __attribute__((packed, aligned(4))) security_group_t; + +typedef struct { + __u32 vni; + __u32 ip; + __u8 direction; +} __attribute__((packed, aligned(4))) port_range_key_t; + +typedef struct { + __u16 port_min1; + __u16 port_max1; + __u16 port_min2; + __u16 port_max2; +} __attribute__((packed, aligned(4))) port_range_t; +#endif + typedef struct { __u32 ip; // IP used for ZGC access __u16 announced; // non-zero indicates the MAC has been announced locally diff --git a/src/xdp/trn_kern.h b/src/xdp/trn_kern.h index 1c98c9a..9742cc1 100644 --- a/src/xdp/trn_kern.h +++ b/src/xdp/trn_kern.h @@ -197,7 +197,7 @@ struct transit_packet { } __attribute__((packed, aligned(8))); struct xdp_hints_src { - __u16 flags; // + __u16 flags; __u32 vni; __u32 saddr; unsigned char h_source[6]; diff --git a/src/xdp/trn_transit_xdp_maps.h b/src/xdp/trn_transit_xdp_maps.h index 5f4ed13..dac1bb2 100644 --- a/src/xdp/trn_transit_xdp_maps.h +++ b/src/xdp/trn_transit_xdp_maps.h @@ -30,7 +30,6 @@ #include "extern/xdpcap_hook.h" #include "trn_datamodel.h" -#define turnOn 0 struct bpf_map_def SEC("maps") jmp_table = { .type = BPF_MAP_TYPE_PROG_ARRAY, @@ -55,6 +54,45 @@ struct bpf_map_def SEC("maps") xsks_map = { .value_size = sizeof(int), .max_entries = 64, /* Assume netdev has no more than 64 queues */ }; +#if connTrack +struct bpf_map_def SEC("maps") contrack_map = { + .type = BPF_MAP_TYPE_LRU_HASH, + .key_size = sizeof(contrack_key_t), + .value_size = sizeof(contrack_t), + .max_entries = TRAN_MAX_NEP, + .map_flags = 0, +}; +BPF_ANNOTATE_KV_PAIR(contrack_map, contrack_key_t, contrack_t); +#endif + +#if sgSupport +struct bpf_map_def SEC("maps") sg_cidr_map = { + .type = BPF_MAP_TYPE_LPM_TRIE, + .key_size = sizeof(sg_cidr_key_t), + .value_size = sizeof(security_group_t), + .max_entries = TRAN_MAX_CIDRS, + .map_flags = BPF_F_NO_PREALLOC, +}; +BPF_ANNOTATE_KV_PAIR(sg_cidr_map, sg_cidr_key_t, security_group_t); + +struct bpf_map_def SEC("maps") security_group_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(security_group_key_t), + .value_size = sizeof(security_group_t), + .max_entries = TRAN_MAX_NEP, + .map_flags = 0, +}; +BPF_ANNOTATE_KV_PAIR(security_group_map, security_group_key_t, security_group_t); + +struct bpf_map_def SEC("maps") port_range_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(port_range_key_t), + .value_size = sizeof(port_range_t), + .max_entries = TRAN_MAX_NEP, + .map_flags = 0, +}; +BPF_ANNOTATE_KV_PAIR(port_range_map, port_range_key_t, port_range_t); +#endif #if turnOn struct bpf_map_def SEC("maps") hosted_eps_if = { @@ -95,6 +133,16 @@ struct bpf_map_def SEC("maps") oam_queue_map = { BPF_ANNOTATE_KV_PAIR_QUEUESTACK(oam_queue_map, flow_ctx_t); #endif +#if cnOn +struct bpf_map_def SEC("maps") oam_queue_map = { + .type = BPF_MAP_TYPE_QUEUE, + .key_size = 0, + .value_size = sizeof(flow_ctx_t), + .max_entries = TRAN_OAM_QUEUE_LEN, +}; +BPF_ANNOTATE_KV_PAIR_QUEUESTACK(oam_queue_map, flow_ctx_t); +#endif + #if turnOn struct bpf_map_def SEC("maps") fwd_flow_cache = { .type = BPF_MAP_TYPE_LRU_HASH, From 6079bcd2d81ae67de84a62d65fde20a967b45e08 Mon Sep 17 00:00:00 2001 From: Wei-Y <83482178+w-yue@users.noreply.github.com> Date: Fri, 20 Jan 2023 23:13:51 +0000 Subject: [PATCH 2/4] sg_cidr_key_t format change. src_ip needs to be added. --- src/include/trn_datamodel.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/include/trn_datamodel.h b/src/include/trn_datamodel.h index 9106bac..d699fc4 100644 --- a/src/include/trn_datamodel.h +++ b/src/include/trn_datamodel.h @@ -194,10 +194,11 @@ typedef struct { typedef struct { __u32 prefixlen; /* up to 32 for AF_INET, 128 for AF_INET6*/ __u32 vni; - __u32 ip; __u16 port; __u8 direction; __u8 protocol; + __u32 src_ip; + __u32 dst_ip; } __attribute__((packed, aligned(4))) sg_cidr_key_t; typedef struct { From 3560a949eb088712ffdc5cffa70b8d49cd4fa160 Mon Sep 17 00:00:00 2001 From: Wei-Y <83482178+w-yue@users.noreply.github.com> Date: Tue, 31 Jan 2023 21:25:30 +0000 Subject: [PATCH 3/4] DP SG support initial logic. --- src/xdp/trn_transit_xdp.c | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/xdp/trn_transit_xdp.c b/src/xdp/trn_transit_xdp.c index 2ad3af2..71e0353 100644 --- a/src/xdp/trn_transit_xdp.c +++ b/src/xdp/trn_transit_xdp.c @@ -128,10 +128,68 @@ static __inline int trn_decapsulate_and_redirect(struct transit_packet *pkt, return bpf_redirect_map(&interfaces_map, ifindex, 0); } +#if sgSupport +static __inline int trn_sg_check(struct transit_packet *pkt) { + + sg_cidr_key_t sgkey; + struct security_group_t *sg_entry = NULL; + + sgkey.vni = pkt->vni; + sgkey.src_ip = pkt->inner_ip->saddr; + sgkey.dst_ip = pkt->inner_ip->daddr; + sgkey.protocol = pkt->inner_ip->protocol; + sgkey.direction = 1; // source -> direction; + + if (sgkey.protocol == IPPROTO_TCP) { + pkt->inner_tcp = (void *)pkt->inner_ip + sizeof(*pkt->inner_ip); + + if (pkt->inner_tcp + 1 > pkt->data_end) { + bpf_debug("[Transit:%d] ABORTED: Bad inner TCP frame\n", + pkt->itf_idx); + return XDP_ABORTED; + } + sgkey.port = pkt->inner_tcp->dest; // check dst first + } else if (sgkey.protocol == IPPROTO_UDP) { + pkt->inner_udp = (void *)pkt->inner_ip + sizeof(*pkt->inner_ip); + + if (pkt->inner_udp + 1 > pkt->data_end) { + bpf_debug("[Transit:%d] ABORTED: Bad inner UDP oframe\n", + pkt->itf_idx); + return XDP_ABORTED; + } + sgkey.port = pkt->inner_udp->dest; + } else { + bpf_debug("[Transit:%d] PASS: Non supported frame, proto: %d\n", + pkt->itf_idx, sgkey.protocol); + return XDP_PASS; + } + + sgkey.prefixlen = SG_IPV4_PREFIX; // ?? + bpf_debug("[Transit:%d] XXXXX port=%d, prefixlen=%d", + sgkey.vni, sgkey.port, sgkey.prefixlen); + bpf_debug(" XXXXX proto=%d, src_ip:0x%x, dst_ip:0x%x ", + sgkey.protocol, + bpf_ntohl(sgkey.src_ip), + bpf_ntohl(sgkey.dst_ip)); + sg_entry = bpf_map_lookup_elem(&sg_cidr_map, &sgkey); + if (sg_entry == NULL) { + // other follow up logic should be added here + bpf_debug("[Transit:%d XXXXX] Drop: no matching sg entry found: %d - %d\n", + pkt->itf_idx, sgkey.port, sgkey.vni); + return XDP_DROP; + } + + bpf_debug("[Transit:%d XXXXX] Pass: matching sg entry found: %d - 0x%x\n", + sgkey.vni, sgkey.port, bpf_ntohl(pkt->inner_ip->daddr)); + return XDP_PASS; +} +#endif + static __inline int trn_process_inner_ip(struct transit_packet *pkt) { endpoint_t *ep; endpoint_key_t epkey; + int action = XDP_PASS; ipv4_flow_t *flow = &pkt->fctx.flow; __u64 csum = 0; __u16 len = 0; @@ -144,6 +202,14 @@ static __inline int trn_process_inner_ip(struct transit_packet *pkt) return XDP_ABORTED; } +#if sgSupport + action = trn_sg_check(pkt); + if (action != XDP_PASS) { + bpf_debug("[Transit:%d XXXX] No SG entry found, drop it: \n", pkt->itf_idx); + return action; + } +#endif + /* Look up target endpoint */ epkey.vni = pkt->vni; epkey.ip = pkt->inner_ip->daddr; From ed45ce8952207ab2fa6beaa633b89271b3da06e1 Mon Sep 17 00:00:00 2001 From: Wei-Y <83482178+w-yue@users.noreply.github.com> Date: Wed, 1 Feb 2023 00:37:56 +0000 Subject: [PATCH 4/4] DP SG support port handling logic. --- src/xdp/trn_transit_xdp.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/xdp/trn_transit_xdp.c b/src/xdp/trn_transit_xdp.c index 71e0353..5e2f610 100644 --- a/src/xdp/trn_transit_xdp.c +++ b/src/xdp/trn_transit_xdp.c @@ -138,7 +138,29 @@ static __inline int trn_sg_check(struct transit_packet *pkt) { sgkey.src_ip = pkt->inner_ip->saddr; sgkey.dst_ip = pkt->inner_ip->daddr; sgkey.protocol = pkt->inner_ip->protocol; - sgkey.direction = 1; // source -> direction; + sgkey.prefixlen = SG_IPV4_PREFIX; // ?? + + // check egress + sgkey.direction = 0; + sgkey.port = 30011; + + bpf_debug("[Transit:%d] XXXXX port=%d, prefixlen=%d", + sgkey.vni, sgkey.port, sgkey.prefixlen); + bpf_debug(" XXXXX proto=%d, src_ip:0x%x, dst_ip:0x%x ", + sgkey.protocol, + bpf_ntohl(sgkey.src_ip), + bpf_ntohl(sgkey.dst_ip)); + sg_entry = bpf_map_lookup_elem(&sg_cidr_map, &sgkey); + if (sg_entry == NULL) { + bpf_debug("[Transit:%d XXXXX] Drop: no matching sg entry found: %d - %d\n", + pkt->itf_idx, sgkey.port, sgkey.vni); + return XDP_DROP; + } + + // check ingress; + sgkey.direction = 1; + sgkey.src_ip = sgkey.dst_ip; + sgkey.dst_ip = pkt->inner_ip->saddr; if (sgkey.protocol == IPPROTO_TCP) { pkt->inner_tcp = (void *)pkt->inner_ip + sizeof(*pkt->inner_ip); @@ -164,7 +186,6 @@ static __inline int trn_sg_check(struct transit_packet *pkt) { return XDP_PASS; } - sgkey.prefixlen = SG_IPV4_PREFIX; // ?? bpf_debug("[Transit:%d] XXXXX port=%d, prefixlen=%d", sgkey.vni, sgkey.port, sgkey.prefixlen); bpf_debug(" XXXXX proto=%d, src_ip:0x%x, dst_ip:0x%x ",