From 7d7c100c4b0103e63480687ad4dece21022f5b1d Mon Sep 17 00:00:00 2001 From: Benjamin Kilimnik Date: Mon, 28 Aug 2023 17:48:57 +0000 Subject: [PATCH 1/4] update tcp_drops to use pre519 and post519 versions of the BPFtrace scripts depending on host Signed-off-by: Benjamin Kilimnik --- src/pxl_scripts/px/tcp_drops/data.pxl | 74 ++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/src/pxl_scripts/px/tcp_drops/data.pxl b/src/pxl_scripts/px/tcp_drops/data.pxl index 54f8dbdd870..6581254dd9b 100644 --- a/src/pxl_scripts/px/tcp_drops/data.pxl +++ b/src/pxl_scripts/px/tcp_drops/data.pxl @@ -20,7 +20,7 @@ import pxtrace import px # Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpdrop.bt -program = """ +pre_519_program = pxtrace.TraceProgram(""" // tcpdrop.bt Trace TCP kernel-dropped packets/segments. // For Linux, uses bpftrace and eBPF. // @@ -83,14 +83,84 @@ kprobe:tcp_drop $statestr); } } +""", +max_kernel='5.18' +) + +post_519_program = pxtrace.TraceProgram( """ +// tcpdrop.bt Trace TCP kernel-dropped packets/segments. +// For Linux, uses bpftrace and eBPF. +// +// Copyright (c) 2018 Dale Hamel. +// Licensed under the Apache License, Version 2.0 (the "License") + +#include +#include + +BEGIN +{ + // See https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h + @tcp_states[1] = "ESTABLISHED"; + @tcp_states[2] = "SYN_SENT"; + @tcp_states[3] = "SYN_RECV"; + @tcp_states[4] = "FIN_WAIT1"; + @tcp_states[5] = "FIN_WAIT2"; + @tcp_states[6] = "TIME_WAIT"; + @tcp_states[7] = "CLOSE"; + @tcp_states[8] = "CLOSE_WAIT"; + @tcp_states[9] = "LAST_ACK"; + @tcp_states[10] = "LISTEN"; + @tcp_states[11] = "CLOSING"; + @tcp_states[12] = "NEW_SYN_RECV"; +} + +tracepoint:skb:kfree_skb +{ + $reason = args->reason; + $skb = (struct sk_buff *)args->skbaddr; + $sk = ((struct sock *) $skb->sk); + $inet_family = $sk->__sk_common.skc_family; + + if ($reason > SKB_DROP_REASON_NOT_SPECIFIED && + ($inet_family == AF_INET || $inet_family == AF_INET6)) { + if ($inet_family == AF_INET) { + $daddr = ntop($sk->__sk_common.skc_daddr); + $saddr = ntop($sk->__sk_common.skc_rcv_saddr); + } else { + $daddr = ntop($sk->__sk_common.skc_v6_daddr.in6_u.u6_addr8); + $saddr = ntop($sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8); + } + $lport = $sk->__sk_common.skc_num; + $dport = $sk->__sk_common.skc_dport; + + // Destination port is big endian, it must be flipped + $dport = bswap($dport); + + $state = $sk->__sk_common.skc_state; + $statestr = @tcp_states[$state]; + + printf(\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\", + nsecs, + pid, + ((struct task_struct*)curtask)->group_leader->start_time / 10000000, + $saddr, + $lport, + $daddr, + $dport, + $statestr); + } +} +""", +min_kernel='5.19' +) def tcp_drops_func(): table_name = 'tcp_drop_table' pxtrace.UpsertTracepoint('tcp_drop_tracer', table_name, - program, + [pre_519_program, post_519_program], pxtrace.kprobe(), "10m") From 8782f261bf3c156a10dec569d43360f9ac2ed746 Mon Sep 17 00:00:00 2001 From: Benjamin Kilimnik Date: Mon, 28 Aug 2023 20:47:18 +0000 Subject: [PATCH 2/4] update bpftrace/tcp_drops Signed-off-by: Benjamin Kilimnik --- src/pxl_scripts/bpftrace/tcp_drops/data.pxl | 74 ++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/src/pxl_scripts/bpftrace/tcp_drops/data.pxl b/src/pxl_scripts/bpftrace/tcp_drops/data.pxl index 54f8dbdd870..6581254dd9b 100644 --- a/src/pxl_scripts/bpftrace/tcp_drops/data.pxl +++ b/src/pxl_scripts/bpftrace/tcp_drops/data.pxl @@ -20,7 +20,7 @@ import pxtrace import px # Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpdrop.bt -program = """ +pre_519_program = pxtrace.TraceProgram(""" // tcpdrop.bt Trace TCP kernel-dropped packets/segments. // For Linux, uses bpftrace and eBPF. // @@ -83,14 +83,84 @@ kprobe:tcp_drop $statestr); } } +""", +max_kernel='5.18' +) + +post_519_program = pxtrace.TraceProgram( """ +// tcpdrop.bt Trace TCP kernel-dropped packets/segments. +// For Linux, uses bpftrace and eBPF. +// +// Copyright (c) 2018 Dale Hamel. +// Licensed under the Apache License, Version 2.0 (the "License") + +#include +#include + +BEGIN +{ + // See https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h + @tcp_states[1] = "ESTABLISHED"; + @tcp_states[2] = "SYN_SENT"; + @tcp_states[3] = "SYN_RECV"; + @tcp_states[4] = "FIN_WAIT1"; + @tcp_states[5] = "FIN_WAIT2"; + @tcp_states[6] = "TIME_WAIT"; + @tcp_states[7] = "CLOSE"; + @tcp_states[8] = "CLOSE_WAIT"; + @tcp_states[9] = "LAST_ACK"; + @tcp_states[10] = "LISTEN"; + @tcp_states[11] = "CLOSING"; + @tcp_states[12] = "NEW_SYN_RECV"; +} + +tracepoint:skb:kfree_skb +{ + $reason = args->reason; + $skb = (struct sk_buff *)args->skbaddr; + $sk = ((struct sock *) $skb->sk); + $inet_family = $sk->__sk_common.skc_family; + + if ($reason > SKB_DROP_REASON_NOT_SPECIFIED && + ($inet_family == AF_INET || $inet_family == AF_INET6)) { + if ($inet_family == AF_INET) { + $daddr = ntop($sk->__sk_common.skc_daddr); + $saddr = ntop($sk->__sk_common.skc_rcv_saddr); + } else { + $daddr = ntop($sk->__sk_common.skc_v6_daddr.in6_u.u6_addr8); + $saddr = ntop($sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8); + } + $lport = $sk->__sk_common.skc_num; + $dport = $sk->__sk_common.skc_dport; + + // Destination port is big endian, it must be flipped + $dport = bswap($dport); + + $state = $sk->__sk_common.skc_state; + $statestr = @tcp_states[$state]; + + printf(\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\", + nsecs, + pid, + ((struct task_struct*)curtask)->group_leader->start_time / 10000000, + $saddr, + $lport, + $daddr, + $dport, + $statestr); + } +} +""", +min_kernel='5.19' +) def tcp_drops_func(): table_name = 'tcp_drop_table' pxtrace.UpsertTracepoint('tcp_drop_tracer', table_name, - program, + [pre_519_program, post_519_program], pxtrace.kprobe(), "10m") From 91e6ec08574f32cc500c06b5e4f72e10b7ef01e4 Mon Sep 17 00:00:00 2001 From: Benjamin Kilimnik Date: Wed, 27 Sep 2023 20:08:46 +0000 Subject: [PATCH 3/4] Add comment that kernel backporting can cause potential compatibility issues Signed-off-by: Benjamin Kilimnik --- src/pxl_scripts/bpftrace/tcp_drops/data.pxl | 2 ++ src/pxl_scripts/px/tcp_drops/data.pxl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/pxl_scripts/bpftrace/tcp_drops/data.pxl b/src/pxl_scripts/bpftrace/tcp_drops/data.pxl index 6581254dd9b..2a1358671cc 100644 --- a/src/pxl_scripts/bpftrace/tcp_drops/data.pxl +++ b/src/pxl_scripts/bpftrace/tcp_drops/data.pxl @@ -20,6 +20,8 @@ import pxtrace import px # Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpdrop.bt +# Due to backported changes (i.e. the kprobe:tcp_drop was removed in older versions of the kernel), +# the old bpftrace script may not work on some older kernels <5.19. pre_519_program = pxtrace.TraceProgram(""" // tcpdrop.bt Trace TCP kernel-dropped packets/segments. // For Linux, uses bpftrace and eBPF. diff --git a/src/pxl_scripts/px/tcp_drops/data.pxl b/src/pxl_scripts/px/tcp_drops/data.pxl index 6581254dd9b..2a1358671cc 100644 --- a/src/pxl_scripts/px/tcp_drops/data.pxl +++ b/src/pxl_scripts/px/tcp_drops/data.pxl @@ -20,6 +20,8 @@ import pxtrace import px # Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpdrop.bt +# Due to backported changes (i.e. the kprobe:tcp_drop was removed in older versions of the kernel), +# the old bpftrace script may not work on some older kernels <5.19. pre_519_program = pxtrace.TraceProgram(""" // tcpdrop.bt Trace TCP kernel-dropped packets/segments. // For Linux, uses bpftrace and eBPF. From d8582c95a346ba4f5a81acdf83c7bb85700fc192 Mon Sep 17 00:00:00 2001 From: Benjamin Kilimnik Date: Fri, 29 Sep 2023 23:36:45 +0000 Subject: [PATCH 4/4] remove duplicate tcp_drops script in pxl_scripts/px/ Signed-off-by: Benjamin Kilimnik --- src/pxl_scripts/px/tcp_drops/data.pxl | 185 --------------------- src/pxl_scripts/px/tcp_drops/manifest.yaml | 3 - src/pxl_scripts/px/tcp_drops/vis.json | 54 ------ 3 files changed, 242 deletions(-) delete mode 100644 src/pxl_scripts/px/tcp_drops/data.pxl delete mode 100644 src/pxl_scripts/px/tcp_drops/manifest.yaml delete mode 100644 src/pxl_scripts/px/tcp_drops/vis.json diff --git a/src/pxl_scripts/px/tcp_drops/data.pxl b/src/pxl_scripts/px/tcp_drops/data.pxl deleted file mode 100644 index 2a1358671cc..00000000000 --- a/src/pxl_scripts/px/tcp_drops/data.pxl +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2018- The Pixie Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -# flake8: noqa:E501 - -import pxtrace -import px - -# Adapted from https://github.com/iovisor/bpftrace/blob/master/tools/tcpdrop.bt -# Due to backported changes (i.e. the kprobe:tcp_drop was removed in older versions of the kernel), -# the old bpftrace script may not work on some older kernels <5.19. -pre_519_program = pxtrace.TraceProgram(""" -// tcpdrop.bt Trace TCP kernel-dropped packets/segments. -// For Linux, uses bpftrace and eBPF. -// -// Copyright (c) 2018 Dale Hamel. -// Licensed under the Apache License, Version 2.0 (the "License") - -#include -#include - -BEGIN -{ - // See https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h - @tcp_states[1] = "ESTABLISHED"; - @tcp_states[2] = "SYN_SENT"; - @tcp_states[3] = "SYN_RECV"; - @tcp_states[4] = "FIN_WAIT1"; - @tcp_states[5] = "FIN_WAIT2"; - @tcp_states[6] = "TIME_WAIT"; - @tcp_states[7] = "CLOSE"; - @tcp_states[8] = "CLOSE_WAIT"; - @tcp_states[9] = "LAST_ACK"; - @tcp_states[10] = "LISTEN"; - @tcp_states[11] = "CLOSING"; - @tcp_states[12] = "NEW_SYN_RECV"; -} - -kprobe:tcp_drop -{ - $sk = ((struct sock *) arg0); - $inet_family = $sk->__sk_common.skc_family; - - $AF_INET = (uint16) 2; - $AF_INET6 = (uint16) 10; - - if ($inet_family == $AF_INET || $inet_family == $AF_INET6) { - if ($inet_family == $AF_INET) { - $daddr = ntop($sk->__sk_common.skc_daddr); - $saddr = ntop($sk->__sk_common.skc_rcv_saddr); - } else { - $daddr = ntop($sk->__sk_common.skc_v6_daddr.in6_u.u6_addr8); - $saddr = ntop($sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8); - } - $sport = $sk->__sk_common.skc_num; - $dport = $sk->__sk_common.skc_dport; - - // Destination port is big endian, it must be flipped - $dport = ($dport >> 8) | (($dport << 8) & 0x00FF00); - - $state = $sk->__sk_common.skc_state; - $statestr = @tcp_states[$state]; - - printf(\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\", - nsecs, - pid, - ((struct task_struct*)curtask)->group_leader->start_time / 10000000, - $saddr, - $sport, - $daddr, - $dport, - $statestr); - } -} -""", -max_kernel='5.18' -) - -post_519_program = pxtrace.TraceProgram( -""" -// tcpdrop.bt Trace TCP kernel-dropped packets/segments. -// For Linux, uses bpftrace and eBPF. -// -// Copyright (c) 2018 Dale Hamel. -// Licensed under the Apache License, Version 2.0 (the "License") - -#include -#include - -BEGIN -{ - // See https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h - @tcp_states[1] = "ESTABLISHED"; - @tcp_states[2] = "SYN_SENT"; - @tcp_states[3] = "SYN_RECV"; - @tcp_states[4] = "FIN_WAIT1"; - @tcp_states[5] = "FIN_WAIT2"; - @tcp_states[6] = "TIME_WAIT"; - @tcp_states[7] = "CLOSE"; - @tcp_states[8] = "CLOSE_WAIT"; - @tcp_states[9] = "LAST_ACK"; - @tcp_states[10] = "LISTEN"; - @tcp_states[11] = "CLOSING"; - @tcp_states[12] = "NEW_SYN_RECV"; -} - -tracepoint:skb:kfree_skb -{ - $reason = args->reason; - $skb = (struct sk_buff *)args->skbaddr; - $sk = ((struct sock *) $skb->sk); - $inet_family = $sk->__sk_common.skc_family; - - if ($reason > SKB_DROP_REASON_NOT_SPECIFIED && - ($inet_family == AF_INET || $inet_family == AF_INET6)) { - if ($inet_family == AF_INET) { - $daddr = ntop($sk->__sk_common.skc_daddr); - $saddr = ntop($sk->__sk_common.skc_rcv_saddr); - } else { - $daddr = ntop($sk->__sk_common.skc_v6_daddr.in6_u.u6_addr8); - $saddr = ntop($sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8); - } - $lport = $sk->__sk_common.skc_num; - $dport = $sk->__sk_common.skc_dport; - - // Destination port is big endian, it must be flipped - $dport = bswap($dport); - - $state = $sk->__sk_common.skc_state; - $statestr = @tcp_states[$state]; - - printf(\"time_:%llu pid:%u pid_start_time:%llu src_ip:%s src_port:%d dst_ip:%s dst_port:%d state:%s\", - nsecs, - pid, - ((struct task_struct*)curtask)->group_leader->start_time / 10000000, - $saddr, - $lport, - $daddr, - $dport, - $statestr); - } -} -""", -min_kernel='5.19' -) - - -def tcp_drops_func(): - table_name = 'tcp_drop_table' - pxtrace.UpsertTracepoint('tcp_drop_tracer', - table_name, - [pre_519_program, post_519_program], - pxtrace.kprobe(), - "10m") - - df = px.DataFrame(table=table_name) - - # Convert IPs to domain names. - df.src = px.pod_id_to_pod_name(px.ip_to_pod_id(df.src_ip)) - df.src = px.select(df.src == '', df.src_ip, df.src) - df.dst = px.nslookup(df.dst_ip) - - # Count drops. - df = df.groupby(['src', 'dst']).agg(drops=('src', px.count)) - - # Filter for a particular service, if desired. - df = df[px.contains(df['dst'], '')] - - # Set a threshold to display, if desired. - df = df[df['drops'] > 0] - - return df diff --git a/src/pxl_scripts/px/tcp_drops/manifest.yaml b/src/pxl_scripts/px/tcp_drops/manifest.yaml deleted file mode 100644 index e9467fcebcc..00000000000 --- a/src/pxl_scripts/px/tcp_drops/manifest.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -short: TCP drops -long: Shows TCP drop counts in the cluster. diff --git a/src/pxl_scripts/px/tcp_drops/vis.json b/src/pxl_scripts/px/tcp_drops/vis.json deleted file mode 100644 index bc8abe39887..00000000000 --- a/src/pxl_scripts/px/tcp_drops/vis.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "variables": [], - "globalFuncs": [ - { - "outputName": "results", - "func": { - "name": "tcp_drops_func", - "args": [] - } - } - ], - "widgets": [ - { - "name": "TCP Drops", - "position": { - "x": 0, - "y": 0, - "w": 12, - "h": 5 - }, - "globalFuncOutputName": "results", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Graph", - "adjacencyList": { - "fromColumn": "src", - "toColumn": "dst" - }, - "edgeWeightColumn": "drops", - "edgeColorColumn": "drops", - "edgeLength": 300, - "edgeThresholds": { - "mediumThreshold": 5, - "highThreshold": 50 - }, - "edgeHoverInfo": [ - "drops" - ] - } - }, - { - "name": "Table", - "position": { - "x": 0, - "y": 5, - "w": 12, - "h": 4 - }, - "globalFuncOutputName": "results", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - } - ] -}