1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __BPF_TCP_HELPERS_H
3 #define __BPF_TCP_HELPERS_H
6 #include <linux/types.h>
7 #include <bpf/bpf_helpers.h>
8 #include <bpf/bpf_core_read.h>
9 #include <bpf/bpf_tracing.h>
11 #define BPF_STRUCT_OPS(name, args...) \
12 SEC("struct_ops/"#name) \
19 #ifndef TCP_CA_NAME_MAX
20 #define TCP_CA_NAME_MAX 16
23 #define tcp_jiffies32 ((__u32)bpf_jiffies64())
26 unsigned char skc_state;
28 } __attribute__((preserve_access_index));
37 struct sock_common __sk_common;
38 #define sk_state __sk_common.skc_state
39 unsigned long sk_pacing_rate;
40 __u32 sk_pacing_status; /* see enum sk_pacing */
41 } __attribute__((preserve_access_index));
45 } __attribute__((preserve_access_index));
47 struct inet_connection_sock {
48 struct inet_sock icsk_inet;
55 __u64 icsk_ca_priv[104 / sizeof(__u64)];
56 } __attribute__((preserve_access_index));
59 struct sock_common __req_common;
60 } __attribute__((preserve_access_index));
63 struct inet_connection_sock inet_conn;
76 __u8 syn_data:1, /* SYN includes data */
77 syn_fastopen:1, /* SYN includes Fast Open option */
78 syn_fastopen_exp:1,/* SYN includes Fast Open exp. option */
79 syn_fastopen_ch:1, /* Active TFO re-enabling probe */
80 syn_data_acked:1,/* data in SYN is acked by SYN-ACK */
81 save_syn:1, /* Save headers of SYN packet */
82 is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */
83 syn_smc:1; /* SYN includes SMC */
84 __u32 max_packets_out;
87 __u64 tcp_mstamp; /* most recent packet received/sent */
89 } __attribute__((preserve_access_index));
91 static __always_inline struct inet_connection_sock *inet_csk(const struct sock *sk)
93 return (struct inet_connection_sock *)sk;
96 static __always_inline void *inet_csk_ca(const struct sock *sk)
98 return (void *)inet_csk(sk)->icsk_ca_priv;
101 static __always_inline struct tcp_sock *tcp_sk(const struct sock *sk)
103 return (struct tcp_sock *)sk;
106 static __always_inline bool before(__u32 seq1, __u32 seq2)
108 return (__s32)(seq1-seq2) < 0;
110 #define after(seq2, seq1) before(seq1, seq2)
113 #define TCP_ECN_QUEUE_CWR 2
114 #define TCP_ECN_DEMAND_CWR 4
115 #define TCP_ECN_SEEN 8
117 enum inet_csk_ack_state_t {
121 ICSK_ACK_PUSHED2 = 8,
122 ICSK_ACK_NOW = 16 /* Send the next ACK immediately (once) */
126 CA_EVENT_TX_START = 0,
127 CA_EVENT_CWND_RESTART = 1,
128 CA_EVENT_COMPLETE_CWR = 2,
130 CA_EVENT_ECN_NO_CE = 4,
131 CA_EVENT_ECN_IS_CE = 5,
138 } __attribute__((preserve_access_index));
141 __u64 prior_mstamp; /* starting timestamp for interval */
142 __u32 prior_delivered; /* tp->delivered at "prior_mstamp" */
143 __s32 delivered; /* number of packets delivered over interval */
144 long interval_us; /* time for tp->delivered to incr "delivered" */
145 __u32 snd_interval_us; /* snd interval for delivered packets */
146 __u32 rcv_interval_us; /* rcv interval for delivered packets */
147 long rtt_us; /* RTT of last (S)ACKed packet (or -1) */
148 int losses; /* number of packets marked lost upon ACK */
149 __u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
150 __u32 prior_in_flight; /* in flight before this ACK */
151 bool is_app_limited; /* is sample from packet with bubble in pipe? */
152 bool is_retrans; /* is sample from retransmission? */
153 bool is_ack_delayed; /* is this (likely) a delayed ACK? */
154 } __attribute__((preserve_access_index));
156 #define TCP_CA_NAME_MAX 16
157 #define TCP_CONG_NEEDS_ECN 0x2
159 struct tcp_congestion_ops {
160 char name[TCP_CA_NAME_MAX];
163 /* initialize private data (optional) */
164 void (*init)(struct sock *sk);
165 /* cleanup private data (optional) */
166 void (*release)(struct sock *sk);
168 /* return slow start threshold (required) */
169 __u32 (*ssthresh)(struct sock *sk);
170 /* do new cwnd calculation (required) */
171 void (*cong_avoid)(struct sock *sk, __u32 ack, __u32 acked);
172 /* call before changing ca_state (optional) */
173 void (*set_state)(struct sock *sk, __u8 new_state);
174 /* call when cwnd event occurs (optional) */
175 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
176 /* call when ack arrives (optional) */
177 void (*in_ack_event)(struct sock *sk, __u32 flags);
178 /* new value of cwnd after loss (required) */
179 __u32 (*undo_cwnd)(struct sock *sk);
180 /* hook for packet ack accounting (optional) */
181 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
182 /* override sysctl_tcp_min_tso_segs */
183 __u32 (*min_tso_segs)(struct sock *sk);
184 /* returns the multiplier used in tcp_sndbuf_expand (optional) */
185 __u32 (*sndbuf_expand)(struct sock *sk);
186 /* call when packets are delivered to update cwnd and pacing rate,
187 * after all the ca_state processing. (optional)
189 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
193 #define min(a, b) ((a) < (b) ? (a) : (b))
194 #define max(a, b) ((a) > (b) ? (a) : (b))
195 #define min_not_zero(x, y) ({ \
196 typeof(x) __x = (x); \
197 typeof(y) __y = (y); \
198 __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
200 static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)
202 return tp->snd_cwnd < tp->snd_ssthresh;
205 static __always_inline bool tcp_is_cwnd_limited(const struct sock *sk)
207 const struct tcp_sock *tp = tcp_sk(sk);
209 /* If in slow start, ensure cwnd grows to twice what was ACKed. */
210 if (tcp_in_slow_start(tp))
211 return tp->snd_cwnd < 2 * tp->max_packets_out;
213 return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
216 static __always_inline bool tcp_cc_eq(const char *a, const char *b)
220 for (i = 0; i < TCP_CA_NAME_MAX; i++) {
230 extern __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked) __ksym;
231 extern void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked) __ksym;
234 struct inet_connection_sock sk;
238 char ca_name[TCP_CA_NAME_MAX];
239 } __attribute__((preserve_access_index));