Lines Matching refs:tm
57 static inline struct net *tm_net(struct tcp_metrics_block *tm) in tm_net() argument
59 return read_pnet(&tm->tcpm_net); in tm_net()
62 static bool tcp_metric_locked(struct tcp_metrics_block *tm, in tcp_metric_locked() argument
65 return tm->tcpm_lock & (1 << idx); in tcp_metric_locked()
68 static u32 tcp_metric_get(struct tcp_metrics_block *tm, in tcp_metric_get() argument
71 return tm->tcpm_vals[idx]; in tcp_metric_get()
74 static void tcp_metric_set(struct tcp_metrics_block *tm, in tcp_metric_set() argument
78 tm->tcpm_vals[idx] = val; in tcp_metric_set()
96 static void tcpm_suck_dst(struct tcp_metrics_block *tm, in tcpm_suck_dst() argument
103 tm->tcpm_stamp = jiffies; in tcpm_suck_dst()
116 tm->tcpm_lock = val; in tcpm_suck_dst()
119 tm->tcpm_vals[TCP_METRIC_RTT] = msval * USEC_PER_MSEC; in tcpm_suck_dst()
122 tm->tcpm_vals[TCP_METRIC_RTTVAR] = msval * USEC_PER_MSEC; in tcpm_suck_dst()
123 tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH); in tcpm_suck_dst()
124 tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND); in tcpm_suck_dst()
125 tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING); in tcpm_suck_dst()
126 tm->tcpm_ts = 0; in tcpm_suck_dst()
127 tm->tcpm_ts_stamp = 0; in tcpm_suck_dst()
129 tm->tcpm_fastopen.mss = 0; in tcpm_suck_dst()
130 tm->tcpm_fastopen.syn_loss = 0; in tcpm_suck_dst()
131 tm->tcpm_fastopen.try_exp = 0; in tcpm_suck_dst()
132 tm->tcpm_fastopen.cookie.exp = false; in tcpm_suck_dst()
133 tm->tcpm_fastopen.cookie.len = 0; in tcpm_suck_dst()
139 static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst) in tcpm_check_stamp() argument
141 if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT))) in tcpm_check_stamp()
142 tcpm_suck_dst(tm, dst, false); in tcpm_check_stamp()
156 struct tcp_metrics_block *tm; in tcpm_new() local
166 tm = __tcp_get_metrics(saddr, daddr, net, hash); in tcpm_new()
167 if (tm == TCP_METRICS_RECLAIM_PTR) { in tcpm_new()
169 tm = NULL; in tcpm_new()
171 if (tm) { in tcpm_new()
172 tcpm_check_stamp(tm, dst); in tcpm_new()
180 for (tm = deref_locked(oldest->tcpm_next); tm; in tcpm_new()
181 tm = deref_locked(tm->tcpm_next)) { in tcpm_new()
182 if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp)) in tcpm_new()
183 oldest = tm; in tcpm_new()
185 tm = oldest; in tcpm_new()
187 tm = kmalloc(sizeof(*tm), GFP_ATOMIC); in tcpm_new()
188 if (!tm) in tcpm_new()
191 write_pnet(&tm->tcpm_net, net); in tcpm_new()
192 tm->tcpm_saddr = *saddr; in tcpm_new()
193 tm->tcpm_daddr = *daddr; in tcpm_new()
195 tcpm_suck_dst(tm, dst, true); in tcpm_new()
198 tm->tcpm_next = tcp_metrics_hash[hash].chain; in tcpm_new()
199 rcu_assign_pointer(tcp_metrics_hash[hash].chain, tm); in tcpm_new()
204 return tm; in tcpm_new()
207 static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth) in tcp_get_encode() argument
209 if (tm) in tcp_get_encode()
210 return tm; in tcp_get_encode()
220 struct tcp_metrics_block *tm; in __tcp_get_metrics() local
223 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in __tcp_get_metrics()
224 tm = rcu_dereference(tm->tcpm_next)) { in __tcp_get_metrics()
225 if (addr_same(&tm->tcpm_saddr, saddr) && in __tcp_get_metrics()
226 addr_same(&tm->tcpm_daddr, daddr) && in __tcp_get_metrics()
227 net_eq(tm_net(tm), net)) in __tcp_get_metrics()
231 return tcp_get_encode(tm, depth); in __tcp_get_metrics()
237 struct tcp_metrics_block *tm; in __tcp_get_metrics_req() local
265 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in __tcp_get_metrics_req()
266 tm = rcu_dereference(tm->tcpm_next)) { in __tcp_get_metrics_req()
267 if (addr_same(&tm->tcpm_saddr, &saddr) && in __tcp_get_metrics_req()
268 addr_same(&tm->tcpm_daddr, &daddr) && in __tcp_get_metrics_req()
269 net_eq(tm_net(tm), net)) in __tcp_get_metrics_req()
272 tcpm_check_stamp(tm, dst); in __tcp_get_metrics_req()
273 return tm; in __tcp_get_metrics_req()
278 struct tcp_metrics_block *tm; in __tcp_get_metrics_tw() local
308 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in __tcp_get_metrics_tw()
309 tm = rcu_dereference(tm->tcpm_next)) { in __tcp_get_metrics_tw()
310 if (addr_same(&tm->tcpm_saddr, &saddr) && in __tcp_get_metrics_tw()
311 addr_same(&tm->tcpm_daddr, &daddr) && in __tcp_get_metrics_tw()
312 net_eq(tm_net(tm), net)) in __tcp_get_metrics_tw()
315 return tm; in __tcp_get_metrics_tw()
322 struct tcp_metrics_block *tm; in tcp_get_metrics() local
352 tm = __tcp_get_metrics(&saddr, &daddr, net, hash); in tcp_get_metrics()
353 if (tm == TCP_METRICS_RECLAIM_PTR) in tcp_get_metrics()
354 tm = NULL; in tcp_get_metrics()
355 if (!tm && create) in tcp_get_metrics()
356 tm = tcpm_new(dst, &saddr, &daddr, hash); in tcp_get_metrics()
358 tcpm_check_stamp(tm, dst); in tcp_get_metrics()
360 return tm; in tcp_get_metrics()
372 struct tcp_metrics_block *tm; in tcp_update_metrics() local
389 tm = tcp_get_metrics(sk, dst, false); in tcp_update_metrics()
390 if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT)) in tcp_update_metrics()
391 tcp_metric_set(tm, TCP_METRIC_RTT, 0); in tcp_update_metrics()
394 tm = tcp_get_metrics(sk, dst, true); in tcp_update_metrics()
396 if (!tm) in tcp_update_metrics()
399 rtt = tcp_metric_get(tm, TCP_METRIC_RTT); in tcp_update_metrics()
406 if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) { in tcp_update_metrics()
411 tcp_metric_set(tm, TCP_METRIC_RTT, rtt); in tcp_update_metrics()
414 if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) { in tcp_update_metrics()
425 var = tcp_metric_get(tm, TCP_METRIC_RTTVAR); in tcp_update_metrics()
431 tcp_metric_set(tm, TCP_METRIC_RTTVAR, var); in tcp_update_metrics()
436 if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { in tcp_update_metrics()
437 val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); in tcp_update_metrics()
439 tcp_metric_set(tm, TCP_METRIC_SSTHRESH, in tcp_update_metrics()
442 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { in tcp_update_metrics()
443 val = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_update_metrics()
445 tcp_metric_set(tm, TCP_METRIC_CWND, in tcp_update_metrics()
451 if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) in tcp_update_metrics()
452 tcp_metric_set(tm, TCP_METRIC_SSTHRESH, in tcp_update_metrics()
454 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { in tcp_update_metrics()
455 val = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_update_metrics()
456 tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1); in tcp_update_metrics()
462 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { in tcp_update_metrics()
463 val = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_update_metrics()
464 tcp_metric_set(tm, TCP_METRIC_CWND, in tcp_update_metrics()
467 if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { in tcp_update_metrics()
468 val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); in tcp_update_metrics()
470 tcp_metric_set(tm, TCP_METRIC_SSTHRESH, in tcp_update_metrics()
473 if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) { in tcp_update_metrics()
474 val = tcp_metric_get(tm, TCP_METRIC_REORDERING); in tcp_update_metrics()
477 tcp_metric_set(tm, TCP_METRIC_REORDERING, in tcp_update_metrics()
481 tm->tcpm_stamp = jiffies; in tcp_update_metrics()
492 struct tcp_metrics_block *tm; in tcp_init_metrics() local
501 tm = tcp_get_metrics(sk, dst, true); in tcp_init_metrics()
502 if (!tm) { in tcp_init_metrics()
507 if (tcp_metric_locked(tm, TCP_METRIC_CWND)) in tcp_init_metrics()
508 tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_init_metrics()
510 val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); in tcp_init_metrics()
521 val = tcp_metric_get(tm, TCP_METRIC_REORDERING); in tcp_init_metrics()
528 crtt = tcp_metric_get(tm, TCP_METRIC_RTT); in tcp_init_metrics()
582 struct tcp_metrics_block *tm; in tcp_peer_is_proven() local
589 tm = __tcp_get_metrics_req(req, dst); in tcp_peer_is_proven()
591 if (tm && in tcp_peer_is_proven()
592 (u32)get_seconds() - tm->tcpm_ts_stamp < TCP_PAWS_MSL && in tcp_peer_is_proven()
593 ((s32)(tm->tcpm_ts - req->ts_recent) > TCP_PAWS_WINDOW || in tcp_peer_is_proven()
599 if (tm && tcp_metric_get(tm, TCP_METRIC_RTT) && tm->tcpm_ts_stamp) in tcp_peer_is_proven()
612 struct tcp_metrics_block *tm; in tcp_fetch_timewait_stamp() local
615 tm = tcp_get_metrics(sk, dst, true); in tcp_fetch_timewait_stamp()
616 if (tm) { in tcp_fetch_timewait_stamp()
619 if ((u32)get_seconds() - tm->tcpm_ts_stamp <= TCP_PAWS_MSL) { in tcp_fetch_timewait_stamp()
620 tp->rx_opt.ts_recent_stamp = tm->tcpm_ts_stamp; in tcp_fetch_timewait_stamp()
621 tp->rx_opt.ts_recent = tm->tcpm_ts; in tcp_fetch_timewait_stamp()
639 struct tcp_metrics_block *tm; in tcp_remember_stamp() local
642 tm = tcp_get_metrics(sk, dst, true); in tcp_remember_stamp()
643 if (tm) { in tcp_remember_stamp()
646 if ((s32)(tm->tcpm_ts - tp->rx_opt.ts_recent) <= 0 || in tcp_remember_stamp()
647 ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL && in tcp_remember_stamp()
648 tm->tcpm_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) { in tcp_remember_stamp()
649 tm->tcpm_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp; in tcp_remember_stamp()
650 tm->tcpm_ts = tp->rx_opt.ts_recent; in tcp_remember_stamp()
661 struct tcp_metrics_block *tm; in tcp_tw_remember_stamp() local
665 tm = __tcp_get_metrics_tw(tw); in tcp_tw_remember_stamp()
666 if (tm) { in tcp_tw_remember_stamp()
671 if ((s32)(tm->tcpm_ts - tcptw->tw_ts_recent) <= 0 || in tcp_tw_remember_stamp()
672 ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL && in tcp_tw_remember_stamp()
673 tm->tcpm_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) { in tcp_tw_remember_stamp()
674 tm->tcpm_ts_stamp = (u32)tcptw->tw_ts_recent_stamp; in tcp_tw_remember_stamp()
675 tm->tcpm_ts = tcptw->tw_ts_recent; in tcp_tw_remember_stamp()
690 struct tcp_metrics_block *tm; in tcp_fastopen_cache_get() local
693 tm = tcp_get_metrics(sk, __sk_dst_get(sk), false); in tcp_fastopen_cache_get()
694 if (tm) { in tcp_fastopen_cache_get()
695 struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen; in tcp_fastopen_cache_get()
717 struct tcp_metrics_block *tm; in tcp_fastopen_cache_set() local
722 tm = tcp_get_metrics(sk, dst, true); in tcp_fastopen_cache_set()
723 if (tm) { in tcp_fastopen_cache_set()
724 struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen; in tcp_fastopen_cache_set()
775 struct tcp_metrics_block *tm) in tcp_metrics_fill_info() argument
780 switch (tm->tcpm_daddr.family) { in tcp_metrics_fill_info()
783 inetpeer_get_addr_v4(&tm->tcpm_daddr)) < 0) in tcp_metrics_fill_info()
786 inetpeer_get_addr_v4(&tm->tcpm_saddr)) < 0) in tcp_metrics_fill_info()
791 inetpeer_get_addr_v6(&tm->tcpm_daddr)) < 0) in tcp_metrics_fill_info()
794 inetpeer_get_addr_v6(&tm->tcpm_saddr)) < 0) in tcp_metrics_fill_info()
802 jiffies - tm->tcpm_stamp) < 0) in tcp_metrics_fill_info()
804 if (tm->tcpm_ts_stamp) { in tcp_metrics_fill_info()
806 (s32) (get_seconds() - tm->tcpm_ts_stamp)) < 0) in tcp_metrics_fill_info()
809 tm->tcpm_ts) < 0) in tcp_metrics_fill_info()
820 u32 val = tm->tcpm_vals[i]; in tcp_metrics_fill_info()
854 tfom_copy[0] = tm->tcpm_fastopen; in tcp_metrics_fill_info()
882 struct tcp_metrics_block *tm) in tcp_metrics_dump_info() argument
892 if (tcp_metrics_fill_info(skb, tm) < 0) in tcp_metrics_dump_info()
912 struct tcp_metrics_block *tm; in tcp_metrics_nl_dump() local
916 for (col = 0, tm = rcu_dereference(hb->chain); tm; in tcp_metrics_nl_dump()
917 tm = rcu_dereference(tm->tcpm_next), col++) { in tcp_metrics_nl_dump()
918 if (!net_eq(tm_net(tm), net)) in tcp_metrics_nl_dump()
922 if (tcp_metrics_dump_info(skb, cb, tm) < 0) { in tcp_metrics_nl_dump()
980 struct tcp_metrics_block *tm; in tcp_metrics_nl_cmd_get() local
1010 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in tcp_metrics_nl_cmd_get()
1011 tm = rcu_dereference(tm->tcpm_next)) { in tcp_metrics_nl_cmd_get()
1012 if (addr_same(&tm->tcpm_daddr, &daddr) && in tcp_metrics_nl_cmd_get()
1013 (!src || addr_same(&tm->tcpm_saddr, &saddr)) && in tcp_metrics_nl_cmd_get()
1014 net_eq(tm_net(tm), net)) { in tcp_metrics_nl_cmd_get()
1015 ret = tcp_metrics_fill_info(msg, tm); in tcp_metrics_nl_cmd_get()
1038 struct tcp_metrics_block *tm; in tcp_metrics_flush_all() local
1045 for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) { in tcp_metrics_flush_all()
1046 if (net_eq(tm_net(tm), net)) { in tcp_metrics_flush_all()
1047 *pp = tm->tcpm_next; in tcp_metrics_flush_all()
1048 kfree_rcu(tm, rcu_head); in tcp_metrics_flush_all()
1050 pp = &tm->tcpm_next; in tcp_metrics_flush_all()
1060 struct tcp_metrics_block *tm; in tcp_metrics_nl_cmd_del() local
1084 for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) { in tcp_metrics_nl_cmd_del()
1085 if (addr_same(&tm->tcpm_daddr, &daddr) && in tcp_metrics_nl_cmd_del()
1086 (!src || addr_same(&tm->tcpm_saddr, &saddr)) && in tcp_metrics_nl_cmd_del()
1087 net_eq(tm_net(tm), net)) { in tcp_metrics_nl_cmd_del()
1088 *pp = tm->tcpm_next; in tcp_metrics_nl_cmd_del()
1089 kfree_rcu(tm, rcu_head); in tcp_metrics_nl_cmd_del()
1092 pp = &tm->tcpm_next; in tcp_metrics_nl_cmd_del()