20 static void update_rx_stats(struct wg_peer *peer, size_t len)  in update_rx_stats()  argument
23 		get_cpu_ptr(peer->device->dev->tstats);  in update_rx_stats()
28 	peer->rx_bytes += len;  in update_rx_stats()
103 	struct wg_peer *peer = NULL;  in wg_receive_handshake_packet()  local
151 		peer = wg_noise_handshake_consume_initiation(message, wg);  in wg_receive_handshake_packet()
152 		if (unlikely(!peer)) {  in wg_receive_handshake_packet()
157 		wg_socket_set_peer_endpoint_from_skb(peer, skb);  in wg_receive_handshake_packet()
159 				    wg->dev->name, peer->internal_id,  in wg_receive_handshake_packet()
160 				    &peer->endpoint.addr);  in wg_receive_handshake_packet()
161 		wg_packet_send_handshake_response(peer);  in wg_receive_handshake_packet()
173 		peer = wg_noise_handshake_consume_response(message, wg);  in wg_receive_handshake_packet()
174 		if (unlikely(!peer)) {  in wg_receive_handshake_packet()
179 		wg_socket_set_peer_endpoint_from_skb(peer, skb);  in wg_receive_handshake_packet()
181 				    wg->dev->name, peer->internal_id,  in wg_receive_handshake_packet()
182 				    &peer->endpoint.addr);  in wg_receive_handshake_packet()
183 		if (wg_noise_handshake_begin_session(&peer->handshake,  in wg_receive_handshake_packet()
184 						     &peer->keypairs)) {  in wg_receive_handshake_packet()
185 			wg_timers_session_derived(peer);  in wg_receive_handshake_packet()
186 			wg_timers_handshake_complete(peer);  in wg_receive_handshake_packet()
193 			wg_packet_send_keepalive(peer);  in wg_receive_handshake_packet()
199 	if (unlikely(!peer)) {  in wg_receive_handshake_packet()
205 	update_rx_stats(peer, skb->len);  in wg_receive_handshake_packet()
208 	wg_timers_any_authenticated_packet_received(peer);  in wg_receive_handshake_packet()
209 	wg_timers_any_authenticated_packet_traversal(peer);  in wg_receive_handshake_packet()
210 	wg_peer_put(peer);  in wg_receive_handshake_packet()
227 static void keep_key_fresh(struct wg_peer *peer)  in keep_key_fresh()  argument
232 	if (peer->sent_lastminute_handshake)  in keep_key_fresh()
236 	keypair = rcu_dereference_bh(peer->keypairs.current_keypair);  in keep_key_fresh()
244 		peer->sent_lastminute_handshake = true;  in keep_key_fresh()
245 		wg_packet_send_queued_handshake_initiation(peer, false);  in keep_key_fresh()
342 static void wg_packet_consume_data_done(struct wg_peer *peer,  in wg_packet_consume_data_done()  argument
346 	struct net_device *dev = peer->device->dev;  in wg_packet_consume_data_done()
350 	wg_socket_set_peer_endpoint(peer, endpoint);  in wg_packet_consume_data_done()
352 	if (unlikely(wg_noise_received_with_keypair(&peer->keypairs,  in wg_packet_consume_data_done()
354 		wg_timers_handshake_complete(peer);  in wg_packet_consume_data_done()
355 		wg_packet_send_staged_packets(peer);  in wg_packet_consume_data_done()
358 	keep_key_fresh(peer);  in wg_packet_consume_data_done()
360 	wg_timers_any_authenticated_packet_received(peer);  in wg_packet_consume_data_done()
361 	wg_timers_any_authenticated_packet_traversal(peer);  in wg_packet_consume_data_done()
365 		update_rx_stats(peer, message_data_len(0));  in wg_packet_consume_data_done()
367 				    dev->name, peer->internal_id,  in wg_packet_consume_data_done()
368 				    &peer->endpoint.addr);  in wg_packet_consume_data_done()
372 	wg_timers_data_received(peer);  in wg_packet_consume_data_done()
411 	routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips,  in wg_packet_consume_data_done()
415 	if (unlikely(routed_peer != peer))  in wg_packet_consume_data_done()
418 	napi_gro_receive(&peer->napi, skb);  in wg_packet_consume_data_done()
419 	update_rx_stats(peer, message_data_len(len_before_trim));  in wg_packet_consume_data_done()
424 				dev->name, skb, peer->internal_id,  in wg_packet_consume_data_done()
425 				&peer->endpoint.addr);  in wg_packet_consume_data_done()
431 			    dev->name, peer->internal_id, &peer->endpoint.addr);  in wg_packet_consume_data_done()
437 			    dev->name, peer->internal_id, &peer->endpoint.addr);  in wg_packet_consume_data_done()
447 	struct wg_peer *peer = container_of(napi, struct wg_peer, napi);  in wg_packet_rx_poll()  local
458 	while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL &&  in wg_packet_rx_poll()
461 		wg_prev_queue_drop_peeked(&peer->rx_queue);  in wg_packet_rx_poll()
471 					    peer->device->dev->name,  in wg_packet_rx_poll()
481 		wg_packet_consume_data_done(peer, skb, &endpoint);  in wg_packet_rx_poll()
486 		wg_peer_put(peer);  in wg_packet_rx_poll()
519 	struct wg_peer *peer = NULL;  in wg_packet_consume_data()  local
526 			&peer);  in wg_packet_consume_data()
530 	if (unlikely(READ_ONCE(peer->is_dead)))  in wg_packet_consume_data()
533 	ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb,  in wg_packet_consume_data()
545 	wg_peer_put(peer);  in wg_packet_consume_data()