Skip to content

Commit 2bc793e

Browse files
Cong WangAlexei Starovoitov
Cong Wang
authored and
Alexei Starovoitov
committed
skmsg: Extract __tcp_bpf_recvmsg() and tcp_bpf_wait_data()
Although these two functions are only used by TCP, they are not specific to TCP at all, both operate on skmsg and ingress_msg, so fit in net/core/skmsg.c very well. And we will need them for non-TCP, so rename and move them to skmsg.c and export them to modules. Signed-off-by: Cong Wang <[email protected]> Signed-off-by: Alexei Starovoitov <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent d7f5711 commit 2bc793e

File tree

5 files changed

+106
-102
lines changed

5 files changed

+106
-102
lines changed

‎include/linux/skmsg.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,10 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
125125
struct sk_msg *msg, u32 bytes);
126126
int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
127127
struct sk_msg *msg, u32 bytes);
128+
int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
129+
long timeo, int *err);
130+
int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
131+
int len, int flags);
128132

129133
static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
130134
{

‎include/net/tcp.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2209,8 +2209,6 @@ void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
22092209

22102210
int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
22112211
int flags);
2212-
int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
2213-
struct msghdr *msg, int len, int flags);
22142212
#endif /* CONFIG_NET_SOCK_MSG */
22152213

22162214
#if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)

‎net/core/skmsg.c

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,104 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
399399
}
400400
EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);
401401

402+
int sk_msg_wait_data(struct sock *sk, struct sk_psock *psock, int flags,
403+
long timeo, int *err)
404+
{
405+
DEFINE_WAIT_FUNC(wait, woken_wake_function);
406+
int ret = 0;
407+
408+
if (sk->sk_shutdown & RCV_SHUTDOWN)
409+
return 1;
410+
411+
if (!timeo)
412+
return ret;
413+
414+
add_wait_queue(sk_sleep(sk), &wait);
415+
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
416+
ret = sk_wait_event(sk, &timeo,
417+
!list_empty(&psock->ingress_msg) ||
418+
!skb_queue_empty(&sk->sk_receive_queue), &wait);
419+
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
420+
remove_wait_queue(sk_sleep(sk), &wait);
421+
return ret;
422+
}
423+
EXPORT_SYMBOL_GPL(sk_msg_wait_data);
424+
425+
/* Receive sk_msg from psock->ingress_msg to @msg. */
426+
int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
427+
int len, int flags)
428+
{
429+
struct iov_iter *iter = &msg->msg_iter;
430+
int peek = flags & MSG_PEEK;
431+
struct sk_msg *msg_rx;
432+
int i, copied = 0;
433+
434+
msg_rx = sk_psock_peek_msg(psock);
435+
while (copied != len) {
436+
struct scatterlist *sge;
437+
438+
if (unlikely(!msg_rx))
439+
break;
440+
441+
i = msg_rx->sg.start;
442+
do {
443+
struct page *page;
444+
int copy;
445+
446+
sge = sk_msg_elem(msg_rx, i);
447+
copy = sge->length;
448+
page = sg_page(sge);
449+
if (copied + copy > len)
450+
copy = len - copied;
451+
copy = copy_page_to_iter(page, sge->offset, copy, iter);
452+
if (!copy)
453+
return copied ? copied : -EFAULT;
454+
455+
copied += copy;
456+
if (likely(!peek)) {
457+
sge->offset += copy;
458+
sge->length -= copy;
459+
if (!msg_rx->skb)
460+
sk_mem_uncharge(sk, copy);
461+
msg_rx->sg.size -= copy;
462+
463+
if (!sge->length) {
464+
sk_msg_iter_var_next(i);
465+
if (!msg_rx->skb)
466+
put_page(page);
467+
}
468+
} else {
469+
/* Lets not optimize peek case if copy_page_to_iter
470+
* didn't copy the entire length lets just break.
471+
*/
472+
if (copy != sge->length)
473+
return copied;
474+
sk_msg_iter_var_next(i);
475+
}
476+
477+
if (copied == len)
478+
break;
479+
} while (i != msg_rx->sg.end);
480+
481+
if (unlikely(peek)) {
482+
msg_rx = sk_psock_next_msg(psock, msg_rx);
483+
if (!msg_rx)
484+
break;
485+
continue;
486+
}
487+
488+
msg_rx->sg.start = i;
489+
if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
490+
msg_rx = sk_psock_dequeue_msg(psock);
491+
kfree_sk_msg(msg_rx);
492+
}
493+
msg_rx = sk_psock_peek_msg(psock);
494+
}
495+
496+
return copied;
497+
}
498+
EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
499+
402500
static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
403501
struct sk_buff *skb)
404502
{

‎net/ipv4/tcp_bpf.c

Lines changed: 2 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -10,80 +10,6 @@
1010
#include <net/inet_common.h>
1111
#include <net/tls.h>
1212

13-
int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
14-
struct msghdr *msg, int len, int flags)
15-
{
16-
struct iov_iter *iter = &msg->msg_iter;
17-
int peek = flags & MSG_PEEK;
18-
struct sk_msg *msg_rx;
19-
int i, copied = 0;
20-
21-
msg_rx = sk_psock_peek_msg(psock);
22-
while (copied != len) {
23-
struct scatterlist *sge;
24-
25-
if (unlikely(!msg_rx))
26-
break;
27-
28-
i = msg_rx->sg.start;
29-
do {
30-
struct page *page;
31-
int copy;
32-
33-
sge = sk_msg_elem(msg_rx, i);
34-
copy = sge->length;
35-
page = sg_page(sge);
36-
if (copied + copy > len)
37-
copy = len - copied;
38-
copy = copy_page_to_iter(page, sge->offset, copy, iter);
39-
if (!copy)
40-
return copied ? copied : -EFAULT;
41-
42-
copied += copy;
43-
if (likely(!peek)) {
44-
sge->offset += copy;
45-
sge->length -= copy;
46-
if (!msg_rx->skb)
47-
sk_mem_uncharge(sk, copy);
48-
msg_rx->sg.size -= copy;
49-
50-
if (!sge->length) {
51-
sk_msg_iter_var_next(i);
52-
if (!msg_rx->skb)
53-
put_page(page);
54-
}
55-
} else {
56-
/* Lets not optimize peek case if copy_page_to_iter
57-
* didn't copy the entire length lets just break.
58-
*/
59-
if (copy != sge->length)
60-
return copied;
61-
sk_msg_iter_var_next(i);
62-
}
63-
64-
if (copied == len)
65-
break;
66-
} while (i != msg_rx->sg.end);
67-
68-
if (unlikely(peek)) {
69-
msg_rx = sk_psock_next_msg(psock, msg_rx);
70-
if (!msg_rx)
71-
break;
72-
continue;
73-
}
74-
75-
msg_rx->sg.start = i;
76-
if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
77-
msg_rx = sk_psock_dequeue_msg(psock);
78-
kfree_sk_msg(msg_rx);
79-
}
80-
msg_rx = sk_psock_peek_msg(psock);
81-
}
82-
83-
return copied;
84-
}
85-
EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
86-
8713
static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
8814
struct sk_msg *msg, u32 apply_bytes, int flags)
8915
{
@@ -237,28 +163,6 @@ static bool tcp_bpf_stream_read(const struct sock *sk)
237163
return !empty;
238164
}
239165

240-
static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
241-
int flags, long timeo, int *err)
242-
{
243-
DEFINE_WAIT_FUNC(wait, woken_wake_function);
244-
int ret = 0;
245-
246-
if (sk->sk_shutdown & RCV_SHUTDOWN)
247-
return 1;
248-
249-
if (!timeo)
250-
return ret;
251-
252-
add_wait_queue(sk_sleep(sk), &wait);
253-
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
254-
ret = sk_wait_event(sk, &timeo,
255-
!list_empty(&psock->ingress_msg) ||
256-
!skb_queue_empty(&sk->sk_receive_queue), &wait);
257-
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
258-
remove_wait_queue(sk_sleep(sk), &wait);
259-
return ret;
260-
}
261-
262166
static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
263167
int nonblock, int flags, int *addr_len)
264168
{
@@ -278,13 +182,13 @@ static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
278182
}
279183
lock_sock(sk);
280184
msg_bytes_ready:
281-
copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
185+
copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
282186
if (!copied) {
283187
int data, err = 0;
284188
long timeo;
285189

286190
timeo = sock_rcvtimeo(sk, nonblock);
287-
data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
191+
data = sk_msg_wait_data(sk, psock, flags, timeo, &err);
288192
if (data) {
289193
if (!sk_psock_queue_empty(psock))
290194
goto msg_bytes_ready;

‎net/tls/tls_sw.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1789,8 +1789,8 @@ int tls_sw_recvmsg(struct sock *sk,
17891789
skb = tls_wait_data(sk, psock, flags, timeo, &err);
17901790
if (!skb) {
17911791
if (psock) {
1792-
int ret = __tcp_bpf_recvmsg(sk, psock,
1793-
msg, len, flags);
1792+
int ret = sk_msg_recvmsg(sk, psock, msg, len,
1793+
flags);
17941794

17951795
if (ret > 0) {
17961796
decrypted += ret;

0 commit comments

Comments
 (0)