diff options
Diffstat (limited to 'net/core')
| -rw-r--r-- | net/core/datagram.c | 47 | ||||
| -rw-r--r-- | net/core/skbuff.c | 6 |
2 files changed, 31 insertions, 22 deletions
diff --git a/net/core/datagram.c b/net/core/datagram.c index e614cfd8e14a..16507b7cc4fb 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -610,16 +610,10 @@ fault: } EXPORT_SYMBOL(skb_copy_datagram_from_iter); -int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, - struct sk_buff *skb, struct iov_iter *from, - size_t length) +int zerocopy_fill_skb_from_iter(struct sk_buff *skb, + struct iov_iter *from, size_t length) { - int frag; - - if (msg && msg->msg_ubuf && msg->sg_from_iter) - return msg->sg_from_iter(sk, skb, from, length); - - frag = skb_shinfo(skb)->nr_frags; + int frag = skb_shinfo(skb)->nr_frags; while (length && iov_iter_count(from)) { struct page *head, *last_head = NULL; @@ -627,7 +621,6 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, int refs, order, n = 0; size_t start; ssize_t copied; - unsigned long truesize; if (frag == MAX_SKB_FRAGS) return -EMSGSIZE; @@ -639,17 +632,9 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, length -= copied; - truesize = PAGE_ALIGN(copied + start); skb->data_len += copied; skb->len += copied; - skb->truesize += truesize; - if (sk && sk->sk_type == SOCK_STREAM) { - sk_wmem_queued_add(sk, truesize); - if (!skb_zcopy_pure(skb)) - sk_mem_charge(sk, truesize); - } else { - refcount_add(truesize, &skb->sk->sk_wmem_alloc); - } + skb->truesize += PAGE_ALIGN(copied + start); head = compound_head(pages[n]); order = compound_order(head); @@ -692,6 +677,30 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, } return 0; } + +int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, + struct sk_buff *skb, struct iov_iter *from, + size_t length) +{ + unsigned long orig_size = skb->truesize; + unsigned long truesize; + int ret; + + if (msg && msg->msg_ubuf && msg->sg_from_iter) + ret = msg->sg_from_iter(skb, from, length); + else + ret = zerocopy_fill_skb_from_iter(skb, from, length); + + truesize = skb->truesize - orig_size; + if (sk && sk->sk_type == SOCK_STREAM) { + sk_wmem_queued_add(sk, truesize); + if (!skb_zcopy_pure(skb)) + sk_mem_charge(sk, truesize); + } else { + refcount_add(truesize, &skb->sk->sk_wmem_alloc); + } + return ret; +} EXPORT_SYMBOL(__zerocopy_sg_from_iter); /** diff --git a/net/core/skbuff.c b/net/core/skbuff.c index eb9a7e65b5c8..0ed4d00d258c 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1871,7 +1871,6 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, struct msghdr *msg, int len, struct ubuf_info *uarg) { - struct ubuf_info *orig_uarg = skb_zcopy(skb); int err, orig_len = skb->len; if (uarg->ops->link_skb) { @@ -1879,6 +1878,8 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, if (err) return err; } else { + struct ubuf_info *orig_uarg = skb_zcopy(skb); + /* An skb can only point to one uarg. This edge case happens * when TCP appends to an skb, but zerocopy_realloc triggered * a new alloc. @@ -1899,8 +1900,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, return err; } - if (!uarg->ops->link_skb) - skb_zcopy_set(skb, uarg, NULL); + skb_zcopy_set(skb, uarg, NULL); return skb->len - orig_len; } EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); |