diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 1fcbd83f7ff2e..420250ca5b3ff 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -132,6 +132,14 @@ struct xenvif_copy_state { struct sk_buff_head *completed; }; +struct persistent_gnt { + struct page *page; + grant_ref_t gnt; + grant_handle_t handle; + bool active; + struct rb_node node; +}; + struct xenvif_queue { /* Per-queue data for xenvif */ unsigned int id; /* Queue ID, 0-based */ char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */ @@ -213,6 +221,12 @@ struct xenvif_queue { /* Per-queue data for xenvif */ u64 credit_window_start; bool rate_limited; + /* Persistent grants */ + struct rb_root persistent_gnts; + unsigned int persistent_gnt_c; + struct gnttab_page_cache persistent_pages; + struct persistent_gnt *tx_pgrants[MAX_PENDING_REQS]; + /* Statistics */ struct xenvif_stats stats; }; @@ -291,6 +305,7 @@ struct xenvif { u8 can_sg:1; u8 ip_csum:1; u8 ipv6_csum:1; + u8 persistent_grants:1; u8 multicast_control:1; /* headroom requested by xen-netfront */ @@ -408,6 +423,7 @@ extern bool provides_xdp_headroom; extern unsigned int rx_drain_timeout_msecs; extern unsigned int rx_stall_timeout_msecs; extern unsigned int xenvif_max_queues; +extern unsigned int xenvif_max_pgrants; extern unsigned int xenvif_hash_cache_size; #ifdef CONFIG_DEBUG_FS @@ -416,12 +432,16 @@ extern struct dentry *xen_netback_dbg_root; void xenvif_skb_zerocopy_prepare(struct xenvif_queue *queue, struct sk_buff *skb); -void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue); +void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue, + unsigned int pending_dealloc); /* Multicast control */ bool xenvif_mcast_match(struct xenvif *vif, const u8 *addr); void xenvif_mcast_addr_list_free(struct xenvif *vif); +/* Persistent grants */ +void xenvif_pgrants_destroy(struct xenvif_queue *queue); + /* Hash */ void xenvif_init_hash(struct xenvif *vif); void xenvif_deinit_hash(struct xenvif *vif); diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index fc3bb63b9ac3e..a5af4d19fe7fd 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -56,7 +56,8 @@ void xenvif_skb_zerocopy_prepare(struct xenvif_queue *queue, atomic_inc(&queue->inflight_packets); } -void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue) +void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue, + unsigned int pending_dealloc) { atomic_dec(&queue->inflight_packets); @@ -64,7 +65,8 @@ void xenvif_skb_zerocopy_complete(struct xenvif_queue *queue) * that if kthread_stop() has already been called, the dealloc thread * does not wait forever with nothing to wake it. */ - wake_up(&queue->dealloc_wq); + if (pending_dealloc) + wake_up(&queue->dealloc_wq); } static int xenvif_schedulable(struct xenvif *vif) @@ -588,12 +590,18 @@ int xenvif_init_queue(struct xenvif_queue *queue) return -ENOMEM; } + if (queue->vif->persistent_grants) { + queue->persistent_gnts.rb_node = NULL; + queue->persistent_gnt_c = 0; + } + for (i = 0; i < MAX_PENDING_REQS; i++) { queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc) { { .callback = xenvif_zerocopy_callback }, { { .ctx = NULL, .desc = i } } }; queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE; + queue->tx_pgrants[i] = NULL; } return 0; @@ -696,6 +704,10 @@ static void xenvif_disconnect_queue(struct xenvif_queue *queue) } xenvif_unmap_frontend_data_rings(queue); + + if (queue->vif->persistent_grants) { + xenvif_pgrants_destroy(queue); + } } int xenvif_connect_data(struct xenvif_queue *queue, diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index 0d51c900c5538..1b92bbe58cc43 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -73,6 +73,15 @@ module_param_named(max_queues, xenvif_max_queues, uint, 0644); MODULE_PARM_DESC(max_queues, "Maximum number of queues per virtual interface"); +// L17 TODO: test performance with higher values like blkback's hardcoded 1056 +/* + * Maximum number of grants to map persistently in netback. + */ +unsigned int xenvif_max_pgrants = XEN_NETIF_TX_RING_SIZE; +module_param_named(max_persistent_grants, xenvif_max_pgrants, int, 0644); +MODULE_PARM_DESC(max_persistent_grants, + "Maximum number of grants to map persistently"); + /* * This is the maximum slots a skb can have. If a guest sends a skb * which exceeds this limit it is considered malicious. @@ -127,9 +136,21 @@ static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue, return (unsigned long)pfn_to_kaddr(idx_to_pfn(queue, idx)); } +static inline unsigned long page_to_kaddr(struct page *page) +{ + return (unsigned long)pfn_to_kaddr(page_to_pfn(page)); +} + #define callback_param(vif, pending_idx) \ (vif->pending_tx_info[pending_idx].callback_struct) +#define foreach_grant_safe(pos, n, rbtree, node) \ + for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \ + (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \ + &(pos)->node != NULL; \ + (pos) = container_of(n, typeof(*(pos)), node), \ + (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL) + /* Find the containing VIF's structure from a pointer in pending_tx_info array */ static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info_msgzc *ubuf) @@ -157,6 +178,205 @@ static inline pending_ring_idx_t pending_index(unsigned i) return i & (MAX_PENDING_REQS-1); } +static struct page *get_free_page(struct xenvif_queue *queue, + int persistent, + u16 pending_idx) +{ + struct page *page; + + if (!persistent || gnttab_page_cache_get(&queue->persistent_pages, &page)) { + page = queue->mmap_pages[pending_idx]; + } + + return page; +} + +static int add_persistent_gnt(struct xenvif_queue *queue, + struct persistent_gnt *pgrant) +{ + struct rb_node **new = NULL, *parent = NULL; + struct persistent_gnt *this; + + if (queue->persistent_gnt_c >= xenvif_max_pgrants) { + pr_alert_ratelimited("trying to add a gref when pgrant limit has been reached\n"); + return -EBUSY; + } + /* Figure out where to put new node */ + new = &queue->persistent_gnts.rb_node; + while (*new) { + this = container_of(*new, struct persistent_gnt, node); + + parent = *new; + if (pgrant->gnt < this->gnt) + new = &((*new)->rb_left); + else if (pgrant->gnt > this->gnt) + new = &((*new)->rb_right); + else { + pr_alert_ratelimited("trying to add a gref that's already in the tree\n"); + return -EINVAL; + } + } + pgrant->active = true; + + /* Add new node and rebalance tree. */ + rb_link_node(&(pgrant->node), parent, new); + rb_insert_color(&(pgrant->node), &queue->persistent_gnts); + queue->persistent_gnt_c++; + //idk what it's for... atomic_inc(&queue->persistent_gnt_in_use); + + return 0; +} + +static struct persistent_gnt *get_persistent_gnt(struct xenvif_queue *queue, + grant_ref_t gref) +{ + struct persistent_gnt *pgrant; + struct rb_node *node = NULL; + + node = queue->persistent_gnts.rb_node; + while (node) { + pgrant = container_of(node, struct persistent_gnt, node); + + if (gref < pgrant->gnt) + node = node->rb_left; + else if (gref > pgrant->gnt) + node = node->rb_right; + else { + if (pgrant->active) { + pr_alert_ratelimited("requesting a grant already in use\n"); + return ERR_PTR(-EBUSY); + } + pgrant->active = true; + //idk what it's for... atomic_inc(&queue->persistent_gnt_in_use); + return pgrant; + } + } + + return NULL; +} + +static void put_persistent_gnt(struct xenvif_queue *queue, + struct persistent_gnt *pgrant) +{ + if (!pgrant->active) + pr_alert_ratelimited("freeing a grant already unused\n"); + + pgrant->active = false; + //idk what it's for... atomic_dec(&ring->persistent_gnt_in_use); +} + +static void free_persistent_gnts(struct xenvif_queue *queue, + struct rb_root *root, + unsigned int num) +{ + struct gnttab_unmap_grant_ref unmap[FATAL_SKB_SLOTS_DEFAULT]; + struct page *pages[FATAL_SKB_SLOTS_DEFAULT]; + struct persistent_gnt *pgrant; + struct rb_node *n; + int segs_to_unmap = 0; + struct gntab_unmap_queue_data unmap_data; + + unmap_data.pages = pages; + unmap_data.unmap_ops = unmap; + unmap_data.kunmap_ops = NULL; + + foreach_grant_safe(pgrant, n, root, node) { + BUG_ON(pgrant->handle == INVALID_GRANT_HANDLE); + + gnttab_set_unmap_op(&unmap[segs_to_unmap], + (unsigned long) pfn_to_kaddr(page_to_pfn(pgrant->page)), + GNTMAP_host_map, + pgrant->handle); + + pages[segs_to_unmap] = pgrant->page; + + if (++segs_to_unmap == FATAL_SKB_SLOTS_DEFAULT || + !rb_next(&pgrant->node)) { + + unmap_data.count = segs_to_unmap; + BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); + gnttab_page_cache_put(&queue->persistent_pages, pages, + segs_to_unmap); + segs_to_unmap = 0; + } + + rb_erase(&pgrant->node, root); + kfree(pgrant); + num--; + } + BUG_ON(num != 0); +} + +static inline void xenvif_pgrant_set(struct xenvif_queue *queue, + u16 pending_idx, + struct persistent_gnt *pgrant) +{ + if (unlikely(queue->tx_pgrants[pending_idx])) { + netdev_err(queue->vif->dev, + "Trying to overwrite an active persistent grant! pending_idx: 0x%x\n", + pending_idx); + BUG(); + } + queue->tx_pgrants[pending_idx] = pgrant; +} + +static inline void xenvif_pgrant_reset(struct xenvif_queue *queue, + u16 pending_idx) +{ + struct persistent_gnt *pgrant = queue->tx_pgrants[pending_idx]; + + if (unlikely(!pgrant)) { + netdev_err(queue->vif->dev, + "Trying to release an inactive persistent_grant! pending_idx: 0x%x\n", + pending_idx); + BUG(); + } + put_persistent_gnt(queue, pgrant); + queue->tx_pgrants[pending_idx] = NULL; +} + +/* + * Create a new persistent grant and add it to the tree. + */ +static struct persistent_gnt *xenvif_pgrant_new(struct xenvif_queue *queue, + struct gnttab_map_grant_ref *gop) +{ + struct persistent_gnt *pgrant; + + pgrant = kmalloc(sizeof(struct persistent_gnt), GFP_KERNEL); + if (!pgrant) + BUG(); // not handling errors yet + // return NULL; + + pgrant->page = virt_to_page(gop->host_addr); + pgrant->gnt = gop->ref; + pgrant->handle = gop->handle; + + if (unlikely(add_persistent_gnt(queue, pgrant))) { + kfree(pgrant); + BUG(); // not handling errors yet + //return NULL; + } + + return pgrant; +} + +/* + * Free all persistent grants data and empty the pool of free pages. + */ +void xenvif_pgrants_destroy(struct xenvif_queue *queue) +{ + if (!RB_EMPTY_ROOT(&queue->persistent_gnts)) + free_persistent_gnts(queue, &queue->persistent_gnts, + queue->persistent_gnt_c); + + BUG_ON(!RB_EMPTY_ROOT(&queue->persistent_gnts)); + queue->persistent_gnts.rb_node = NULL; + queue->persistent_gnt_c = 0; + + gnttab_page_cache_shrink(&queue->persistent_pages, 0 /* All */); +} + void xenvif_kick_thread(struct xenvif_queue *queue) { wake_up(&queue->wq); @@ -337,19 +557,14 @@ struct xenvif_tx_cb { #define copy_count(skb) (XENVIF_TX_CB(skb)->copy_count) static inline void xenvif_tx_create_map_op(struct xenvif_queue *queue, - u16 pending_idx, struct xen_netif_tx_request *txp, - unsigned int extra_count, + struct page *page, struct gnttab_map_grant_ref *mop) { - queue->pages_to_map[mop-queue->tx_map_ops] = queue->mmap_pages[pending_idx]; - gnttab_set_map_op(mop, idx_to_kaddr(queue, pending_idx), + queue->pages_to_map[mop-queue->tx_map_ops] = page; + gnttab_set_map_op(mop, page_to_kaddr(page), GNTMAP_host_map | GNTMAP_readonly, txp->gref, queue->vif->domid); - - memcpy(&queue->pending_tx_info[pending_idx].req, txp, - sizeof(*txp)); - queue->pending_tx_info[pending_idx].extra_count = extra_count; } static inline struct sk_buff *xenvif_alloc_skb(unsigned int size) @@ -390,6 +605,9 @@ static void xenvif_get_requests(struct xenvif_queue *queue, struct gnttab_copy *cop = queue->tx_copy_ops + *copy_ops; struct gnttab_map_grant_ref *gop = queue->tx_map_ops + *map_ops; struct xen_netif_tx_request *txp = first; + struct page *page; + int use_pgrants = queue->vif->persistent_grants; + struct persistent_gnt *pgrant = NULL; nr_slots = shinfo->nr_frags + frag_overflow + 1; @@ -408,10 +626,10 @@ static void xenvif_get_requests(struct xenvif_queue *queue, cop->dest.domid = DOMID_SELF; cop->dest.offset = (offset_in_page(skb->data + - skb_headlen(skb) - - data_len)) & ~XEN_PAGE_MASK; + skb_headlen(skb) - + data_len)) & ~XEN_PAGE_MASK; cop->dest.u.gmfn = virt_to_gfn(skb->data + skb_headlen(skb) - - data_len); + - data_len); /* Don't cross local page boundary! */ if (cop->dest.offset + amount > XEN_PAGE_SIZE) { @@ -425,6 +643,7 @@ static void xenvif_get_requests(struct xenvif_queue *queue, index = pending_index(queue->pending_cons); pending_idx = queue->pending_ring[index]; + callback_param(queue, pending_idx).ctx = NULL; copy_pending_idx(skb, copy_count(skb)) = pending_idx; if (!split) @@ -435,7 +654,6 @@ static void xenvif_get_requests(struct xenvif_queue *queue, if (amount == txp->size) { /* The copy op covered the full tx_request */ - memcpy(&queue->pending_tx_info[pending_idx].req, txp, sizeof(*txp)); queue->pending_tx_info[pending_idx].extra_count = @@ -445,6 +663,7 @@ static void xenvif_get_requests(struct xenvif_queue *queue, txp = txfrags; else txp++; + queue->pending_cons++; nr_slots--; } else { @@ -457,6 +676,7 @@ static void xenvif_get_requests(struct xenvif_queue *queue, } } + /* Create map ops for the skb frags */ for (shinfo->nr_frags = 0; nr_slots > 0 && shinfo->nr_frags < MAX_SKB_FRAGS; nr_slots--) { if (unlikely(!txp->size)) { @@ -467,11 +687,24 @@ static void xenvif_get_requests(struct xenvif_queue *queue, index = pending_index(queue->pending_cons++); pending_idx = queue->pending_ring[index]; - xenvif_tx_create_map_op(queue, pending_idx, txp, - txp == first ? extra_count : 0, gop); + + if (use_pgrants) + pgrant = get_persistent_gnt(queue, txp->gref); + + if (!use_pgrants || !pgrant) { + page = get_free_page(queue, use_pgrants, pending_idx); + xenvif_tx_create_map_op(queue, txp, page, gop++); + } else { + /* No map op needed for this frag */ + xenvif_pgrant_set(queue, pending_idx, pgrant); + } + + memcpy(&queue->pending_tx_info[pending_idx].req, txp, sizeof(*txp)); + queue->pending_tx_info[pending_idx].extra_count = + (txp == first) ? extra_count : 0; + frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); ++shinfo->nr_frags; - ++gop; if (txp == first) txp = txfrags; @@ -479,6 +712,7 @@ static void xenvif_get_requests(struct xenvif_queue *queue, txp++; } + /* Create map ops for the nskb frags */ if (nr_slots > 0) { shinfo = skb_shinfo(nskb); @@ -493,12 +727,24 @@ static void xenvif_get_requests(struct xenvif_queue *queue, index = pending_index(queue->pending_cons++); pending_idx = queue->pending_ring[index]; - xenvif_tx_create_map_op(queue, pending_idx, txp, 0, - gop); + + if (use_pgrants) + pgrant = get_persistent_gnt(queue, txp->gref); + + if (!use_pgrants || !pgrant) { + page = get_free_page(queue, use_pgrants, pending_idx); + xenvif_tx_create_map_op(queue, txp, page, gop++); + } else { + /* No map op needed for this frag */ + xenvif_pgrant_set(queue, pending_idx, pgrant); + } + + memcpy(&queue->pending_tx_info[pending_idx].req, txp, sizeof(*txp)); + queue->pending_tx_info[pending_idx].extra_count = 0; + frag_set_pending_idx(&frags[shinfo->nr_frags], pending_idx); ++shinfo->nr_frags; - ++gop; } if (shinfo->nr_frags) { @@ -565,6 +811,7 @@ static int xenvif_tx_check_gop(struct xenvif_queue *queue, const bool sharedslot = nr_frags && frag_get_pending_idx(&shinfo->frags[0]) == copy_pending_idx(skb, copy_count(skb) - 1); + struct persistent_gnt *pgrant; int i, err = 0; for (i = 0; i < copy_count(skb); i++) { @@ -572,7 +819,6 @@ static int xenvif_tx_check_gop(struct xenvif_queue *queue, /* Check status of header. */ pending_idx = copy_pending_idx(skb, i); - newerr = (*gopp_copy)->status; /* Split copies need to be handled together. */ @@ -607,6 +853,16 @@ static int xenvif_tx_check_gop(struct xenvif_queue *queue, int j, newerr; pending_idx = frag_get_pending_idx(&shinfo->frags[i]); + // if there is a pgrant at pending_idx, set grant handle and continue + pgrant = queue->tx_pgrants[pending_idx]; + if (pgrant) { + xenvif_grant_handle_set(queue, + pending_idx, + pgrant->handle); + // don't skip current map op to check + gop_map--; + continue; + } /* Check error status: if okay then remember grant handle. */ newerr = gop_map->status; @@ -615,6 +871,13 @@ static int xenvif_tx_check_gop(struct xenvif_queue *queue, xenvif_grant_handle_set(queue, pending_idx, gop_map->handle); + // if the page used to map is a persistent page, make a new pgrant + // and set the pgrant to tx_pgrants at pending_idx + if (virt_to_page(gop_map->host_addr) != queue->mmap_pages[pending_idx]) { + pgrant = xenvif_pgrant_new(queue, gop_map); + xenvif_pgrant_set(queue, pending_idx, pgrant); + } + /* Had a previous error? Invalidate this fragment. */ if (unlikely(err)) { xenvif_idx_unmap(queue, pending_idx); @@ -694,6 +957,7 @@ static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb) skb_frag_t *frag = shinfo->frags + i; struct xen_netif_tx_request *txp; struct page *page; + struct persistent_gnt *pgrant; u16 pending_idx; pending_idx = frag_get_pending_idx(frag); @@ -710,14 +974,17 @@ static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb) prev_pending_idx = pending_idx; txp = &queue->pending_tx_info[pending_idx].req; - page = virt_to_page(idx_to_kaddr(queue, pending_idx)); + // if there is a pgrant at pending_idx, get the page from the pgrant + pgrant = queue->tx_pgrants[pending_idx]; + page = pgrant ? pgrant->page : queue->mmap_pages[pending_idx]; __skb_fill_page_desc(skb, i, page, txp->offset, txp->size); skb->len += txp->size; skb->data_len += txp->size; skb->truesize += txp->size; /* Take an extra reference to offset network stack's put_page */ - get_page(queue->mmap_pages[pending_idx]); + // use already existing var *page* instead + get_page(page); } } @@ -1293,7 +1560,7 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base, bool zerocopy_success) { unsigned long flags; - pending_ring_idx_t index; + pending_ring_idx_t index, dealloc_prod_save; struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); struct xenvif_queue *queue = ubuf_to_queue(ubuf); @@ -1301,9 +1568,18 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base, * from each other. */ spin_lock_irqsave(&queue->callback_lock, flags); + dealloc_prod_save = queue->dealloc_prod; do { u16 pending_idx = ubuf->desc; ubuf = (struct ubuf_info_msgzc *) ubuf->ctx; + + if (queue->tx_pgrants[pending_idx]) { + // release stuff without unmapping + xenvif_pgrant_reset(queue, pending_idx); + xenvif_grant_handle_reset(queue, pending_idx); + xenvif_idx_release(queue, pending_idx, XEN_NETIF_RSP_OKAY); + continue; + } BUG_ON(queue->dealloc_prod - queue->dealloc_cons >= MAX_PENDING_REQS); index = pending_index(queue->dealloc_prod); @@ -1320,7 +1596,8 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base, queue->stats.tx_zerocopy_success++; else queue->stats.tx_zerocopy_fail++; - xenvif_skb_zerocopy_complete(queue); + // if dealloc_prod didn't move, skip only the wake_up call (see function) + xenvif_skb_zerocopy_complete(queue, dealloc_prod_save - queue->dealloc_prod); } static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue) @@ -1401,6 +1678,7 @@ int xenvif_tx_action(struct xenvif_queue *queue, int budget) xenvif_tx_build_gops(queue, budget, &nr_cops, &nr_mops); + // L17 TODO it may be normal when we will memcpy using pgrants in copy loop if (nr_cops == 0) return 0; diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index c1ba4294f3647..a6c4d9a218a9b 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -972,6 +972,9 @@ static int read_xenbus_vif_flags(struct backend_info *be) vif->ipv6_csum = !!xenbus_read_unsigned(dev->otherend, "feature-ipv6-csum-offload", 0); + vif->persistent_grants = xenvif_max_pgrants && + xenbus_read_unsigned(dev->otherend, "feature-persistent", 0); + read_xenbus_frontend_xdp(be, dev); return 0; @@ -1116,6 +1119,12 @@ static int netback_probe(struct xenbus_device *dev, if (err) pr_debug("Error writing feature-split-event-channels\n"); + /* Persistent grants support, this is an optional feature */ + err = xenbus_printf(XBT_NIL, dev->nodename, + "feature-persistent", "%d", xenvif_max_pgrants > 0); + if (err) + pr_debug("Error writing feature-persistent\n"); + /* Multi-queue support: This is an optional feature. */ err = xenbus_printf(XBT_NIL, dev->nodename, "multi-queue-max-queues", "%u", xenvif_max_queues); diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 8425226c09f0d..81d09fd25d0e0 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -174,6 +174,9 @@ struct netfront_info { bool netback_has_xdp_headroom; bool netfront_xdp_enabled; + /* Persistent grants */ + bool persistent_grants; + /* Is device behaving sane? */ bool broken; @@ -431,16 +434,19 @@ static bool xennet_tx_buf_gc(struct netfront_queue *queue) queue->tx_link[id] = TX_LINK_NONE; skb = queue->tx_skbs[id]; queue->tx_skbs[id] = NULL; - if (unlikely(!gnttab_end_foreign_access_ref( - queue->grant_tx_ref[id]))) { - dev_alert(dev, - "Grant still in use by backend domain\n"); - goto err; + + if (!queue->info->persistent_grants) { + if (unlikely(!gnttab_end_foreign_access_ref( + queue->grant_tx_ref[id]))) { + dev_alert(dev, + "Grant still in use by backend domain\n"); + goto err; + } + gnttab_release_grant_reference( + &queue->gref_tx_head, queue->grant_tx_ref[id]); + queue->grant_tx_ref[id] = INVALID_GRANT_REF; + queue->grant_tx_page[id] = NULL; } - gnttab_release_grant_reference( - &queue->gref_tx_head, queue->grant_tx_ref[id]); - queue->grant_tx_ref[id] = INVALID_GRANT_REF; - queue->grant_tx_page[id] = NULL; add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); dev_kfree_skb_irq(skb); } @@ -484,18 +490,34 @@ static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset, id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); - ref = gnttab_claim_grant_reference(&queue->gref_tx_head); - WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); - gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, - gfn, GNTMAP_readonly); + /* Reuse claimed grants */ + if (queue->grant_tx_ref[id] == INVALID_GRANT_REF) { + ref = gnttab_claim_grant_reference(&queue->gref_tx_head); + WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); + + if (queue->info->persistent_grants) + gfn = pfn_to_gfn(page_to_xen_pfn(queue->grant_tx_page[id])); + + gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, + gfn, GNTMAP_readonly); + + queue->grant_tx_ref[id] = ref; + } + + if (queue->info->persistent_grants) { + /* Reuse granted pages */ + memcpy(pfn_to_kaddr(page_to_pfn(queue->grant_tx_page[id])), + pfn_to_kaddr(page_to_pfn(page)) + offset, len); + offset = 0; // worried that offsets will introduce bleeding + } else { + queue->grant_tx_page[id] = page; + } queue->tx_skbs[id] = skb; - queue->grant_tx_page[id] = page; - queue->grant_tx_ref[id] = ref; info->tx_local.id = id; - info->tx_local.gref = ref; + info->tx_local.gref = queue->grant_tx_ref[id]; info->tx_local.offset = offset; info->tx_local.size = len; info->tx_local.flags = 0; @@ -637,8 +659,6 @@ static int xennet_xdp_xmit_one(struct net_device *dev, tx_stats->packets++; u64_stats_update_end(&tx_stats->syncp); - xennet_tx_buf_gc(queue); - return 0; } @@ -848,9 +868,6 @@ static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev tx_stats->packets++; u64_stats_update_end(&tx_stats->syncp); - /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ - xennet_tx_buf_gc(queue); - if (!netfront_tx_slot_available(queue)) netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); @@ -2040,7 +2057,8 @@ static int xennet_init_queue(struct netfront_queue *queue) for (i = 0; i < NET_TX_RING_SIZE; i++) { queue->tx_link[i] = i + 1; queue->grant_tx_ref[i] = INVALID_GRANT_REF; - queue->grant_tx_page[i] = NULL; + queue->grant_tx_page[i] = queue->info->persistent_grants ? + alloc_page(GFP_NOIO) : NULL; } queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; @@ -2268,6 +2286,10 @@ static int talk_to_netback(struct xenbus_device *dev, info->bounce = !xennet_trusted || !xenbus_read_unsigned(dev->nodename, "trusted", 1); + /* Check if backend supports persistent grants */ + info->persistent_grants = !!xenbus_read_unsigned(info->xbdev->otherend, + "feature-persistent", 0); + /* Check if backend supports multiple queues */ max_queues = xenbus_read_unsigned(info->xbdev->otherend, "multi-queue-max-queues", 1); @@ -2391,6 +2413,13 @@ static int talk_to_netback(struct xenbus_device *dev, goto abort_transaction; } + // L17 TODO: decide if having a persistent mod param like netback is useful + err = xenbus_write(xbt, dev->nodename, "feature-persistent", "1"); + if (err) { + message = "writing feature-persistent"; + goto abort_transaction; + } + err = xenbus_transaction_end(xbt, 0); if (err) { if (err == -EAGAIN)