aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/virtio
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/virtio')
-rw-r--r--sys/dev/virtio/block/virtio_blk.c237
-rw-r--r--sys/dev/virtio/gpu/virtio_gpu.c51
-rw-r--r--sys/dev/virtio/network/if_vtnet.c828
-rw-r--r--sys/dev/virtio/network/if_vtnetvar.h35
-rw-r--r--sys/dev/virtio/p9fs/virtio_p9fs.c39
-rw-r--r--sys/dev/virtio/pci/virtio_pci.c7
-rw-r--r--sys/dev/virtio/pci/virtio_pci_modern.c49
-rw-r--r--sys/dev/virtio/scsi/virtio_scsi.c24
-rw-r--r--sys/dev/virtio/scsi/virtio_scsi.h9
-rw-r--r--sys/dev/virtio/scsi/virtio_scsivar.h1
-rw-r--r--sys/dev/virtio/virtio.c12
-rw-r--r--sys/dev/virtio/virtio.h16
-rw-r--r--sys/dev/virtio/virtio_config.h28
-rw-r--r--sys/dev/virtio/virtio_ring.h27
-rw-r--r--sys/dev/virtio/virtqueue.c218
15 files changed, 1298 insertions, 283 deletions
diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c
index 5eb681128e9c..96846eb0529a 100644
--- a/sys/dev/virtio/block/virtio_blk.c
+++ b/sys/dev/virtio/block/virtio_blk.c
@@ -59,10 +59,15 @@ struct vtblk_request {
struct vtblk_softc *vbr_sc;
bus_dmamap_t vbr_mapp;
+ struct virtio_blk_outhdr *vbr_hdr;
+ vm_paddr_t vbr_hdr_paddr;
+ bus_dmamap_t vbr_hdr_mapp;
+ uint8_t *vbr_ack;
+ vm_paddr_t vbr_ack_paddr;
+ bus_dmamap_t vbr_ack_mapp;
+
/* Fields after this point are zeroed for each request. */
- struct virtio_blk_outhdr vbr_hdr;
struct bio *vbr_bp;
- uint8_t vbr_ack;
uint8_t vbr_requeue_on_error;
uint8_t vbr_busdma_wait;
int vbr_error;
@@ -78,6 +83,8 @@ enum vtblk_cache_mode {
struct vtblk_softc {
device_t vtblk_dev;
struct mtx vtblk_mtx;
+ struct mtx vtblk_hdr_mtx;
+ struct mtx vtblk_ack_mtx;
uint64_t vtblk_features;
uint32_t vtblk_flags;
#define VTBLK_FLAG_INDIRECT 0x0001
@@ -91,6 +98,8 @@ struct vtblk_softc {
struct virtqueue *vtblk_vq;
struct sglist *vtblk_sglist;
bus_dma_tag_t vtblk_dmat;
+ bus_dma_tag_t vtblk_hdr_dmat;
+ bus_dma_tag_t vtblk_ack_dmat;
struct disk *vtblk_disk;
struct bio_queue_head vtblk_bioq;
@@ -385,7 +394,7 @@ vtblk_attach(device_t dev)
maxphys, /* max request size */
sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS, /* max # segments */
maxphys, /* maxsegsize */
- 0, /* flags */
+ BUS_DMA_COHERENT, /* flags */
busdma_lock_mutex, /* lockfunc */
&sc->vtblk_mtx, /* lockarg */
&sc->vtblk_dmat);
@@ -394,6 +403,46 @@ vtblk_attach(device_t dev)
goto fail;
}
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ (sc->vtblk_flags & VTBLK_FLAG_BUSDMA_ALIGN) ? PAGE_SIZE :
+ sizeof(struct virtio_blk_outhdr), /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ sizeof(struct virtio_blk_outhdr), /* max request size */
+ 1, /* max # segments */
+ sizeof(struct virtio_blk_outhdr), /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &sc->vtblk_hdr_mtx, /* lockarg */
+ &sc->vtblk_hdr_dmat);
+ if (error) {
+ device_printf(dev, "cannot create hdr bus dma tag\n");
+ goto fail;
+ }
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ (sc->vtblk_flags & VTBLK_FLAG_BUSDMA_ALIGN) ? PAGE_SIZE :
+ sizeof(uint8_t), /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ sizeof(uint8_t), /* max request size */
+ 1, /* max # segments */
+ sizeof(uint8_t), /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &sc->vtblk_ack_mtx, /* lockarg */
+ &sc->vtblk_ack_dmat);
+ if (error) {
+ device_printf(dev, "cannot create ack bus dma tag\n");
+ goto fail;
+ }
+
#ifdef __powerpc__
/*
* Virtio uses physical addresses rather than bus addresses, so we
@@ -401,6 +450,8 @@ vtblk_attach(device_t dev)
* present, this is only a thing on the powerpc architectures.
*/
bus_dma_tag_set_iommu(sc->vtblk_dmat, NULL, NULL);
+ bus_dma_tag_set_iommu(sc->vtblk_hdr_dmat, NULL, NULL);
+ bus_dma_tag_set_iommu(sc->vtblk_ack_dmat, NULL, NULL);
#endif
error = vtblk_alloc_virtqueue(sc);
@@ -452,6 +503,16 @@ vtblk_detach(device_t dev)
sc->vtblk_disk = NULL;
}
+ if (sc->vtblk_ack_dmat != NULL) {
+ bus_dma_tag_destroy(sc->vtblk_ack_dmat);
+ sc->vtblk_ack_dmat = NULL;
+ }
+
+ if (sc->vtblk_hdr_dmat != NULL) {
+ bus_dma_tag_destroy(sc->vtblk_hdr_dmat);
+ sc->vtblk_hdr_dmat = NULL;
+ }
+
if (sc->vtblk_dmat != NULL) {
bus_dma_tag_destroy(sc->vtblk_dmat);
sc->vtblk_dmat = NULL;
@@ -839,11 +900,88 @@ vtblk_create_disk(struct vtblk_softc *sc)
disk_create(dp, DISK_VERSION);
}
+static void
+vtblk_ack_load_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
+ int error)
+{
+ struct vtblk_request *req;
+
+ if (error != 0)
+ return;
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ req = (struct vtblk_request *)arg;
+ req->vbr_ack_paddr = segs[0].ds_addr;
+}
+
+static void
+vtblk_hdr_load_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
+ int error)
+{
+ struct vtblk_request *req;
+
+ if (error != 0)
+ return;
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ req = (struct vtblk_request *)arg;
+ req->vbr_hdr_paddr = segs[0].ds_addr;
+}
+
+static int
+vtblk_create_request(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+ req->vbr_sc = sc;
+
+ if (bus_dmamap_create(sc->vtblk_dmat, 0, &req->vbr_mapp))
+ goto error_free;
+
+ if (bus_dmamem_alloc(sc->vtblk_hdr_dmat, (void **)&req->vbr_hdr,
+ BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &req->vbr_hdr_mapp))
+ goto error_destroy;
+
+ if (bus_dmamem_alloc(sc->vtblk_ack_dmat, (void **)&req->vbr_ack,
+ BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &req->vbr_ack_mapp))
+ goto error_hdr_free;
+
+ MPASS(sglist_count(req->vbr_hdr, sizeof(*req->vbr_hdr)) == 1);
+ MPASS(sglist_count(req->vbr_ack, sizeof(*req->vbr_ack)) == 1);
+
+ if (bus_dmamap_load(sc->vtblk_hdr_dmat, req->vbr_hdr_mapp,
+ req->vbr_hdr, sizeof(struct virtio_blk_outhdr),
+ vtblk_hdr_load_callback, req, BUS_DMA_NOWAIT))
+ goto error_ack_free;
+
+ if (bus_dmamap_load(sc->vtblk_ack_dmat, req->vbr_ack_mapp,
+ req->vbr_ack, sizeof(uint8_t), vtblk_ack_load_callback,
+ req, BUS_DMA_NOWAIT))
+ goto error_hdr_unload;
+
+ return (0);
+
+error_hdr_unload:
+ bus_dmamap_unload(sc->vtblk_hdr_dmat, req->vbr_hdr_mapp);
+error_ack_free:
+ bus_dmamem_free(sc->vtblk_ack_dmat, req->vbr_ack, req->vbr_ack_mapp);
+error_hdr_free:
+ bus_dmamem_free(sc->vtblk_hdr_dmat, req->vbr_hdr, req->vbr_hdr_mapp);
+error_destroy:
+ bus_dmamap_destroy(sc->vtblk_dmat, req->vbr_mapp);
+error_free:
+
+ return (ENOMEM);
+}
+
static int
vtblk_request_prealloc(struct vtblk_softc *sc)
{
struct vtblk_request *req;
int i, nreqs;
+ int error;
nreqs = virtqueue_size(sc->vtblk_vq);
@@ -860,20 +998,19 @@ vtblk_request_prealloc(struct vtblk_softc *sc)
if (req == NULL)
return (ENOMEM);
- req->vbr_sc = sc;
- if (bus_dmamap_create(sc->vtblk_dmat, 0, &req->vbr_mapp)) {
+ error = vtblk_create_request(sc, req);
+ if (error) {
free(req, M_DEVBUF);
- return (ENOMEM);
+ return (error);
}
- MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1);
- MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1);
-
sc->vtblk_request_count++;
vtblk_request_enqueue(sc, req);
}
- return (0);
+ error = vtblk_create_request(sc, &sc->vtblk_dump_request);
+
+ return (error);
}
static void
@@ -885,8 +1022,13 @@ vtblk_request_free(struct vtblk_softc *sc)
while ((req = vtblk_request_dequeue(sc)) != NULL) {
sc->vtblk_request_count--;
+ bus_dmamap_unload(sc->vtblk_ack_dmat, req->vbr_ack_mapp);
+ bus_dmamem_free(sc->vtblk_ack_dmat, req->vbr_ack,
+ req->vbr_ack_mapp);
+ bus_dmamap_unload(sc->vtblk_hdr_dmat, req->vbr_hdr_mapp);
+ bus_dmamem_free(sc->vtblk_hdr_dmat, req->vbr_hdr,
+ req->vbr_hdr_mapp);
bus_dmamap_destroy(sc->vtblk_dmat, req->vbr_mapp);
- free(req, M_DEVBUF);
}
KASSERT(sc->vtblk_request_count == 0,
@@ -901,8 +1043,10 @@ vtblk_request_dequeue(struct vtblk_softc *sc)
req = TAILQ_FIRST(&sc->vtblk_req_free);
if (req != NULL) {
TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
- bzero(&req->vbr_hdr, sizeof(struct vtblk_request) -
- offsetof(struct vtblk_request, vbr_hdr));
+ bzero(req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
+ *req->vbr_ack = 0;
+ bzero(&req->vbr_bp, sizeof(struct vtblk_request) -
+ offsetof(struct vtblk_request, vbr_bp));
}
return (req);
@@ -965,32 +1109,35 @@ vtblk_request_bio(struct vtblk_softc *sc)
bp = bioq_takefirst(bioq);
req->vbr_bp = bp;
- req->vbr_ack = -1;
- req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
+ *req->vbr_ack = -1;
+ req->vbr_hdr->ioprio = vtblk_gtoh32(sc, 1);
switch (bp->bio_cmd) {
case BIO_FLUSH:
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);
- req->vbr_hdr.sector = 0;
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);
+ req->vbr_hdr->sector = 0;
break;
case BIO_READ:
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN);
- req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN);
+ req->vbr_hdr->sector = vtblk_gtoh64(sc, bp->bio_offset /
+ VTBLK_BSIZE);
break;
case BIO_WRITE:
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);
- req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);
+ req->vbr_hdr->sector = vtblk_gtoh64(sc, bp->bio_offset /
+ VTBLK_BSIZE);
break;
case BIO_DELETE:
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_DISCARD);
- req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE);
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_DISCARD);
+ req->vbr_hdr->sector = vtblk_gtoh64(sc, bp->bio_offset /
+ VTBLK_BSIZE);
break;
default:
panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
}
if (bp->bio_flags & BIO_ORDERED)
- req->vbr_hdr.type |= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER);
+ req->vbr_hdr->type |= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER);
return (req);
}
@@ -1072,13 +1219,17 @@ vtblk_request_execute_cb(void * callback_arg, bus_dma_segment_t * segs,
goto out;
}
ordered = 1;
- req->vbr_hdr.type &= vtblk_gtoh32(sc,
+ req->vbr_hdr->type &= vtblk_gtoh32(sc,
~VIRTIO_BLK_T_BARRIER);
}
}
+ bus_dmamap_sync(sc->vtblk_hdr_dmat, req->vbr_hdr_mapp,
+ BUS_DMASYNC_PREWRITE);
+
sglist_reset(sg);
- sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
+ sglist_append_phys(sg, req->vbr_hdr_paddr,
+ sizeof(struct virtio_blk_outhdr));
if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
/*
@@ -1125,8 +1276,11 @@ vtblk_request_execute_cb(void * callback_arg, bus_dma_segment_t * segs,
}
}
+ bus_dmamap_sync(sc->vtblk_ack_dmat, req->vbr_ack_mapp,
+ BUS_DMASYNC_PREREAD);
+
writable++;
- sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
+ sglist_append_phys(sg, req->vbr_ack_paddr, sizeof(uint8_t));
readable = sg->sg_nseg - writable;
if (req->vbr_mapp != NULL) {
@@ -1168,7 +1322,10 @@ vtblk_request_error(struct vtblk_request *req)
{
int error;
- switch (req->vbr_ack) {
+ bus_dmamap_sync(req->vbr_sc->vtblk_ack_dmat, req->vbr_ack_mapp,
+ BUS_DMASYNC_POSTREAD);
+
+ switch (*req->vbr_ack) {
case VIRTIO_BLK_S_OK:
error = 0;
break;
@@ -1409,10 +1566,10 @@ vtblk_ident(struct vtblk_softc *sc)
if (req == NULL)
return;
- req->vbr_ack = -1;
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID);
- req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
- req->vbr_hdr.sector = 0;
+ *req->vbr_ack = -1;
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID);
+ req->vbr_hdr->ioprio = vtblk_gtoh32(sc, 1);
+ req->vbr_hdr->sector = 0;
req->vbr_bp = &buf;
g_reset_bio(&buf);
@@ -1547,10 +1704,10 @@ vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset,
req = &sc->vtblk_dump_request;
req->vbr_sc = sc;
- req->vbr_ack = -1;
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);
- req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
- req->vbr_hdr.sector = vtblk_gtoh64(sc, offset / VTBLK_BSIZE);
+ *req->vbr_ack = -1;
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);
+ req->vbr_hdr->ioprio = vtblk_gtoh32(sc, 1);
+ req->vbr_hdr->sector = vtblk_gtoh64(sc, offset / VTBLK_BSIZE);
req->vbr_bp = &buf;
g_reset_bio(&buf);
@@ -1570,10 +1727,10 @@ vtblk_dump_flush(struct vtblk_softc *sc)
req = &sc->vtblk_dump_request;
req->vbr_sc = sc;
- req->vbr_ack = -1;
- req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);
- req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);
- req->vbr_hdr.sector = 0;
+ *req->vbr_ack = -1;
+ req->vbr_hdr->type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);
+ req->vbr_hdr->ioprio = vtblk_gtoh32(sc, 1);
+ req->vbr_hdr->sector = 0;
req->vbr_bp = &buf;
g_reset_bio(&buf);
diff --git a/sys/dev/virtio/gpu/virtio_gpu.c b/sys/dev/virtio/gpu/virtio_gpu.c
index 668eb170304a..b95bae9fe286 100644
--- a/sys/dev/virtio/gpu/virtio_gpu.c
+++ b/sys/dev/virtio/gpu/virtio_gpu.c
@@ -445,21 +445,33 @@ vtgpu_alloc_virtqueue(struct vtgpu_softc *sc)
}
static int
-vtgpu_req_resp(struct vtgpu_softc *sc, void *req, size_t reqlen,
- void *resp, size_t resplen)
+vtgpu_req_resp2(struct vtgpu_softc *sc, void *req1, size_t req1len,
+ void *req2, size_t req2len, void *resp, size_t resplen)
{
struct sglist sg;
- struct sglist_seg segs[2];
- int error;
+ struct sglist_seg segs[3];
+ int error, rcount;
- sglist_init(&sg, 2, segs);
+ sglist_init(&sg, 3, segs);
- error = sglist_append(&sg, req, reqlen);
+ rcount = 1;
+ error = sglist_append(&sg, req1, req1len);
if (error != 0) {
device_printf(sc->vtgpu_dev,
- "Unable to append the request to the sglist: %d\n", error);
+ "Unable to append the request to the sglist: %d\n",
+ error);
return (error);
}
+ if (req2 != NULL) {
+ error = sglist_append(&sg, req2, req2len);
+ if (error != 0) {
+ device_printf(sc->vtgpu_dev,
+ "Unable to append the request to the sglist: %d\n",
+ error);
+ return (error);
+ }
+ rcount++;
+ }
error = sglist_append(&sg, resp, resplen);
if (error != 0) {
device_printf(sc->vtgpu_dev,
@@ -467,7 +479,7 @@ vtgpu_req_resp(struct vtgpu_softc *sc, void *req, size_t reqlen,
error);
return (error);
}
- error = virtqueue_enqueue(sc->vtgpu_ctrl_vq, resp, &sg, 1, 1);
+ error = virtqueue_enqueue(sc->vtgpu_ctrl_vq, resp, &sg, rcount, 1);
if (error != 0) {
device_printf(sc->vtgpu_dev, "Enqueue failed: %d\n", error);
return (error);
@@ -480,6 +492,13 @@ vtgpu_req_resp(struct vtgpu_softc *sc, void *req, size_t reqlen,
}
static int
+vtgpu_req_resp(struct vtgpu_softc *sc, void *req, size_t reqlen,
+ void *resp, size_t resplen)
+{
+ return (vtgpu_req_resp2(sc, req, reqlen, NULL, 0, resp, resplen));
+}
+
+static int
vtgpu_get_display_info(struct vtgpu_softc *sc)
{
struct {
@@ -559,9 +578,15 @@ static int
vtgpu_attach_backing(struct vtgpu_softc *sc)
{
struct {
+ /*
+ * Split the backing and mem request arguments as some
+ * hypervisors, e.g. Parallels Desktop, don't work when
+ * they are enqueued together.
+ */
struct {
struct virtio_gpu_resource_attach_backing backing;
- struct virtio_gpu_mem_entry mem[1];
+ char pad;
+ struct virtio_gpu_mem_entry mem;
} req;
char pad;
struct virtio_gpu_ctrl_hdr resp;
@@ -577,11 +602,11 @@ vtgpu_attach_backing(struct vtgpu_softc *sc)
s.req.backing.resource_id = htole32(VTGPU_RESOURCE_ID);
s.req.backing.nr_entries = htole32(1);
- s.req.mem[0].addr = htole64(sc->vtgpu_fb_info.fb_pbase);
- s.req.mem[0].length = htole32(sc->vtgpu_fb_info.fb_size);
+ s.req.mem.addr = htole64(sc->vtgpu_fb_info.fb_pbase);
+ s.req.mem.length = htole32(sc->vtgpu_fb_info.fb_size);
- error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp,
- sizeof(s.resp));
+ error = vtgpu_req_resp2(sc, &s.req.backing, sizeof(s.req.backing),
+ &s.req.mem, sizeof(s.req.mem), &s.resp, sizeof(s.resp));
if (error != 0)
return (error);
diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c
index 19775416f8d4..e9683698decb 100644
--- a/sys/dev/virtio/network/if_vtnet.c
+++ b/sys/dev/virtio/network/if_vtnet.c
@@ -96,6 +96,17 @@
#define VTNET_ETHER_ALIGN ETHER_ALIGN
#endif
+/*
+ * Worst case offset to ensure header doesn't share any cache lines with
+ * payload.
+ */
+#define VTNET_RX_BUFFER_HEADER_OFFSET 128
+
+struct vtnet_rx_buffer_header {
+ bus_addr_t addr;
+ bus_dmamap_t dmap;
+};
+
static int vtnet_modevent(module_t, int, void *);
static int vtnet_probe(device_t);
@@ -135,7 +146,7 @@ static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int);
static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *);
static int vtnet_rxq_new_buf(struct vtnet_rxq *);
#if defined(INET) || defined(INET6)
-static int vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *,
+static void vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *,
bool, int, struct virtio_net_hdr *);
static void vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *,
int);
@@ -208,11 +219,14 @@ static void vtnet_init_locked(struct vtnet_softc *, int);
static void vtnet_init(void *);
static void vtnet_free_ctrl_vq(struct vtnet_softc *);
-static void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
+static int vtnet_exec_ctrl_cmd(struct vtnet_softc *, uint8_t *,
struct sglist *, int, int);
static int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
static int vtnet_ctrl_guest_offloads(struct vtnet_softc *, uint64_t);
static int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t);
+static int vtnet_ctrl_announce_ack_cmd(struct vtnet_softc *);
+static bool vtnet_announce_pending(struct vtnet_softc *);
+static void vtnet_announce(void *, int);
static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, uint8_t, bool);
static int vtnet_set_promisc(struct vtnet_softc *, bool);
static int vtnet_set_allmulti(struct vtnet_softc *, bool);
@@ -273,11 +287,6 @@ static int vtnet_csum_disable = 0;
SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN,
&vtnet_csum_disable, 0, "Disables receive and send checksum offload");
-static int vtnet_fixup_needs_csum = 0;
-SYSCTL_INT(_hw_vtnet, OID_AUTO, fixup_needs_csum, CTLFLAG_RDTUN,
- &vtnet_fixup_needs_csum, 0,
- "Calculate valid checksum for NEEDS_CSUM packets");
-
static int vtnet_tso_disable = 0;
SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN,
&vtnet_tso_disable, 0, "Disables TSO");
@@ -389,6 +398,17 @@ MODULE_DEPEND(vtnet, netmap, 1, 1, 1);
VIRTIO_SIMPLE_PNPINFO(vtnet, VIRTIO_ID_NETWORK, "VirtIO Networking Adapter");
+static struct vtnet_rx_buffer_header *
+vtnet_mbuf_to_rx_buffer_header(struct vtnet_softc *sc, struct mbuf *m)
+{
+ if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0)
+ return (struct vtnet_rx_buffer_header *)((uintptr_t)m->m_data -
+ VTNET_RX_BUFFER_HEADER_OFFSET - VTNET_ETHER_ALIGN);
+ else
+ return (struct vtnet_rx_buffer_header *)((uintptr_t)m->m_data -
+ VTNET_RX_BUFFER_HEADER_OFFSET);
+}
+
static int
vtnet_modevent(module_t mod __unused, int type, void *unused __unused)
{
@@ -451,6 +471,7 @@ vtnet_attach(device_t dev)
VTNET_CORE_LOCK_INIT(sc);
callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0);
+ TASK_INIT(&sc->vtnet_announce_task, 0, vtnet_announce, sc);
vtnet_load_tunables(sc);
vtnet_alloc_interface(sc);
@@ -462,6 +483,106 @@ vtnet_attach(device_t dev)
goto fail;
}
+ mtx_init(&sc->vtnet_rx_mtx, device_get_nameunit(dev),
+ "VirtIO Net RX lock", MTX_DEF);
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ 1, /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ MJUM9BYTES, /* max request size */
+ 1, /* max # segments */
+ MJUM9BYTES, /* maxsegsize - worst case */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &sc->vtnet_rx_mtx, /* lockarg */
+ &sc->vtnet_rx_dmat);
+ if (error) {
+ device_printf(dev, "cannot create bus_dma_tag\n");
+ goto fail;
+ }
+
+ mtx_init(&sc->vtnet_tx_mtx, device_get_nameunit(dev),
+ "VirtIO Net TX lock", MTX_DEF);
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ 1, /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ sc->vtnet_tx_nsegs * MJUM9BYTES, /* max request size */
+ sc->vtnet_tx_nsegs, /* max # segments */
+ MJUM9BYTES, /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &sc->vtnet_tx_mtx, /* lockarg */
+ &sc->vtnet_tx_dmat);
+ if (error) {
+ device_printf(dev, "cannot create bus_dma_tag\n");
+ goto fail;
+ }
+
+ mtx_init(&sc->vtnet_hdr_mtx, device_get_nameunit(dev),
+ "VirtIO Net header lock", MTX_DEF);
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ sizeof(uint16_t), /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ PAGE_SIZE, /* max request size */
+ 1, /* max # segments */
+ PAGE_SIZE, /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &sc->vtnet_hdr_mtx, /* lockarg */
+ &sc->vtnet_hdr_dmat);
+ if (error) {
+ device_printf(dev, "cannot create bus_dma_tag\n");
+ goto fail;
+ }
+
+ mtx_init(&sc->vtnet_ack_mtx, device_get_nameunit(dev),
+ "VirtIO Net ACK lock", MTX_DEF);
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ sizeof(uint8_t), /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ sizeof(uint8_t), /* max request size */
+ 1, /* max # segments */
+ sizeof(uint8_t), /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &sc->vtnet_ack_mtx, /* lockarg */
+ &sc->vtnet_ack_dmat);
+ if (error) {
+ device_printf(dev, "cannot create bus_dma_tag\n");
+ goto fail;
+ }
+
+#ifdef __powerpc__
+ /*
+ * Virtio uses physical addresses rather than bus addresses, so we
+ * need to ask busdma to skip the iommu physical->bus mapping. At
+ * present, this is only a thing on the powerpc architectures.
+ */
+ bus_dma_tag_set_iommu(sc->vtnet_rx_dmat, NULL, NULL);
+ bus_dma_tag_set_iommu(sc->vtnet_tx_dmat, NULL, NULL);
+ bus_dma_tag_set_iommu(sc->vtnet_hdr_dmat, NULL, NULL);
+ bus_dma_tag_set_iommu(sc->vtnet_ack_dmat, NULL, NULL);
+#endif
+
error = vtnet_alloc_rx_filters(sc);
if (error) {
device_printf(dev, "cannot allocate Rx filters\n");
@@ -525,6 +646,8 @@ vtnet_detach(device_t dev)
ether_ifdetach(ifp);
}
+ taskqueue_drain(taskqueue_thread, &sc->vtnet_announce_task);
+
#ifdef DEV_NETMAP
netmap_detach(ifp);
#endif
@@ -629,6 +752,8 @@ vtnet_config_change(device_t dev)
VTNET_CORE_LOCK(sc);
vtnet_update_link_status(sc);
+ if (vtnet_announce_pending(sc))
+ taskqueue_enqueue(taskqueue_thread, &sc->vtnet_announce_task);
if (sc->vtnet_link_active != 0)
vtnet_tx_start_all(sc);
VTNET_CORE_UNLOCK(sc);
@@ -641,7 +766,7 @@ vtnet_negotiate_features(struct vtnet_softc *sc)
{
device_t dev;
uint64_t features, negotiated_features;
- int no_csum;
+ int error, no_csum;
dev = sc->vtnet_dev;
features = virtio_bus_is_modern(dev) ? VTNET_MODERN_FEATURES :
@@ -719,7 +844,19 @@ vtnet_negotiate_features(struct vtnet_softc *sc)
sc->vtnet_features = negotiated_features;
sc->vtnet_negotiated_features = negotiated_features;
- return (virtio_finalize_features(dev));
+ error = virtio_finalize_features(dev);
+ if (error != 0 && (features & VTNET_OFFLOAD_FEATURES) != 0) {
+ device_printf(dev,
+ "retrying feature negotiation without offloads\n");
+ features &= ~VTNET_OFFLOAD_FEATURES;
+ negotiated_features &= ~VTNET_OFFLOAD_FEATURES;
+ sc->vtnet_flags &= ~VTNET_FLAG_LRO_NOMRG;
+ sc->vtnet_features = negotiated_features;
+ sc->vtnet_negotiated_features = negotiated_features;
+ error = virtio_reinit(dev, features);
+ }
+
+ return (error);
}
static int
@@ -1158,10 +1295,6 @@ vtnet_setup_interface(struct vtnet_softc *sc)
if_setcapabilitiesbit(ifp, IFCAP_RXCSUM, 0);
if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0);
- if (vtnet_tunable_int(sc, "fixup_needs_csum",
- vtnet_fixup_needs_csum) != 0)
- sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM;
-
/* Support either "hardware" or software LRO. */
if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
}
@@ -1554,6 +1687,11 @@ static struct mbuf *
vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
{
struct mbuf *m_head, *m_tail, *m;
+ struct vtnet_rx_buffer_header *vthdr;
+ bus_dma_segment_t segs[1];
+ bus_dmamap_t dmap;
+ int nsegs;
+ int err;
int i, size;
m_head = NULL;
@@ -1571,13 +1709,43 @@ vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
}
m->m_len = size;
+ vthdr = (struct vtnet_rx_buffer_header *)m->m_data;
+
+ /* Reserve space for header */
+ m_adj(m, VTNET_RX_BUFFER_HEADER_OFFSET);
+
/*
* Need to offset the mbuf if the header we're going to add
* will misalign.
*/
- if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) {
+ if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0)
m_adj(m, VTNET_ETHER_ALIGN);
+
+ err = bus_dmamap_create(sc->vtnet_rx_dmat, 0, &dmap);
+ if (err) {
+ printf("Failed to create dmamap, err :%d\n",
+ err);
+ m_freem(m);
+ return (NULL);
+ }
+
+ nsegs = 0;
+ err = bus_dmamap_load_mbuf_sg(sc->vtnet_rx_dmat, dmap, m, segs,
+ &nsegs, BUS_DMA_NOWAIT);
+ if (err != 0) {
+ printf("Failed to map mbuf into DMA visible memory, err: %d\n",
+ err);
+ m_freem(m);
+ bus_dmamap_destroy(sc->vtnet_rx_dmat, dmap);
+ return (NULL);
}
+ KASSERT(nsegs == 1,
+ ("%s: unexpected number of DMA segments for rx buffer: %d",
+ __func__, nsegs));
+
+ vthdr->addr = segs[0].ds_addr;
+ vthdr->dmap = dmap;
+
if (m_head != NULL) {
m_tail->m_next = m;
m_tail = m;
@@ -1603,7 +1771,7 @@ vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *rxq, struct mbuf *m0,
int len, clustersz, nreplace, error;
sc = rxq->vtnrx_sc;
- clustersz = sc->vtnet_rx_clustersz;
+ clustersz = sc->vtnet_rx_clustersz - VTNET_RX_BUFFER_HEADER_OFFSET;
/*
* Need to offset the mbuf if the header we're going to add will
* misalign, account for that here.
@@ -1718,9 +1886,12 @@ vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len)
static int
vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)
{
+ struct vtnet_rx_buffer_header *hdr;
struct vtnet_softc *sc;
struct sglist *sg;
int header_inlined, error;
+ bus_addr_t paddr;
+ struct mbuf *mp;
sc = rxq->vtnrx_sc;
sg = rxq->vtnrx_sg;
@@ -1733,28 +1904,38 @@ vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m)
header_inlined = vtnet_modern(sc) ||
(sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */
+ hdr = vtnet_mbuf_to_rx_buffer_header(sc, m);
+ paddr = hdr->addr;
+
/*
* Note: The mbuf has been already adjusted when we allocate it if we
* have to do strict alignment.
*/
- if (header_inlined)
- error = sglist_append_mbuf(sg, m);
- else {
- struct vtnet_rx_header *rxhdr =
- mtod(m, struct vtnet_rx_header *);
+ if (header_inlined) {
+ error = sglist_append_phys(sg, paddr, m->m_len);
+ } else {
MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr));
/* Append the header and remaining mbuf data. */
- error = sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size);
+ error = sglist_append_phys(sg, paddr, sc->vtnet_hdr_size);
if (error)
return (error);
- error = sglist_append(sg, &rxhdr[1],
+ error = sglist_append_phys(sg,
+ paddr + sizeof(struct vtnet_rx_header),
m->m_len - sizeof(struct vtnet_rx_header));
if (error)
return (error);
- if (m->m_next != NULL)
- error = sglist_append_mbuf(sg, m->m_next);
+ mp = m->m_next;
+ while (mp) {
+ hdr = vtnet_mbuf_to_rx_buffer_header(sc, mp);
+ paddr = hdr->addr;
+ error = sglist_append_phys(sg, paddr, mp->m_len);
+ if (error)
+ return (error);
+
+ mp = mp->m_next;
+ }
}
if (error)
@@ -1784,12 +1965,10 @@ vtnet_rxq_new_buf(struct vtnet_rxq *rxq)
}
#if defined(INET) || defined(INET6)
-static int
+static void
vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, bool isipv6,
int protocol, struct virtio_net_hdr *hdr)
{
- struct vtnet_softc *sc;
-
/*
* The packet is likely from another VM on the same host or from the
* host that itself performed checksum offloading so Tx/Rx is basically
@@ -1800,57 +1979,18 @@ vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, bool isipv6,
("%s: unsupported IP protocol %d", __func__, protocol));
/*
- * If the user don't want us to fix it up here by computing the
- * checksum, just forward the order to compute the checksum by setting
+ * Just forward the order to compute the checksum by setting
* the corresponding mbuf flag (e.g., CSUM_TCP).
*/
- sc = rxq->vtnrx_sc;
- if ((sc->vtnet_flags & VTNET_FLAG_FIXUP_NEEDS_CSUM) == 0) {
- switch (protocol) {
- case IPPROTO_TCP:
- m->m_pkthdr.csum_flags |=
- (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP);
- break;
- case IPPROTO_UDP:
- m->m_pkthdr.csum_flags |=
- (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP);
- break;
- }
- m->m_pkthdr.csum_data = hdr->csum_offset;
- return (0);
- }
-
- /*
- * Compute the checksum in the driver so the packet will contain a
- * valid checksum. The checksum is at csum_offset from csum_start.
- */
- int csum_off, csum_end;
- uint16_t csum;
-
- csum_off = hdr->csum_start + hdr->csum_offset;
- csum_end = csum_off + sizeof(uint16_t);
-
- /* Assume checksum will be in the first mbuf. */
- if (m->m_len < csum_end || m->m_pkthdr.len < csum_end) {
- sc->vtnet_stats.rx_csum_bad_offset++;
- return (1);
+ switch (protocol) {
+ case IPPROTO_TCP:
+ m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP);
+ break;
+ case IPPROTO_UDP:
+ m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP);
+ break;
}
-
- /*
- * Like in_delayed_cksum()/in6_delayed_cksum(), compute the
- * checksum and write it at the specified offset. We could
- * try to verify the packet: csum_start should probably
- * correspond to the start of the TCP/UDP header.
- *
- * BMV: Need to properly handle UDP with zero checksum. Is
- * the IPv4 header checksum implicitly validated?
- */
- csum = in_cksum_skip(m, m->m_pkthdr.len, hdr->csum_start);
- *(uint16_t *)(mtodo(m, csum_off)) = csum;
- m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
- m->m_pkthdr.csum_data = 0xFFFF;
-
- return (0);
+ m->m_pkthdr.csum_data = hdr->csum_offset;
}
static void
@@ -1934,8 +2074,7 @@ vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,
}
if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
- return (vtnet_rxq_csum_needs_csum(rxq, m, isipv6, protocol,
- hdr));
+ vtnet_rxq_csum_needs_csum(rxq, m, isipv6, protocol, hdr);
else /* VIRTIO_NET_HDR_F_DATA_VALID */
vtnet_rxq_csum_data_valid(rxq, m, protocol);
@@ -1982,6 +2121,7 @@ vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
m_tail = m_head;
while (--nbufs > 0) {
+ struct vtnet_rx_buffer_header *vthdr;
struct mbuf *m;
uint32_t len;
@@ -1991,6 +2131,10 @@ vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
goto fail;
}
+ vthdr = vtnet_mbuf_to_rx_buffer_header(sc, m);
+ bus_dmamap_sync(sc->vtnet_rx_dmat, vthdr->dmap,
+ BUS_DMASYNC_POSTREAD);
+
if (vtnet_rxq_new_buf(rxq) != 0) {
rxq->vtnrx_stats.vrxs_iqdrops++;
vtnet_rxq_discard_buf(rxq, m);
@@ -1999,6 +2143,9 @@ vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs)
goto fail;
}
+ bus_dmamap_unload(sc->vtnet_rx_dmat, vthdr->dmap);
+ bus_dmamap_destroy(sc->vtnet_rx_dmat, vthdr->dmap);
+
if (m->m_len < len)
len = m->m_len;
@@ -2059,8 +2206,16 @@ vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
}
}
- m->m_pkthdr.flowid = rxq->vtnrx_id;
- M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
+ if (sc->vtnet_act_vq_pairs == 1) {
+ /*
+ * When RSS is not needed (one active rx queue), let the upper
+ * layer know and react.
+ */
+ M_HASHTYPE_CLEAR(m);
+ } else {
+ m->m_pkthdr.flowid = rxq->vtnrx_id;
+ M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
+ }
if (hdr->flags &
(VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) {
@@ -2103,6 +2258,7 @@ static int
vtnet_rxq_eof(struct vtnet_rxq *rxq)
{
struct virtio_net_hdr lhdr, *hdr;
+ struct vtnet_rx_buffer_header *vthdr;
struct vtnet_softc *sc;
if_t ifp;
struct virtqueue *vq;
@@ -2118,14 +2274,31 @@ vtnet_rxq_eof(struct vtnet_rxq *rxq)
CURVNET_SET(if_getvnet(ifp));
while (count-- > 0) {
- struct mbuf *m;
+ struct mbuf *m, *mp;
uint32_t len, nbufs, adjsz;
+ uint32_t synced;
m = virtqueue_dequeue(vq, &len);
if (m == NULL)
break;
deq++;
+ mp = m;
+
+ /*
+ * Sync all mbufs in this packet. There will only be a single
+ * mbuf unless LRO is in use.
+ */
+ synced = 0;
+ while (mp && synced < len) {
+ vthdr = vtnet_mbuf_to_rx_buffer_header(sc, mp);
+ bus_dmamap_sync(sc->vtnet_rx_dmat, vthdr->dmap,
+ BUS_DMASYNC_POSTREAD);
+
+ synced += mp->m_len;
+ mp = mp->m_next;
+ }
+
if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) {
rxq->vtnrx_stats.vrxs_ierrors++;
vtnet_rxq_discard_buf(rxq, m);
@@ -2159,6 +2332,18 @@ vtnet_rxq_eof(struct vtnet_rxq *rxq)
continue;
}
+ mp = m;
+ synced = 0;
+ while (mp && synced < len) {
+ vthdr = vtnet_mbuf_to_rx_buffer_header(sc, mp);
+
+ bus_dmamap_unload(sc->vtnet_rx_dmat, vthdr->dmap);
+ bus_dmamap_destroy(sc->vtnet_rx_dmat, vthdr->dmap);
+
+ synced += mp->m_len;
+ mp = mp->m_next;
+ }
+
m->m_pkthdr.len = len;
m->m_pkthdr.rcvif = ifp;
m->m_pkthdr.csum_flags = 0;
@@ -2385,6 +2570,14 @@ vtnet_txq_free_mbufs(struct vtnet_txq *txq)
while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
if (kring == NULL) {
+ bus_dmamap_unload(txq->vtntx_sc->vtnet_tx_dmat,
+ txhdr->dmap);
+ bus_dmamap_destroy(txq->vtntx_sc->vtnet_tx_dmat,
+ txhdr->dmap);
+ bus_dmamap_unload(txq->vtntx_sc->vtnet_tx_dmat,
+ txhdr->hdr_dmap);
+ bus_dmamap_destroy(txq->vtntx_sc->vtnet_tx_dmat,
+ txhdr->hdr_dmap);
m_freem(txhdr->vth_mbuf);
uma_zfree(vtnet_tx_header_zone, txhdr);
}
@@ -2554,15 +2747,36 @@ drop:
return (NULL);
}
+static void
+vtnet_txq_enqueue_callback(void *arg, bus_dma_segment_t *segs,
+ int nsegs, int error)
+{
+ vm_paddr_t *hdr_paddr;
+
+ if (error != 0)
+ return;
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ hdr_paddr = (vm_paddr_t *)arg;
+ *hdr_paddr = segs[0].ds_addr;
+}
+
static int
vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
struct vtnet_tx_header *txhdr)
{
+ bus_dma_segment_t segs[VTNET_TX_SEGS_MAX];
+ int nsegs;
struct vtnet_softc *sc;
struct virtqueue *vq;
struct sglist *sg;
struct mbuf *m;
int error;
+ vm_paddr_t hdr_paddr;
+ bus_dmamap_t hdr_dmap;
+ bus_dmamap_t dmap;
+ int i;
sc = txq->vtntx_sc;
vq = txq->vtntx_vq;
@@ -2570,15 +2784,55 @@ vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
m = *m_head;
sglist_reset(sg);
- error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
+
+ error = bus_dmamap_create(sc->vtnet_tx_dmat, 0, &hdr_dmap);
+ if (error)
+ goto fail;
+
+ error = bus_dmamap_load(sc->vtnet_tx_dmat, hdr_dmap, &txhdr->vth_uhdr,
+ sc->vtnet_hdr_size, vtnet_txq_enqueue_callback, &hdr_paddr,
+ BUS_DMA_NOWAIT);
+ if (error)
+ goto fail_hdr_dmamap_destroy;
+
+ error = sglist_append_phys(sg, hdr_paddr, sc->vtnet_hdr_size);
if (error != 0 || sg->sg_nseg != 1) {
KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d",
__func__, error, sg->sg_nseg));
- goto fail;
+ goto fail_hdr_dmamap_unload;
}
- error = sglist_append_mbuf(sg, m);
+ bus_dmamap_sync(sc->vtnet_tx_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
+
+ error = bus_dmamap_create(sc->vtnet_tx_dmat, 0, &dmap);
+ if (error)
+ goto fail_hdr_dmamap_unload;
+
+ nsegs = 0;
+ error = bus_dmamap_load_mbuf_sg(sc->vtnet_tx_dmat, dmap, m, segs,
+ &nsegs, BUS_DMA_NOWAIT);
+ if (error != 0)
+ goto fail_dmamap_destroy;
+ KASSERT(nsegs <= sc->vtnet_tx_nsegs,
+ ("%s: unexpected number of DMA segments for tx buffer: %d (max %d)",
+ __func__, nsegs, sc->vtnet_tx_nsegs));
+
+ bus_dmamap_sync(sc->vtnet_tx_dmat, dmap, BUS_DMASYNC_PREWRITE);
+
+ for (i = 0; i < nsegs && !error; i++)
+ error = sglist_append_phys(sg, segs[i].ds_addr, segs[i].ds_len);
+
if (error) {
+ sglist_reset(sg);
+ bus_dmamap_unload(sc->vtnet_tx_dmat, dmap);
+
+ error = sglist_append_phys(sg, hdr_paddr, sc->vtnet_hdr_size);
+ if (error != 0 || sg->sg_nseg != 1) {
+ KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d",
+ __func__, error, sg->sg_nseg));
+ goto fail_dmamap_destroy;
+ }
+
m = m_defrag(m, M_NOWAIT);
if (m == NULL) {
sc->vtnet_stats.tx_defrag_failed++;
@@ -2588,16 +2842,41 @@ vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head,
*m_head = m;
sc->vtnet_stats.tx_defragged++;
- error = sglist_append_mbuf(sg, m);
+ nsegs = 0;
+ error = bus_dmamap_load_mbuf_sg(sc->vtnet_tx_dmat, dmap, m,
+ segs, &nsegs, BUS_DMA_NOWAIT);
+ if (error != 0)
+ goto fail_dmamap_destroy;
+ KASSERT(nsegs <= sc->vtnet_tx_nsegs,
+ ("%s: unexpected number of DMA segments for tx buffer: %d (max %d)",
+ __func__, nsegs, sc->vtnet_tx_nsegs));
+
+ bus_dmamap_sync(sc->vtnet_tx_dmat, dmap, BUS_DMASYNC_PREWRITE);
+
+ for (i = 0; i < nsegs && !error; i++)
+ error = sglist_append_phys(sg, segs[i].ds_addr,
+ segs[i].ds_len);
+
if (error)
- goto fail;
+ goto fail_dmamap_unload;
}
txhdr->vth_mbuf = m;
+ txhdr->dmap = dmap;
+ txhdr->hdr_dmap = hdr_dmap;
+
error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0);
return (error);
+fail_dmamap_unload:
+ bus_dmamap_unload(sc->vtnet_tx_dmat, dmap);
+fail_dmamap_destroy:
+ bus_dmamap_destroy(sc->vtnet_tx_dmat, dmap);
+fail_hdr_dmamap_unload:
+ bus_dmamap_unload(sc->vtnet_tx_dmat, hdr_dmap);
+fail_hdr_dmamap_destroy:
+ bus_dmamap_destroy(sc->vtnet_tx_dmat, hdr_dmap);
fail:
m_freem(*m_head);
*m_head = NULL;
@@ -2877,6 +3156,7 @@ vtnet_txq_tq_intr(void *xtxq, int pending __unused)
static int
vtnet_txq_eof(struct vtnet_txq *txq)
{
+ struct vtnet_softc *sc;
struct virtqueue *vq;
struct vtnet_tx_header *txhdr;
struct mbuf *m;
@@ -2886,6 +3166,8 @@ vtnet_txq_eof(struct vtnet_txq *txq)
deq = 0;
VTNET_TXQ_LOCK_ASSERT(txq);
+ sc = txq->vtntx_sc;
+
while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
m = txhdr->vth_mbuf;
deq++;
@@ -2895,6 +3177,11 @@ vtnet_txq_eof(struct vtnet_txq *txq)
if (m->m_flags & M_MCAST)
txq->vtntx_stats.vtxs_omcasts++;
+ bus_dmamap_unload(sc->vtnet_tx_dmat, txhdr->dmap);
+ bus_dmamap_destroy(sc->vtnet_tx_dmat, txhdr->dmap);
+ bus_dmamap_unload(sc->vtnet_tx_dmat, txhdr->hdr_dmap);
+ bus_dmamap_destroy(sc->vtnet_tx_dmat, txhdr->hdr_dmap);
+
m_freem(m);
uma_zfree(vtnet_tx_header_zone, txhdr);
}
@@ -3553,10 +3840,43 @@ vtnet_free_ctrl_vq(struct vtnet_softc *sc)
}
static void
-vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
- struct sglist *sg, int readable, int writable)
+vtnet_load_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
+ int error)
+{
+ bus_addr_t *paddr;
+
+ if (error != 0)
+ return;
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ paddr = (bus_addr_t *)arg;
+ *paddr = segs[0].ds_addr;
+}
+
+static int
+vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, uint8_t *ack, struct sglist *sg,
+ int readable, int writable)
{
+ bus_dmamap_t ack_dmap;
+ bus_addr_t ack_paddr;
struct virtqueue *vq;
+ int error;
+
+ error = bus_dmamap_create(sc->vtnet_ack_dmat, 0, &ack_dmap);
+ if (error)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_ack_dmat, ack_dmap, ack,
+ sizeof(uint8_t), vtnet_load_callback, &ack_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto error_destroy;
+
+ bus_dmamap_sync(sc->vtnet_ack_dmat, ack_dmap, BUS_DMASYNC_PREWRITE);
+
+ error = sglist_append_phys(sg, ack_paddr, sizeof(uint8_t));
+ if (error)
+ goto error_unload;
vq = sc->vtnet_ctrl_vq;
@@ -3564,152 +3884,317 @@ vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie,
VTNET_CORE_LOCK_ASSERT(sc);
if (!virtqueue_empty(vq))
- return;
+ goto error_unload;
/*
* Poll for the response, but the command is likely completed before
* returning from the notify.
*/
- if (virtqueue_enqueue(vq, cookie, sg, readable, writable) == 0) {
+ if (virtqueue_enqueue(vq, (void *)ack, sg, readable, writable) == 0) {
virtqueue_notify(vq);
virtqueue_poll(vq, NULL);
}
+
+ bus_dmamap_sync(sc->vtnet_ack_dmat, ack_dmap, BUS_DMASYNC_POSTREAD);
+
+error_unload:
+ bus_dmamap_unload(sc->vtnet_ack_dmat, ack_dmap);
+error_destroy:
+ bus_dmamap_destroy(sc->vtnet_ack_dmat, ack_dmap);
+error_out:
+ return (error);
}
static int
vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
{
struct sglist_seg segs[3];
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
struct sglist sg;
struct {
struct virtio_net_ctrl_hdr hdr __aligned(2);
uint8_t pad1;
uint8_t addr[ETHER_ADDR_LEN] __aligned(8);
uint8_t pad2;
- uint8_t ack;
} s;
+ uint8_t ack;
int error;
- error = 0;
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s,
+ sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto error_destroy_hdr;
+
MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_MAC);
s.hdr.class = VIRTIO_NET_CTRL_MAC;
s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
bcopy(hwaddr, &s.addr[0], ETHER_ADDR_LEN);
- s.ack = VIRTIO_NET_ERR;
+ ack = VIRTIO_NET_ERR;
+ bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
sglist_init(&sg, nitems(segs), segs);
- error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &s.addr[0], ETHER_ADDR_LEN);
- error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
- MPASS(error == 0 && sg.sg_nseg == nitems(segs));
+ error |= sglist_append_phys(&sg, hdr_paddr,
+ sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append_phys(&sg,
+ hdr_paddr + ((uintptr_t)&s.addr - (uintptr_t)&s),
+ ETHER_ADDR_LEN);
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
if (error == 0)
- vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
- return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+error_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+error_out:
+ return (error);
}
static int
vtnet_ctrl_guest_offloads(struct vtnet_softc *sc, uint64_t offloads)
{
struct sglist_seg segs[3];
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
struct sglist sg;
struct {
struct virtio_net_ctrl_hdr hdr __aligned(2);
uint8_t pad1;
uint64_t offloads __aligned(8);
uint8_t pad2;
- uint8_t ack;
} s;
+ uint8_t ack;
int error;
- error = 0;
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s,
+ sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto error_destroy_hdr;
+
MPASS(sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
s.hdr.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS;
s.hdr.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
s.offloads = vtnet_gtoh64(sc, offloads);
- s.ack = VIRTIO_NET_ERR;
+ ack = VIRTIO_NET_ERR;
+ bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
sglist_init(&sg, nitems(segs), segs);
- error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &s.offloads, sizeof(uint64_t));
- error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
- MPASS(error == 0 && sg.sg_nseg == nitems(segs));
+ error |= sglist_append_phys(&sg, hdr_paddr,
+ sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append_phys(&sg,
+ hdr_paddr + ((uintptr_t)&s.offloads - (uintptr_t)&s),
+ sizeof(uint64_t));
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
if (error == 0)
- vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
- return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+error_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+error_out:
+ return (error);
}
static int
vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs)
{
struct sglist_seg segs[3];
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
struct sglist sg;
struct {
struct virtio_net_ctrl_hdr hdr __aligned(2);
uint8_t pad1;
struct virtio_net_ctrl_mq mq __aligned(2);
uint8_t pad2;
- uint8_t ack;
} s;
+ uint8_t ack;
int error;
- error = 0;
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s,
+ sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto error_destroy_hdr;
+
MPASS(sc->vtnet_flags & VTNET_FLAG_MQ);
s.hdr.class = VIRTIO_NET_CTRL_MQ;
s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
s.mq.virtqueue_pairs = vtnet_gtoh16(sc, npairs);
- s.ack = VIRTIO_NET_ERR;
+ ack = VIRTIO_NET_ERR;
+ bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
+
+ sglist_init(&sg, nitems(segs), segs);
+ error |= sglist_append_phys(&sg, hdr_paddr,
+ sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append_phys(&sg,
+ hdr_paddr + ((uintptr_t)&s.mq - (uintptr_t)&s),
+ sizeof(struct virtio_net_ctrl_mq));
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
+
+ if (error == 0)
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
+
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+error_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+error_out:
+ return (error);
+}
+
+static int
+vtnet_ctrl_announce_ack_cmd(struct vtnet_softc *sc)
+{
+ struct sglist_seg segs[2];
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
+ struct sglist sg;
+ struct virtio_net_ctrl_hdr hdr __aligned(2);
+ uint8_t ack;
+ int error;
+
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error != 0)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &hdr,
+ sizeof(hdr), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error != 0)
+ goto error_destroy_hdr;
+
+ hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
+ hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
+ ack = VIRTIO_NET_ERR;
+ bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
sglist_init(&sg, nitems(segs), segs);
- error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq));
- error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
- MPASS(error == 0 && sg.sg_nseg == nitems(segs));
+ error = sglist_append_phys(&sg, hdr_paddr, sizeof(hdr));
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
if (error == 0)
- vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
+
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+error_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+error_out:
+ return (error);
+}
+
+static bool
+vtnet_announce_pending(struct vtnet_softc *sc)
+{
+ uint16_t status;
+
+ if ((sc->vtnet_features & VIRTIO_NET_F_GUEST_ANNOUNCE) == 0 ||
+ (sc->vtnet_features & VIRTIO_NET_F_CTRL_VQ) == 0 ||
+ (sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0)
+ return (false);
+
+ status = virtio_read_dev_config_2(sc->vtnet_dev,
+ offsetof(struct virtio_net_config, status));
- return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
+ return ((status & VIRTIO_NET_S_ANNOUNCE) != 0);
+}
+
+static void
+vtnet_announce(void *xsc, int pending __unused)
+{
+ struct vtnet_softc *sc;
+ if_t ifp;
+
+ sc = xsc;
+ ifp = sc->vtnet_ifp;
+
+ if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
+ return;
+
+ CURVNET_SET(if_getvnet(ifp));
+ EVENTHANDLER_INVOKE(iflladdr_event, ifp);
+ CURVNET_RESTORE();
+
+ VTNET_CORE_LOCK(sc);
+ if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0 &&
+ vtnet_ctrl_announce_ack_cmd(sc) != 0)
+ device_printf(sc->vtnet_dev, "cannot ack announcement\n");
+ VTNET_CORE_UNLOCK(sc);
}
static int
vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, uint8_t cmd, bool on)
{
struct sglist_seg segs[3];
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
struct sglist sg;
struct {
struct virtio_net_ctrl_hdr hdr __aligned(2);
uint8_t pad1;
uint8_t onoff;
uint8_t pad2;
- uint8_t ack;
} s;
+ uint8_t ack;
int error;
- error = 0;
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s,
+ sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto error_destroy_hdr;
+
MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX);
s.hdr.class = VIRTIO_NET_CTRL_RX;
s.hdr.cmd = cmd;
s.onoff = on;
- s.ack = VIRTIO_NET_ERR;
+ ack = VIRTIO_NET_ERR;
+ bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
sglist_init(&sg, nitems(segs), segs);
- error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
- error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
- MPASS(error == 0 && sg.sg_nseg == nitems(segs));
+ error |= sglist_append_phys(&sg, hdr_paddr,
+ sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append_phys(&sg,
+ hdr_paddr + ((uintptr_t)&s.onoff - (uintptr_t)&s),
+ sizeof(uint8_t));
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
if (error == 0)
- vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
- return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+error_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+error_out:
+ return (error);
}
static int
@@ -3780,6 +4265,10 @@ vtnet_rx_filter_mac(struct vtnet_softc *sc)
struct virtio_net_ctrl_hdr hdr __aligned(2);
struct vtnet_mac_filter *filter;
struct sglist_seg segs[4];
+ bus_dmamap_t filter_dmap;
+ bus_addr_t filter_paddr;
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
struct sglist sg;
if_t ifp;
bool promisc, allmulti;
@@ -3819,6 +4308,25 @@ vtnet_rx_filter_mac(struct vtnet_softc *sc)
if (promisc && allmulti)
goto out;
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error)
+ goto out_error;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &hdr,
+ sizeof(hdr), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto out_destroy_hdr;
+
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &filter_dmap);
+ if (error)
+ goto out_unload_hdr;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, filter,
+ sizeof(*filter), vtnet_load_callback, &filter_paddr,
+ BUS_DMA_NOWAIT);
+ if (error)
+ goto out_destroy_filter;
+
filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt);
filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt);
@@ -3827,19 +4335,33 @@ vtnet_rx_filter_mac(struct vtnet_softc *sc)
ack = VIRTIO_NET_ERR;
sglist_init(&sg, nitems(segs), segs);
- error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &filter->vmf_unicast,
+ error |= sglist_append_phys(&sg, hdr_paddr,
+ sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append_phys(&sg,
+ filter_paddr + ((uintptr_t)&filter->vmf_unicast -
+ (uintptr_t)filter),
sizeof(uint32_t) + ucnt * ETHER_ADDR_LEN);
- error |= sglist_append(&sg, &filter->vmf_multicast,
+ error |= sglist_append_phys(&sg,
+ filter_paddr + ((uintptr_t)&filter->vmf_multicast -
+ (uintptr_t)filter),
sizeof(uint32_t) + mcnt * ETHER_ADDR_LEN);
- error |= sglist_append(&sg, &ack, sizeof(uint8_t));
- MPASS(error == 0 && sg.sg_nseg == nitems(segs));
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
if (error == 0)
- vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
- if (ack != VIRTIO_NET_OK)
- if_printf(ifp, "error setting host MAC filter table\n");
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, filter_dmap);
+out_destroy_filter:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, filter_dmap);
+out_unload_hdr:
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+out_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+out_error:
+ if (error != 0)
+ if_printf(ifp, "error setting host MAC filter table\n");
out:
if (promisc && vtnet_set_promisc(sc, true) != 0)
if_printf(ifp, "cannot enable promiscuous mode\n");
@@ -3851,34 +4373,53 @@ static int
vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag)
{
struct sglist_seg segs[3];
+ bus_dmamap_t hdr_dmap;
+ bus_addr_t hdr_paddr;
struct sglist sg;
struct {
struct virtio_net_ctrl_hdr hdr __aligned(2);
uint8_t pad1;
uint16_t tag __aligned(2);
uint8_t pad2;
- uint8_t ack;
} s;
+ uint8_t ack;
int error;
- error = 0;
+ error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap);
+ if (error)
+ goto error_out;
+
+ error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s,
+ sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT);
+ if (error)
+ goto error_destroy_hdr;
+
MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);
s.hdr.class = VIRTIO_NET_CTRL_VLAN;
s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
s.tag = vtnet_gtoh16(sc, tag);
- s.ack = VIRTIO_NET_ERR;
+ ack = VIRTIO_NET_ERR;
+ bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE);
sglist_init(&sg, nitems(segs), segs);
- error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
- error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
- error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
- MPASS(error == 0 && sg.sg_nseg == nitems(segs));
+ error |= sglist_append_phys(&sg, hdr_paddr,
+ sizeof(struct virtio_net_ctrl_hdr));
+ error |= sglist_append_phys(&sg,
+ hdr_paddr + ((uintptr_t)&s.tag - (uintptr_t)&s),
+ sizeof(uint16_t));
+ MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1);
if (error == 0)
- vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
+ error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1);
+ if (error == 0)
+ error = (ack == VIRTIO_NET_OK ? 0 : EIO);
- return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
+ bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap);
+error_destroy_hdr:
+ bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap);
+error_out:
+ return (error);
}
static void
@@ -4346,9 +4887,6 @@ vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx,
SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto",
CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_ipproto,
"Received checksum offloaded buffer with incorrect IP protocol");
- SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset",
- CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_offset,
- "Received checksum offloaded buffer with incorrect offset");
SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_inaccessible_ipproto",
CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_inaccessible_ipproto,
"Received checksum offloaded buffer with inaccessible IP protocol");
diff --git a/sys/dev/virtio/network/if_vtnetvar.h b/sys/dev/virtio/network/if_vtnetvar.h
index eb5e6784b07f..d5313c9af3cc 100644
--- a/sys/dev/virtio/network/if_vtnetvar.h
+++ b/sys/dev/virtio/network/if_vtnetvar.h
@@ -183,6 +183,7 @@ struct vtnet_softc {
uint64_t vtnet_negotiated_features;
struct vtnet_statistics vtnet_stats;
struct callout vtnet_tick_ch;
+ struct task vtnet_announce_task;
struct ifmedia vtnet_media;
eventhandler_tag vtnet_vlan_attach;
eventhandler_tag vtnet_vlan_detach;
@@ -190,6 +191,18 @@ struct vtnet_softc {
struct mtx vtnet_mtx;
char vtnet_mtx_name[16];
uint8_t vtnet_hwaddr[ETHER_ADDR_LEN];
+
+ bus_dma_tag_t vtnet_rx_dmat;
+ struct mtx vtnet_rx_mtx;
+
+ bus_dma_tag_t vtnet_tx_dmat;
+ struct mtx vtnet_tx_mtx;
+
+ bus_dma_tag_t vtnet_hdr_dmat;
+ struct mtx vtnet_hdr_mtx;
+
+ bus_dma_tag_t vtnet_ack_dmat;
+ struct mtx vtnet_ack_mtx;
};
/* vtnet flag descriptions for use with printf(9) %b identifier. */
#define VTNET_FLAGS_BITS \
@@ -273,6 +286,10 @@ struct vtnet_tx_header {
} vth_uhdr;
struct mbuf *vth_mbuf;
+
+ bus_dmamap_t dmap;
+
+ bus_dmamap_t hdr_dmap;
};
/*
@@ -320,8 +337,10 @@ CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
VIRTIO_NET_F_MTU | \
VIRTIO_NET_F_CTRL_VQ | \
VIRTIO_NET_F_CTRL_RX | \
+ VIRTIO_NET_F_CTRL_RX_EXTRA | \
VIRTIO_NET_F_CTRL_MAC_ADDR | \
VIRTIO_NET_F_CTRL_VLAN | \
+ VIRTIO_NET_F_GUEST_ANNOUNCE | \
VIRTIO_NET_F_CSUM | \
VIRTIO_NET_F_HOST_TSO4 | \
VIRTIO_NET_F_HOST_TSO6 | \
@@ -353,6 +372,22 @@ CTASSERT(sizeof(struct vtnet_mac_filter) <= PAGE_SIZE);
#define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)
+/*
+ * Union of the offload-related features offered by the driver. As per spec,
+ * a device is permitted to reject an otherwise valid subset of its offered
+ * features by failing FEATURES_OK (v1.3 ยง2.2.2). Offloads are where this
+ * happens in practice, so feature negotiation retries without this entire
+ * group when the device rejects the first feature set.
+ *
+ * Must cover every offload-related bit in VTNET_COMMON_FEATURES.
+ */
+#define VTNET_OFFLOAD_FEATURES \
+ (VIRTIO_NET_F_CSUM | \
+ VIRTIO_NET_F_GUEST_CSUM | \
+ VIRTIO_NET_F_CTRL_GUEST_OFFLOADS | \
+ VTNET_TSO_FEATURES | \
+ VTNET_LRO_FEATURES)
+
#define VTNET_MIN_MTU 68
#define VTNET_MAX_MTU 65536
#define VTNET_MAX_RX_SIZE 65550
diff --git a/sys/dev/virtio/p9fs/virtio_p9fs.c b/sys/dev/virtio/p9fs/virtio_p9fs.c
index aa84d3970698..7a5d703513c0 100644
--- a/sys/dev/virtio/p9fs/virtio_p9fs.c
+++ b/sys/dev/virtio/p9fs/virtio_p9fs.c
@@ -76,7 +76,7 @@ struct vt9p_softc {
/* Global channel list, Each channel will correspond to a mount point */
static STAILQ_HEAD( ,vt9p_softc) global_chan_list =
STAILQ_HEAD_INITIALIZER(global_chan_list);
-struct mtx global_chan_list_mtx;
+static struct mtx global_chan_list_mtx;
MTX_SYSINIT(global_chan_list_mtx, &global_chan_list_mtx, "9pglobal", MTX_DEF);
static struct virtio_feature_desc virtio_9p_feature_desc[] = {
@@ -84,12 +84,7 @@ static struct virtio_feature_desc virtio_9p_feature_desc[] = {
{ 0, NULL }
};
-/* We don't currently allow canceling of virtio requests */
-static int
-vt9p_cancel(void *handle, struct p9_req_t *req)
-{
- return (1);
-}
+VIRTIO_SIMPLE_PNPINFO(virtio_p9fs, VIRTIO_ID_9P, "VirtIO 9P Transport");
SYSCTL_NODE(_vfs, OID_AUTO, 9p, CTLFLAG_RW, 0, "9P File System Protocol");
@@ -110,7 +105,7 @@ SYSCTL_UINT(_vfs_9p, OID_AUTO, ackmaxidle, CTLFLAG_RW, &vt9p_ackmaxidle, 0,
static int
vt9p_req_wait(struct vt9p_softc *chan, struct p9_req_t *req)
{
- KASSERT(req->tc->tag != req->rc->tag,
+ KASSERT(req->tc.tag != req->rc.tag,
("%s: request %p already completed", __func__, req));
if (msleep(req, VT9P_MTX(chan), 0, "chan lock", vt9p_ackmaxidle * hz)) {
@@ -122,7 +117,7 @@ vt9p_req_wait(struct vt9p_softc *chan, struct p9_req_t *req)
"for an ack from host\n", vt9p_ackmaxidle);
return (EIO);
}
- KASSERT(req->tc->tag == req->rc->tag,
+ KASSERT(req->tc.tag == req->rc.tag,
("%s spurious event on request %p", __func__, req));
return (0);
}
@@ -155,7 +150,7 @@ vt9p_request(void *handle, struct p9_req_t *req)
req_retry:
sglist_reset(sg);
/* Handle out VirtIO ring buffers */
- error = sglist_append(sg, req->tc->sdata, req->tc->size);
+ error = sglist_append(sg, req->tc.sdata, req->tc.size);
if (error != 0) {
P9_DEBUG(ERROR, "%s: sglist append failed\n", __func__);
VT9P_UNLOCK(chan);
@@ -163,7 +158,7 @@ req_retry:
}
readable = sg->sg_nseg;
- error = sglist_append(sg, req->rc->sdata, req->rc->capacity);
+ error = sglist_append(sg, req->rc.sdata, req->rc.capacity);
if (error != 0) {
P9_DEBUG(ERROR, "%s: sglist append failed\n", __func__);
VT9P_UNLOCK(chan);
@@ -224,7 +219,7 @@ vt9p_intr_complete(void *xsc)
VT9P_LOCK(chan);
again:
while ((curreq = virtqueue_dequeue(vq, NULL)) != NULL) {
- curreq->rc->tag = curreq->tc->tag;
+ curreq->rc.tag = curreq->tc.tag;
wakeup_one(curreq);
}
if (virtqueue_enable_intr(vq) != 0) {
@@ -257,13 +252,7 @@ vt9p_alloc_virtqueue(struct vt9p_softc *sc)
static int
vt9p_probe(device_t dev)
{
-
- /* If the virtio device type is a 9P device, then we claim and attach it */
- if (virtio_get_device_type(dev) != VIRTIO_ID_9P)
- return (ENXIO);
- device_set_desc(dev, "VirtIO 9P Transport");
-
- return (BUS_PROBE_DEFAULT);
+ return (VIRTIO_SIMPLE_PROBE(dev, virtio_p9fs));
}
static void
@@ -447,7 +436,6 @@ static struct p9_trans_module vt9p_trans = {
.create = vt9p_create,
.close = vt9p_close,
.request = vt9p_request,
- .cancel = vt9p_cancel,
};
static device_method_t vt9p_mthds[] = {
@@ -468,16 +456,20 @@ static int
vt9p_modevent(module_t mod, int type, void *unused)
{
int error;
+ static int loaded = 0;
error = 0;
switch (type) {
case MOD_LOAD:
- p9_init_zones();
- p9_register_trans(&vt9p_trans);
+ if (loaded++ == 0) {
+ p9_register_trans(&vt9p_trans);
+ }
break;
case MOD_UNLOAD:
- p9_destroy_zones();
+ if (--loaded == 0) {
+ p9_unregister_trans(&vt9p_trans);
+ }
break;
case MOD_SHUTDOWN:
break;
@@ -485,6 +477,7 @@ vt9p_modevent(module_t mod, int type, void *unused)
error = EOPNOTSUPP;
break;
}
+
return (error);
}
diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c
index b7b34b448f6e..2d33d99014ae 100644
--- a/sys/dev/virtio/pci/virtio_pci.c
+++ b/sys/dev/virtio/pci/virtio_pci.c
@@ -97,8 +97,13 @@ MODULE_VERSION(virtio_pci, 1);
MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
+SYSCTL_DECL(_hw_virtio);
+SYSCTL_NODE(_hw_virtio, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "VirtIO PCI driver parameters");
+
int vtpci_disable_msix = 0;
-TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
+SYSCTL_INT(_hw_virtio_pci, OID_AUTO, disable_msix, CTLFLAG_RDTUN,
+ &vtpci_disable_msix, 0, "If set to 1, disables MSI-X.");
static uint8_t
vtpci_read_isr(struct vtpci_common *cn)
diff --git a/sys/dev/virtio/pci/virtio_pci_modern.c b/sys/dev/virtio/pci/virtio_pci_modern.c
index eb1d5a1e6989..6732f9ee610f 100644
--- a/sys/dev/virtio/pci/virtio_pci_modern.c
+++ b/sys/dev/virtio/pci/virtio_pci_modern.c
@@ -33,6 +33,7 @@
#include <sys/bus.h>
#include <sys/lock.h>
#include <sys/kernel.h>
+#include <sys/sysctl.h>
#include <sys/module.h>
#include <machine/bus.h>
@@ -188,8 +189,12 @@ static void vtpci_modern_write_device_8(struct vtpci_modern_softc *,
bus_size_t, uint64_t);
/* Tunables. */
-static int vtpci_modern_transitional = 0;
-TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional);
+SYSCTL_DECL(_hw_virtio_pci);
+
+static int vtpci_modern_transitional = 1;
+SYSCTL_INT(_hw_virtio_pci, OID_AUTO, transitional, CTLFLAG_RDTUN,
+ &vtpci_modern_transitional, 0,
+ "If 0, a transitional VirtIO device is used in legacy mode; otherwise, in modern mode.");
static device_method_t vtpci_modern_methods[] = {
/* Device interface. */
@@ -436,6 +441,14 @@ vtpci_modern_negotiate_features(device_t dev, uint64_t child_features)
*/
child_features |= VIRTIO_F_VERSION_1;
+ /*
+ * Accept per-virtqueue reset if the device offers it: negotiating
+ * the feature carries no obligation for a driver that never uses
+ * it, while declining capability-only transport features can make
+ * strict devices refuse the entire feature set.
+ */
+ child_features |= host_features & VIRTIO_F_RING_RESET;
+
features = vtpci_negotiate_features(&sc->vtpci_common,
child_features, host_features);
vtpci_modern_write_features(sc, features);
@@ -459,7 +472,10 @@ vtpci_modern_finalize_features(device_t dev)
status = vtpci_modern_get_status(sc);
if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) {
- device_printf(dev, "desired features were not accepted\n");
+ device_printf(dev, "desired features were not accepted "
+ "(host %#jx, written %#jx)\n",
+ (uintmax_t)sc->vtpci_common.vtpci_host_features,
+ (uintmax_t)sc->vtpci_common.vtpci_features);
return (ENOTSUP);
}
@@ -660,16 +676,13 @@ vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst,
*(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset);
break;
case 2:
- *(uint16_t *) dst = virtio_htog16(true,
- vtpci_modern_read_device_2(sc, offset));
+ *(uint16_t *) dst = vtpci_modern_read_device_2(sc, offset);
break;
case 4:
- *(uint32_t *) dst = virtio_htog32(true,
- vtpci_modern_read_device_4(sc, offset));
+ *(uint32_t *) dst = vtpci_modern_read_device_4(sc, offset);
break;
case 8:
- *(uint64_t *) dst = virtio_htog64(true,
- vtpci_modern_read_device_8(sc, offset));
+ *(uint64_t *) dst = vtpci_modern_read_device_8(sc, offset);
break;
default:
panic("%s: device %s invalid device read length %d offset %d",
@@ -695,17 +708,17 @@ vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src,
vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src);
break;
case 2: {
- uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src);
+ uint16_t val = *(const uint16_t *) src;
vtpci_modern_write_device_2(sc, offset, val);
break;
}
case 4: {
- uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src);
+ uint32_t val = *(const uint32_t *) src;
vtpci_modern_write_device_4(sc, offset, val);
break;
}
case 8: {
- uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src);
+ uint64_t val = *(const uint64_t *) src;
vtpci_modern_write_device_8(sc, offset, val);
break;
}
@@ -1307,15 +1320,13 @@ vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off)
static uint16_t
vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off)
{
- return virtio_htog16(true,
- bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off));
+ return bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off);
}
static uint32_t
vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off)
{
- return virtio_htog32(true,
- bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off));
+ return bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off);
}
static void
@@ -1329,16 +1340,14 @@ static void
vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off,
uint16_t val)
{
- bus_write_2(&sc->vtpci_common_res_map.vtrm_map,
- off, virtio_gtoh16(true, val));
+ bus_write_2(&sc->vtpci_common_res_map.vtrm_map, off, val);
}
static void
vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off,
uint32_t val)
{
- bus_write_4(&sc->vtpci_common_res_map.vtrm_map,
- off, virtio_gtoh32(true, val));
+ bus_write_4(&sc->vtpci_common_res_map.vtrm_map, off, val);
}
static void
diff --git a/sys/dev/virtio/scsi/virtio_scsi.c b/sys/dev/virtio/scsi/virtio_scsi.c
index 857da56ba426..a61c92485ccc 100644
--- a/sys/dev/virtio/scsi/virtio_scsi.c
+++ b/sys/dev/virtio/scsi/virtio_scsi.c
@@ -1787,6 +1787,27 @@ vtscsi_transport_reset_event(struct vtscsi_softc *sc,
}
static void
+vtscsi_param_change_event(struct vtscsi_softc *sc,
+ struct virtio_scsi_event *event)
+{
+ target_id_t target_id;
+ lun_id_t lun_id;
+
+ vtscsi_get_request_lun(event->lun, &target_id, &lun_id);
+
+ if (VIRTIO_SCSI_ASC(event->reason) == VIRTIO_SCSI_CAPACITY_CHNG_ASC &&
+ VIRTIO_SCSI_ASCQ(event->reason) == VIRTIO_SCSI_CAPACITY_CHNG_ASCQ) {
+ /* Disk capacity has changed. */
+ vtscsi_announce(sc, AC_INQ_CHANGED, target_id, lun_id);
+ } else {
+ device_printf(sc->vtscsi_dev,
+ "unhandled PARAM_CHANGE event. ASC: %02x, ASCQ: %02x\n",
+ VIRTIO_SCSI_ASC(event->reason),
+ VIRTIO_SCSI_ASCQ(event->reason));
+ }
+}
+
+static void
vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event)
{
int error __diagused;
@@ -1796,6 +1817,9 @@ vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event)
case VIRTIO_SCSI_T_TRANSPORT_RESET:
vtscsi_transport_reset_event(sc, event);
break;
+ case VIRTIO_SCSI_T_PARAM_CHANGE:
+ vtscsi_param_change_event(sc, event);
+ break;
default:
device_printf(sc->vtscsi_dev,
"unhandled event: %d\n", event->event);
diff --git a/sys/dev/virtio/scsi/virtio_scsi.h b/sys/dev/virtio/scsi/virtio_scsi.h
index 02e13c6f81b7..66c72304f3f1 100644
--- a/sys/dev/virtio/scsi/virtio_scsi.h
+++ b/sys/dev/virtio/scsi/virtio_scsi.h
@@ -162,6 +162,15 @@ struct virtio_scsi_config {
#define VIRTIO_SCSI_T_ASYNC_NOTIFY 2
#define VIRTIO_SCSI_T_PARAM_CHANGE 3
+/*
+ * SCSI ASC/ASCQ Sense Codes
+ * https://www.t10.org/lists/asc-num.txt
+ */
+#define VIRTIO_SCSI_ASC(x) ((x) & 0xff)
+#define VIRTIO_SCSI_ASCQ(x) (((x) >> 8) & 0xff)
+#define VIRTIO_SCSI_CAPACITY_CHNG_ASC 0x2a
+#define VIRTIO_SCSI_CAPACITY_CHNG_ASCQ 0x09
+
/* Reasons of transport reset event */
#define VIRTIO_SCSI_EVT_RESET_HARD 0
#define VIRTIO_SCSI_EVT_RESET_RESCAN 1
diff --git a/sys/dev/virtio/scsi/virtio_scsivar.h b/sys/dev/virtio/scsi/virtio_scsivar.h
index 85d2990d8bf4..b5f3c59727cc 100644
--- a/sys/dev/virtio/scsi/virtio_scsivar.h
+++ b/sys/dev/virtio/scsi/virtio_scsivar.h
@@ -139,6 +139,7 @@ struct vtscsi_request {
/* Features desired/implemented by this driver. */
#define VTSCSI_FEATURES \
(VIRTIO_SCSI_F_HOTPLUG | \
+ VIRTIO_SCSI_F_CHANGE | \
VIRTIO_RING_F_INDIRECT_DESC)
#define VTSCSI_MTX(_sc) &(_sc)->vtscsi_mtx
diff --git a/sys/dev/virtio/virtio.c b/sys/dev/virtio/virtio.c
index 75d65ba4a8c8..a032bfc20499 100644
--- a/sys/dev/virtio/virtio.c
+++ b/sys/dev/virtio/virtio.c
@@ -30,6 +30,7 @@
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
+#include <sys/sysctl.h>
#include <sys/module.h>
#include <sys/sbuf.h>
@@ -86,10 +87,21 @@ static struct virtio_feature_desc virtio_common_feature_desc[] = {
{ VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */
{ VIRTIO_F_VERSION_1, "Version1" },
{ VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" },
+ { VIRTIO_F_RING_PACKED, "RingPacked" },
+ { VIRTIO_F_IN_ORDER, "InOrder" },
+ { VIRTIO_F_ORDER_PLATFORM, "OrderPlatform" },
+ { VIRTIO_F_SR_IOV, "SRIOV" },
+ { VIRTIO_F_NOTIFICATION_DATA, "NotificationData" },
+ { VIRTIO_F_NOTIF_CONFIG_DATA, "NotifConfigData" },
+ { VIRTIO_F_RING_RESET, "RingReset" },
+ { VIRTIO_F_ADMIN_VQ, "AdminVq" },
{ 0, NULL }
};
+SYSCTL_NODE(_hw, OID_AUTO, virtio, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "VirtIO driver parameters");
+
const char *
virtio_device_name(uint16_t devid)
{
diff --git a/sys/dev/virtio/virtio.h b/sys/dev/virtio/virtio.h
index 855dafd63033..1b8dc630e339 100644
--- a/sys/dev/virtio/virtio.h
+++ b/sys/dev/virtio/virtio.h
@@ -53,13 +53,15 @@ struct vq_alloc_info;
/*
* VirtIO instance variables indices.
*/
-#define VIRTIO_IVAR_DEVTYPE 1
-#define VIRTIO_IVAR_FEATURE_DESC 2
-#define VIRTIO_IVAR_VENDOR 3
-#define VIRTIO_IVAR_DEVICE 4
-#define VIRTIO_IVAR_SUBVENDOR 5
-#define VIRTIO_IVAR_SUBDEVICE 6
-#define VIRTIO_IVAR_MODERN 7
+enum {
+ VIRTIO_IVAR_DEVTYPE = BUS_IVARS_PRIVATE,
+ VIRTIO_IVAR_FEATURE_DESC,
+ VIRTIO_IVAR_VENDOR,
+ VIRTIO_IVAR_DEVICE,
+ VIRTIO_IVAR_SUBVENDOR,
+ VIRTIO_IVAR_SUBDEVICE,
+ VIRTIO_IVAR_MODERN
+};
struct virtio_feature_desc {
uint64_t vfd_val;
diff --git a/sys/dev/virtio/virtio_config.h b/sys/dev/virtio/virtio_config.h
index 02765490ed27..ce6e96a223d4 100644
--- a/sys/dev/virtio/virtio_config.h
+++ b/sys/dev/virtio/virtio_config.h
@@ -63,10 +63,10 @@
* The guest should never negotiate this feature; it
* is used to detect faulty drivers.
*/
-#define VIRTIO_F_BAD_FEATURE (1UL << 30)
+#define VIRTIO_F_BAD_FEATURE (1UL << 30)
/* v1.0 compliant. */
-#define VIRTIO_F_VERSION_1 (1ULL << 32)
+#define VIRTIO_F_VERSION_1 (1ULL << 32)
/*
* If clear - device has the IOMMU bypass quirk feature.
@@ -77,6 +77,30 @@
*/
#define VIRTIO_F_IOMMU_PLATFORM (1ULL << 33)
+/* Support for packed virtqueues. */
+#define VIRTIO_F_RING_PACKED (1ULL << 34)
+
+/* Device may behave as if IN_ORDER was negotiated. */
+#define VIRTIO_F_IN_ORDER (1ULL << 35)
+
+/* Memory accesses are ordered by the platform. */
+#define VIRTIO_F_ORDER_PLATFORM (1ULL << 36)
+
+/* Device supports Single Root I/O Virtualization. */
+#define VIRTIO_F_SR_IOV (1ULL << 37)
+
+/* Driver passes extra data in device notifications. */
+#define VIRTIO_F_NOTIFICATION_DATA (1ULL << 38)
+
+/* Driver uses the data provided by the device as a virtqueue identifier. */
+#define VIRTIO_F_NOTIF_CONFIG_DATA (1ULL << 39)
+
+/* Driver can reset a queue individually. */
+#define VIRTIO_F_RING_RESET (1ULL << 40)
+
+/* Device supports an administration virtqueue. */
+#define VIRTIO_F_ADMIN_VQ (1ULL << 41)
+
/*
* Some VirtIO feature bits (currently bits 28 through 34) are
* reserved for the transport being used (eg. virtio_ring), the
diff --git a/sys/dev/virtio/virtio_ring.h b/sys/dev/virtio/virtio_ring.h
index 15348a77d8a3..7ac78f0defed 100644
--- a/sys/dev/virtio/virtio_ring.h
+++ b/sys/dev/virtio/virtio_ring.h
@@ -88,6 +88,10 @@ struct vring {
struct vring_desc *desc;
struct vring_avail *avail;
struct vring_used *used;
+
+ vm_paddr_t desc_paddr;
+ vm_paddr_t avail_paddr;
+ vm_paddr_t used_paddr;
};
/* Alignment requirements for vring elements.
@@ -145,15 +149,24 @@ vring_size(unsigned int num, unsigned long align)
}
static inline void
-vring_init(struct vring *vr, unsigned int num, uint8_t *p,
+vring_init(struct vring *vr, unsigned int num, uint8_t *p, vm_paddr_t paddr,
unsigned long align)
{
- vr->num = num;
- vr->desc = (struct vring_desc *) p;
- vr->avail = (struct vring_avail *) (p +
- num * sizeof(struct vring_desc));
- vr->used = (void *)
- (((unsigned long) &vr->avail->ring[num] + align-1) & ~(align-1));
+ unsigned long avail_offset;
+ unsigned long used_offset;
+
+ avail_offset = num * sizeof(struct vring_desc);
+ used_offset = (avail_offset + sizeof(struct vring_avail) +
+ sizeof(uint16_t) * num + align - 1) & ~(align - 1);
+
+ vr->num = num;
+ vr->desc = (struct vring_desc *) p;
+ vr->avail = (struct vring_avail *) (p + avail_offset);
+ vr->used = (struct vring_used *) (p + used_offset);
+
+ vr->desc_paddr = paddr;
+ vr->avail_paddr = paddr + avail_offset;
+ vr->used_paddr = paddr + used_offset;
}
/*
diff --git a/sys/dev/virtio/virtqueue.c b/sys/dev/virtio/virtqueue.c
index 41e01549c8b2..10b5179bd3d5 100644
--- a/sys/dev/virtio/virtqueue.c
+++ b/sys/dev/virtio/virtqueue.c
@@ -35,6 +35,7 @@
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
+#include <sys/mutex.h>
#include <sys/sdt.h>
#include <sys/sglist.h>
#include <vm/vm.h>
@@ -55,6 +56,8 @@
struct virtqueue {
device_t vq_dev;
+ struct mtx vq_ring_mtx;
+ struct mtx vq_indirect_mtx;
uint16_t vq_queue_index;
uint16_t vq_nentries;
uint32_t vq_flags;
@@ -83,15 +86,22 @@ struct virtqueue {
uint16_t vq_used_cons_idx;
void *vq_ring_mem;
+ bus_dmamap_t vq_ring_mapp;
+ vm_paddr_t vq_ring_paddr;
+
int vq_indirect_mem_size;
int vq_alignment;
int vq_ring_size;
char vq_name[VIRTQUEUE_MAX_NAME_SZ];
+ bus_dma_tag_t vq_ring_dmat;
+ bus_dma_tag_t vq_indirect_dmat;
+
struct vq_desc_extra {
void *cookie;
struct vring_desc *indirect;
vm_paddr_t indirect_paddr;
+ bus_dmamap_t mapp;
uint16_t ndescs;
} vq_descx[0];
};
@@ -148,6 +158,21 @@ SDT_PROBE_DEFINE1(virtqueue, , enqueue_segments, return, "uint16_t");
#define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val)
#define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val)
+static void
+virtqueue_ring_load_callback(void *arg, bus_dma_segment_t *segs,
+ int nsegs, int error)
+{
+ struct virtqueue *vq;
+
+ if (error != 0)
+ return;
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ vq = (struct virtqueue *)arg;
+ vq->vq_ring_paddr = segs[0].ds_addr;
+}
+
int
virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
bus_size_t notify_offset, int align, vm_paddr_t highaddr,
@@ -199,19 +224,58 @@ virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
+ vq->vq_ring_size = round_page(vring_size(size, align));
+
+ mtx_init(&vq->vq_ring_mtx, device_get_nameunit(dev),
+ "VirtIO Queue Lock", MTX_DEF);
+
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ align, /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ vq->vq_ring_size, /* max request size */
+ 1, /* max # segments */
+ vq->vq_ring_size, /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &vq->vq_ring_mtx, /* lockarg */
+ &vq->vq_ring_dmat);
+ if (error) {
+ device_printf(dev, "cannot create bus_dma_tag\n");
+ goto fail;
+ }
+
+#ifdef __powerpc__
+ /*
+ * Virtio uses physical addresses rather than bus addresses, so we
+ * need to ask busdma to skip the iommu physical->bus mapping. At
+ * present, this is only a thing on the powerpc architectures.
+ */
+ bus_dma_tag_set_iommu(vq->vq_ring_dmat, NULL, NULL);
+#endif
+
if (info->vqai_maxindirsz > 1) {
error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
if (error)
goto fail;
}
- vq->vq_ring_size = round_page(vring_size(size, align));
- vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
- M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
- if (vq->vq_ring_mem == NULL) {
- device_printf(dev,
- "cannot allocate memory for virtqueue ring\n");
- error = ENOMEM;
+ error = bus_dmamem_alloc(vq->vq_ring_dmat, &vq->vq_ring_mem,
+ BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &vq->vq_ring_mapp);
+ if (error) {
+ device_printf(dev, "bus_dmamem_alloc failed\n");
+ goto fail;
+ }
+
+ error = bus_dmamap_load(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ vq->vq_ring_mem, vq->vq_ring_size, virtqueue_ring_load_callback,
+ vq, BUS_DMA_NOWAIT);
+ if (error) {
+ device_printf(dev, "vq->vq_ring_mapp load failed\n");
goto fail;
}
@@ -227,12 +291,29 @@ fail:
return (error);
}
+static void
+virtqueue_indirect_load_callback(void *arg, bus_dma_segment_t *segs,
+ int nsegs, int error)
+{
+ struct vq_desc_extra *dxp;
+
+ if (error != 0)
+ return;
+
+ KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
+
+ dxp = (struct vq_desc_extra *)arg;
+ dxp->indirect_paddr = segs[0].ds_addr;
+}
+
static int
virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
{
device_t dev;
struct vq_desc_extra *dxp;
int i, size;
+ int error;
+ int align;
dev = vq->vq_dev;
@@ -254,16 +335,61 @@ virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
vq->vq_indirect_mem_size = size;
vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
+ mtx_init(&vq->vq_indirect_mtx, device_get_nameunit(dev),
+ "VirtIO Indirect Queue Lock", MTX_DEF);
+
+ align = size;
+ error = bus_dma_tag_create(
+ bus_get_dma_tag(dev), /* parent */
+ roundup_pow_of_two(align), /* alignment */
+ 0, /* boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ size, /* max request size */
+ 1, /* max # segments */
+ size, /* maxsegsize */
+ BUS_DMA_COHERENT, /* flags */
+ busdma_lock_mutex, /* lockfunc */
+ &vq->vq_indirect_mtx, /* lockarg */
+ &vq->vq_indirect_dmat);
+ if (error) {
+ device_printf(dev, "cannot create indirect bus_dma_tag\n");
+ return (error);
+ }
+
+#ifdef __powerpc__
+ /*
+ * Virtio uses physical addresses rather than bus addresses, so we
+ * need to ask busdma to skip the iommu physical->bus mapping. At
+ * present, this is only a thing on the powerpc architectures.
+ */
+ bus_dma_tag_set_iommu(vq->vq_indirect_dmat, NULL, NULL);
+#endif
+
for (i = 0; i < vq->vq_nentries; i++) {
dxp = &vq->vq_descx[i];
- dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
- if (dxp->indirect == NULL) {
- device_printf(dev, "cannot allocate indirect list\n");
- return (ENOMEM);
+ error = bus_dmamem_alloc(vq->vq_indirect_dmat,
+ (void **)&dxp->indirect,
+ BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &dxp->mapp);
+ if (error) {
+ panic("dxp->mapp alloc failed\n");
+ return (error);
+ }
+
+ error = bus_dmamap_load(vq->vq_indirect_dmat, dxp->mapp,
+ dxp->indirect, size, virtqueue_indirect_load_callback, dxp,
+ BUS_DMA_NOWAIT);
+ if (error) {
+ panic("dxp->mapp load failed\n");
+ bus_dmamem_free(vq->vq_indirect_dmat, dxp->indirect,
+ dxp->mapp);
+ dxp->indirect = NULL;
+ return (error);
}
- dxp->indirect_paddr = vtophys(dxp->indirect);
virtqueue_init_indirect_list(vq, dxp->indirect);
}
@@ -282,7 +408,8 @@ virtqueue_free_indirect(struct virtqueue *vq)
if (dxp->indirect == NULL)
break;
- free(dxp->indirect, M_DEVBUF);
+ bus_dmamap_unload(vq->vq_indirect_dmat, dxp->mapp);
+ bus_dmamem_free(vq->vq_indirect_dmat, dxp->indirect, dxp->mapp);
dxp->indirect = NULL;
dxp->indirect_paddr = 0;
}
@@ -360,9 +487,14 @@ virtqueue_free(struct virtqueue *vq)
virtqueue_free_indirect(vq);
if (vq->vq_ring_mem != NULL) {
- free(vq->vq_ring_mem, M_DEVBUF);
+ bus_dmamap_unload(vq->vq_ring_dmat, vq->vq_ring_mapp);
+ bus_dmamem_free(vq->vq_ring_dmat, vq->vq_ring_mem,
+ vq->vq_ring_mapp);
vq->vq_ring_size = 0;
- vq->vq_ring_mem = NULL;
+ }
+
+ if (vq->vq_ring_dmat != NULL) {
+ bus_dma_tag_destroy(vq->vq_ring_dmat);
}
free(vq, M_DEVBUF);
@@ -371,29 +503,25 @@ virtqueue_free(struct virtqueue *vq)
vm_paddr_t
virtqueue_paddr(struct virtqueue *vq)
{
-
- return (vtophys(vq->vq_ring_mem));
+ return (vq->vq_ring_paddr);
}
vm_paddr_t
virtqueue_desc_paddr(struct virtqueue *vq)
{
-
- return (vtophys(vq->vq_ring.desc));
+ return (vq->vq_ring.desc_paddr);
}
vm_paddr_t
virtqueue_avail_paddr(struct virtqueue *vq)
{
-
- return (vtophys(vq->vq_ring.avail));
+ return (vq->vq_ring.avail_paddr);
}
vm_paddr_t
virtqueue_used_paddr(struct virtqueue *vq)
{
-
- return (vtophys(vq->vq_ring.used));
+ return (vq->vq_ring.used_paddr);
}
uint16_t
@@ -434,9 +562,12 @@ virtqueue_full(struct virtqueue *vq)
void
virtqueue_notify(struct virtqueue *vq)
{
-
/* Ensure updated avail->idx is visible to host. */
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
+#if defined(__i386__) || defined(__amd64__)
mb();
+#endif
if (vq_ring_must_notify_host(vq))
vq_ring_notify_host(vq);
@@ -448,6 +579,9 @@ virtqueue_nused(struct virtqueue *vq)
{
uint16_t used_idx, nused;
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_POSTREAD);
+
used_idx = vq_htog16(vq, vq->vq_ring.used->idx);
nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
@@ -459,6 +593,8 @@ virtqueue_nused(struct virtqueue *vq)
int
virtqueue_intr_filter(struct virtqueue *vq)
{
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_POSTREAD);
if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
return (0);
@@ -487,6 +623,9 @@ virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
{
uint16_t ndesc, avail_idx;
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_POSTREAD);
+
avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
@@ -518,6 +657,9 @@ virtqueue_disable_intr(struct virtqueue *vq)
}
vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
+
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
}
int
@@ -561,6 +703,9 @@ virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
sg, readable, writable);
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
+
vq->vq_desc_head_idx = idx;
vq->vq_free_cnt -= needed;
if (vq->vq_free_cnt == 0)
@@ -580,6 +725,9 @@ virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
void *cookie;
uint16_t used_idx, desc_idx;
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_POSTREAD);
+
if (vq->vq_used_cons_idx ==
vq_htog16(vq, atomic_load_16(&vq->vq_ring.used->idx)))
return (NULL);
@@ -666,11 +814,14 @@ vq_ring_init(struct virtqueue *vq)
size = vq->vq_nentries;
vr = &vq->vq_ring;
- vring_init(vr, size, ring_mem, vq->vq_alignment);
+ vring_init(vr, size, ring_mem, vq->vq_ring_paddr, vq->vq_alignment);
for (i = 0; i < size - 1; i++)
vr->desc[i].next = vq_gtoh16(vq, i + 1);
vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
+
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
}
static void
@@ -694,6 +845,9 @@ vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
/* Keep pending count until virtqueue_notify(). */
vq->vq_queued_cnt++;
+
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
}
static uint16_t
@@ -777,6 +931,10 @@ vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
vq_ring_enqueue_segments(vq, dxp->indirect, 0,
sg, readable, writable);
+ bus_dmamap_sync(vq->vq_indirect_dmat, dxp->mapp, BUS_DMASYNC_PREWRITE);
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
+
vq->vq_desc_head_idx = vq_htog16(vq, dp->next);
vq->vq_free_cnt--;
if (vq->vq_free_cnt == 0)
@@ -803,7 +961,11 @@ vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
}
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
+#if defined(__i386__) || defined(__amd64__)
mb();
+#endif
/*
* Enough items may have already been consumed to meet our threshold
@@ -821,6 +983,9 @@ vq_ring_must_notify_host(struct virtqueue *vq)
{
uint16_t new_idx, prev_idx, event_idx, flags;
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_POSTREAD);
+
if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
prev_idx = new_idx - vq->vq_queued_cnt;
@@ -876,4 +1041,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
*/
dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);
vq->vq_desc_head_idx = desc_idx;
+
+ bus_dmamap_sync(vq->vq_ring_dmat, vq->vq_ring_mapp,
+ BUS_DMASYNC_PREWRITE);
}
generated by cgit v1.3 (git 2.53.0) at 2026-08-12 12:13:17 +0000