aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlan Somers <asomers@FreeBSD.org>2020-08-26 02:44:35 +0000
committerAlan Somers <asomers@FreeBSD.org>2020-08-26 02:44:35 +0000
commit7d874f0f36aaa894b5bb85336b7e7f955b208d30 (patch)
tree60d91d592a6904df049b342337dfeb34f93f8669
parente6f6d0c9bcbf7942c390f65062054ec4784ce5b8 (diff)
downloadsrc-7d874f0f36aaa894b5bb85336b7e7f955b208d30.tar.gz
src-7d874f0f36aaa894b5bb85336b7e7f955b208d30.zip
geli: use unmapped I/O
Use unmapped I/O for geli. Unlike most geom providers, geli needs to manipulate data on every read or write. Previously it would always map bios. On my 16-core, dual socket server using geli atop md(4) devices, with 512B sectors, this change increases geli IOPs by about 3x. Note that geli still can't use unmapped I/O when data integrity verification is enabled (but it could, with a little more work). And it can't use unmapped I/O in combination with ZFS, because ZFS uses mapped bios. Reviewed by: markj, kib, jhb, mjg, mat, bcr (manpages) MFC after: 1 week Sponsored by: Axcient Differential Revision: https://reviews.freebsd.org/D25671
Notes
Notes: svn path=/head/; revision=364800
-rw-r--r--sys/geom/eli/g_eli.c11
-rw-r--r--sys/geom/eli/g_eli_privacy.c65
2 files changed, 68 insertions, 8 deletions
diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index b246b83c1d7b..842fce324722 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -49,6 +49,8 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <sys/vnode.h>
+#include <machine/vmparam.h>
+
#include <vm/uma.h>
#include <geom/geom.h>
@@ -972,6 +974,15 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
*/
pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
+ if (CRYPTO_HAS_VMPAGE) {
+ /*
+ * On DMAP architectures we can use unmapped I/O. But don't
+ * use it with data integrity verification. That code hasn't
+ * been written yet.
+ */
+ if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0)
+ pp->flags |= G_PF_ACCEPT_UNMAPPED;
+ }
pp->mediasize = sc->sc_mediasize;
pp->sectorsize = sc->sc_sectorsize;
LIST_FOREACH(gap, &bpp->aliases, ga_next)
diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c
index 7bd8eb29cc28..adb353441e3d 100644
--- a/sys/geom/eli/g_eli_privacy.c
+++ b/sys/geom/eli/g_eli_privacy.c
@@ -63,6 +63,28 @@ __FBSDID("$FreeBSD$");
MALLOC_DECLARE(M_ELI);
/*
+ * Copy data from a (potentially unmapped) bio to a kernelspace buffer.
+ *
+ * The buffer must have at least as much room as bp->bio_length.
+ */
+static void
+g_eli_bio_copyin(struct bio *bp, void *kaddr)
+{
+ struct uio uio;
+ struct iovec iov[1];
+
+ iov[0].iov_base = kaddr;
+ iov[0].iov_len = bp->bio_length;
+ uio.uio_iov = iov;
+ uio.uio_iovcnt = 1;
+ uio.uio_offset = 0;
+ uio.uio_resid = bp->bio_length;
+ uio.uio_segflg = UIO_SYSSPACE;
+ uio.uio_rw = UIO_READ;
+ uiomove_fromphys(bp->bio_ma, bp->bio_ma_offset, bp->bio_length, &uio);
+}
+
+/*
* The function is called after we read and decrypt data.
*
* g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
@@ -98,8 +120,7 @@ g_eli_crypto_read_done(struct cryptop *crp)
*/
if (bp->bio_inbed < bp->bio_children)
return (0);
- free(bp->bio_driver2, M_ELI);
- bp->bio_driver2 = NULL;
+
if (bp->bio_error != 0) {
G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
bp->bio_error);
@@ -167,6 +188,11 @@ g_eli_crypto_write_done(struct cryptop *crp)
return (0);
}
cbp->bio_data = bp->bio_driver2;
+ /*
+ * Clear BIO_UNMAPPED, which was inherited from where we cloned the bio
+ * in g_eli_start, because we manually set bio_data
+ */
+ cbp->bio_flags &= ~BIO_UNMAPPED;
cbp->bio_done = g_eli_write_done;
cp = LIST_FIRST(&gp->consumer);
cbp->bio_to = cp->provider;
@@ -236,10 +262,12 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
{
struct g_eli_softc *sc;
struct cryptop *crp;
+ vm_page_t *pages;
u_int i, nsec, secsize;
off_t dstoff;
- u_char *data;
+ u_char *data = NULL;
int error;
+ int pages_offset;
G_ELI_LOGREQ(3, bp, "%s", __func__);
@@ -258,16 +286,37 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
if (bp->bio_cmd == BIO_WRITE) {
data = malloc(bp->bio_length, M_ELI, M_WAITOK);
bp->bio_driver2 = data;
- bcopy(bp->bio_data, data, bp->bio_length);
- } else
- data = bp->bio_data;
+ /*
+ * This copy could be eliminated by using crypto's output
+ * buffer, instead of using a single overwriting buffer.
+ */
+ if ((bp->bio_flags & BIO_UNMAPPED) != 0)
+ g_eli_bio_copyin(bp, data);
+ else
+ bcopy(bp->bio_data, data, bp->bio_length);
+ } else {
+ if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
+ pages = bp->bio_ma;
+ pages_offset = bp->bio_ma_offset;
+ } else {
+ data = bp->bio_data;
+ }
+ }
for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
crp = crypto_getreq(wr->w_sid, M_WAITOK);
- crypto_use_buf(crp, data, secsize);
+ if (data) {
+ crypto_use_buf(crp, data, secsize);
+ data += secsize;
+ } else {
+ MPASS(pages != NULL);
+ crypto_use_vmpage(crp, pages, secsize, pages_offset);
+ pages_offset += secsize;
+ pages += pages_offset >> PAGE_SHIFT;
+ pages_offset &= PAGE_MASK;
+ }
crp->crp_opaque = (void *)bp;
- data += secsize;
if (bp->bio_cmd == BIO_WRITE) {
crp->crp_op = CRYPTO_OP_ENCRYPT;
crp->crp_callback = g_eli_crypto_write_done;