aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBrooks Davis <brooks@FreeBSD.org>2026-06-21 15:47:53 +0000
committerBrooks Davis <brooks@FreeBSD.org>2026-06-21 16:13:27 +0000
commitfc9b28879e8935cc830b83d8a36d85b9b98b276a (patch)
tree69a7318947d7e1c60dddf51cf60b66b469287a18
parent2ee2ef607f6087757efd62b814cbf2400ef58ef2 (diff)
uiomove_*: centralize the copy function selection
Add a uiomove_step() for the central set of switch statements which choose between userspace and kernel and if data is going to or from the iovec. Refactor uiomove_fromphys loops to unconditionally free per-iteration resources and drop gotos. While here, switch from bcopy to memcpy. Reviewed by: kib Suggested by: emaste Sponsored by: Innovate UK Differential Revision: https://reviews.freebsd.org/D57688
Diffstat
-rw-r--r--sys/amd64/amd64/uio_machdep.c34
-rw-r--r--sys/arm/arm/uio_machdep.c33
-rw-r--r--sys/arm64/arm64/uio_machdep.c36
-rw-r--r--sys/i386/i386/uio_machdep.c33
-rw-r--r--sys/kern/subr_uio.c65
-rw-r--r--sys/powerpc/powerpc/uio_machdep.c33
-rw-r--r--sys/riscv/riscv/uio_machdep.c36
-rw-r--r--sys/sys/uio.h1
8 files changed, 56 insertions, 215 deletions
diff --git a/sys/amd64/amd64/uio_machdep.c b/sys/amd64/amd64/uio_machdep.c
index 11e6ad2b1da9..9fdfc898172b 100644
--- a/sys/amd64/amd64/uio_machdep.c
+++ b/sys/amd64/amd64/uio_machdep.c
@@ -96,38 +96,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
&ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
cp = (char *)vaddr + page_offset;
}
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
if (__predict_false(mapped)) {
pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
&vaddr, 1, true);
mapped = false;
}
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -135,10 +111,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
- if (__predict_false(mapped))
- pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
- true);
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/arm/arm/uio_machdep.c b/sys/arm/arm/uio_machdep.c
index 6b8e4352e06f..7df6104b25fc 100644
--- a/sys/arm/arm/uio_machdep.c
+++ b/sys/arm/arm/uio_machdep.c
@@ -91,36 +91,10 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
cnt = min(cnt, PAGE_SIZE - page_offset);
sf = sf_buf_alloc(ma[offset >> PAGE_SHIFT], 0);
cp = (char*)sf_buf_kva(sf) + page_offset;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error) {
- sf_buf_free(sf);
- goto out;
- }
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
sf_buf_free(sf);
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -128,7 +102,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/arm64/arm64/uio_machdep.c b/sys/arm64/arm64/uio_machdep.c
index 976055a69491..77f596498c45 100644
--- a/sys/arm64/arm64/uio_machdep.c
+++ b/sys/arm64/arm64/uio_machdep.c
@@ -92,38 +92,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
&ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
cp = (char *)vaddr + page_offset;
}
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
if (__predict_false(mapped)) {
pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
&vaddr, 1, true);
mapped = false;
}
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -131,12 +107,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
- if (__predict_false(mapped)) {
- panic("ARM64TODO: uiomove_fromphys");
- pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
- true);
- }
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/i386/i386/uio_machdep.c b/sys/i386/i386/uio_machdep.c
index b41460aef5b4..674de877d2a1 100644
--- a/sys/i386/i386/uio_machdep.c
+++ b/sys/i386/i386/uio_machdep.c
@@ -91,38 +91,11 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
sched_pin();
sf = sf_buf_alloc(ma[offset >> PAGE_SHIFT], SFB_CPUPRIVATE);
cp = (char *)sf_buf_kva(sf) + page_offset;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error) {
- sf_buf_free(sf);
- sched_unpin();
- goto out;
- }
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
sf_buf_free(sf);
sched_unpin();
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c
index fa5865fbe938..fea1395a7f77 100644
--- a/sys/kern/subr_uio.c
+++ b/sys/kern/subr_uio.c
@@ -200,6 +200,40 @@ uiomove_nofault(void *cp, int n, struct uio *uio)
return (uiomove_faultflag(cp, n, uio, 1));
}
+int
+uiomove_step(void *cp, void *base, size_t len, struct uio *uio)
+{
+ int error = 0;
+
+ switch (uio->uio_segflg) {
+ case UIO_USERSPACE:
+ maybe_yield();
+ switch (uio->uio_rw) {
+ case UIO_READ:
+ error = copyout(cp, base, len);
+ break;
+ case UIO_WRITE:
+ error = copyin(base, cp, len);
+ break;
+ }
+ break;
+ case UIO_SYSSPACE:
+ switch (uio->uio_rw) {
+ case UIO_READ:
+ memcpy(base, cp, len);
+ break;
+ case UIO_WRITE:
+ memcpy(cp, base, len);
+ break;
+ }
+ break;
+ case UIO_NOCOPY:
+ break;
+ }
+
+ return (error);
+}
+
static int
uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
{
@@ -246,34 +280,9 @@ uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
if (cnt > n)
cnt = n;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
-
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
+ if (error != 0)
+ goto out;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
diff --git a/sys/powerpc/powerpc/uio_machdep.c b/sys/powerpc/powerpc/uio_machdep.c
index 63911963279a..05d06a95f544 100644
--- a/sys/powerpc/powerpc/uio_machdep.c
+++ b/sys/powerpc/powerpc/uio_machdep.c
@@ -97,36 +97,10 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
sf = sf_buf_alloc(m, 0);
cp = (char*)sf_buf_kva(sf) + page_offset;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error) {
- sf_buf_free(sf);
- goto out;
- }
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
sf_buf_free(sf);
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -134,7 +108,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/riscv/riscv/uio_machdep.c b/sys/riscv/riscv/uio_machdep.c
index f171feb1a4bd..77f596498c45 100644
--- a/sys/riscv/riscv/uio_machdep.c
+++ b/sys/riscv/riscv/uio_machdep.c
@@ -92,38 +92,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
&ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
cp = (char *)vaddr + page_offset;
}
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
if (__predict_false(mapped)) {
pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
&vaddr, 1, true);
mapped = false;
}
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -131,12 +107,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
- if (__predict_false(mapped)) {
- panic("TODO 3");
- pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
- true);
- }
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/sys/uio.h b/sys/sys/uio.h
index 58bb7b13b253..217f65365cb0 100644
--- a/sys/sys/uio.h
+++ b/sys/sys/uio.h
@@ -98,6 +98,7 @@ int uiomove_fromphys(struct vm_page *ma[], vm_offset_t offset, int n,
struct uio *uio);
int uiomove_nofault(void *cp, int n, struct uio *uio);
int uiomove_object(struct vm_object *obj, off_t obj_size, struct uio *uio);
+int uiomove_step(void *cp, void *base, size_t cnt, struct uio *uio);
#else /* !_KERNEL */
generated by cgit v1.2.3 (git 2.25.1) at 2026-07-14 16:43:07 +0000