libc: fix access mode tests in fmemopen(3)

Previously a stream opened as read-only could be written to.  Add a test
case for the fix.

Also correct another incorrect access mode check that worked by
accident, and improve the tests for that.

PR:		281953
Reported by:	Erkki Moorits, fuz
Reviewed by:	fuz, khng (earlier)
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D47265

(cherry picked from commit 0953460ce149e6f384aafbcb1e6213dfbf8f6a16)
(cherry picked from commit 6b9f7133aba44189d9625c352bc2c2a59baf18ef)
(cherry picked from commit 4fbd6e0e3ca8e69d2d3789ecda6e4dd76c34e06a)

2 files changed, 38 insertions, 5 deletions

diff --git a/lib/libc/stdio/fmemopen.c b/lib/libc/stdio/fmemopen.c
index 295592ac896f..d6f83722bb6f 100644
--- a/lib/libc/stdio/fmemopen.c
+++ b/lib/libc/stdio/fmemopen.c
@@ -75,10 +75,9 @@ fmemopen(void * __restrict buf, size_t size, const char * __restrict mode)
 	}
 
 	/*
-	 * There's no point in requiring an automatically allocated buffer
-	 * in write-only mode.
+	 * An automatically allocated buffer is only allowed in read-write mode.
 	 */
-	if (!(flags & O_RDWR) && buf == NULL) {
+	if ((flags & O_ACCMODE) != O_RDWR && buf == NULL) {
 		errno = EINVAL;
 		return (NULL);
 	}
@@ -137,9 +136,10 @@ fmemopen(void * __restrict buf, size_t size, const char * __restrict mode)
 		break;
 	}
 
+	/* Disable read in O_WRONLY mode, and write in O_RDONLY mode. */
 	f = funopen(ck,
-	    flags & O_WRONLY ? NULL : fmemopen_read, 
-	    flags & O_RDONLY ? NULL : fmemopen_write,
+	    (flags & O_ACCMODE) == O_WRONLY ? NULL : fmemopen_read,
+	    (flags & O_ACCMODE) == O_RDONLY ? NULL : fmemopen_write,
 	    fmemopen_seek, fmemopen_close);
 
 	if (f == NULL) {
diff --git a/lib/libc/tests/stdio/fmemopen2_test.c b/lib/libc/tests/stdio/fmemopen2_test.c
index d68150d19594..2839ee04bb52 100644
--- a/lib/libc/tests/stdio/fmemopen2_test.c
+++ b/lib/libc/tests/stdio/fmemopen2_test.c
@@ -133,9 +133,11 @@ ATF_TC_BODY(test_autoalloc, tc)
 	/* Open a FILE * using a wrong mode */
 	fp = fmemopen(NULL, 512, "r");
 	ATF_REQUIRE(fp == NULL);
+	ATF_REQUIRE(errno == EINVAL);
 
 	fp = fmemopen(NULL, 512, "w");
 	ATF_REQUIRE(fp == NULL);
+	ATF_REQUIRE(errno == EINVAL);
 }
 
 ATF_TC_WITHOUT_HEAD(test_data_length);
@@ -272,6 +274,36 @@ ATF_TC_BODY(test_size_0, tc)
 	ATF_REQUIRE(errno == EINVAL);
 }
 
+/*
+ * PR281953 - ensure we cannot write in read-only only mode, and cannot read in
+ * write-only mode.
+ */
+ATF_TC_WITHOUT_HEAD(test_rdonly_wronly);
+ATF_TC_BODY(test_rdonly_wronly, tc)
+{
+	FILE *fp;
+	char buf[16];
+	char buf_orig[16] = "input data";
+	char buf_write[16] = "write";
+	size_t sz;
+
+	memcpy(buf, buf_orig, sizeof(buf));
+	fp = fmemopen(buf, sizeof(buf), "r");
+	ATF_REQUIRE(fp != NULL);
+	sz = fwrite(buf_write, 1, strlen(buf_write), fp);
+	ATF_REQUIRE(sz == 0);
+	ATF_REQUIRE(errno == EBADF);
+	ATF_REQUIRE(memcmp(buf, buf_orig, sizeof(buf)) == 0);
+	fclose(fp);
+
+	fp = fmemopen(buf_orig, sizeof(buf), "w");
+	ATF_REQUIRE(fp != NULL);
+	sz = fread(buf, sizeof(buf), 1, fp);
+	ATF_REQUIRE(sz == 0);
+	ATF_REQUIRE(errno == EBADF);
+	fclose(fp);
+}
+
 ATF_TP_ADD_TCS(tp)
 {
 
@@ -281,6 +313,7 @@ ATF_TP_ADD_TCS(tp)
 	ATF_TP_ADD_TC(tp, test_binary);
 	ATF_TP_ADD_TC(tp, test_append_binary_pos);
 	ATF_TP_ADD_TC(tp, test_size_0);
+	ATF_TP_ADD_TC(tp, test_rdonly_wronly);
 
 	return (atf_no_error());
 }