In current implementation, thread cancellation is done in signal handler,

which does not know what is the state of interrupted system call, for
example, open() system call opened a file and the thread is still cancelled,
result is descriptor leak, there are other problems which can cause resource
leak or undeterminable side effect when a thread is cancelled. However, this
is no longer true in new implementation.

  In defering mode, a thread is canceled if cancellation request is pending and
later the thread enters a cancellation point, otherwise, a later
pthread_cancel() just causes SIGCANCEL to be sent to the target thread, and
causes target thread to abort system call, userland code in libthr then checks
cancellation state, and cancels the thread if needed. For example, the
cancellation point open(), the thread may be canceled at start,
but later, if it opened a file descriptor, it is not canceled, this avoids
file handle leak. Another example is read(), a thread may be canceled at start
of the function, but later, if it read some bytes from a socket, the thread
is not canceled, the caller then can decide if it should still enable cancelling
or disable it and continue reading data until it thinks it has read all
bytes of a packet, and keeps a protocol stream in health state, if user ignores
partly reading of a packet without disabling cancellation, then second iteration
of read loop cause the thread to be cancelled.
An exception is that the close() cancellation point always closes a file handle
despite whether the thread is cancelled or not.

  The old mechanism is still kept, for a functions which is not so easily to
fix a cancellation problem, the rough mechanism is used.

Reviewed by: kib@

6 files changed, 284 insertions, 99 deletions

diff --git a/lib/libthr/thread/thr_cancel.c b/lib/libthr/thread/thr_cancel.c
index bf93fdcae4da..943d53c39e92 100644
--- a/lib/libthr/thread/thr_cancel.c
+++ b/lib/libthr/thread/thr_cancel.c
@@ -42,7 +42,7 @@ static inline void
 testcancel(struct pthread *curthread)
 {
 	if (__predict_false(SHOULD_CANCEL(curthread) &&
-	    !THR_IN_CRITICAL(curthread) && curthread->cancel_defer == 0))
+	    !THR_IN_CRITICAL(curthread)))
 		_pthread_exit(PTHREAD_CANCELED);
 }
 
@@ -143,40 +143,44 @@ _pthread_testcancel(void)
 void
 _thr_cancel_enter(struct pthread *curthread)
 {
-	if (curthread->cancel_enable) {
-		curthread->cancel_point++;
+	curthread->cancel_point++;
+	if (curthread->cancel_enable)
 		testcancel(curthread);
+}
+
+void
+_thr_cancel_enter_defer(struct pthread *curthread, int maycancel)
+{
+	curthread->cancel_defer++;
+	curthread->cancel_point++;
+	if (__predict_false(SHOULD_CANCEL(curthread) &&
+	    !THR_IN_CRITICAL(curthread))) {
+		if (!maycancel)
+			thr_wake(curthread->tid);
+		else
+			_pthread_exit(PTHREAD_CANCELED);
 	}
 }
 
 void
 _thr_cancel_leave(struct pthread *curthread)
 {
-	if (curthread->cancel_enable)
-		curthread->cancel_point--;
+	curthread->cancel_point--;
 }
 
 void
-_thr_cancel_enter_defer(struct pthread *curthread)
+_thr_cancel_leave2(struct pthread *curthread, int maycancel)
 {
-	if (curthread->cancel_enable) {
-		curthread->cancel_point++;
+	if (curthread->cancel_enable && maycancel)
 		testcancel(curthread);
-		curthread->cancel_defer++;
-	}
+	curthread->cancel_point--;
 }
 
 void
-_thr_cancel_leave_defer(struct pthread *curthread, int check)
+_thr_cancel_leave_defer(struct pthread *curthread, int maycancel)
 {
-	if (curthread->cancel_enable) {
-		if (!check) {
-			curthread->cancel_point--;
-			curthread->cancel_defer--;
-		} else {
-			curthread->cancel_defer--;
-			testcancel(curthread);
-			curthread->cancel_point--;
-		}
-	}
+	if (curthread->cancel_enable && maycancel)
+		testcancel(curthread);
+	curthread->cancel_point--;
+	curthread->cancel_defer--;
 }
diff --git a/lib/libthr/thread/thr_cond.c b/lib/libthr/thread/thr_cond.c
index 85eecaaf1ee5..95970d9d0fcd 100644
--- a/lib/libthr/thread/thr_cond.c
+++ b/lib/libthr/thread/thr_cond.c
@@ -162,6 +162,14 @@ cond_cancel_handler(void *arg)
 	_mutex_cv_lock(info->mutex, info->count);
 }
 
+/*
+ * Cancellation behaivor:
+ *   Thread may be canceled at start, if thread is canceled, it means it
+ *   did not get a wakeup from pthread_cond_signal(), otherwise, it is
+ *   not canceled.
+ *   Thread cancellation never cause wakeup from pthread_cond_signal()
+ *   to be lost.
+ */
 static int
 cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
 	const struct timespec *abstime, int cancel)
@@ -180,6 +188,8 @@ cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
 	    (ret = init_static(curthread, cond)) != 0))
 		return (ret);
 
+	_thr_testcancel(curthread);
+
 	cv = *cond;
 	THR_UMUTEX_LOCK(curthread, &cv->c_lock);
 	ret = _mutex_cv_unlock(mutex, &info.count);
@@ -200,10 +210,10 @@ cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
 
 	if (cancel) {
 		THR_CLEANUP_PUSH(curthread, cond_cancel_handler, &info);
-		_thr_cancel_enter_defer(curthread);
+		_thr_cancel_enter_defer(curthread, 0);
 		ret = _thr_ucond_wait(&cv->c_kerncv, &cv->c_lock, tsp, 1);
 		info.cond = NULL;
-		_thr_cancel_leave_defer(curthread, ret);
+		_thr_cancel_leave_defer(curthread, (ret != 0));
 		THR_CLEANUP_POP(curthread, 0);
 	} else {
 		ret = _thr_ucond_wait(&cv->c_kerncv, &cv->c_lock, tsp, 0);
diff --git a/lib/libthr/thread/thr_join.c b/lib/libthr/thread/thr_join.c
index 139c74c7925c..8201abafeae9 100644
--- a/lib/libthr/thread/thr_join.c
+++ b/lib/libthr/thread/thr_join.c
@@ -68,6 +68,10 @@ _pthread_timedjoin_np(pthread_t pthread, void **thread_return,
 	return (join_common(pthread, thread_return, abstime));
 }
 
+/*
+ * Cancellation behavior:
+ *   if the thread is canceled, joinee is not recycled.
+ */
 static int
 join_common(pthread_t pthread, void **thread_return,
 	const struct timespec *abstime)
@@ -103,10 +107,11 @@ join_common(pthread_t pthread, void **thread_return,
 	THREAD_LIST_UNLOCK(curthread);
 
 	THR_CLEANUP_PUSH(curthread, backout_join, pthread);
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 
 	tid = pthread->tid;
 	while (pthread->tid != TID_TERMINATED) {
+		_thr_testcancel(curthread);
 		if (abstime != NULL) {
 			clock_gettime(CLOCK_REALTIME, &ts);
 			TIMESPEC_SUB(&ts2, abstime, &ts);
@@ -122,7 +127,7 @@ join_common(pthread_t pthread, void **thread_return,
 			break;
 	}
 
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, 0);
 	THR_CLEANUP_POP(curthread, 0);
 
 	if (ret == ETIMEDOUT) {
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index 415c1ecc4ecd..465f725cd42d 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -644,7 +644,8 @@ void	_thread_printf(int, const char *, ...) __hidden;
 void	_thr_spinlock_init(void) __hidden;
 void	_thr_cancel_enter(struct pthread *) __hidden;
 void	_thr_cancel_leave(struct pthread *) __hidden;
-void	_thr_cancel_enter_defer(struct pthread *) __hidden;
+void	_thr_cancel_leave2(struct pthread *, int) __hidden;
+void	_thr_cancel_enter_defer(struct pthread *, int) __hidden;
 void	_thr_cancel_leave_defer(struct pthread *, int) __hidden;
 void	_thr_testcancel(struct pthread *) __hidden;
 void	_thr_signal_block(struct pthread *) __hidden;
diff --git a/lib/libthr/thread/thr_sig.c b/lib/libthr/thread/thr_sig.c
index efd8cb492849..eed6fdbba095 100644
--- a/lib/libthr/thread/thr_sig.c
+++ b/lib/libthr/thread/thr_sig.c
@@ -67,8 +67,6 @@ sigcancel_handler(int sig __unused,
 {
 	struct pthread *curthread = _get_curthread();
 
-	if (curthread->cancel_defer && curthread->cancel_pending)
-		thr_wake(curthread->tid);
 	curthread->in_sigcancel_handler++;
 	_thr_ast(curthread);
 	curthread->in_sigcancel_handler--;
@@ -77,13 +75,50 @@ sigcancel_handler(int sig __unused,
 void
 _thr_ast(struct pthread *curthread)
 {
-	if (!THR_IN_CRITICAL(curthread)) {
-		_thr_testcancel(curthread);
-		if (__predict_false((curthread->flags &
-		    (THR_FLAGS_NEED_SUSPEND | THR_FLAGS_SUSPENDED))
-			== THR_FLAGS_NEED_SUSPEND))
-			_thr_suspend_check(curthread);
+
+	if (THR_IN_CRITICAL(curthread))
+		return;
+
+	if (curthread->cancel_pending && curthread->cancel_enable
+		&& !curthread->cancelling) {
+		if (curthread->cancel_async) {
+			/*
+		 	 * asynchronous cancellation mode, act upon
+			 * immediately.
+		 	 */
+			_pthread_exit(PTHREAD_CANCELED);
+		} else {
+			/*
+		 	 * Otherwise, we are in defer mode, and we are at
+			 * cancel point, tell kernel to not block the current
+			 * thread on next cancelable system call.
+			 * 
+			 * There are two cases we should call thr_wake() to 
+			 * turn on TDP_WAKEUP in kernel:
+			 * 1) we are going to call a cancelable system call,
+			 *    non-zero cancel_point means we are already in
+			 *    cancelable state, next system call is cancelable.
+			 * 2) because _thr_ast() may be called by
+			 *    THR_CRITICAL_LEAVE() which is used by rtld rwlock
+			 *    and any libthr internal locks, when rtld rwlock
+			 *    is used, it is mostly caused my an unresolved PLT.
+			 *    those routines may clear the TDP_WAKEUP flag by
+			 *    invoking some system calls, in those cases, we
+			 *    also should reenable the flag.
+		 	 */
+			if (curthread->cancel_point) {
+				if (curthread->cancel_defer)
+					thr_wake(curthread->tid);
+				else
+					_pthread_exit(PTHREAD_CANCELED);
+			}
+		}
 	}
+
+	if (__predict_false((curthread->flags &
+	    (THR_FLAGS_NEED_SUSPEND | THR_FLAGS_SUSPENDED))
+		== THR_FLAGS_NEED_SUSPEND))
+		_thr_suspend_check(curthread);
 }
 
 void
@@ -296,6 +331,11 @@ _sigtimedwait(const sigset_t *set, siginfo_t *info,
 	return (ret);
 }
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, if thread got signal,
+ *   it is not canceled.
+ */
 int
 __sigtimedwait(const sigset_t *set, siginfo_t *info,
 	const struct timespec * timeout)
@@ -311,9 +351,9 @@ __sigtimedwait(const sigset_t *set, siginfo_t *info,
 		pset = &newset;
 	} else
 		pset = set;
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_sigtimedwait(pset, info, timeout);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, (ret == -1));
 	return (ret);
 }
 
@@ -335,6 +375,11 @@ _sigwaitinfo(const sigset_t *set, siginfo_t *info)
 	return (ret);
 }
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, if thread got signal,
+ *   it is not canceled.
+ */ 
 int
 __sigwaitinfo(const sigset_t *set, siginfo_t *info)
 {
@@ -350,9 +395,9 @@ __sigwaitinfo(const sigset_t *set, siginfo_t *info)
 	} else
 		pset = set;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_sigwaitinfo(pset, info);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	return (ret);
 }
 
@@ -374,6 +419,11 @@ _sigwait(const sigset_t *set, int *sig)
 	return (ret);
 }
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, if thread got signal,
+ *   it is not canceled.
+ */ 
 int
 __sigwait(const sigset_t *set, int *sig)
 {
@@ -389,8 +439,8 @@ __sigwait(const sigset_t *set, int *sig)
 	} else 
 		pset = set;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_sigwait(pset, sig);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, (ret != 0));
 	return (ret);
 }
diff --git a/lib/libthr/thread/thr_syscalls.c b/lib/libthr/thread/thr_syscalls.c
index ad3f091470a3..73fa56fcc8f3 100644
--- a/lib/libthr/thread/thr_syscalls.c
+++ b/lib/libthr/thread/thr_syscalls.c
@@ -158,6 +158,10 @@ ssize_t	__writev(int, const struct iovec *, int);
 
 __weak_reference(__accept, accept);
 
+/*
+ * Cancellation behavior:
+ *   If thread is canceled, no socket is created.
+ */
 int
 __accept(int s, struct sockaddr *addr, socklen_t *addrlen)
 {
@@ -165,9 +169,9 @@ __accept(int s, struct sockaddr *addr, socklen_t *addrlen)
 	int ret;
 
 	curthread = _get_curthread();
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_accept(s, addr, addrlen);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
  	return (ret);
 }
@@ -190,61 +194,84 @@ __aio_suspend(const struct aiocb * const iocbs[], int niocb, const struct
 
 __weak_reference(__close, close);
 
+/*
+ * Cancellation behavior:
+ *   According to manual of close(), the file descriptor is always deleted.
+ *   Here, thread is only canceled after the system call, so the file
+ *   descriptor is always deleted despite whether the thread is canceled
+ *   or not.
+ */
 int
 __close(int fd)
 {
 	struct pthread	*curthread = _get_curthread();
 	int	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 0);
 	ret = __sys_close(fd);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, 1);
 	
 	return (ret);
 }
 
 __weak_reference(__connect, connect);
 
+/*
+ * Cancellation behavior:
+ *   If the thread is canceled, connection is not made.
+ */
 int
 __connect(int fd, const struct sockaddr *name, socklen_t namelen)
 {
 	struct pthread *curthread = _get_curthread();
 	int ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 0);
 	ret = __sys_connect(fd, name, namelen);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
  	return (ret);
 }
 
 __weak_reference(___creat, creat);
 
+/*
+ * Cancellation behavior:
+ *   If thread is canceled, file is not created.
+ */
 int
 ___creat(const char *path, mode_t mode)
 {
 	struct pthread *curthread = _get_curthread();
 	int ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __creat(path, mode);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	
 	return ret;
 }
 
 __weak_reference(__fcntl, fcntl);
 
+/*
+ * Cancellation behavior:
+ *   According to specification, only F_SETLKW is a cancellation point.
+ *   Thread is only canceled at start, or canceled if the system call
+ *   is failure, this means the function does not generate side effect
+ *   if it is canceled.
+ */
 int
 __fcntl(int fd, int cmd,...)
 {
 	struct pthread *curthread = _get_curthread();
 	int	ret;
 	va_list	ap;
-	
+
 	va_start(ap, cmd);
 	switch (cmd) {
 	case F_DUPFD:
+	case F_DUP2FD:
 		ret = __sys_fcntl(fd, cmd, va_arg(ap, int));
 		break;
 	case F_SETFD:
@@ -255,16 +282,15 @@ __fcntl(int fd, int cmd,...)
 	case F_GETFL:
 		ret = __sys_fcntl(fd, cmd);
 		break;
-
 	case F_OSETLKW:
 	case F_SETLKW:
-		_thr_cancel_enter(curthread);
+		_thr_cancel_enter_defer(curthread, 1);
 #ifdef SYSCALL_COMPAT
 		ret = __fcntl_compat(fd, cmd, va_arg(ap, void *));
 #else
 		ret = __sys_fcntl(fd, cmd, va_arg(ap, void *));
 #endif
-		_thr_cancel_leave(curthread);
+		_thr_cancel_leave_defer(curthread, ret == -1);
 		break;
 	default:
 #ifdef SYSCALL_COMPAT
@@ -280,30 +306,38 @@ __fcntl(int fd, int cmd,...)
 
 __weak_reference(__fsync, fsync);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled after system call.
+ */
 int
 __fsync(int fd)
 {
 	struct pthread *curthread = _get_curthread();
 	int	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 0);
 	ret = __sys_fsync(fd);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, 1);
 
 	return (ret);
 }
 
 __weak_reference(__msync, msync);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled after system call.
+ */
 int
 __msync(void *addr, size_t len, int flags)
 {
 	struct pthread *curthread = _get_curthread();
 	int	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 0);
 	ret = __sys_msync(addr, len, flags);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, 1);
 
 	return ret;
 }
@@ -326,6 +360,10 @@ __nanosleep(const struct timespec *time_to_sleep,
 
 __weak_reference(__open, open);
 
+/*
+ * Cancellation behavior:
+ *   If the thread is canceled, file is not opened.
+ */
 int
 __open(const char *path, int flags,...)
 {
@@ -334,8 +372,6 @@ __open(const char *path, int flags,...)
 	int	mode = 0;
 	va_list	ap;
 
-	_thr_cancel_enter(curthread);
-	
 	/* Check if the file is being created: */
 	if (flags & O_CREAT) {
 		/* Get the creation mode: */
@@ -344,15 +380,19 @@ __open(const char *path, int flags,...)
 		va_end(ap);
 	}
 	
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_open(path, flags, mode);
-
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
 	return ret;
 }
 
 __weak_reference(__openat, openat);
 
+/*
+ * Cancellation behavior:
+ *   If the thread is canceled, file is not opened.
+ */
 int
 __openat(int fd, const char *path, int flags, ...)
 {
@@ -361,7 +401,6 @@ __openat(int fd, const char *path, int flags, ...)
 	int	mode = 0;
 	va_list	ap;
 
-	_thr_cancel_enter(curthread);
 	
 	/* Check if the file is being created: */
 	if (flags & O_CREAT) {
@@ -371,30 +410,40 @@ __openat(int fd, const char *path, int flags, ...)
 		va_end(ap);
 	}
 	
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_openat(fd, path, flags, mode);
-
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
 	return ret;
 }
 
 __weak_reference(__poll, poll);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns something,
+ *   the thread is not canceled.
+ */
 int
 __poll(struct pollfd *fds, unsigned int nfds, int timeout)
 {
 	struct pthread *curthread = _get_curthread();
 	int ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_poll(fds, nfds, timeout);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
 	return ret;
 }
 
 __weak_reference(___pselect, pselect);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns something,
+ *   the thread is not canceled.
+ */
 int 
 ___pselect(int count, fd_set *rfds, fd_set *wfds, fd_set *efds, 
 	const struct timespec *timo, const sigset_t *mask)
@@ -402,45 +451,59 @@ ___pselect(int count, fd_set *rfds, fd_set *wfds, fd_set *efds,
 	struct pthread *curthread = _get_curthread();
 	int ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_pselect(count, rfds, wfds, efds, timo, mask);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
 	return (ret);
 }
 
 __weak_reference(__read, read);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call got some data, 
+ *   the thread is not canceled.
+ */
 ssize_t
 __read(int fd, void *buf, size_t nbytes)
 {
 	struct pthread *curthread = _get_curthread();
 	ssize_t	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_read(fd, buf, nbytes);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 
 	return ret;
 }
 
 __weak_reference(__readv, readv);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call got some data, 
+ *   the thread is not canceled.
+ */
 ssize_t
 __readv(int fd, const struct iovec *iov, int iovcnt)
 {
 	struct pthread *curthread = _get_curthread();
 	ssize_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_readv(fd, iov, iovcnt);
-	_thr_cancel_leave(curthread);
-
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	return ret;
 }
 
 __weak_reference(__recvfrom, recvfrom);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call got some data, 
+ *   the thread is not canceled.
+ */
 ssize_t
 __recvfrom(int s, void *b, size_t l, int f, struct sockaddr *from,
     socklen_t *fl)
@@ -448,28 +511,38 @@ __recvfrom(int s, void *b, size_t l, int f, struct sockaddr *from,
 	struct pthread *curthread = _get_curthread();
 	ssize_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_recvfrom(s, b, l, f, from, fl);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	return (ret);
 }
 
 __weak_reference(__recvmsg, recvmsg);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call got some data, 
+ *   the thread is not canceled.
+ */
 ssize_t
 __recvmsg(int s, struct msghdr *m, int f)
 {
 	struct pthread *curthread = _get_curthread();
 	ssize_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_recvmsg(s, m, f);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	return (ret);
 }
 
 __weak_reference(__select, select);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns something,
+ *   the thread is not canceled.
+ */
 int 
 __select(int numfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
 	struct timeval *timeout)
@@ -477,28 +550,38 @@ __select(int numfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
 	struct pthread *curthread = _get_curthread();
 	int ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_select(numfds, readfds, writefds, exceptfds, timeout);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	return ret;
 }
 
 __weak_reference(__sendmsg, sendmsg);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call sent
+ *   data, the thread is not canceled.
+ */
 ssize_t
 __sendmsg(int s, const struct msghdr *m, int f)
 {
 	struct pthread *curthread = _get_curthread();
 	ssize_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_sendmsg(s, m, f);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret <= 0);
 	return (ret);
 }
 
 __weak_reference(__sendto, sendto);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call sent some
+ *   data, the thread is not canceled.
+ */
 ssize_t
 __sendto(int s, const void *m, size_t l, int f, const struct sockaddr *t,
     socklen_t tl)
@@ -506,9 +589,9 @@ __sendto(int s, const void *m, size_t l, int f, const struct sockaddr *t,
 	struct pthread *curthread = _get_curthread();
 	ssize_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_sendto(s, m, l, f, t, tl);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret <= 0);
 	return (ret);
 }
 
@@ -544,16 +627,20 @@ ___system(const char *string)
 
 __weak_reference(___tcdrain, tcdrain);
 
+/*
+ * Cancellation behavior:
+ *   If thread is canceled, the system call is not completed,
+ *   this means not all bytes were drained.
+ */
 int
 ___tcdrain(int fd)
 {
 	struct pthread *curthread = _get_curthread();
 	int	ret;
 	
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __tcdrain(fd);
-	_thr_cancel_leave(curthread);
-
+	_thr_cancel_leave_defer(curthread, ret == -1);
 	return (ret);
 }
 
@@ -574,90 +661,118 @@ ___usleep(useconds_t useconds)
 
 __weak_reference(___wait, wait);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns
+ *   a child pid, the thread is not canceled.
+ */
 pid_t
 ___wait(int *istat)
 {
 	struct pthread *curthread = _get_curthread();
 	pid_t	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __wait(istat);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret <= 0);
 
 	return ret;
 }
 
 __weak_reference(__wait3, wait3);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns
+ *   a child pid, the thread is not canceled.
+ */
 pid_t
 __wait3(int *status, int options, struct rusage *rusage)
 {
 	struct pthread *curthread = _get_curthread();
 	pid_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = _wait4(WAIT_ANY, status, options, rusage);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret <= 0);
 
 	return (ret);
 }
 
 __weak_reference(__wait4, wait4);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns
+ *   a child pid, the thread is not canceled.
+ */
 pid_t
 __wait4(pid_t pid, int *status, int options, struct rusage *rusage)
 {
 	struct pthread *curthread = _get_curthread();
 	pid_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_wait4(pid, status, options, rusage);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret <= 0);
 
 	return ret;
 }
 
 __weak_reference(___waitpid, waitpid);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the system call returns
+ *   a child pid, the thread is not canceled.
+ */
 pid_t
 ___waitpid(pid_t wpid, int *status, int options)
 {
 	struct pthread *curthread = _get_curthread();
 	pid_t	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __waitpid(wpid, status, options);
-	_thr_cancel_leave(curthread);
+	_thr_cancel_leave_defer(curthread, ret <= 0);
 	
 	return ret;
 }
 
 __weak_reference(__write, write);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the thread wrote some data,
+ *   it is not canceled.
+ */
 ssize_t
 __write(int fd, const void *buf, size_t nbytes)
 {
 	struct pthread *curthread = _get_curthread();
 	ssize_t	ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_write(fd, buf, nbytes);
-	_thr_cancel_leave(curthread);
-
+	_thr_cancel_leave_defer(curthread, (ret <= 0));
 	return ret;
 }
 
 __weak_reference(__writev, writev);
 
+/*
+ * Cancellation behavior:
+ *   Thread may be canceled at start, but if the thread wrote some data,
+ *   it is not canceled.
+ */
 ssize_t
 __writev(int fd, const struct iovec *iov, int iovcnt)
 {
 	struct pthread *curthread = _get_curthread();
 	ssize_t ret;
 
-	_thr_cancel_enter(curthread);
+	_thr_cancel_enter_defer(curthread, 1);
 	ret = __sys_writev(fd, iov, iovcnt);
-	_thr_cancel_leave(curthread);
-
+	_thr_cancel_leave_defer(curthread, (ret <= 0));
 	return ret;
 }