Implemented complete support for test target.

You can now `make test' on any port to run test sequence, no-op by default.
If a port defines TEST_TARGET, it'll run sub-make with specified target,
usually `check' or `test', useful if upstream supports that. The port may
instead define custom do-test target, as well as usual satellite targets:

  {pre,do,post}-test, {pre,do,post}-test-OPT, {pre,do,post}-test-OPT-off

`make test' builds and stages port first, so test may use both WRKDIR and
STAGEDIR, and both BUILD and RUN depends are available for test target.
Additionally, TEST_DEPENDS is now properly supported and may be used to
define additional depends specifically for testing.

Framework may define default tests for specific cases. For instance,
perl5.mk and cran.mk already provide default test target on their own.

This commit also converts my ports which have tests to this new framework.

Approved by:	portmgr (bapt)
Differential Revision:	D3680

11 files changed, 155 insertions, 45 deletions

diff --git a/CHANGES b/CHANGES
index e70274fd0e40..c151cc3d5a04 100644
--- a/CHANGES
+++ b/CHANGES
@@ -10,6 +10,26 @@ in the release notes and/or placed into UPDATING.
 
 All ports committers are allowed to commit to this file.
 
+20150928:
+AUTHOR: amdmi3@FreeBSD.org
+
+  Implemented complete support for test target.
+
+  You can now `make test' on any port to run test sequence, no-op by default.
+  If a port defines TEST_TARGET, it'll run sub-make with specified target,
+  usually `check' or `test' if upstream supports that. The port may also
+  define custom do-test target, as well as usual satellite targets:
+
+    {pre,do,post}-test, {pre,do,post}-test-OPT, {pre,do,post}-test-OPT-off
+
+  `make test' builds and stages port first, so test may use both WRKDIR and
+  STAGEDIR, and both BUILD and RUN depends are available for test target.
+  Additionally, TEST_DEPENDS is now properly supported and may be used to
+  define additional depends specifically for testing.
+
+  Framework may define default tests for specific cases. For instance,
+  perl5.mk and cran.mk provide default test target on their own.
+
 20150926:
 AUTHOR: bapt@FreeBSD.org
 
diff --git a/Mk/Scripts/depends-list.sh b/Mk/Scripts/depends-list.sh
new file mode 100644
index 000000000000..8409607873f2
--- /dev/null
+++ b/Mk/Scripts/depends-list.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+# MAINTAINER: portmgr@FreeBSD.org
+# $FreeBSD$
+
+set -e
+
+. ${dp_SCRIPTSDIR}/functions.sh
+
+validate_env dp_ALLDEPENDS dp_PORTSDIR dp_PKGNAME
+
+set -u
+
+check_dep() {
+	for _dep ; do
+		myifs=${IFS}
+		IFS=:
+		set -- ${_dep}
+		IFS=${myifs}
+
+		case "${2}" in
+			/*) d=${2} ;;
+			*) d=${dp_PORTSDIR}/${2} ;;
+		esac
+
+		case " ${checked} " in
+			*\ ${d}\ *) continue ;; # Already checked
+		esac
+		checked="${checked} ${d}"
+		if [ ! -d ${d} ]; then
+			echo "${dp_PKGNAME}: \"${d}\" non-existent -- dependency list incomplete" >&2
+			continue
+		fi
+		echo ${d}
+	done
+}
+
+checked=
+check_dep ${dp_ALLDEPENDS}
diff --git a/Mk/Uses/cran.mk b/Mk/Uses/cran.mk
index dd6006409330..20f16a0500e1 100644
--- a/Mk/Uses/cran.mk
+++ b/Mk/Uses/cran.mk
@@ -41,14 +41,14 @@ WRKSRC?=	${WRKDIR}/${PORTNAME}
 NO_BUILD=	yes
 R_COMMAND=	${LOCALBASE}/bin/R
 
-.if !target(regression-test)
+.if !target(do-test)
 R_POSTCMD_CHECK_OPTIONS?=	--timings
 
 .if !exists(${LOCALBASE}/bin/pdflatex)
 R_POSTCMD_CHECK_OPTIONS+=	--no-manual --no-rebuild-vignettes
 .endif
 
-regression-test: build
+do-test:
 	@cd ${WRKDIR} ; ${SETENV} ${MAKE_ENV} _R_CHECK_FORCE_SUGGESTS_=FALSE \
 	${R_COMMAND} ${R_PRECMD_CHECK_OPTIONS} CMD check \
 	${R_POSTCMD_CHECK_OPTIONS} ${PORTNAME}
diff --git a/Mk/Uses/perl5.mk b/Mk/Uses/perl5.mk
index 818954fb2c22..58201234d712 100644
--- a/Mk/Uses/perl5.mk
+++ b/Mk/Uses/perl5.mk
@@ -308,19 +308,14 @@ fix-perl-things:
 	@${RM} -f ${STAGEDIR}${PREFIX}/lib/perl5/${PERL_VER}/${PERL_ARCH}/perllocal.pod* || :
 	@${RMDIR} -p ${STAGEDIR}${PREFIX}/lib/perl5/${PERL_VER}/${PERL_ARCH} 2>/dev/null || :
 
-.if !target(regression-test)
-TEST_ARGS?=	${MAKE_ARGS}
-TEST_ENV?=	${MAKE_ENV}
+.if !target(do-test)
 TEST_TARGET?=	test
 TEST_WRKSRC?=	${BUILD_WRKSRC}
-.if !target(test)
-test: regression-test
-.endif # test
-regression-test: build
+do-test:
 .if ${USE_PERL5:Mmodbuild*}
-	-cd ${TEST_WRKSRC}/ && ${SETENV} ${TEST_ENV} ${PERL5} ${PL_BUILD} ${TEST_TARGET} ${TEST_ARGS}
+	cd ${TEST_WRKSRC}/ && ${SETENV} ${TEST_ENV} ${PERL5} ${PL_BUILD} ${TEST_TARGET} ${TEST_ARGS}
 .elif ${USE_PERL5:Mconfigure}
-	-cd ${TEST_WRKSRC}/ && ${SETENV} ${TEST_ENV} ${MAKE_CMD} ${TEST_ARGS} ${TEST_TARGET}
+	cd ${TEST_WRKSRC}/ && ${SETENV} ${TEST_ENV} ${MAKE_CMD} ${TEST_ARGS} ${TEST_TARGET}
 .endif # USE_PERL5:Mmodbuild*
 .endif # regression-test
 .endif # defined(_POSTMKINCLUDED)
diff --git a/Mk/bsd.options.mk b/Mk/bsd.options.mk
index 594e8b4432bb..b299e548f43e 100644
--- a/Mk/bsd.options.mk
+++ b/Mk/bsd.options.mk
@@ -162,6 +162,7 @@ _OPTIONS_TARGETS=	fetch:300:pre fetch:500:do fetch:700:post \
 			configure:300:pre configure:500:do configure:700:post \
 			build:300:pre build:500:do build:700:post \
 			install:300:pre install:500:do install:700:post  \
+			test:300:pre test:500:do test:700:post  \
 			package:300:pre package:500:do package:700:post \
 			stage:800:post
 
diff --git a/Mk/bsd.port.mk b/Mk/bsd.port.mk
index 9f21f7da7bab..da50034a4526 100644
--- a/Mk/bsd.port.mk
+++ b/Mk/bsd.port.mk
@@ -294,6 +294,17 @@ FreeBSD_MAINTAINER=	portmgr@FreeBSD.org
 #				  if a particular version is desired.
 # LIB_DEPENDS	- A list of "lib:dir[:target]" tuples of other ports this
 #				  package depends on.  "lib" is the name of a shared library.
+# TEST_DEPENDS	- A list of "path:dir[:target]" tuples of other ports this
+#				  package depends on in the "test" stage.  "path" is the
+#				  name of a file if it starts with a slash (/), an
+#				  executable otherwise.  make will test for the existence
+#				  (if it is a full pathname) or search for it in your
+#				  $PATH (if it is an executable) and go into "dir" to do
+#				  a "make all install" if it's not found.  If the third
+#				  field ("target") exists, it will be used instead of
+#				  ${DEPENDS_TARGET}.  The first field also supports a
+#				  package name with a version range, in the form package>=1.2
+#				  if a particular version is desired.
 # DEPENDS_TARGET
 #				- The default target to execute when a port is calling a
 #				  dependency.
@@ -648,6 +659,9 @@ FreeBSD_MAINTAINER=	portmgr@FreeBSD.org
 # run-depends-list
 #				- Show all directories which are run-dependencies
 #				  for this port.
+# test-depends-list
+#				- Show all directories which are test-dependencies
+#				  for this port.
 #
 # extract		- Unpacks ${DISTFILES} into ${WRKDIR}.
 # patch			- Apply any provided patches to the source.
@@ -659,6 +673,7 @@ FreeBSD_MAINTAINER=	portmgr@FreeBSD.org
 #				  flag.
 # deinstall		- Remove the installation.
 # deinstall-all	- Remove all installations with the same PKGORIGIN.
+# test			- Run tests for the port.
 # package		- Create a package from an _installed_ port.
 # package-recursive
 #				- Create a package for a port and _all_ of its dependencies.
@@ -715,6 +730,7 @@ FreeBSD_MAINTAINER=	portmgr@FreeBSD.org
 #
 # NO_BUILD		- Use a dummy (do-nothing) build target.
 # NO_INSTALL	- Use a dummy (do-nothing) install target.
+# NO_TEST		- Use a dummy (do-nothing) test target.
 #
 # Here are some variables used in various stages.
 #
@@ -843,6 +859,18 @@ FreeBSD_MAINTAINER=	portmgr@FreeBSD.org
 #				- Disable CCACHE support for example for certain ports if
 #				  CCACHE is enabled.  User settable.
 #
+# For test:
+#
+# TEST_TARGET	- Target for sub-make in test stage. If not defined,
+#				  no default test target is provided.
+#				  Default: (none)
+# TEST_WRKSRC	- Directory to do test in (default: ${WRKSRC}).
+# TEST_ENV		- Additional environment vars passed to sub-make in test
+#				  stage
+#				  Default: ${MAKE_ENV}
+# TEST_ARGS		- Any extra arguments to sub-make in test stage
+#				  Default: ${MAKE_ARGS}
+#
 # For install:
 #
 # INSTALL_TARGET
@@ -1423,7 +1451,7 @@ UID!=	${ID} -u
 DESTDIRNAME?=	DESTDIR
 
 # setup empty variables for USES targets
-.for target in sanity fetch extract patch configure build install package stage
+.for target in sanity fetch extract patch configure build install test package stage
 _USES_${target}?=
 .endfor
 
@@ -1478,6 +1506,9 @@ PKG_NOTES+=	no_provide_shlib
 PKG_NOTE_no_provide_shlib=	yes
 .endif
 
+TEST_ARGS?=		${MAKE_ARGS}
+TEST_ENV?=		${MAKE_ENV}
+
 PKG_ENV+=		PORTSDIR=${PORTSDIR}
 CONFIGURE_ENV+=	XDG_DATA_HOME=${WRKDIR} \
 				XDG_CONFIG_HOME=${WRKDIR} \
@@ -1547,6 +1578,7 @@ PATCH_WRKSRC?=	${WRKSRC}
 CONFIGURE_WRKSRC?=	${WRKSRC}
 BUILD_WRKSRC?=	${WRKSRC}
 INSTALL_WRKSRC?=${WRKSRC}
+TEST_WRKSRC?=	${WRKSRC}
 
 PLIST_SUB+=	OSREL=${OSREL} PREFIX=%D LOCALBASE=${LOCALBASE} \
 			RESETPREFIX=${PREFIX}
@@ -1933,6 +1965,7 @@ REINPLACE_CMD?=	${SED} ${REINPLACE_ARGS}
 EXTRACT_COOKIE?=	${WRKDIR}/.extract_done.${PORTNAME}.${PREFIX:S/\//_/g}
 CONFIGURE_COOKIE?=	${WRKDIR}/.configure_done.${PORTNAME}.${PREFIX:S/\//_/g}
 INSTALL_COOKIE?=	${WRKDIR}/.install_done.${PORTNAME}.${PREFIX:S/\//_/g}
+TEST_COOKIE?=		${WRKDIR}/.test_done.${PORTNAME}.${PREFIX:S/\//_/g}
 BUILD_COOKIE?=		${WRKDIR}/.build_done.${PORTNAME}.${PREFIX:S/\//_/g}
 PATCH_COOKIE?=		${WRKDIR}/.patch_done.${PORTNAME}.${PREFIX:S/\//_/g}
 PACKAGE_COOKIE?=	${WRKDIR}/.package_done.${PORTNAME}.${PREFIX:S/\//_/g}
@@ -2793,7 +2826,7 @@ IGNORECMD=	${ECHO_MSG} "===>  ${PKGNAME} "${IGNORE:Q}.;exit 1
 .endif
 
 _TARGETS=	check-sanity fetch checksum extract patch configure all build \
-			install reinstall package stage restage
+			install reinstall test package stage restage
 
 .for target in ${_TARGETS}
 .if !target(${target})
@@ -2919,6 +2952,12 @@ build: configure
 	@${TOUCH} ${TOUCH_FLAGS} ${BUILD_COOKIE}
 .endif
 
+# Disable test
+.if defined(NO_TEST) && !target(test)
+test: stage
+	@${TOUCH} ${TOUCH_FLAGS} ${TEST_COOKIE}
+.endif
+
 # Disable package
 .if defined(NO_PACKAGE) && !target(package)
 package:
@@ -3454,6 +3493,23 @@ do-install:
 	@(cd ${INSTALL_WRKSRC} && ${SETENV} ${MAKE_ENV} ${FAKEROOT} ${MAKE_CMD} ${MAKE_FLAGS} ${MAKEFILE} ${MAKE_ARGS} ${INSTALL_TARGET})
 .endif
 
+# Test
+
+.if !target(do-test) && defined(TEST_TARGET)
+DO_MAKE_TEST?=	${SETENV} ${TEST_ENV} ${MAKE_CMD} ${MAKE_FLAGS} ${MAKEFILE} ${TEST_ARGS:C,^${DESTDIRNAME}=.*,,g}
+do-test:
+	@(cd ${TEST_WRKSRC}; if ! ${DO_MAKE_TEST} ${TEST_TARGET}; then \
+		if [ -n "${TEST_FAIL_MESSAGE}" ] ; then \
+			${ECHO_MSG} "===> Tests failed unexpectedly."; \
+			(${ECHO_CMD} "${TEST_FAIL_MESSAGE}") | ${FMT} 75 79 ; \
+			fi; \
+		${FALSE}; \
+		fi)
+.elif !target(do-test)
+do-test:
+	@${DO_NADA}
+.endif
+
 # Package
 
 .if !target(do-package)
@@ -3740,6 +3796,8 @@ stage-message:
 	@${ECHO_MSG} "===>  Staging for ${PKGNAME}"
 install-message:
 	@${ECHO_MSG} "===>  Installing for ${PKGNAME}"
+test-message:
+	@${ECHO_MSG} "===>  Testing for ${PKGNAME}"
 package-message:
 	@${ECHO_MSG} "===>  Building package for ${PKGNAME}"
 
@@ -4260,7 +4318,7 @@ package-noinstall: package
 .if !target(depends)
 depends: pkg-depends extract-depends patch-depends lib-depends fetch-depends build-depends run-depends
 
-.for deptype in PKG EXTRACT PATCH FETCH BUILD LIB RUN
+.for deptype in PKG EXTRACT PATCH FETCH BUILD LIB RUN TEST
 ${deptype:tl}-depends:
 .if defined(${deptype}_DEPENDS) && !defined(NO_DEPENDS)
 	@${SETENV} \
@@ -4291,7 +4349,7 @@ ${deptype:tl}-depends:
 
 # Dependency lists: both build and runtime, recursive.  Print out directory names.
 
-_UNIFIED_DEPENDS=${PKG_DEPENDS} ${EXTRACT_DEPENDS} ${PATCH_DEPENDS} ${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS} ${RUN_DEPENDS}
+_UNIFIED_DEPENDS=${PKG_DEPENDS} ${EXTRACT_DEPENDS} ${PATCH_DEPENDS} ${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS} ${RUN_DEPENDS} ${TEST_DEPENDS}
 _DEPEND_SPECIALS=	${_UNIFIED_DEPENDS:M*\:*\:*:C,^[^:]*:([^:]*):.*$,\1,}
 
 all-depends-list:
@@ -4458,6 +4516,18 @@ RUN-DEPENDS-LIST= \
 		fi; \
 	done | ${SORT} -u
 
+test-depends-list:
+.if defined(TEST_DEPENDS)
+	@${TEST-DEPENDS-LIST}
+.endif
+
+TEST-DEPENDS-LIST= \
+	${SETENV} dp_ALLDEPENDS="${TEST_DEPENDS}" \
+			dp_PORTSDIR="${PORTSDIR}" \
+			dp_PKGNAME="${PKGNAME}" \
+			dp_SCRIPTSDIR="${SCRIPTSDIR}" \
+			${SH} ${SCRIPTSDIR}/depends-list.sh
+
 # Package (recursive runtime) dependency list.  Print out both directory names
 # and package names.
 
@@ -5556,7 +5626,7 @@ show-dev-errors:
 # Please note that the order of the following targets is important, and
 # should not be modified.
 
-_TARGETS_STAGES=	SANITY PKG FETCH EXTRACT PATCH CONFIGURE BUILD INSTALL PACKAGE STAGE
+_TARGETS_STAGES=	SANITY PKG FETCH EXTRACT PATCH CONFIGURE BUILD INSTALL TEST PACKAGE STAGE
 
 # Define the SEQ of actions to take when each target is ran, and which targets
 # it depends on before running its SEQ.
@@ -5620,6 +5690,10 @@ _STAGE_SEQ=		050:stage-message 100:stage-dir 150:run-depends \
 .if defined(DEVELOPER)
 _STAGE_SEQ+=	995:stage-qa
 .endif
+_TEST_DEP=		stage
+_TEST_SEQ=		100:test-message 150:test-depends 300:pre-test 500:do-test \
+				800:post-test \
+				${_OPTIONS_test} ${_USES_test}
 _INSTALL_DEP=	stage
 _INSTALL_SEQ=	100:install-message 150:run-depends 151:lib-depends \
 				200:check-already-installed
@@ -5670,7 +5744,7 @@ _${_t}_REAL_SUSEQ+=	${s}
 # See above *_SEQ and *_DEP. The _DEP will run before this defined target is
 # ran. The _SEQ will run as this target once _DEP is satisfied.
 
-.for target in extract patch configure build stage install package
+.for target in extract patch configure build stage install test package
 
 # Check if config dialog needs to show and execute it if needed. If is it not
 # needed (_OPTIONS_OK), then just depend on the cookie which is defined later
diff --git a/astro/osmium/Makefile b/astro/osmium/Makefile
index 261d5e25691c..49dcc925bac1 100644
--- a/astro/osmium/Makefile
+++ b/astro/osmium/Makefile
@@ -46,9 +46,6 @@ do-build-DOXYGEN-on:
 	cd ${WRKSRC} && ${SETENV} ${MAKE_ENV} \
 		${MAKE} ${MAKE_FLAGS} ${MAKEFILE} ${MAKR_ARGS} doc
 
-regression-test:
-	cd ${WRKSRC}/test && ${SETENV} ${MAKE_ENV} ${SH} run_tests.sh
-
 do-install:
 	cd ${WRKSRC}/include && ${COPYTREE_SHARE} '${PORTNAME} ${PORTNAME}.hpp' \
 		${STAGEDIR}${PREFIX}/include/
@@ -58,4 +55,7 @@ do-install:
 do-install-DOXYGEN-on:
 	cd ${WRKSRC}/doc && ${COPYTREE_SHARE} html ${STAGEDIR}${DOCSDIR}/
 
+do-test:
+	cd ${WRKSRC}/test && ${SETENV} ${MAKE_ENV} ${SH} run_tests.sh
+
 .include <bsd.port.mk>
diff --git a/devel/pire/Makefile b/devel/pire/Makefile
index 466735231403..a62c438f325d 100644
--- a/devel/pire/Makefile
+++ b/devel/pire/Makefile
@@ -20,22 +20,9 @@ GNU_CONFIGURE=	yes
 INSTALL_TARGET=	install-strip
 USES=		autoreconf bison gmake libtool
 USE_LDCONFIG=	yes
+TEST_TARGET=	check
 
 ONLY_FOR_ARCHS=	amd64 i386 ia64
 ONLY_FOR_ARCHS_REASON=	not yet ported to big-endian platforms
 
-OPTIONS_DEFINE=	UNITTEST
-UNITTEST_DESC=	Compile with unittest support
-
-.include <bsd.port.options.mk>
-
-.if ${PORT_OPTIONS:MUNITTEST} || defined(PACKAGE_BUILDING)
-BUILD_DEPENDS+=	cppunit-config:${PORTSDIR}/devel/cppunit
-CONFIGURE_ARGS+=--with-cppunit-prefix=${LOCALBASE}
-
-regression-test: build
-	@cd ${WRKSRC} && ${SETENV} ${MAKE_ENV} ${MAKE_CMD} ${MAKE_FLAGS} ${MAKEFILE} ${MAKE_ARGS} check || \
-		(${CAT} ${WRKSRC}/tests/test-suite.log; false)
-.endif
-
 .include <bsd.port.mk>
diff --git a/devel/sdl2pp/Makefile b/devel/sdl2pp/Makefile
index e0f1d0d12f57..34882aef5365 100644
--- a/devel/sdl2pp/Makefile
+++ b/devel/sdl2pp/Makefile
@@ -19,6 +19,7 @@ USES=		compiler:c++11-lib cmake
 CMAKE_ARGS=	-DSDL2PP_ENABLE_LIVE_TESTS=OFF \
 		-DSDL2PP_WITH_WERROR=ON
 USE_SDL=	sdl2 image2 mixer2 ttf2
+TEST_TARGET=	test
 
 PORTDOCS=	*
 
@@ -40,7 +41,4 @@ post-install:
 post-install-DOXYGEN-on:
 	cd ${CONFIGURE_WRKSRC} && ${COPYTREE_SHARE} doxygen ${STAGEDIR}${DOCSDIR}/
 
-regression-test: build
-	cd ${WRKSRC} && ${DO_MAKE_BUILD} test
-
 .include <bsd.port.post.mk>
diff --git a/games/spring/Makefile b/games/spring/Makefile
index 0b3807956a2e..be891c1d9583 100644
--- a/games/spring/Makefile
+++ b/games/spring/Makefile
@@ -52,6 +52,9 @@ LEI_PREFIX=	${LOCALBASE}
 # Do not exctract bundled copies of header files for 3rd-party packages:
 EXTRACT_AFTER_ARGS=--no-same-owner --no-same-permissions --exclude ${WRKSRC:T}/include
 
+# The check-target fails right now: https://springrts.com/mantis/view.php?id=4736
+TEST_TARGET=	check
+
 PORTDOCS=	*
 PORTDATA=	*
 
@@ -75,10 +78,6 @@ CXXFLAGS+=	-Wno-deprecated		# Too much noise
 
 PR_DOWNLOADER_LIB_DEPENDS=libcurl.so:${PORTSDIR}/ftp/curl
 
-# The check-target fails right now: https://springrts.com/mantis/view.php?id=4736
-check test xregression-test: build
-	${MAKE} -C ${WRKSRC} check
-
 .include <bsd.port.pre.mk>
 
 .if ${OPSYS} == FreeBSD && ${OSVERSION} < 1000000
diff --git a/graphics/glosm/Makefile b/graphics/glosm/Makefile
index 1218069c1ed6..011ed9a82e70 100644
--- a/graphics/glosm/Makefile
+++ b/graphics/glosm/Makefile
@@ -18,15 +18,13 @@ USE_GL=		gl
 USE_SDL=	sdl
 USE_GITHUB=	yes
 GH_ACCOUNT=	AMDmi3
+TEST_TARGET=	test
 
 PORTDOCS=	README ChangeLog
 
 OPTIONS_DEFINE=	DOCS
 
-regression-test:
-	cd ${BUILD_WRKSRC} && ctest
-
-post-install:
+post-install-DOCS-on:
 	${MKDIR} ${STAGEDIR}${DOCSDIR}
 .for f in ${PORTDOCS}
 	${INSTALL_DATA} ${WRKSRC}/${f} ${STAGEDIR}${DOCSDIR}/