implement experimental pure-sh testsuite runner and generation

Makefile.in

@@ -547,3 +547,7 @@ test check: test_compile
 test_compile: $(LOCAL_INSTALL_PATH)
 	$(Q)$(MAKE) -C test compile \
 		$(if $(O),top_builddir=$(O)/)
+
+test_gen: $(LOCAL_INSTALL_PATH)
+	$(Q)$(MAKE) -C test gen \
+		$(if $(O),top_builddir=$(O)/)

test/Makefile

@@ -47,6 +47,10 @@ test check all: run
 
 run: subdirs_run
 
+gen:
+	-rm -f $(top_builddir)/test/uclibcng-testrunner.in
+	$(MAKE) run UCLIBCNG_GENERATE_TESTRUNNER=1
+
 compile: $(top_builddir)$(LOCAL_INSTALL_PATH) subdirs_compile
 
 $(top_builddir)$(LOCAL_INSTALL_PATH):
@@ -68,6 +72,7 @@ $(patsubst %, _dir_%, $(DIRS)) : dummy
 
 $(patsubst %, _dirrun_%, $(DIRS)) : dummy
 	$(Q)$(MAKE) -C $(patsubst _dirrun_%, %, $@) run \
+		UCLIBCNG_TEST_SUBDIR=$(strip $(patsubst _dirrun_%, %, $@)) \
 		KCONFIG_CONFIG=$(KCONFIG_CONFIG)
 
 $(patsubst %, _dircompile_%, $(DIRS)) : dummy

test/Test.mak

@@ -4,6 +4,8 @@
 #
 # Licensed under the LGPL v2.1, see the file COPYING.LIB in this tarball.
 
+shellescape='$(subst ','\'',$(1))'
+
 ifeq ($(TESTS),)
 TESTS := $(patsubst %.c,%,$(wildcard *.c))
 endif
@@ -76,7 +78,7 @@ define exec_test
 		test -z "$$expected_ret" && export expected_ret=0 ; \
 	if ! test $$ret -eq $$expected_ret ; then \
 		echo "ret == $$ret ; expected_ret == $$expected_ret" ; \
-		echo "The output of failed test is:"; \
+		echo "The output of the failed test is:"; \
 		cat "$(binary_name).out"; \
 		exit 1 ; \
 	fi
@@ -89,11 +91,21 @@ run: $(RUN_TARGETS)
 $(addsuffix .exe,$(U_TARGETS)): SIMULATOR:=$(SIMULATOR_uclibc)
 $(addsuffix .exe,$(G_TARGETS)): SIMULATOR:=$(SIMULATOR_glibc)
 $(RUN_TARGETS):
+ifeq (1,$(UCLIBCNG_GENERATE_TESTRUNNER))
+	expected_ret="$(RET_$(tst_src_name))"; echo \
+	    "$${expected_ret:-0}" \
+	    $(call shellescape,$(tst_src_name)) \
+	    $(call shellescape,$(binary_name)) \
+	    $(call shellescape,$(UCLIBCNG_TEST_SUBDIR)) \
+	    $(call shellescape,$(WRAPPER) $(WRAPPER_$(tst_src_name)) ./$(binary_name) $(OPTS) $(OPTS_$(tst_src_name))) \
+	    >>$(top_builddir)/test/uclibcng-testrunner.in
+else
 	$(exec_test)
 	$(diff_test)
 ifeq ($(UCLIBC_ONLY),)
 	$(uclibc_glibc_diff_test)
 endif
+endif
 
 compile: $(COMPILE_TARGETS)
 

test/uclibcng-testrunner.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+#-
+# Copyright (c) 2015
+#	Thorsten "mirabilos" Glaser <tg@mirbsd.org>
+#
+# Provided that these terms and disclaimer and all copyright notices
+# are retained or reproduced in an accompanying document, permission
+# is granted to deal in this work without restriction, including un-
+# limited rights to use, publicly perform, distribute, sell, modify,
+# merge, give away, or sublicence.
+#
+# This work is provided "AS IS" and WITHOUT WARRANTY of any kind, to
+# the utmost extent permitted by applicable law, neither express nor
+# implied; without malicious intent or gross negligence. In no event
+# may a licensor, author or contributor be held liable for indirect,
+# direct, other damage, loss, or other issues arising in any way out
+# of dealing in the work, even if advised of the possibility of such
+# damage or existence of a defect, except proven that it results out
+# of said person's immediate fault when using the work as intended.
+#-
+# Testsuite runner
+
+die() {
+	echo >&2 E: "$*"
+	exit 1
+}
+
+test -s uclibcng-testrunner.in || die uclibcng-testrunner.in not found
+
+nfail=0
+npass=0
+while read expected_ret tst_src_name binary_name subdir cmd; do
+	printf '.... %s\r' "$binary_name"
+	(cd $subdir && eval "$cmd" >$binary_name.out 2>&1) </dev/null
+	ret=$?
+	test $ret = "$expected_ret" || {
+		echo "FAIL $binary_name got $ret expected $expected_ret"
+		nfail=`expr $nfail + 1`
+		sed 's/^/	/' <$subdir/$binary_name.out
+		continue
+	}
+	for x in $binary_name.out $test_src_name.out -; do
+		if test x"$x" = x"-"; then
+			echo "PASS $binary_name"
+			npass=`expr $npass + 1`
+			break
+		fi
+		test -e "$subdir/$x.good" || continue
+		if d=`diff -u "$subdir/$binary_name.out" "$subdir/$x.good"`; then
+			echo "PASS $binary_name"
+			npass=`expr $npass + 1`
+		else
+			echo "FAIL $binary_name expected output differs"
+			nfail=`expr $nfail + 1`
+			echo "$d" | sed 's/^/       /'
+		fi
+		break
+	done
+done <uclibcng-testrunner.in
+echo Total failed: $nfail
+echo Total passed: $npass
+test $nfail = 0