Nicer test framework -- more internals in the .C, nicer and more correct

test result output.

3 files changed, 552 insertions, 104 deletions

diff --git a/tests/demo.c b/tests/demo.c
index 2b874b9e..cfa879da 100644
--- a/tests/demo.c
+++ b/tests/demo.c
@@ -15,19 +15,35 @@ test_main("Things work", {
 		test_so(y == 3);
 	});
 
-test_convey("Bounce", {
-	test_convey("Arithmetic", {
-		test_so(y == 2);
-		test_convey("Addition", {
-			test_so(x + y == 3);
-			test_so(x + y + y == 5);
-			y = 5;
-			test_so(x + y == 6);
+	test_convey("Operations (Outer)", {
+		test_convey("Arithmetic", {
+			test_so(y == 2);
+			test_convey("Addition", {
+				test_so(x + y == 3);
+				test_so(x + y + y == 5);
+				test_so(x == 9);
+				y = 5;
+				test_so(x + y == 6);
+			});
+			test_convey("Subtraction", {
+				test_so(x - y == -1);
+				test_so(y - x == 1);
+			});
+		});
+	});
+
+	test_convey("Middle test is skipped", {
+		test_convey("Start", {
+			test_so(1 == 1);
+		});
+		test_convey("Middle (Skip?)", {
+			test_so(9 - 1 == 8);
+			test_skip();
+			test_so(0 == 1);
 		});
-		test_convey("Subtraction", {
-			test_so(x - y == -1);
-			test_so(y - x == 1);
+		test_convey("Ending", {
+			test_so(2 == 2);
 		});
 	});
-});
+
 })
diff --git a/tests/test.c b/tests/test.c
index 2a881307..6aa32efe 100644
--- a/tests/test.c
+++ b/tests/test.c
@@ -23,47 +23,353 @@
 #include <stddef.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <stdint.h>
 #include <setjmp.h>
 #include <stdarg.h>
+#include <string.h>
+#include <locale.h>
+#include <langinfo.h>
+#include <unistd.h>
+#include <time.h>
 
 #include "test.h"
 
 test_ctx_t *T_C = NULL;
 
-void
-test_ctx_print_start(test_ctx_t *ctx)
+static const char *sym_pass = ".";
+static const char *sym_skip = "?";
+static const char *sym_fail = "X";
+static const char *sym_fatal = "!";
+static const char *color_none = "";
+static const char *color_green = "";
+static const char *color_red = "";
+static const char *color_yellow = "";
+
+static int verbose = 0;
+static int nasserts = 0;
+static int nskips = 0;
+static const char *color_asserts = "";
+
+typedef struct tperfcnt {
+	uint64_t	pc_count;
+	uint64_t	pc_rate;
+} tperfcnt_t;
+
+typedef struct tlog {
+	const char	*l_buf;
+	size_t		l_size;
+	size_t		l_length;
+} tlog_t;
+
+typedef struct tctx {
+	char		t_name[256];
+	struct tctx	*t_parent;
+	struct tctx	*t_root;
+	int		t_level;
+	int		t_done;
+	int		t_started;
+	jmp_buf		*t_jmp;
+
+	void		(*t_cleanup)(void *);
+	void		*t_cleanup_arg;
+
+	int		t_nloops;
+
+	int		t_fatal;
+	int		t_fail;
+	int		t_skip;
+	int		t_printed;
+	tperfcnt_t	t_starttime;
+	tperfcnt_t	t_endtime;
+	tlog_t		t_fatallog;
+	tlog_t		t_faillog;
+	tlog_t		t_debuglog;
+} tctx_t;
+
+#define		PARENT(t)	((t_ctx_t *)(t->t_parent->t_data))
+
+static void test_print_result(tctx_t *);
+static void test_getperfcnt(tperfcnt_t *);
+static uint64_t test_perfdelta(tperfcnt_t *, tperfcnt_t *, int *, int *);
+
+void 
+test_print_result(tctx_t *t)
+{
+	int secs, usecs;
+
+	if ((t->t_root == t) && !t->t_printed) {
+
+		t->t_printed = 1;
+
+		test_getperfcnt(&t->t_endtime);
+		test_perfdelta(&t->t_starttime, &t->t_endtime, &secs, &usecs);
+
+		if (!verbose) {
+			(void) printf("%-8s%-52s%4d.%03ds\n",
+				t->t_fatal ? "fatal" :
+				t->t_fail ? "fail" : "ok",
+				t->t_name, secs, usecs / 1000);
+		} else {
+			printf("\n\n%s%d assertions thus far%s",
+				color_asserts, nasserts, color_none);
+			if (nskips) {
+				printf(" %s(one or more sections skipped)%s",
+					color_yellow, color_none);
+			}
+			printf("\n\n--- %s: %s (%d.%02d)\n",
+				t->t_fatal ? "FATAL" :
+				t->t_fail ? "FAIL" :
+				"PASS", t->t_name, secs, usecs / 10000);
+		}
+
+		/* XXX: EMIT LOGS */
+	}
+}
+
+int
+test_ctx_init(test_ctx_t *ctx, test_ctx_t *parent, const char *name)
+{
+	tctx_t *t;
+
+	if ((t = ctx->T_data) != NULL) {
+		if (t->t_done) {
+			test_print_result(t);
+			return (1);	/* all done, skip */
+		}
+		return (0);	/* continue onward */
+	}
+	ctx->T_data = (t = calloc(1, sizeof (tctx_t)));
+	if (t == NULL) {
+		/* PANIC */
+		return (1);
+	}
+	t->t_jmp = &ctx->T_jmp;
+
+	(void) snprintf(t->t_name, sizeof(t->t_name)-1, "%s", name);
+	if (parent != NULL) {
+		t->t_parent = parent->T_data;
+		t->t_root = t->t_parent->t_root;
+		t->t_level = t->t_parent->t_level + 1;
+	} else {
+		t->t_parent = t;
+		t->t_root = t;
+	}
+	return (0);
+}
+
+/*
+ * This is called right after setjmp.  The jumped being true indicates
+ * that setjmp returned true, and we are popping the stack.
+ */
+int
+test_ctx_loop(test_ctx_t *ctx, int jumped)
 {
+	tctx_t *t;
 	int i;
-	if (ctx->T_printed) {
+	if ((t = ctx->T_data) == NULL) {
+		return (1);
+	}
+	if (jumped != 0) {
+		if (t->t_cleanup != NULL) {
+			t->t_cleanup(t->t_cleanup_arg);
+		}
+		if ((t->t_parent != t) && (t->t_parent != NULL)) {
+			longjmp(*t->t_parent->t_jmp, 1);
+		}
+		if (t->t_done) {
+			test_print_result(t);
+			return (1);
+		}
+	}
+
+	if (!t->t_started) {
+		t->t_started = 1;
+
+		if (verbose) {
+			if (t->t_root == t) {
+				printf("\n=== RUN: %s\n", t->t_name);
+			} else {
+				printf("\n");
+				for (i = 0; i < t->t_level; i++) {
+					printf("  ");
+				}
+				printf("%s ", t->t_name);
+				fflush(stdout);
+			}
+		}
+
+		test_getperfcnt(&t->t_starttime);
+	}
+	/* Reset TC for the following code. */
+	T_C = ctx;
+	return (0);
+}
+
+void
+test_ctx_fini(test_ctx_t *ctx, int *rvp)
+{
+	tctx_t *t;
+	if ((t = ctx->T_data) == NULL) {
 		return;
 	}
-	ctx->T_printed = 1;
-	if (ctx->T_root) {
-		printf("\n=== RUN: %s\n", ctx->T_name);
-	} else {
-		printf("\n");
-		for (i = 0; i < ctx->T_level; i++) {
-			printf("  ");
+	t->t_done = 1;
+	if (rvp != NULL) {
+		/* exit code 1 is reserved for usage errors */
+		if (t->t_fatal) {
+			*rvp = 3;
+		} else if (t->t_fail) {
+			*rvp = 2;
+		} else {
+			*rvp = 0;
 		}
-		printf("%s ", ctx->T_name);
-		fflush(stdout);
+	}
+	longjmp(*t->t_jmp, 1);
+}
+
+void
+test_ctx_skip(test_ctx_t *ctx)
+{
+	tctx_t *t = ctx->T_data;
+	if (verbose) {
+		(void) printf("%s%s%s", color_none, sym_skip, color_none);
+	}
+	t->t_done = 1;	/* This forces an end */
+	nskips++;
+	longjmp(*t->t_jmp, 1);
+}
+
+void
+test_assert_fail(test_ctx_t *ctx, const char *cond, const char *file, int line)
+{
+	tctx_t *t;
+	if ((t = ctx->T_data) == NULL) {
+		/* PANIC? */
+		/* XXX: */
+	}
+	nasserts++;
+	if (verbose) {
+		(void) printf("%s%s%s", color_yellow, sym_fail, color_none);
+	}
+	if (t->t_root != t) {
+		t->t_root->t_fail++;
+	}
+	color_asserts = color_yellow;
+	t->t_fail++;
+	t->t_done = 1;	/* This forces an end */
+	longjmp(*t->t_jmp, 1);
+}
+
+void
+test_assert_pass(test_ctx_t *ctx, const char *cond, const char *file, int line)
+{
+	tctx_t *t;
+	if ((t = ctx->T_data) == NULL) {
+		/* PANIC? */
+	}
+	nasserts++;
+	if (verbose) {
+		(void) printf("%s%s%s", color_green, sym_pass, color_none);
 	}
 }
 
 void
-test_ctx_print_result(test_ctx_t *ctx)
+test_assert_skip(test_ctx_t *ctx, const char *cond, const char *file, int line)
 {
-	if (ctx->T_root) {
-		printf("\n\n--- %s: %s\n",
-			ctx->T_fatal ? "FATAL" :
-			ctx->T_fail ? "FAIL" :
-			ctx->T_skip ? "SKIP" :
-			"PASS", ctx->T_name);
+	tctx_t *t;
+	if ((t = ctx->T_data) == NULL) {
+		/* PANIC? */
+	}
+	nskips++;
+	if (verbose) {
+		(void) printf("%s%s%s", color_none, sym_pass, color_none);
 	}
 }
 
 void
-test_ctx_log(test_ctx_t ctx, const char *fmt, ...)
+test_assert_fatal(test_ctx_t *ctx, const char *cond, const char *file, int line)
+{
+	tctx_t *t;
+	if ((t = ctx->T_data) == NULL) {
+		/* PANIC? */
+		/* XXX: */
+	}
+	nasserts++;
+	if (verbose) {
+		(void) printf("%s%s%s", color_red, sym_fail, color_none);
+	}
+	if (t->t_root != t) {
+		t->t_root->t_fatal++;
+	}
+	color_asserts = color_red;
+	t->t_fail++;
+	t->t_done = 1;	/* This forces an end */
+	longjmp(*t->t_jmp, 1);
+}
+
+static void
+test_getperfcnt(tperfcnt_t *pc)
+{
+#if defined(_WIN32)
+	LARGE_INTEGER pcnt, pfreq;
+	QueryPerformanceCounter(&pcnt);
+	QueryPerformanceFrequency(&pfreq);
+	pc->pc_count = pcnt.QuadPart;
+	pc->pc_rate = pfreq.QuadPart;
+#elif defined(CLOCK_MONOTONIC)
+	uint64_t usecs;
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	pc->pc_count = ts.tv_sec * 1000000000;
+	pc->pc_count += ts.tv_nsec;
+	pc->pc_rate = 1000000000;
+#else
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	pc->pc_count = tv.tv_secs * 1000000;
+	pc->pc_count += tv.tv_usec;
+	pc->pc_rate = 1000000;
+#endif
+}
+
+/*
+ * Calculates the seconds and usecs between two values.  Returns the
+ * entire usecs as a 64-bit integer.
+ */
+uint64_t
+test_perfdelta(tperfcnt_t *start, tperfcnt_t *end, int *secp, int *usecp)
+{
+	uint64_t delta, rate, sec, usec;
+
+	delta = end->pc_count - start->pc_count;
+	rate = start->pc_rate;
+
+	sec = delta / rate;
+	delta -= (sec * rate);
+
+	/*
+	 * done this way we avoid dividing rate by 1M -- and the above
+	 * ensures we don't wrap.
+	 */
+	usec = (delta * 1000000) / rate;
+
+	if (secp) {
+		*secp = (int)sec;
+	}
+	if (usecp) {
+		*usecp = (int)usec;
+	}
+	return ((sec * 1000000) + usec);
+}
+
+void
+test_ctx_log(test_ctx_t *ctx, const char *fmt, ...)
+{
+}
+
+void
+test_ctx_vlog(test_ctx_t *ctx, const char *fmt, va_list va)
 {
 }
 
@@ -77,10 +383,103 @@ test_ctx_fatal(test_ctx_t *ctx, const char *fmt, ...)
 	printf("\n");
 }
 
+#if 0
+void
+test_ctx_emit_pass(test_ctx_t *ctx)
+{
+	nasserts++;
+	if (verbose) {
+		(void) printf("%s%s%s", color_green, sym_pass, color_none);
+	}
+}
+
+void
+test_ctx_emit_fail(test_ctx_t *ctx)
+{
+	nasserts++;
+	if (verbose) {
+		(void) printf("%s%s%s", color_yellow, sym_fail, color_none);
+	}
+	while (!ctx->T_root) {
+		ctx = ctx->T_parent;
+	}
+	color_asserts = color_yellow;
+	ctx->T_fail++;
+}
+
+void
+test_ctx_emit_fatal(test_ctx_t *ctx)
+{
+	if (verbose) {
+		(void) printf("%s%s%s", color_red, sym_fatal, color_none);
+	}
+	nasserts++;
+	while (!ctx->T_root) {
+		ctx = ctx->T_parent;
+	}
+	ctx->T_fatal++;
+	color_asserts = color_red;
+}
+
+void
+test_ctx_emit_skipped(test_ctx_t *ctx)
+{
+	if (verbose) {
+		(void) printf("%s%s%s", color_none, sym_skip, color_none);
+	}
+	nskips++;
+}
+#endif
+
 extern int test_main_impl(void);
 
+static void
+pretty(void)
+{
+#ifndef _WIN32
+	/* Windows console doesn't do Unicode (consistently). */
+	const char *codeset;
+	const char *term;
+
+	(void) setlocale(LC_ALL, "");
+	codeset = nl_langinfo(CODESET);
+	if ((codeset == NULL) || (strcmp(codeset, "UTF-8") != 0)) {
+		return;
+	}
+	sym_pass = "✔";
+	sym_fail = "✘";
+	sym_fatal = "🔥";
+	sym_skip = "⚠";
+
+	term = getenv("TERM");
+	if (isatty(1) && (term != NULL)) {
+		if ((strstr(term, "xterm") != NULL) ||
+		    (strstr(term, "ansi") != NULL) ||
+		    (strstr(term, "color") != NULL)) {
+		    	color_none = "\e[0m";
+		    	color_green = "\e[32m";
+		    	color_yellow = "\e[33m";
+		    	color_red = "\e[31m";
+		    	color_asserts = color_green;
+		}
+	}
+#endif
+}
+
 int
 main(int argc, char **argv)
 {
+	int i;
+
+	/* Poor man's getopt.  Very poor. */
+	for (i = 1; i < argc; i++) {
+		if (argv[i][0] != '-') {
+			break;
+		}
+		if (strcmp(argv[i], "-v") == 0) {
+			verbose = 1;
+		}
+	}
+	pretty();
 	test_main_impl();
 }
\ No newline at end of file
diff --git a/tests/test.h b/tests/test.h
index 3be760b2..c712cb3f 100644
--- a/tests/test.h
+++ b/tests/test.h
@@ -30,6 +30,7 @@
 #include <stdio.h>
 #include <setjmp.h>
 #include <stdarg.h>
+#include <sys/time.h>
 
 /*
  * This test framework allows one to write tests as a form of assertion,
@@ -68,109 +69,141 @@
  * is a version for C programs.
  */
 
+/*
+ * This structure has to be exposed in order to expose the buffer used for
+ * setjmp.  It's members should never be accessed directly.  These should be
+ * allocated statically in the routine(s) that need custom contexts.  The
+ * framework creates a context automatically for each convey scope.
+ */
 typedef struct test_ctx {
-	const char *T_name;
 	jmp_buf T_jmp;
-	void (*T_cleanup_fn)(void *);
-	void *T_cleanup_arg;
-	int T_fail;
-	int T_fatal;
-	int T_skip;
-	int T_asserts;
-	int T_good;
-	int T_bad;
-	int T_done;
-	int T_root;
-	int T_level;
-	int T_printed;
-	struct test_ctx *T_parent;
+	void 	*T_data;
 } test_ctx_t;
 
 extern test_ctx_t *T_C;
 
-#define test_ctx_do(T_xparent, T_xname, T_xcode)			\
+/* These functions are not for use by tests -- they are used internally. */
+extern int test_ctx_init(test_ctx_t *, test_ctx_t *, const char *);
+extern int test_ctx_loop(test_ctx_t *, int);
+extern void test_ctx_fini(test_ctx_t *, int *);
+extern void test_assert_pass(test_ctx_t *, const char *, const char *, int);
+extern void test_assert_skip(test_ctx_t *, const char *, const char *, int);
+extern void test_assert_fail(test_ctx_t *, const char *, const char *, int);
+extern void test_assert_fatal(test_ctx_t *, const char *, const char *, int);
+
+
+/*
+ * test_ctx_do is a helper function not to be called directly by user
+ * code.  It has to be here exposed, in order for setjmp() to work.  Do
+ * not call it directly, instead use one of the other macros.
+ */
+#define test_ctx_do(T_xparent, T_xname, T_xcode, T_rvp)			\
 do {									\
 	static test_ctx_t T_ctx;					\
-	T_ctx.T_name = T_xname;						\
-	T_ctx.T_parent = T_xparent;					\
-	if (T_xparent != NULL) {					\
-		T_ctx.T_level = T_xparent->T_level + 1;			\
-		T_ctx.T_root = 0;					\
-	} else {							\
-		T_ctx.T_root = 1;					\
-		T_ctx.T_level = 0;					\
-	}								\
-	if (T_ctx.T_done) {						\
-		test_ctx_print_result(&T_ctx);				\
+	int T_jumped;							\
+	if (test_ctx_init(&T_ctx, T_xparent, T_xname) != 0) {		\
 		break;							\
 	}								\
-	if (setjmp(T_ctx.T_jmp) != 0) {					\
-		if (T_ctx.T_cleanup_fn != NULL) {			\
-			T_ctx.T_cleanup_fn(T_ctx.T_cleanup_arg);	\
-		}							\
-		if (!T_ctx.T_root) {					\
-			longjmp(T_ctx.T_parent->T_jmp, 1);		\
-		}							\
-		if (T_ctx.T_done) {					\
-			test_ctx_print_result(&T_ctx);			\
-			break;						\
-		}							\
+	T_jumped = setjmp(T_ctx.T_jmp);					\
+	if (test_ctx_loop(&T_ctx, T_jumped) != 0) {			\
+		break;							\
 	}								\
-	test_ctx_print_start(&T_ctx);					\
 	do {								\
-		T_C = &T_ctx;						\
-		{ T_xcode }						\
-		T_ctx.T_done = 1;					\
+		T_xcode							\
 	} while (0);							\
-	longjmp(T_ctx.T_jmp, 1);					\
-} while (0);
+	test_ctx_fini(&T_ctx, T_rvp);					\
+} while (0)
 
+/*
+ * test_ctx_convey creates a "convey" using a custom parent context.
+ * The idea is that this new context creates a new "root" context,
+ * which can be useful for running tests in separate threads, or for
+ * passing context around explicitly.  It should rarely be needed.
+ */
+#define test_ctx_convey(T_xparent, T_xname, T_xcode)			\
+	test_ctx_do(T_xparent, T_xname, T_xcode, NULL)
+
+/*
+ * test_main is used to wrap the top-level of your test suite, and is
+ * used in lieu of a normal main() function.
+ */
 #define test_main(name, code)						\
 int test_main_impl(void) {						\
 	static test_ctx_t ctx;						\
-	test_ctx_t *T_null = NULL;					\
-	ctx.T_root = 1;							\
-	test_ctx_t *T_xctx = &ctx;					\
-	test_ctx_do(T_null, name, code);				\
-	return (ctx.T_fatal ? 2 : ctx.T_fail ? 1 : 0);			\
-}
-
-#define test_ctx_convey(T_xparent, T_xname, T_xcode)			\
-{									\
-	test_ctx_do( T_xparent, T_xname, T_xcode); 		\
+	int rv;								\
+	test_ctx_do(NULL, name, code, &rv);				\
+	return (rv);							\
 }
 
-#define	test_convey(T_xname, T_xcode)	test_ctx_convey(T_C, T_xname, T_xcode)
-
-
+/*
+ * test_ctx_assert and test_ctx_so allow you to run assertions against
+ * an explicit context.  You shouldn't need these.
+ */
 #define	test_ctx_assert(T_xctx, T_cond)					\
 	if (!(T_cond)) {						\
-		T_xctx->T_bad++;					\
-		test_ctx_fatal(T_xctx, "%s: %d: Assertion failed: %s",	\
-			__FILE__, __LINE__, #T_cond);			\
+		test_assert_fatal(T_xctx, #T_cond, __FILE__, __LINE__);	\
 	} else {							\
-		printf(".");						\
-		T_xctx->T_good++;					\
+		test_assert_pass(T_xctx, #T_cond, __FILE__, __LINE__);	\
 	}
 
-#define	test_assert(T_cond)	test_ctx_assert(T_C, T_cond)
-
 #define test_ctx_so(T_xctx, T_cond)					\
 	if (!(T_cond)) {						\
-		printf("X");						\
-		T_xctx->T_bad++;					\
+		test_assert_fail(T_xctx, #T_cond, __FILE__, __LINE__);	\
 	} else {							\
-		printf(".");						\
-		T_xctx->T_good++;					\
+		test_assert_pass(T_xctx, #T_cond, __FILE__, __LINE__);	\
 	}
 
-#define test_so(T_cond)	test_ctx_so(T_C, T_cond)
+/*
+ * test_ctx_skip can be used to skip further processing in the context.
+ */
+extern void test_ctx_skip(test_ctx_t *);
+
+/*
+ * These are convenience versions that use the "global" context.
+ * Note that normally convey() will create its own contexts.  These
+ * are the macros you should use.
+ */
+
+/*
+ * test_convey(name, <code>) starts a convey context, with <code> as
+ * the body.  The <code> is its scope, and may be called repeatedly
+ * within the body of a loop.
+ */
+#define	test_convey(T_xn, T_xc)	test_ctx_convey(T_C, T_xn, T_xc)
 
+/*
+ * test_assert is just like assert(3), but applies to the context.
+ * Failures in these are treated as "fatal" failures; and get higher
+ * alerting than other other kinds of failures.  The entire test run
+ * all the way to the root context is aborted.  (Other root contexts
+ * can run, however.)
+ */
 #define	test_assert(T_cond)	test_ctx_assert(T_C, T_cond)
 
-extern void test_ctx_print_start(test_ctx_t *);
-extern void test_ctx_print_result(test_ctx_t *);
-extern void test_ctx_fatal(test_ctx_t *ctx, const char *fmt, ...);
-extern int test_main_impl(void);
+/*
+ * test_so() is like test_assert, but failures here only abort the current
+ * test.
+ */
+#define test_so(T_cond)		test_ctx_so(T_C, T_cond)
+
+/*
+ * test_skip() just stops processing of the rest of the current context,
+ * and records that processing was skipped.
+ */
+#define	test_skip()		test_ctx_skip(T_C)
+
+/*
+ * test_skip_assert() is used to skip processing of a single assertion.
+ * Further processing in the same context continues.
+ */
+#define	test_skip_assert(T_cnd)	test_assert_skip(T_C, T_cnd)
+
+/*
+ * test_skip_convey() is used to skip a convey context.  This is intended
+ * to permit changing "test_convey", to "test_skip_convey".  This is logged,
+ * and the current convey context continues processing.
+ */
+#define	test_skip_convey(X_n, X_c)	/* TBD */
+
 
 #endif	/* TEST_TEST_H */