diff --git a/utest.h b/utest.h
index 11e4a51..d932e1b 100644
--- a/utest.h
+++ b/utest.h
@@ -204,6 +204,9 @@
#define UTEST_NULL 0
#endif
+#ifndef UTEST_DETECT_USE_COLOUR
+#define UTEST_COLOUR_OUTPUT() true
+#else
#ifdef _MSC_VER
/*
io.h contains definitions for some structures with natural padding. This is
@@ -224,6 +227,7 @@
#define UTEST_COLOUR_OUTPUT() (isatty(STDOUT_FILENO))
#endif
#endif
+#endif
static UTEST_INLINE void *utest_realloc(void *const pointer, size_t new_size) {
void *const new_pointer = realloc(pointer, new_size);
@@ -271,12 +275,23 @@
utest_testcase_t func;
size_t index;
char *name;
+ char *output;
+ size_t iterations;
+ utest_int64_t ns;
+ // Store stats for benchmark tests
+ utest_int64_t best_avg_ns;
+ utest_int64_t min_ns;
+ utest_int64_t max_ns;
+ double best_deviation;
+ double best_confidence;
};
struct utest_state_s {
struct utest_test_state_s *tests;
size_t tests_length;
FILE *output;
+ size_t buffer_size;
+ char *output_buffer;
};
/* extern to the global state utest needs to execute */
@@ -299,20 +314,6 @@
#pragma clang diagnostic ignored "-Wvariadic-macros"
#pragma clang diagnostic ignored "-Wc++98-compat-pedantic"
#endif
-#define UTEST_PRINTF(...) \
- if (utest_state.output) { \
- fprintf(utest_state.output, __VA_ARGS__); \
- } \
- printf(__VA_ARGS__)
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
-
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wvariadic-macros"
-#pragma clang diagnostic ignored "-Wc++98-compat-pedantic"
-#endif
#ifdef _MSC_VER
#define UTEST_SNPRINTF(BUFFER, N, ...) _snprintf_s(BUFFER, N, N, __VA_ARGS__)
@@ -320,6 +321,19 @@
#define UTEST_SNPRINTF(...) snprintf(__VA_ARGS__)
#endif
+#define UTEST_PRINTF(...) \
+ if (utest_state.output) { \
+ size_t new_bytes = UTEST_SNPRINTF(0, 0, __VA_ARGS__); \
+ if (new_bytes > 0) { \
+ utest_state.output_buffer = (char*)realloc(utest_state.output_buffer, \
+ utest_state.buffer_size+new_bytes); \
+ utest_state.buffer_size += UTEST_SNPRINTF(utest_state.output_buffer \
+ + utest_state.buffer_size - 1, \
+ utest_state.buffer_size+new_bytes, __VA_ARGS__); \
+ } \
+ } \
+ printf(__VA_ARGS__)
+
#ifdef __clang__
#pragma clang diagnostic pop
#endif
@@ -759,10 +773,39 @@
utest_state.tests[index].func = &utest_##SET##_##NAME; \
utest_state.tests[index].name = name; \
utest_state.tests[index].index = 0; \
+ utest_state.tests[index].output = 0; \
+ utest_state.tests[index].iterations = 0; \
+ utest_state.tests[index].ns = 0LL; \
UTEST_SNPRINTF(name, name_size, "%s", name_part); \
} \
void utest_run_##SET##_##NAME(int *utest_result)
+#define UTEST_BENCHMARK(SET, NAME, ITERATIONS) \
+ UTEST_EXTERN struct utest_state_s utest_state; \
+ static void utest_run_b_##SET##_##NAME(int *utest_result, size_t iterations);\
+ static void utest_b_##SET##_##NAME(int *utest_result, size_t iterations) { \
+ utest_run_b_##SET##_##NAME(utest_result, iterations); \
+ } \
+ UTEST_INITIALIZER(utest_register_b_##SET##_##NAME) { \
+ const size_t index = utest_state.tests_length++; \
+ const char *name_part = #SET "." #NAME; \
+ const size_t name_size = strlen(name_part) + 1; \
+ char *name = UTEST_PTR_CAST(char *, malloc(name_size)); \
+ utest_state.tests = UTEST_PTR_CAST( \
+ struct utest_test_state_s *, \
+ utest_realloc(UTEST_PTR_CAST(void *, utest_state.tests), \
+ sizeof(struct utest_test_state_s) * \
+ utest_state.tests_length)); \
+ utest_state.tests[index].func = &utest_b_##SET##_##NAME; \
+ utest_state.tests[index].name = name; \
+ utest_state.tests[index].index = 0; \
+ utest_state.tests[index].output = 0; \
+ utest_state.tests[index].iterations = 1; \
+ utest_state.tests[index].ns = 0LL; \
+ UTEST_SNPRINTF(name, name_size, "%s", name_part); \
+ } \
+ void utest_run_b_##SET##_##NAME(int *utest_result, size_t ITERATIONS)
+
#define UTEST_F_SETUP(FIXTURE) \
static void utest_f_setup_##FIXTURE(int *utest_result, \
struct FIXTURE *utest_fixture)
@@ -800,6 +843,9 @@
utest_state.tests_length)); \
utest_state.tests[index].func = &utest_f_##FIXTURE##_##NAME; \
utest_state.tests[index].name = name; \
+ utest_state.tests[index].output = 0; \
+ utest_state.tests[index].iterations = 0; \
+ utest_state.tests[index].ns = 0LL; \
UTEST_SNPRINTF(name, name_size, "%s", name_part); \
} \
void utest_run_##FIXTURE##_##NAME(int *utest_result, \
@@ -843,6 +889,9 @@
utest_state.tests[index].func = &utest_i_##FIXTURE##_##NAME##_##INDEX; \
utest_state.tests[index].index = i; \
utest_state.tests[index].name = name; \
+ utest_state.tests[index].output = 0; \
+ utest_state.tests[index].iterations = 0; \
+ utest_state.tests[index].ns = 0LL; \
iUp = UTEST_CAST(utest_uint64_t, i); \
UTEST_SNPRINTF(name, name_size, "%s/%" UTEST_PRIu64, name_part, iUp); \
} \
@@ -996,19 +1045,20 @@
printf("%s[==========]%s Running %" UTEST_PRIu64 " test cases.\n",
colours[GREEN], colours[RESET], UTEST_CAST(utest_uint64_t, ran_tests));
- if (utest_state.output) {
- fprintf(utest_state.output, "\n");
- fprintf(utest_state.output,
- "\n",
- UTEST_CAST(utest_uint64_t, ran_tests));
- fprintf(utest_state.output,
- "\n",
- UTEST_CAST(utest_uint64_t, ran_tests));
- }
-
for (index = 0; index < utest_state.tests_length; index++) {
int result = 0;
utest_int64_t ns = 0;
+ utest_int64_t mndex = 0;
+ size_t iterations = 0;
+ size_t arg1 = 0;
+ const size_t min_iterations = 10;
+ const size_t max_iterations = 5000;
+ utest_int64_t best_avg_ns = 0;
+ utest_int64_t min_ns = 1000000000000LL;
+ utest_int64_t max_ns = 0;
+ double best_deviation = 0;
+ double best_confidence = 101.0;
+ const double max_confidence = 1.0;
if (utest_should_filter_test(filter, utest_state.tests[index].name)) {
continue;
@@ -1017,20 +1067,92 @@
printf("%s[ RUN ]%s %s\n", colours[GREEN], colours[RESET],
utest_state.tests[index].name);
- if (utest_state.output) {
- fprintf(utest_state.output, "",
- utest_state.tests[index].name);
- }
+ /* if iterations is not zero, this is a benchmark test */
+ arg1 = (utest_state.tests[index].iterations) ? 1 : utest_state.tests[index].index;
+ /* Time once to work out the base number of iterations to use. */
ns = utest_ns();
errno = 0;
- utest_state.tests[index].func(&result, utest_state.tests[index].index);
+ utest_state.tests[index].func(&result, arg1);
ns = utest_ns() - ns;
- if (utest_state.output) {
- fprintf(utest_state.output, "\n");
+
+ if (utest_state.tests[index].iterations) {
+ result = 1;
+ iterations = (100 * 1000 * 1000) / ns;
+ iterations = iterations < min_iterations ? min_iterations : iterations;
+ iterations = iterations > max_iterations ? max_iterations : iterations;
+
+ for (mndex = 0; (mndex < 100) && (result != 0); mndex++) {
+ int unused_result = 0;
+ utest_int64_t kndex = 0;
+ utest_int64_t avg_ns = 0;
+ double deviation = 0;
+ double confidence = 0;
+ double vsum = 0.0;
+ double v2sum = 0.0;
+ double ns_dbl = 0.0;
+
+ iterations = iterations * (UTEST_CAST(utest_int64_t, mndex) + 1);
+ iterations = iterations > max_iterations ? max_iterations : iterations;
+
+ for (kndex = 0; kndex < iterations; kndex++) {
+ ns = utest_ns();
+ utest_state.tests[index].func(&unused_result, 1);
+ ns = utest_ns() - ns;
+
+ ns_dbl = UTEST_CAST(double, ns);
+ avg_ns += ns;
+ vsum += ns_dbl;
+ v2sum += ns_dbl * ns_dbl;
+ if (ns > max_ns) {
+ max_ns = ns;
+ }
+ if (ns < min_ns) {
+ min_ns = ns;
+ }
+ }
+
+ avg_ns /= iterations;
+ v2sum /= iterations;
+ vsum /= iterations;
+ deviation = v2sum - vsum * vsum;
+ deviation = sqrt(deviation);// / UTEST_CAST(double, iterations));
+
+ /* Confidence is the 99% confidence index - whose magic value is 2.576. */
+ confidence = 2.576 * deviation / sqrt(UTEST_CAST(double, iterations));
+ confidence = (confidence / UTEST_CAST(double, avg_ns)) * 100.0;
+ deviation = (deviation / UTEST_CAST(double, avg_ns)) * 100.0;
+
+ /* If we've found a more confident solution, use that. */
+ result = confidence > max_confidence;
+
+ /* If the deviation beats our previous best, record it. */
+ if (confidence < best_confidence) {
+ best_avg_ns = avg_ns;
+ best_deviation = deviation;
+ best_confidence = confidence;
+ }
+ }
+
+ if (result) {
+ printf("confidence interval %f%% exceeds maximum permitted %f%%\n",
+ best_confidence, max_confidence);
+ }
+
+ // save the stats
+ utest_state.tests[index].best_avg_ns = best_avg_ns;
+ utest_state.tests[index].min_ns = min_ns;
+ utest_state.tests[index].max_ns = max_ns;
+ utest_state.tests[index].best_deviation = best_deviation;
+ utest_state.tests[index].best_confidence = best_confidence;
}
+
+ // save how long it took
+ iterations = utest_state.tests[index].iterations;
+ utest_state.tests[index].ns = (iterations) ? (ns / iterations) : ns;
+
if (0 != result) {
const size_t failed_testcase_index = failed_testcases_length++;
failed_testcases = UTEST_PTR_CAST(
@@ -1038,12 +1160,28 @@
sizeof(size_t) * failed_testcases_length));
failed_testcases[failed_testcase_index] = index;
failed++;
- printf("%s[ FAILED ]%s %s (%" UTEST_PRId64 "ns)\n", colours[RED],
- colours[RESET], utest_state.tests[index].name, ns);
+ printf("%s[ FAILED ]%s %s ", colours[RED],
+ colours[RESET], utest_state.tests[index].name);
} else {
- printf("%s[ OK ]%s %s (%" UTEST_PRId64 "ns)\n", colours[GREEN],
- colours[RESET], utest_state.tests[index].name, ns);
+ printf("%s[ OK ]%s %s ", colours[GREEN],
+ colours[RESET], utest_state.tests[index].name);
}
+
+ // print timing info
+ if (iterations) {
+ printf("(mean %" UTEST_PRId64 "ns, between %lld to %lld, stddev %f, confidence interval +- %f%%)",
+ best_avg_ns, min_ns, max_ns, best_deviation, best_confidence);
+ } else {
+ printf("(%" UTEST_PRId64 "ns)", ns);
+ }
+ printf("\n");
+
+ // save the output with the test (test takes ownership of that memory)
+ if (utest_state.output_buffer && utest_state.buffer_size) {
+ utest_state.tests[index].output = utest_state.output_buffer;
+ }
+ utest_state.buffer_size = 0;
+ utest_state.output_buffer = 0;
}
printf("%s[==========]%s %" UTEST_PRIu64 " test cases ran.\n", colours[GREEN],
@@ -1060,13 +1198,40 @@
}
}
+ // Output the results as xml
if (utest_state.output) {
- fprintf(utest_state.output, "\n\n");
+ fprintf(utest_state.output, "\n");
+ fprintf(utest_state.output, "\n",
+ UTEST_CAST(utest_uint64_t, ran_tests), UTEST_CAST(utest_uint64_t, failed));
+ fprintf(utest_state.output, " \n",
+ UTEST_CAST(utest_uint64_t, ran_tests), UTEST_CAST(utest_uint64_t, failed));
+ for (index = 0; index < utest_state.tests_length; index++) {
+ if (!utest_should_filter_test(filter, utest_state.tests[index].name)) {
+ fprintf(utest_state.output, " \n \n",
+ utest_state.tests[index].output, utest_state.tests[index].output);
+ fprintf(utest_state.output, " \n");
+ } else {
+ fprintf(utest_state.output, " />\n");
+ }
+ }
+ }
+ fprintf(utest_state.output, " \n\n");
}
cleanup:
for (index = 0; index < utest_state.tests_length; index++) {
free(UTEST_PTR_CAST(void *, utest_state.tests[index].name));
+ if (utest_state.tests[index].output) {
+ free(UTEST_PTR_CAST(void *, utest_state.tests[index].output));
+ }
}
free(UTEST_PTR_CAST(void *, failed_testcases));
@@ -1085,7 +1250,7 @@
data without having to use the UTEST_MAIN macro, thus allowing them to write
their own main() function.
*/
-#define UTEST_STATE() struct utest_state_s utest_state = {0, 0, 0}
+#define UTEST_STATE() struct utest_state_s utest_state = {0, 0, 0, 1, 0}
/*
define a main() function to call into utest.h and start executing tests! A