path: root/cleanbench.c

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include <math.h>
#include <limits.h>
#include <time.h>

#include "cleanbench.h"
#include "randnum.h"

static bool bench_with_confidence(int benchmark, double* average, double* std_dev, int* runs);
static bool calc_confidence(double scores[], int runs, double* c_half_interval, double* average, double* std_dev);

#define NUM_TESTS 10

enum {
	NUMSORT,
	FPEMULATION,
	IDEA,
	HUFFMAN,
	STRINGSORT,
	BITFIELD,
	ASSIGNMENT,
	FOURIER,
	NEURAL,
	LINEAR
} tests_t;

int
main()
{
        const char* benchmark_name[] = {
                "NUMERIC SORT    ",
                "FP EMULATION    ",
                "IDEA            ",
                "HUFFMAN         ",
                "STRING SORT     ",
                "BITFIELD        ",
                "ASSIGNMENT      ",
                "FOURIER         ",
                "NEURAL NET      ",
                "LU DECOMPOSITION"
        };

        /*
        ** Indexes -- Baseline is DELL Pentium XP90
        ** 11/28/94
        */
        const double old_index[] = {
              38.993,   /* Numeric sort */
               2.084,   /* FP Emulation */
              65.382,   /* IDEA */
              36.062,   /* Huffman */
               2.238,   /* String sort */
             5829704,   /* Bitfield */
              0.2628,   /* Assignment */
             879.278,   /* Fourier */
              0.6225,   /* Neural Net */
             19.3031    /* LU Decomposition */
        };

        /*
        ** Indices -- Baseline is a AMD K6-233, 32MB RAM (60ns SDRAM),512k L2 cache,
        ** Linux kernel 2.0.32, libc-5.4.38, gcc-2.7.2.3)
        ** Nov/30/97
        */
        const double linux_index[] = {
              118.73, 	/* Numeric sort */
              9.0314,	/* FP Emulation */
              220.21,	/* IDEA */
              112.93,	/* Huffman */
              14.459,	/* String sort */
            27910000,	/* Bitfield */
              1.0132,	/* Assignment */
              1565.5,	/* Fourier */
              1.4799,   /* Neural Net */
              26.732    /* LU Decomposition */
        };

        double          linux_memindex = 1.0;   /* Linux memory index */
        double          linux_intindex = 1.0;   /* Linux integer index */
        double          linux_fpindex = 1.0;    /* Linux floating-point index */
        double          intindex = 1.0;         /* Old Integer index */
        double          fpindex = 1.0;          /* Old Floating-point index */
        double          average;                /* Average of benchmark results */
        double          std_dev;                /* Standard deviation of benchmark results */
        int             runs;                   /* # of runs */
        int             benchmark = FOURIER;

        puts(   "TEST                : Iterations/sec.  : Old Index   : New Index\n"
                "                    :                  : Pentium 90  : AMD K6/233\n"
                "--------------------:------------------:-------------:------------");

        for (; benchmark < NUM_TESTS; benchmark++) {
                printf("%s    :", benchmark_name[benchmark]);

                if (!bench_with_confidence(benchmark, &average, &std_dev, &runs)) {
                        printf( "\n** WARNING: The current benchmark result is NOT 95 %% statistically certain.\n"
                                "** WARNING: The variation among the individual results is too large.\n"
                                "                    :");
                }
                printf(" %15.5g  :  %9.2f  :  %9.2f\n", average, average / old_index[benchmark], average / linux_index[benchmark]);

		if (benchmark >= FOURIER) {
                        fpindex *= average /old_index[benchmark];
                        linux_fpindex *= average / linux_index[benchmark];
		} else {
                        intindex *= average / old_index[benchmark];
                        if (benchmark <= HUFFMAN) {
                                linux_intindex *= average / linux_index[benchmark];
                        } else {
                                linux_memindex *= average / linux_index[benchmark];
                        }
		}
        }
        printf( "==========================ORIGINAL BYTEMARK RESULTS==========================\n"
                "INTEGER INDEX       : %.3f\n"
                "FLOATING-POINT INDEX: %.3f\n"
                "Baseline (MSDOS)    : Pentium 90, 256 KB L2-cache, Watcom compiler 10.0\n"
                "===========================LINUX BENCHMARK RESULTS===========================\n",
                pow(intindex, .142857), pow(fpindex, .33333));
	hardware();
#include "sysinfo.c"
        printf( "MEMORY INDEX        : %.3f\n"
                "INTEGER INDEX       : %.3f\n"
                "FLOATING-POINT INDEX: %.3f\n"
                "Baseline (Linux)    : AMD K6/233, 512 KB L2-cache, gcc 2.7.2.3, libc-5.4.38\n",
                pow(linux_memindex, .3333333333), pow(linux_intindex, .25), pow(linux_fpindex, .3333333333));
        return 0;
}

/**************************
** bench_with_confidence **
***************************
** Given a benchmark, this routine repeatedly calls that benchmark,
** seeking to collect and replace scores to get 5 that meet the
** confidence criteria.
**
** The above is mathematically questionable, as the statistical theory
** depends on independent observations, and if we exchange data points
** depending on what we already have then this certainly violates
** independence of the observations. Hence I changed this so that at
** most 30 observations are done, but none are deleted as we go
** along. We simply do more runs and hope to get a big enough sample
** size so that things stabilize. Uwe F. Mayer
**
** Return true; false on failure.  Returns average
** and standard deviation through argument list if successful.
*/
static bool
bench_with_confidence(int benchmark, double* average, double* std_dev, int* runs)
{
        double (*Do[])(void) = {
                DoNumSort,
                DoEmFloat,
                DoIDEA,
                DoHuffman,
                DoStringSort,
                DoBitops,
                DoAssign,
                DoFourier,
                DoNNET,
                DoLU
        };

        double score[30];            /* Need at least 5 scores, use at most 30 */
        double c_half_interval;         /* Confidence half interval */
        int i;                          /* Index */

        /*
        ** Get first 5 scores.  Then begin confidence testing.
        */
        for (i = 0; i < 5; i++) {
                score[i] = (*Do[benchmark])();
        }
        *runs = 5;            /* Show 5 attempts */

        /*
        ** The system allows a maximum of 30 tries before it gives
        ** up.  Since we've done 5 already, we'll allow 25 more.
        */

        /*
        ** Enter loop to test for confidence criteria.
        */
        while(1) {
                /* Calculate confidence. Should always return true */
                if (!calc_confidence(score, *runs, &c_half_interval, average, std_dev)) {
                        return false;
                }

                /*
                ** Is the length of the half interval 5% or less of average?
                ** If so, we can go home.  Otherwise, we have to continue.
                */
                if (c_half_interval / (*average) <= 0.05) {
                        break;
                }

                if (*runs == 30) {
                        return false;
                }

                score[*runs] = (*Do[benchmark])();
                *runs += 1;
        }

        return true;
}

 /********************
** calc_confidence **
*********************
** Given a set of scores, calculate the confidence
** half-interval.  We'll also return the sample average
** and sample standard deviation.
** NOTE: This routines presumes a confidence of 95% and
** a confidence coefficient of .95
** returns false if there is an error, otherwise true
*/
static bool calc_confidence(double scores[], int runs, double* c_half_interval, double* average, double* std_dev)
{
        /* Here is a list of the student-t distribution up to 29 degrees of
        ** freedom. The value at 0 is bogus, as there is no value for zero
        ** degrees of freedom. */
        const double t_distribution[30] = {
                0.0,12.706, 4.303, 3.182, 2.776,
              2.571, 2.447, 2.365, 2.306, 2.262,
              2.228, 2.201, 2.179, 2.160, 2.145,
              2.131, 2.120, 2.110, 2.101, 2.093,
              2.086, 2.080, 2.074, 2.069, 2.064,
              2.060, 2.056, 2.052, 2.048, 2.045
        };

        int i;

        if ((runs<2) || (runs>30)) {
                fputs("Internal error: calc_confidence called with an illegal number of scores", stderr);
                return false;
        }

        /* First, calculate average.*/
        *average = 0.0;
        for (i = 0; i < runs; i++) {
                *average += scores[i];
        }
        *average /=  (double)runs;

        /* Get standard deviation */
        *std_dev = 0.0;
        for (i = 0; i < runs; i++) {
                *std_dev += (scores[i] - (*average)) * (scores[i] - (*average));
        }
        *std_dev /= (double)(runs - 1);
        *std_dev = sqrt(*std_dev);

        /* Now calculate the length of the confidence half-interval.  For a
        ** confidence level of 95% our confidence coefficient gives us a
        ** multiplying factor of the upper .025 quartile of a t distribution
        ** with runs-1 degrees of freedom, and dividing by sqrt(number of
        ** observations). See any introduction to statistics.
        */
        *c_half_interval = t_distribution[runs - 1] * (*std_dev) / sqrt((double)runs);

        return true;
}