path: root/cleanbench.c

#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include <math.h>

#include "cleanbench.h"
#include "hardware.h"

static int bench_with_confidence(int fid, double *mean, double *stdev, unsigned long *numtries);
static int calc_confidence(double scores[], int num_scores, double *c_half_interval,double *smean, double *sdev);
static double getScore(int fid);

#define NUMTESTS 10

enum {
	NUMSORT,
	STRINGSORT,
	BITFIELD,
	FPEMULATION,
	FOURIER,
	ASSIGNMENT,
	IDEA,
	HUFFMAN,
	NEURAL,
	LINEAR
} tests_t;

int
main(int argc, char *argv[]) 
{
        const char* test_name[] = {
                "NUMERIC SORT    ",
                "STRING SORT     ",
                "BITFIELD        ",
                "FP EMULATION    ",
                "FOURIER         ",
                "ASSIGNMENT      ",
                "IDEA            ",
                "HUFFMAN         ",
                "NEURAL NET      ",
                "LU DECOMPOSITION"
        };

        /*
        ** Indexes -- Baseline is DELL Pentium XP90
        ** 11/28/94
        */
        const double bindex[] = {
              38.993,   /* Numeric sort */
               2.238,   /* String sort */
             5829704,   /* Bitfield */
               2.084,   /* FP Emulation */
             879.278,   /* Fourier */
              0.2628,   /* Assignment */
              65.382,   /* IDEA */
              36.062,   /* Huffman */
              0.6225,   /* Neural Net */
             19.3031    /* LU Decomposition */
        };

        /*
        ** Indices -- Baseline is a AMD K6-233, 32MB RAM (60ns SDRAM),512k L2 cache,
        ** Linux kernel 2.0.32, libc-5.4.38, gcc-2.7.2.3)
        ** Nov/30/97
        */
        const double linux_bindex[] = {
              118.73, 	/* Numeric sort */
              14.459,	/* String sort */
            27910000,	/* Bitfield */
              9.0314,	/* FP Emulation */
              1565.5,	/* Fourier */
              1.0132,	/* Assignment */
              220.21,	/* IDEA */
              112.93,	/* Huffman */
              1.4799,   /* Neural Net */
              26.732    /* LU Decomposition */
        };      

        double          linux_memindex = 1.0;     /* Linux memory index (mainly integer operations)*/
        double          linux_intindex = 1.0;     /* Linux integer index */
        double          linux_fpindex = 1.0;      /* Linux floating-point index */
        double          intindex = 1.0;        /* Integer index */
        double          fpindex = 1.0;         /* Floating-point index */
        double          bmean;           /* Benchmark mean */
        double          bstdev;          /* Benchmark stdev */
        unsigned long   bnumrun;  /* # of runs */
        int test;                  /* Index */

        puts(   "TEST                : Iterations/sec.  : Old Index   : New Index\n"
                "                    :                  : Pentium 90  : AMD K6/233\n"
                "--------------------:------------------:-------------:------------");

        for (test = NUMSORT; test < NUMTESTS; test++) {
                printf("%s    :", test_name[test]);
                
                if (!bench_with_confidence(test, &bmean, &bstdev, &bnumrun)) {
                        printf( "\n** WARNING: The current test result is NOT 95 %% statistically certain.\n"
                                "** WARNING: The variation among the individual results is too large.\n"
                                "                    :");
                }
                printf(" %15.5g  :  %9.2f  :  %9.2f\n", bmean, bmean / bindex[test], bmean / linux_bindex[test]);

		if ((test == FOURIER) || (test == NEURAL) || (test == LINEAR)) {
                        fpindex *= bmean / bindex[test];
                        linux_fpindex *= bmean / linux_bindex[test];
		} else {
                        intindex *= bmean / bindex[test];
                        if ((test == NUMSORT) || (test == FPEMULATION) || (test == IDEA) || (test == HUFFMAN)) {
                                linux_intindex *= bmean / linux_bindex[test];
                        } else {
                                linux_memindex *= bmean / linux_bindex[test];
                        }
		}
        }
        printf( "==========================ORIGINAL BYTEMARK RESULTS==========================\n"
                "INTEGER INDEX       : %.3f\n"
                "FLOATING-POINT INDEX: %.3f\n"
                "Baseline (MSDOS)    : Pentium 90, 256 KB L2-cache, Watcom compiler 10.0\n"
                "==============================LINUX DATA BELOW===============================\n",
                pow(intindex, .142857), pow(fpindex, .33333));
	hardware();
#include "sysinfoc.c"
        printf( "MEMORY INDEX        : %.3f\n"
                "INTEGER INDEX       : %.3f\n"
                "FLOATING-POINT INDEX: %.3f\n"
                "Baseline (Linux)    : AMD K6/233, 512 KB L2-cache, gcc 2.7.2.3, libc-5.4.38\n",
                pow(linux_memindex, .3333333333), pow(linux_intindex, .25), pow(linux_fpindex, .3333333333));
        return 0;
}

/**************************
** bench_with_confidence **
***************************
** Given a benchmark id that indicates a function, this routine
** repeatedly calls that benchmark, seeking to collect and replace
** scores to get 5 that meet the confidence criteria.
**
** The above is mathematically questionable, as the statistical theory
** depends on independent observations, and if we exchange data points
** depending on what we already have then this certainly violates
** independence of the observations. Hence I changed this so that at
** most 30 observations are done, but none are deleted as we go
** along. We simply do more runs and hope to get a big enough sample
** size so that things stabilize. Uwe F. Mayer
**
** Return TRUE if ok, FALSE if failure.  Returns mean
** and std. deviation of results if successful.
*/
static int bench_with_confidence(int benchmark,
        double *mean,                   /* Mean of scores */
        double *stdev,                  /* Standard deviation */
        unsigned long *numtries)                /* # of attempts */
{
void (*funcpointer[])(void) =
{       DoNumSort,
        DoStringSort,
        DoBitops,
        DoEmFloat,
        DoFourier,
        DoAssign,
        DoIDEA,
        DoHuffman,
        DoNNET,
        DoLU };

double myscores[30];            /* Need at least 5 scores, use at most 30 */
double c_half_interval;         /* Confidence half interval */
int i;                          /* Index */
/* double newscore; */          /* For improving confidence interval */

/*
** Get first 5 scores.  Then begin confidence testing.
*/
for (i=0;i<5;i++)
{       (*funcpointer[benchmark])();
        myscores[i]=getScore(benchmark);
}
*numtries=5;            /* Show 5 attempts */

/*
** The system allows a maximum of 30 tries before it gives
** up.  Since we've done 5 already, we'll allow 25 more.
*/

/*
** Enter loop to test for confidence criteria.
*/
while(1)
{
        /*
        ** Calculate confidence. Should always return TRUE
        */
        if (0!=calc_confidence(myscores,
		*numtries,
                &c_half_interval,
                mean,
                stdev)) return FALSE;

        /*
        ** Is the length of the half interval 5% or less of mean?
        ** If so, we can go home.  Otherwise, we have to continue.
        */
        if(c_half_interval/ (*mean) <= (double)0.05)
                break;

	/* We now simply add a new test run and hope that the runs
           finally stabilize, Uwe F. Mayer */
	if(*numtries==30) return FALSE;
	(*funcpointer[benchmark])();
	myscores[*numtries]=getScore(benchmark);
	*numtries+=1;
}

        return TRUE;
}

/********************
** calc_confidence **
*********************
** Given a set of numtries scores, calculate the confidence
** half-interval.  We'll also return the sample mean and sample
** standard deviation.
** NOTE: This routines presumes a confidence of 95% and
** a confidence coefficient of .95
** returns 0 if there is an error, otherwise -1
*/
static int calc_confidence(double scores[], /* Array of scores */
		int num_scores,             /* number of scores in array */
                double *c_half_interval,    /* Confidence half-int */
                double *smean,              /* Standard mean */
                double *sdev)               /* Sample stand dev */
{
/* Here is a list of the student-t distribution up to 29 degrees of
   freedom. The value at 0 is bogus, as there is no value for zero
   degrees of freedom. */
double student_t[30]={0.0 , 12.706 , 4.303 , 3.182 , 2.776 , 2.571 ,
                             2.447 , 2.365 , 2.306 , 2.262 , 2.228 ,
                             2.201 , 2.179 , 2.160 , 2.145 , 2.131 ,
                             2.120 , 2.110 , 2.101 , 2.093 , 2.086 ,
                             2.080 , 2.074 , 2.069 , 2.064 , 2.060 ,
		             2.056 , 2.052 , 2.048 , 2.045 };
int i;          /* Index */
if ((num_scores<2) || (num_scores>30)) {
  puts("Internal error: calc_confidence called with an illegal number of scores");
  return TRUE;
}
/*
** First calculate mean.
*/
*smean=(double)0.0;
for(i=0;i<num_scores;i++){
  *smean+=scores[i];
}
*smean/=(double)num_scores;

/* Get standard deviation */
*sdev=(double)0.0;
for(i=0;i<num_scores;i++) {
  *sdev+=(scores[i]-(*smean))*(scores[i]-(*smean));
}
*sdev/=(double)(num_scores-1);
*sdev=sqrt(*sdev);

/* Now calculate the length of the confidence half-interval.  For a
** confidence level of 95% our confidence coefficient gives us a
** multiplying factor of the upper .025 quartile of a t distribution
** with num_scores-1 degrees of freedom, and dividing by sqrt(number of
** observations). See any introduction to statistics.
*/
*c_half_interval=student_t[num_scores-1] * (*sdev) / sqrt((double)num_scores);
return FALSE;
}

/*************
** getScore **
**************
** Return the score for a particular benchmark.
*/
static double getScore(int benchmark) {
        double (*getScore[])(void) = {
                getNumSortScore,
                getStringSortScore,
                getBitfieldScore,
                getEmFloatScore,
                getFourierScore,
                getAssignmentScore,
                getIDEAScore,
                getHuffmanScore,
                getNNETScore,
                getLinearScore
        };

        return (*getScore[benchmark])();
}