Add ability to compare against reference counts with deviations

This commit extends the ability of the COMPARE_COUNTS test facility.
This addresses issue #1 on the tracker.

Previously, COMPARE_COUNTS would check for an exact match of counts compared
to the values in a reference file in a column by column fashion. This
commit adds the ability to compare test data against reference data
allowing for a certain tolerance specified via

"absDeviation" = [A1, A2, A3, ...]

or

"relDeviation" = [R1, R2, R3, ....]

Only one of absDeviation or relDeviation can be specified per
check.

Here the Ai are integers specifying the allowed absolute deviation of
column i with respect to the reference data set. Similarly, the
Ri are floats specifying the relative deviation with regard
to the reference data set. E.g. if the reference data set looks
like

1e-6 100 50
2e-6 110 55
3e-6 120 60
...

and

"absDeviation" = [10, 5]

Then the following observed output would pass the test

1e-6 108 53
2e-6 112 56
3e-6 118 58

Whereas the following observed output would fail in row 2

1e-6 108 53
2e-6 113 61
3e-6 124 64

since 61 is outside 55 +/- 5.

Similarly for

"relDeviation" = [0.1, 0.1]

the first test above would pass while the second would not since
61 is outside 55 +/- (55 * 0.1) = 55 +/- 5.5.

src/jsonParser/jsonParser.go

@@ -107,12 +107,17 @@ type testPatternMatch struct {
 	NumMatches   int    // number of expected pattern matches
 }
 
-// testCompareCounts pertains to checks comparing data against exact reference
-// counts
+// testCompareCounts pertains to checks comparing data against reference
+// counts. If absDeviation or relDeviation are provided the actual data
+// is compared to the reference data taking into account the relative or
+// absolute deviation. If absDeviation or relDeviation are not provided
+// they are assumed to be 0. Both absDeviation and relDeviation are arrays with
+// one value per data column. Any non-specified columns are assumed to be zero,
+// any additional values are ignored.
 type testCompareCounts struct {
-	ReferenceFile string  // name of file with reference counts to compare against
-	Delay         float64 // delay in seconds at which checkpoint should happen
-	Margin        float64 // acceptable margin for checkpoint delay in seconds
+	ReferenceFile string    // name of file with reference counts to compare against
+	AbsDeviation  []int     // allowed absolute deviation from reference, one per column
+	RelDeviation  []float64 // allowed relative deviation from reference, one per column
 }
 
 // testMeans pertaints to checks testing that data values have a certain mean
@@ -172,6 +177,8 @@ type testASCIIVizOutput struct {
 // testCheckPoint does basic timing tests involving checkpoints
 type testCheckPoint struct {
 	BaseName string
+	Delay    float64 // delay in seconds at which checkpoint should happen
+	Margin   float64 // acceptable margin for checkpoint delay in seconds
 }
 
 // testDREAMMV3MeshCommon contains common items used in DREAMM V3 molecule
@@ -297,19 +304,3 @@ func ReadConfig() (*Config, error) {
 	}
 	return &myConf, nil
 }
-
-/*
-// RunStatus encapsulates the status of running N mdl files which make
-// up a single test case
-// NOTE: a run might fail for a number of reasons, e.g., during preparation of
-// a run and patching in stderr, or during running of MCell itself. If running
-// MCell failed we try to figure out the exit code.
-// NOTE: This function should not be part of jsonParser (engine would be better
-// or perhaps a shared include)
-type RunStatus struct {
-	Success       bool // indicates if prepping/running the simulation succeeded
-	ExitMessage   string
-	StdErrContent string
-	ExitCode      int // this is only used if mcell was actually run
-}
-*/

src/misc/misc.go

@@ -327,3 +327,12 @@ func CheckDREAMMV3GroupedItem(filePath string, haveItemProperty, noItem bool) er
 	}
 	return nil
 }
+
+// below are some useful math functions
+
+// Abs returns the absolute value of integer i
+func Abs(i int) int {
+	var isize uint = strconv.IntSize - 1
+	t := i >> isize
+	return t ^ (i + t)
+}

src/tester/tester.go

@@ -181,15 +181,21 @@ func Run(test *TestData, result chan *TestResult) {
 			}
 
 		case "COMPARE_COUNTS":
+			// only one of absDeviation or relDeviation can be defined
+			if (len(c.AbsDeviation) > 0) && (len(c.RelDeviation) > 0) {
+				testErr = fmt.Errorf("absDeviation and relDeviation are mutually exclusive")
+				break
+			}
+
 			referencePath := filepath.Join(test.Path, c.ReferenceFile)
 			refData, err := file.ReadCounts(referencePath, c.HaveHeader)
 			if err != nil {
 				testErr = err
 				break
 			}
 			for i, d := range data {
-				if testErr = compareCounts(d, refData, dataPaths[i], c.MinTime,
-					c.MaxTime); testErr != nil {
+				if testErr = compareCounts(d, refData, c.AbsDeviation, c.RelDeviation,
+					dataPaths[i], c.MinTime, c.MaxTime); testErr != nil {
 					break
 				}
 			}
@@ -356,8 +362,8 @@ func fileMatchPattern(filePath string, matchPattern string,
 
 // compareCounts checks that the test data matches the provided column counts
 // exactly
-func compareCounts(data, refData *file.Columns, dataPath string, minTime,
-	maxTime float64) error {
+func compareCounts(data, refData *file.Columns, absDev []int, relDev []float64,
+	dataPath string, minTime, maxTime float64) error {
 
 	if len(refData.Times) != len(data.Times) {
 		return fmt.Errorf(
@@ -372,15 +378,29 @@ func compareCounts(data, refData *file.Columns, dataPath string, minTime,
 	}
 
 	numCols := len(data.Counts)
+	// pad absDev and relDev arrays with zeros if necessary
+	for i := len(absDev); i < numCols; i++ {
+		absDev = append(absDev, 0)
+	}
+	for i := len(relDev); i < numCols; i++ {
+		relDev = append(relDev, 0.0)
+	}
+
 	for r, time := range data.Times {
 		if (minTime > 0 && time < minTime) || (maxTime > 0 && time > maxTime) {
 			continue
 		}
 
 		for c := 0; c < numCols; c++ {
-			if data.Counts[c][r] != refData.Counts[c][r] {
-				return fmt.Errorf("in %s: reference and actual data differ in row %d and col %d",
-					dataPath, r, c)
+			// determine allowed deviation if definied via absDeviation or relDeviation
+			dev := absDev[c]
+			if dev == 0 {
+				dev = int(relDev[c] * float64(refData.Counts[c][r]))
+			}
+			if misc.Abs(data.Counts[c][r]-refData.Counts[c][r]) > dev {
+				return fmt.Errorf("in %s: reference and actual data differ in row %d "+
+					"and col %d (exptected: %d +/- %d actual value: %d)", dataPath, r, c,
+					refData.Counts[c][r], dev, data.Counts[c][r])
 			}
 		}
 	}
@@ -838,7 +858,8 @@ func checkExpressions(filePath string) error {
 // diffFileContent matches the content of datafile with the one provided in
 // the template file. The template file can contain format string parameters
 // which will be filled with the template parameters as requested by the
-// test file.
+// test file. Currently available template parameters are:
+//   TODAY_DAY: todays weekday name (Monday, Tuesday, ....)
 func diffFileContent(path, dataPath, templateFile string,
 	templateParams []string) error {