Convert existing test.bsh into a regression test

The existing test.bash script has been converted to a regression test
that:
  - Verifies that in default mode (GCOV) we calcualte the correct
    coverage for a simple c binary
  - Verifies that in lcov mode  we correctly parse the .info file for a
    simple c binary

Additional tests can be created by copying the "test-src/simple"
directory and:
  - Updating the source file & Makefile with the new test code
  - Updating the expected.json with the expected coverage.
    NOTE: If gcov and lcov provide different results, this can be
          configured in the config.sh by specifying separate
          expected.json files
  - [optional] Providing a config.sh file in the directory to configure
    optional flags. (See valid options listed at the top of
    test-utils/testDir.sh)
  - Adding the directory to test.bash

Author: LAHumphreys

.gitignore

@@ -25,6 +25,9 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+*_test_output
+*.gcov
+
 
 # Translations
 *.mo

.travis.yml

@@ -7,6 +7,11 @@ python:
     - "3.4"
     - "3.5"
 
+addons:
+  apt:
+    packages:
+      - lcov
+
 install:
     - python setup.py --quiet install
 

test-src/simple/Makefile

@@ -0,0 +1,8 @@
+foo: foo.c
+	gcc --coverage -o foo foo.c
+
+clean:
+	rm -f foo *.gcno *.gcda output *.gcov output.json
+
+test: foo
+	./foo

test-src/simple/expected.json

@@ -0,0 +1,6 @@
+{
+    "source_files": [{
+        "name": "foo.c",
+        "coverage": [1, 1, 1, 0, null, null, null, null, 0, null, 1, 0, null, null, 1, null],
+        "source_digest": "500a03d2484de8cc2b9ed06615d7f02a"}]
+}

test-src/simple/foo.c

@@ -0,0 +1,16 @@
+int main() {
+    int a = 1;
+    if(a == 2) {
+        a = 3;
+        /* LCOV_EXCL_START */
+        a = 4;
+        a = 5;
+        /* LCOV_EXCL_STOP */
+        a = 6;
+    }
+    if(a == 7) {
+        a = 8;
+        a = 9; /* LCOV_EXCL_LINE */
+    }
+    return 0;
+}

test-utils/check_output.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import json
+import sys
+
+expectedJson="expected.json"
+if sys.argv[1] == "-e":
+    expectedJson = sys.argv[2]
+
+
+expected=json.loads(open(expectedJson).read())
+output=json.loads(open("output.json").read())
+
+if 'source_files' not in expected:
+    print ("expected.json does not contain a 'source_files' array")
+    print(expected)
+    exit (1)
+
+if 'source_files' not in output:
+    print ("output.json does not contain a 'source_files' array")
+    print(output)
+    exit (1)
+
+expected_files={}
+output_files={}
+
+for f in expected['source_files']:
+    expected_files[f['name']] = f
+for f in output['source_files']:
+    output_files[f['name']] = f
+
+#
+# Check for missing files
+#
+for f in expected_files:
+    if f not in output_files:
+        print ("File '%s' is missing from output.json" %f)
+        exit(1)
+#
+# Check for rogue files
+#
+for f in output_files:
+    if f not in expected_files:
+        print ("File '%s' is reported in output.json, but was not expected!" %f)
+        exit(1)
+
+#
+# Finally check each file has the expected output...
+#
+for f in expected_files:
+    exp_file = expected_files[f]
+    out_file = output_files[f]
+
+    exp_lines=exp_file['coverage']
+    out_lines=out_file['coverage']
+    if len(exp_lines) != len(out_lines):
+        print ("File '%s': expected %d lines, but actually has %s" %(f, len(exp_lines), len(out_lines)))
+        exit(1)
+
+    for i in range(len(out_lines)):
+        if exp_lines[i] != out_lines[i]:
+            print ("File '%s':%d, expected %s hits, but actually has %s" %(f, i+1, exp_lines[i], out_lines[i]))
+            exit(1)
+
+
+