Run the regresion tests over multiple gcc versions

Whilst investigating issues with dodgey coverage being reported to
coveralls, it was found that the issue was dependent on the version of
g++/gcov used. Extending the existing test suite to multiple gcc
versions shows this up.

The issue is that in version 8, gcov changed the format of its output,
which makes the assumptions made in the current parser invalid. Tools
such as LCOV have already migrated to use gcov's new JSON format to work
around this issue.

For now the affected tests are marked as known failure to prevent a
failing build.

Author: LAHumphreys

.gitignore

@@ -25,7 +25,7 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
-*_test_output
+*_test_output*
 *.gcov
 output.json
 *.gcno

.travis.yml

@@ -9,12 +9,25 @@ python:
 
 addons:
   apt:
+    sources:
+      - ubuntu-toolchain-r-test
     packages:
       - lcov
+      - g++-4.8
+      - g++-5
+      - g++-6
+      - g++-7
+      - g++-8
+      - g++-9
+      - g++-9
+# Required for latest lcov build - TODO: remove the need for this...
+      - libperlio-gzip-perl
+      - libjson-perl
 
 install:
     - python setup.py --quiet install
 
 script:
     - nosetests
-    - ./test.bash
+    - git clone https://github.com/linux-test-project/lcov.git
+    - PATH="$PWD/lcov/bin:$PATH" ./test.bash --gcc-version 4.8 --gcc-version 5 --gcc-version 6 --gcc-version 7 --gcc-version 8 --gcc-version 9

test-src/excluded_files/Makefile

@@ -1,14 +1,14 @@
 part_unused.o: part_unused.h part_unused.cpp
-	g++ --coverage -c -o part_unused.o part_unused.cpp
+	$(CXX) --coverage -c -o part_unused.o part_unused.cpp
 
 toy.o: extra_lib/toy.cpp extra_lib/toy.h
-	g++ --coverage -c -o toy.o extra_lib/toy.cpp
+	$(CXX) --coverage -c -o toy.o extra_lib/toy.cpp
 
 a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
-	g++ --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+	$(CXX) --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
 
 foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
-	g++ --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+	$(CXX) --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o

test-src/included_files/Makefile

@@ -1,14 +1,14 @@
 part_unused.o: part_unused.h part_unused.cpp
-	g++ --coverage -c -o part_unused.o part_unused.cpp
+	$(CXX) --coverage -c -o part_unused.o part_unused.cpp
 
 toy.o: extra_lib/toy.cpp extra_lib/toy.h
-	g++ --coverage -c -o toy.o extra_lib/toy.cpp
+	$(CXX) --coverage -c -o toy.o extra_lib/toy.cpp
 
 a_toy.o: another_extra_lib/another_toy.cpp another_extra_lib/another_toy.h
-	g++ --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
+	$(CXX) --coverage -c -o a_toy.o another_extra_lib/another_toy.cpp
 
 foo: foo.cpp part_unused.o toy.o a_toy.o extra_lib/toy.h another_extra_lib/another_toy.h
-	g++ --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
+	$(CXX) --coverage -o foo foo.cpp part_unused.o toy.o a_toy.o
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json part_unused.o toy.o a_toy.o

test-src/missing_files/Makefile

@@ -1,7 +1,7 @@
 unused.o: unused.h unused.cpp
-	gcc --coverage -c -o unused.o unused.cpp
+	$(CXX) --coverage -c -o unused.o unused.cpp
 foo: foo.c unused.o
-	gcc --coverage -o foo foo.c
+	$(CXX) --coverage -o foo foo.c
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json unused.o

test-src/missing_files/config.sh

@@ -6,9 +6,9 @@ gcov_expected_output="expected_gcov.json"
 # Override to change the file that contains the expected output for the lcov run
 lcov_expected_output="expected_lcov.json"
 
-gcc_version=$(g++ -v 2>&1 | awk '/gcc version/ {print $3}')
+gcc_version=$($CXX -v 2>&1 | awk '/gcc version/ {print $3}')
 if [[ "$gcc_version" < "5.5" ]]; then
-    echo "LEGACY G++ detected: skipping GCOV missing files test"
+    echo "LEGACY G++ detected ($gcc_version: $CXX): skipping GCOV missing files test"
     modes="lcov"
 else
     modes="gcov lcov"

test-src/out_of_tree/build_tree/Makefile

@@ -2,16 +2,16 @@ SOURCE_TREE=../source_tree
 INCLUDE_FLAGS=-I$(SOURCE_TREE)/llib
 
 lib.o: $(SOURCE_TREE)/llib/lib.cpp $(SOURCE_TREE)/llib/lib.h
-	g++ --coverage -c -o $@ $(SOURCE_TREE)/llib/lib.cpp
+	$(CXX) --coverage -c -o $@ $(SOURCE_TREE)/llib/lib.cpp
 
 lib.a: lib.o
 	ar cr $@ $+
 
 a: $(SOURCE_TREE)/A/a.cpp lib.a
-	g++ --coverage $(INCLUDE_FLAGS) -o a $+
+	$(CXX) --coverage $(INCLUDE_FLAGS) -o a $+
 
 b: $(SOURCE_TREE)/B/b.cpp lib.a
-	g++ --coverage $(INCLUDE_FLAGS) -o b $+
+	$(CXX) --coverage $(INCLUDE_FLAGS) -o b $+
 
 prepare_test: a b
 

test-src/simple/Makefile

@@ -1,5 +1,5 @@
 foo: foo.c
-	gcc --coverage -o foo foo.c
+	$(CXX) --coverage -o foo foo.c
 
 clean:
 	rm -f foo *.gcno *.gcda output *.gcov output.json

test-src/static_lib/Makefile

@@ -1,14 +1,14 @@
 llib/lib.o: llib/lib.cpp llib/lib.h
-	g++ --coverage -c -o $@ llib/lib.cpp
+	$(CXX) --coverage -c -o $@ llib/lib.cpp
 
 llib/lib.a: llib/lib.o
 	ar cr $@ $+
 
 A/a: A/a.cpp llib/lib.a
-	cd A && g++ --coverage -I../llib -o a a.cpp ../llib/lib.a
+	cd A && $(CXX) --coverage -I../llib -o a a.cpp ../llib/lib.a
 
 B/b: B/b.cpp llib/lib.a
-	cd B && g++ --coverage -I../llib -o b b.cpp ../llib/lib.a
+	cd B && $(CXX) --coverage -I../llib -o b b.cpp ../llib/lib.a
 
 prepare_test: A/a B/b
 

test-src/static_lib/config.sh

@@ -1 +1,7 @@
 #!/bin/bash
+gcc_version=$($CXX -v 2>&1 | awk '/gcc version/ {print $3}')
+if [[ "$gcc_version" < "8" ]]; then
+    gcov_fail_reason=""
+else
+    gcov_fail_reason="cpp-coveralls parser does not support modern gcov's output"
+fi