testrunner: bail out on unknown options

.github/workflows/CI-unixish.yml

@@ -679,6 +679,13 @@ jobs:
           test -z "$(nm processexecutor.o)"
           # TODO: test NO_* defines
 
+      - name: Test testrunner inclusion/exclusion
+        run: |
+          ! ./testrunner -d TestUtils | grep -v TestUtils > /dev/null
+          ! ./testrunner -d TestUtils::trim | grep -v TestUtils::trim > /dev/null
+          ! ./testrunner -d -x TestUtils | grep TestUtils > /dev/null
+          ! ./testrunner -d -x TestUtils:trim | grep TestUtils:trim > /dev/null
+
       - name: Show all ignored files
         if: false  # TODO: currently lists all the contents of ignored folders - we only need what actually matched
         run: |

test/fixture.cpp

@@ -97,19 +97,22 @@ bool TestFixture::prepareTest(const char testname[])
     prepareTestInternal();
 
     // Check if tests should be executed
-    if (testToRun.empty() || testToRun == testname) {
-        // Tests will be executed - prepare them
-        mTestname = testname;
-        ++countTests;
-        if (quiet_tests) {
-            std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
-            std::fflush(stdout);
-        } else {
-            std::cout << classname << "::" << mTestname << std::endl;
-        }
-        return !dry_run;
+    if (!testsToRun.empty()) {
+        const bool match = testsToRun.count(testname);
+        if ((match && exclude_tests) || (!match && !exclude_tests))
+            return false;
+    }
+
+    // Tests will be executed - prepare them
+    mTestname = testname;
+    ++countTests;
+    if (quiet_tests) {
+        std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
+        std::fflush(stdout);
+    } else {
+        std::cout << classname << "::" << mTestname << std::endl;
     }
-    return false;
+    return !dry_run;
 }
 
 void TestFixture::teardownTest()
@@ -350,9 +353,9 @@ void TestFixture::printHelp()
         "    -x                   Exclude the specified tests.\n";
 }
 
-void TestFixture::run(const std::string &str)
+void TestFixture::run(const std::set<std::string> &tests)
 {
-    testToRun = str;
+    testsToRun = tests;
     try {
         if (quiet_tests) {
             std::cout << '\n' << classname << ':';
@@ -380,6 +383,7 @@ void TestFixture::processOptions(const options& args)
 {
     quiet_tests = args.quiet();
     dry_run = args.dry_run();
+    exclude_tests = args.exclude_tests();
     exename = args.exe();
 }
 
@@ -388,27 +392,27 @@ std::size_t TestFixture::runTests(const options& args)
     countTests = 0;
     errmsg.str("");
 
+    const auto& which_tests = args.which_tests();
+    const auto exclude_tests = args.exclude_tests();
+
     // TODO: bail out when given class/test is not found?
-    for (std::string classname : args.which_test()) {
-        std::string testname;
-        const std::string::size_type pos = classname.find("::");
-        if (pos != std::string::npos) {
-            // TODO: excluding indiviual tests is not supported yet
-            testname = classname.substr(pos + 2);
-            classname.erase(pos);
+    for (TestInstance * test : TestRegistry::theInstance().tests())
+    {
+        std::set<std::string> tests;
+        if (!which_tests.empty()) {
+            const auto it = which_tests.find(test->classname);
+            const bool match = it != which_tests.cend();
+            if (match && exclude_tests && it->second.empty()) // only bailout when the whole fixture is excluded
+                continue;
+            if (!match && !exclude_tests)
+                continue;
+            if (match)
+                tests = it->second;
         }
 
-        for (TestInstance * test : TestRegistry::theInstance().tests()) {
-            if (!classname.empty()) {
-                const bool match = test->classname == classname;
-                if ((match && args.exclude_tests()) || (!match && !args.exclude_tests()))
-                    continue;
-            }
-
-            TestFixture* fixture = test->create();
-            fixture->processOptions(args);
-            fixture->run(testname);
-        }
+        TestFixture* fixture = test->create();
+        fixture->processOptions(args);
+        fixture->run(tests);
     }
 
     if (args.summary() && !args.dry_run()) {

test/fixture.h

@@ -34,6 +34,7 @@
 #include <exception>
 #include <list>
 #include <memory>
+#include <set>
 #include <sstream>
 #include <stdexcept>
 #include <string>
@@ -56,9 +57,10 @@ class TestFixture : public ErrorLogger {
 
 protected:
     std::string exename;
-    std::string testToRun;
+    std::set<std::string> testsToRun;
     bool quiet_tests{};
     bool dry_run{};
+    bool exclude_tests{};
     bool mNewTemplate{};
 
     virtual void run() = 0;
@@ -290,7 +292,7 @@ class TestFixture : public ErrorLogger {
     {
         (void) metric;
     }
-    void run(const std::string &str);
+    void run(const std::set<std::string> &tests);
 
 public:
     static void printHelp();

test/main.cpp

@@ -22,6 +22,7 @@
 #include "fixture.h"
 
 #include <cstdlib>
+#include <iostream>
 
 int main(int argc, char *argv[])
 {
@@ -34,6 +35,13 @@ int main(int argc, char *argv[])
         TestFixture::printHelp();
         return EXIT_SUCCESS;
     }
+    if (!args.errors().empty()) {
+        for (const auto& error : args.errors())
+        {
+            std::cout << "error: " << error << '\n';
+        }
+        return EXIT_FAILURE;
+    }
     const std::size_t failedTestsCount = TestFixture::runTests(args);
     return (failedTestsCount == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
 }

test/options.cpp

@@ -17,23 +17,38 @@
 #include "options.h"
 
 options::options(int argc, const char* const argv[])
-    : mWhichTests(argv + 1, argv + argc)
-    ,mQuiet(mWhichTests.count("-q") != 0)
-    ,mHelp(mWhichTests.count("-h") != 0 || mWhichTests.count("--help"))
-    ,mSummary(mWhichTests.count("-n") == 0)
-    ,mDryRun(mWhichTests.count("-d") != 0)
-    ,mExcludeTests(mWhichTests.count("-x") != 0)
-    ,mExe(argv[0])
+    : mExe(argv[0])
 {
-    for (auto it = mWhichTests.cbegin(); it != mWhichTests.cend();) {
-        if (!it->empty() && (((*it)[0] == '-') || (it->find("::") != std::string::npos && mWhichTests.count(it->substr(0, it->find("::"))))))
-            it = mWhichTests.erase(it);
-        else
-            ++it;
-    }
-
-    if (mWhichTests.empty()) {
-        mWhichTests.insert("");
+    const std::set<std::string> args(argv + 1, argv + argc);
+    for (const auto& arg : args) {
+        if (arg.empty())
+            continue; // empty argument
+        if (arg[0] == '-') {
+            if (arg == "-q")
+                mQuiet = true;
+            else if (arg == "-h" || arg == "--help")
+                mHelp = true;
+            else if (arg == "-n")
+                mSummary = false;
+            else if (arg == "-d")
+                mDryRun = true;
+            else if (arg == "-x")
+                mExcludeTests = true;
+            else
+                mErrors.emplace_back("unknown option '" + arg + "'");
+            continue; // command-line switch
+        }
+        const auto pos = arg.find("::");
+        if (pos == std::string::npos) {
+            mWhichTests[arg] = {}; // run whole fixture
+            continue;
+        }
+        const std::string fixture = arg.substr(0, pos);
+        const auto it = mWhichTests.find(fixture);
+        if (it != mWhichTests.cend() && it->second.empty())
+            continue; // whole fixture is already included
+        const std::string test = arg.substr(pos+2);
+        mWhichTests[fixture].emplace(test); // run individual test
     }
 }
 
@@ -57,7 +72,7 @@ bool options::dry_run() const
     return mDryRun;
 }
 
-const std::set<std::string>& options::which_test() const
+const std::map<std::string, std::set<std::string>>& options::which_tests() const
 {
     return mWhichTests;
 }
@@ -71,3 +86,8 @@ bool options::exclude_tests() const
 {
     return mExcludeTests;
 }
+
+const std::vector<std::string>& options::errors() const
+{
+    return mErrors;
+}

test/options.h

@@ -17,8 +17,10 @@
 #ifndef OPTIONS_H
 #define OPTIONS_H
 
+#include <map>
 #include <set>
 #include <string>
+#include <vector>
 
 /**
  * @brief Class to parse command-line parameters for ./testrunner .
@@ -39,8 +41,11 @@ class options {
     bool dry_run() const;
     /** Exclude provided lists of tests. */
     bool exclude_tests() const;
-    /** Which test should be run. Empty string means 'all tests' */
-    const std::set<std::string>& which_test() const;
+    /** Which tests should be run. */
+    const std::map<std::string, std::set<std::string>>& which_tests() const;
+
+    /** Errors encountered during option processing. */
+    const std::vector<std::string>& errors() const;
 
     const std::string& exe() const;
 
@@ -49,12 +54,13 @@ class options {
     options& operator =(const options&) = delete;
 
 private:
-    std::set<std::string> mWhichTests;
-    const bool mQuiet;
-    const bool mHelp;
-    const bool mSummary;
-    const bool mDryRun;
-    const bool mExcludeTests;
+    std::map<std::string, std::set<std::string>> mWhichTests;
+    std::vector<std::string> mErrors;
+    bool mQuiet{};
+    bool mHelp{};
+    bool mSummary{true};
+    bool mDryRun{};
+    bool mExcludeTests{};
     std::string mExe;
 };