Merge pull request #149 from BerkeleyLab/selective-test-execution

Test reordering and selective test execution

Author: rouson

fpm.toml

@@ -1,9 +1,9 @@
 name = "inference-engine"
-version = "0.11.0"
+version = "0.11.1"
 license = "see LICENSE.txt"
 author = "Damian Rouson, Tan Nguyen, Jordan Welsman, David Torres, Brad Richardson, Katherine Rasmussen, Federica Villani"
 maintainer = "rouson@lbl.gov"
 
 [dependencies]
 assert = {git = "https://github.com/sourceryinstitute/assert", tag = "1.6.0"}
-sourcery = {git = "https://github.com/sourceryinstitute/sourcery", tag = "4.6.1"}
+sourcery = {git = "https://github.com/sourceryinstitute/sourcery", tag = "4.8.0"}

test/asymmetric_engine_test_m.F90

@@ -5,7 +5,8 @@ module asymmetric_engine_test_m
 
   ! External dependencies
   use assert_m, only : assert
-  use sourcery_m, only : string_t, test_t, test_result_t
+  use sourcery_m, only : &
+    test_t, test_result_t, vector_test_description_t, test_description_substring, string_t, vector_function_strategy_t
 
   ! Internal dependencies
   use inference_engine_m, only : inference_engine_t, tensor_t
@@ -22,6 +23,11 @@ module asymmetric_engine_test_m
     procedure, nopass :: results
   end type
 
+  type, extends(vector_function_strategy_t) :: xor_and_2nd_input_t
+  contains
+    procedure, nopass :: vector_function => xor_and_2nd_input_truth_table 
+  end type
+
 contains
 
   pure function subject() result(specimen)
@@ -31,34 +37,28 @@ pure function subject() result(specimen)
 
   function results() result(test_results)
     type(test_result_t), allocatable :: test_results(:)
-
-    character(len=*), parameter :: longest_description = &
-          "mapping (false,false) to false"
-#ifdef _CRAYFTN
-    character(len=len(longest_description)), allocatable :: descriptions(:)
-    logical, allocatable :: outcomes(:)
-    descriptions = [ "mapping (true,true) to false", "mapping (true,false) to false", &
-                     "mapping (false,true) to true", "mapping (false,false) to false" ]
-    outcomes = xor_and_2nd_input_truth_table()
-#else
-    associate( &
-      descriptions => &
-        [ character(len=len(longest_description)) :: &
-          "mapping (true,true) to false", &
-          "mapping (true,false) to false", &
-          "mapping (false,true) to true", &
-          "mapping (false,false) to false" &
-        ], &
-      outcomes => &
-        [ xor_and_2nd_input_truth_table() & 
-        ] & 
+    type(xor_and_2nd_input_t) xor_and_2nd_input
+
+    associate( vector_test_description => vector_test_description_t( &
+      [ string_t("mapping (true,true) to false"), & 
+        string_t("mapping (true,false) to false"), &
+        string_t("mapping (false,true) to true"), &
+        string_t("mapping (false,false) to false") &
+      ], xor_and_2nd_input) &
     )
-#endif
-      call assert(size(descriptions) == size(outcomes),"asymetric_engine_test_m(results): size(descriptions) == size(outcomes)")
-      test_results = test_result_t(descriptions, outcomes)
-#ifndef _CRAYFTN
+      associate(substring_in_subject => index(subject(), test_description_substring) /= 0)
+        associate(substring_in_description => vector_test_description%contains_text(test_description_substring))
+          if (substring_in_subject) then
+            test_results = vector_test_description%run()
+          else if (any(substring_in_description)) then
+            test_results = vector_test_description%run()
+            test_results = pack(test_results, test_results%description_contains(string_t(test_description_substring)))
+           else
+            test_results = [test_result_t::]
+          end if
+        end associate
+      end associate
     end associate
-#endif
   end function
 
   function xor_and_2nd_input_network() result(inference_engine)

test/hyperparameters_test_m.F90

@@ -4,9 +4,11 @@ module hyperparameters_test_m
   !! Test hyperparameters_t object I/O and construction
 
   ! External dependencies
-  use assert_m, only : assert
-  use sourcery_m, only : string_t, test_t, test_result_t, file_t
   use inference_engine_m, only : hyperparameters_t
+  use sourcery_m, only : test_t, test_result_t, test_description_t, test_description_substring, string_t
+#ifdef __GFORTRAN__
+  use sourcery_m, only : test_function_i
+#endif
 
   ! Internal dependencies
   use hyperparameters_m, only : hyperparameters_t
@@ -30,24 +32,32 @@ pure function subject() result(specimen)
   end function
 
   function results() result(test_results)
+    type(test_description_t), allocatable :: test_descriptions(:)
     type(test_result_t), allocatable :: test_results(:)
 
-    character(len=*), parameter :: longest_description = &
-          "component-wise construction followed by conversion to and from JSON"
+#ifndef __GFORTRAN__
+    test_descriptions = [ & 
+      test_description_t( &
+        string_t("component-wise construction followed by conversion to and from JSON"), &
+        write_then_read_hyperparameters) &
+    ]
+#else
+    procedure(test_function_i), pointer :: check_write_then_read_ptr
+    check_write_then_read_ptr => write_then_read_hyperparameters
 
+    test_descriptions = [ &
+      test_description_t( &
+        string_t("component-wise construction followed by conversion to and from JSON"), &
+        check_write_then_read_ptr) &
+    ]
+#endif
     associate( &
-      descriptions => &
-        [ character(len=len(longest_description)) :: &
-          "component-wise construction followed by conversion to and from JSON" &
-        ], &
-      outcomes => &
-        [ write_then_read_hyperparameters() & 
-        ] & 
+      substring_in_subject => index(subject(), test_description_substring) /= 0, &
+      substring_in_description => test_descriptions%contains_text(string_t(test_description_substring)) &
     )
-      call assert(size(descriptions) == size(outcomes),"hyperparameters_test_m(results): size(descriptions) == size(outcomes)")
-      test_results = test_result_t(descriptions, outcomes)
+      test_descriptions = pack(test_descriptions, substring_in_subject .or. substring_in_description)
     end associate
-       
+    test_results = test_descriptions%run()
   end function
 
   function write_then_read_hyperparameters() result(test_passes)

test/inference_engine_test_m.f90 test/inference_engine_test_m.F90test/inference_engine_test_m.f90 renamed to test/inference_engine_test_m.F90

@@ -6,7 +6,10 @@ module inference_engine_test_m
   ! External dependencies
   use assert_m, only : assert
   use kind_parameters_m, only : rkind
-  use sourcery_m, only : string_t, test_t, test_result_t, file_t
+  use sourcery_m, only : test_t, test_result_t, test_description_t, test_description_substring, string_t, file_t
+#ifdef __GFORTRAN__
+  use sourcery_m, only : test_function_i
+#endif
 
   ! Internal dependencies
   use inference_engine_m, only : inference_engine_t, tensor_t, difference_t
@@ -31,25 +34,33 @@ pure function subject() result(specimen)
 
   function results() result(test_results)
     type(test_result_t), allocatable :: test_results(:)
-
-    character(len=*), parameter :: longest_description = &
-          "converting a network with 2 hidden layers to and from JSON format"
-
+    type(test_description_t), allocatable :: test_descriptions(:)
+
+#ifndef __GFORTRAN__
+    test_descriptions = [ &
+      test_description_t("performing elemental inference with 1 hidden layer", elemental_infer_with_1_hidden_layer_xor_net), &
+      test_description_t("performing elemental inference with 2 hidden layers", elemental_infer_with_2_hidden_layer_xor_net), &
+      test_description_t("converting a network with 2 hidden layers to and from JSON format", multi_hidden_layer_net_to_from_json) &
+    ]
+#else
+    procedure(test_function_i), pointer :: elemental_infer_1_ptr, elemental_infer_2_ptr, multi_hidden_ptr
+    elemental_infer_1_ptr => elemental_infer_with_1_hidden_layer_xor_net
+    elemental_infer_2_ptr => elemental_infer_with_2_hidden_layer_xor_net
+    multi_hidden_ptr => multi_hidden_layer_net_to_from_json
+
+    test_descriptions = [ &
+      test_description_t("performing elemental inference with 1 hidden layer", elemental_infer_1_ptr), &
+      test_description_t("performing elemental inference with 2 hidden layers", elemental_infer_2_ptr), &
+      test_description_t("converting a network with 2 hidden layers to and from JSON format", multi_hidden_ptr) &
+    ]
+#endif
     associate( &
-      descriptions => &
-        [ character(len=len(longest_description)) :: &
-          "performing elemental inference with 1 hidden layer", &
-          "performing elemental inference with 2 hidden layers", &
-          "converting a network with 2 hidden layers to and from JSON format"  &
-        ], &
-      outcomes => &
-        [ elemental_infer_with_1_hidden_layer_xor_net(), elemental_infer_with_2_hidden_layer_xor_net(), &
-          multi_hidden_layer_net_to_from_json() &
-        ] &
+      substring_in_subject => index(subject(), test_description_substring) /= 0, &
+      substring_in_description => test_descriptions%contains_text(string_t(test_description_substring)) &
     )
-      call assert(size(descriptions) == size(outcomes), "inference_engine_test(results): size(descriptions) == size(outcomes)")
-      test_results = test_result_t(descriptions, outcomes)
+      test_descriptions = pack(test_descriptions, substring_in_subject .or. substring_in_description)
     end associate
+    test_results = test_descriptions%run()
   end function
 
   function single_hidden_layer_xor_network() result(inference_engine)
@@ -98,7 +109,7 @@ function distinct_parameters() result(inference_engine)
   end function
 
   function multi_hidden_layer_net_to_from_json() result(test_passes)
-    logical, allocatable :: test_passes
+    logical test_passes
     type(inference_engine_t) inference_engine, from_json
     type(file_t) json_file
     type(difference_t) difference
@@ -112,7 +123,7 @@ function multi_hidden_layer_net_to_from_json() result(test_passes)
   end function
 
   function elemental_infer_with_1_hidden_layer_xor_net() result(test_passes)
-    logical, allocatable :: test_passes(:)
+    logical test_passes
     type(inference_engine_t) inference_engine
 
     inference_engine = single_hidden_layer_xor_network()
@@ -125,15 +136,15 @@ function elemental_infer_with_1_hidden_layer_xor_net() result(test_passes)
       associate(array_of_inputs => [tensor_t([true,true]), tensor_t([true,false]), tensor_t([false,true]), tensor_t([false,false])])
         truth_table = inference_engine%infer(array_of_inputs)
       end associate
-      test_passes = [ &
+      test_passes = all( &
         abs(truth_table(1)%values() - false) < tolerance .and. abs(truth_table(2)%values() - true) < tolerance .and. &
         abs(truth_table(3)%values() - true) < tolerance .and. abs(truth_table(4)%values() - false) < tolerance &
-      ]
+      )
     end block
   end function
 
   function elemental_infer_with_2_hidden_layer_xor_net() result(test_passes)
-    logical, allocatable :: test_passes(:)
+    logical test_passes
     type(inference_engine_t) inference_engine
 
     inference_engine = multi_layer_xor_network()
@@ -146,10 +157,10 @@ function elemental_infer_with_2_hidden_layer_xor_net() result(test_passes)
       associate(array_of_inputs => [tensor_t([true,true]), tensor_t([true,false]), tensor_t([false,true]), tensor_t([false,false])])
         truth_table = inference_engine%infer(array_of_inputs)
       end associate
-      test_passes = [ &
+      test_passes = all( &
         abs(truth_table(1)%values() - false) < tolerance .and. abs(truth_table(2)%values() - true) < tolerance .and. &
         abs(truth_table(3)%values() - true) < tolerance .and. abs(truth_table(4)%values() - false) < tolerance &
-      ]
+      )
     end block
   end function
 

test/main.f90

@@ -8,6 +8,7 @@ program main
   use network_configuration_test_m, only : network_configuration_test_t
   use training_configuration_test_m, only : training_configuration_test_t
   use tensor_range_test_m, only : tensor_range_test_t
+  use sourcery_m, only : command_line_t
   implicit none
 
   type(inference_engine_test_t) inference_engine_test
@@ -21,15 +22,28 @@ program main
 
   integer :: passes=0, tests=0
 
+  print_usage_if_help_requested: &
+  block
+    type(command_line_t) command_line
+    character(len=*), parameter :: usage = &
+      new_line('') // new_line('') // &
+      'Usage: fpm test -- [--help] | [--contains <substring>]' // &
+      new_line('') // new_line('') // &
+      'where square brackets ([]) denote optional arguments, a pipe (|) separates alternative arguments,' // new_line('') // &
+      'angular brackets (<>) denote a user-provided value, and passing a substring limits execution to' // new_line('') // &
+      'the tests with test subjects or test descriptions containing the user-specified substring.' // new_line('')
+    if (command_line%argument_present([character(len=len("--help"))::"--help","-h"])) stop usage
+  end block print_usage_if_help_requested
+
   call cpu_time(t_start)
   call random_init(repeatable=.true.,image_distinct=.true.)
   call hyperparameters_test%report(passes, tests)
   call network_configuration_test%report(passes, tests)
   call training_configuration_test%report(passes, tests)
+  call tensor_range_test%report(passes, tests)
   call asymmetric_engine_test%report(passes, tests)
   call inference_engine_test%report(passes, tests)
   call trainable_engine_test%report(passes, tests)
-  call tensor_range_test%report(passes, tests)
   call cpu_time(t_finish)
 
   print *

test/network_configuration_test_m.F90

@@ -4,13 +4,14 @@ module network_configuration_test_m
   !! Test network_configuration_t object I/O and construction
 
   ! External dependencies
-  use assert_m, only : assert
-  use sourcery_m, only : string_t, test_t, test_result_t, file_t
   use inference_engine_m, only : network_configuration_t
+  use sourcery_m, only : test_t, test_result_t, test_description_t, test_description_substring, string_t, file_t
+#ifdef __GFORTRAN__
+  use sourcery_m, only : test_function_i
+#endif
 
   ! Internal dependencies
   use network_configuration_m, only : network_configuration_t
-
   implicit none
 
   private
@@ -31,24 +32,31 @@ pure function subject() result(specimen)
 
   function results() result(test_results)
     type(test_result_t), allocatable :: test_results(:)
+    type(test_description_t), allocatable :: test_descriptions(:)
 
-    character(len=*), parameter :: longest_description = &
-          "component-wise construction followed by conversion to and from JSON"
+#ifndef __GFORTRAN__
+    test_descriptions = [ & 
+      test_description_t( &
+        string_t("component-wise construction followed by conversion to and from JSON"), &
+        write_then_read_network_configuration) &
+    ]
+#else
+    procedure(test_function_i), pointer :: check_write_then_read_ptr
+    check_write_then_read_ptr => write_then_read_network_configuration
 
+    test_descriptions = [ &
+      test_description_t( &
+        string_t("component-wise construction followed by conversion to and from JSON"), &
+        check_write_then_read_ptr) &
+    ]
+#endif
     associate( &
-      descriptions => &
-        [ character(len=len(longest_description)) :: &
-          "component-wise construction followed by conversion to and from JSON" &
-        ], &
-      outcomes => &
-        [ write_then_read_network_configuration() & 
-        ] & 
+      substring_in_subject => index(subject(), test_description_substring) /= 0, &
+      substring_in_description => test_descriptions%contains_text(string_t(test_description_substring)) &
     )
-      call assert(size(descriptions) == size(outcomes), &
-        "network_configuration_test_m(results): size(descriptions) == size(outcomes)")
-      test_results = test_result_t(descriptions, outcomes)
+      test_descriptions = pack(test_descriptions, substring_in_subject .or. substring_in_description)
     end associate
-       
+    test_results = test_descriptions%run()
   end function
 
   function write_then_read_network_configuration() result(test_passes)