Dev -> Master (#204)

* Added RelativeDifferenceScore

* fixup! Added RelativeDifferenceScore

* make judging from features more convenient, this method expects uses to have pre-existing code that performs something like judge generate prediction. Prediction is then just treated as a settable object attribute

* Refactored cached prediction logic for tests

* Restored feature_judge for backwards compatibility

* Drop support for Python 3.5

* Merged

Co-authored-by: Russell Jarvis <[email protected]>

Author: rgerkin

sciunit/scores/complete.py

@@ -8,7 +8,7 @@
 
 import numpy as np
 import quantities as pq
-
+from typing import Union
 from sciunit import errors, utils
 
 from .base import Score
@@ -241,8 +241,83 @@ def norm_score(self) -> float:
         return 1 - 2 * math.fabs(0.5 - cdf)
 
     def __str__(self):
-        return "Ratio = %.2f" % self.score
+        return 'Ratio = %.2f' % self.score
+    
+    
+class RelativeDifferenceScore(Score):
+    """A relative difference between prediction and observation.
+    
+    The absolute value of the difference between the prediction and the
+    observation is divided by a reference value with the same units. This
+    reference scale should be chosen for each test such that normalization
+    produces directly comparable scores across tests. For example, if 5 volts
+    represents a medium size difference for TestA, and 10 seconds represents a
+    medium size difference for TestB, then 5 volts and 10 seconds should be
+    used for this reference scale in TestA and TestB, respectively. The
+    attribute `scale` can be passed to the compute method or set for the whole
+    class in advance. Otherwise, a scale of 1 (in the units of the
+    observation and prediction) will be used.
+    """
+
+    _allowed_types = (float,)
+
+    _description = ('The relative difference between the prediction and the observation')
+
+    _best = 0.0  # A RelativeDifferenceScore of 0.0 is best
+
+    _worst = np.inf
+    
+    scale = None
+
+    def _check_score(self, score):
+        if score < 0.0:
+            raise errors.InvalidScoreError(("RelativeDifferenceScore was initialized with "
+                                            "a score of %f, but a RelativeDifferenceScore "
+                                            "must be non-negative.") % score)
 
+    @classmethod
+    def compute(cls, observation: Union[dict, float, int, pq.Quantity],
+                     prediction: Union[dict, float, int, pq.Quantity],
+                     key=None,
+                     scale: Union[float, int, pq.Quantity, None] = None) -> 'RelativeDifferenceScore':
+        """Compute the relative difference between the observation and a prediction.
+
+        Returns:
+            RelativeDifferenceScore: A relative difference between an observation and a prediction.
+        """
+        assert isinstance(observation, (dict, float, int, pq.Quantity))
+        assert isinstance(prediction, (dict, float, int, pq.Quantity))
+        
+        obs, pred = cls.extract_means_or_values(observation, prediction,
+                                                key=key)
+        
+        scale = scale or cls.scale or (obs/float(obs))
+        assert type(obs) is type(scale)
+        assert type(obs) is type(pred)
+        if isinstance(obs, pq.Quantity):
+            assert obs.units == pred.units, \
+                "Prediction must have the same units as the observation"
+            assert obs.units == scale.units, \
+                "RelativeDifferenceScore.Scale must have the same units as the observation"
+        assert scale > 0, \
+            "RelativeDifferenceScore.scale must be positive (not %g)" % scale
+        value = np.abs(pred - obs) / scale
+        value = utils.assert_dimensionless(value)
+        return RelativeDifferenceScore(value)
+
+    @property
+    def norm_score(self) -> float:
+        """Return 1.0 for a ratio of 0.0, falling to 0.0 for extremely large values.
+
+        Returns:
+            float: The value of the norm score.
+        """
+        x = self.score
+        return 1 / (1+x)
+
+    def __str__(self):
+        return 'Relative Difference = %.2f' % self.score
+      
 
 class RelativeDifferenceScore(Score):
     """A relative difference between prediction and observation.

sciunit/tests.py

@@ -420,8 +420,21 @@ def _judge(
 
         # 6.
         self._bind_score(score, model, self.observation, prediction)
-
+        
         return score
+    
+    
+    def feature_judge(
+        self,
+        model: Model,
+        skip_incapable: bool = False,
+        stop_on_error: bool = True,
+        deep_error: bool = False,
+    ) -> Score:
+        """For backwards compatibility"""
+        return self.judge(model, skip_incapable=skip_incapable, stop_on_error=stop_on_error,
+                          deep_error=deep_error, cached_prediction=True)
+
 
     def feature_judge(
         self,
@@ -445,7 +458,7 @@ def judge(
         skip_incapable: bool = False,
         stop_on_error: bool = True,
         deep_error: bool = False,
-        cached_prediction: bool = False,
+        cached_prediction: bool = False
     ) -> Score:
         """Generate a score for the provided model (public method).