Confidence interval

Author: mgraffg

.devcontainer/devcontainer.json

@@ -1,10 +1,7 @@
-{"image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+{"image": "mcr.microsoft.com/devcontainers/python:3.13",
 "features": {
     "ghcr.io/rocker-org/devcontainer-features/quarto-cli": 
-        {"installChromium": true, "installTinyTex": true},
-    "ghcr.io/rocker-org/devcontainer-features/apt-packages:1": 
-        {"packages": "ca-certificates,fonts-liberation,libasound2,libatk-bridge2.0-0,libatk1.0-0,libc6,libcairo2,libcups2,libdbus-1-3,libexpat1,libfontconfig1,libgbm1,libgcc1,libglib2.0-0,libgtk-3-0,libnspr4,libnss3,libpango-1.0-0,libpangocairo-1.0-0,libstdc++6,libx11-6,libx11-xcb1,libxcb1,libxcomposite1,libxcursor1,libxdamage1,libxext6,libxfixes3,libxi6,libxrandr2,libxrender1,libxss1,libxtst6,lsb-release,wget,xdg-utils"},    
-    "ghcr.io/rocker-org/devcontainer-features/miniforge:2": {}
+        {"installChromium": true, "installTinyTex": true}
     },
-"postCreateCommand": "conda env create --file environment.yml"
+"postCreateCommand": "python -m pip install -r requirements.txt"
 }

.gitignore

@@ -160,4 +160,5 @@ cython_debug/
 #.idea/
 
 .vscode/settings.json
-quarto/CompStats_files/
+quarto/CompStats_files/
+quarto/

CompStats/__init__.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = '0.1.13'
+__version__ = '0.1.14'
 from CompStats.bootstrap import StatisticSamples
 from CompStats.measurements import CI, SE, difference_p_value
 from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference

CompStats/interface.py

@@ -19,7 +19,7 @@
 from CompStats.bootstrap import StatisticSamples
 from CompStats.utils import progress_bar
 from CompStats import measurements
-from CompStats.measurements import SE
+from CompStats.measurements import SE, CI
 from CompStats.utils import dataframe
 
 
@@ -270,7 +270,7 @@ def best(self):
                 best = data.argmin()
         self._best = keys[best]
         return self._best
-    
+
     @best.setter
     def best(self, value):
         self._best = value
@@ -279,7 +279,7 @@ def best(self, value):
     def sorting_func(self):
         """Rank systems when multiple performances are used"""
         return self._sorting_func
-    
+
     @sorting_func.setter
     def sorting_func(self, value):
         self._sorting_func = value
@@ -315,7 +315,7 @@ def statistic(self):
         else:
             self._statistic = dict(data)
         return self._statistic
-    
+
     @statistic.setter
     def statistic(self, value):
         """statistic setter"""
@@ -346,6 +346,30 @@ def se(self):
             return list(output.values())[0]
         return output
 
+    @property
+    def ci(self):
+        """Confidence interval
+    
+        >>> from sklearn.svm import LinearSVC
+        >>> from sklearn.datasets import load_iris
+        >>> from sklearn.model_selection import train_test_split
+        >>> from CompStats.interface import Perf
+        >>> X, y = load_iris(return_X_y=True)
+        >>> _ = train_test_split(X, y, test_size=0.3)
+        >>> X_train, X_val, y_train, y_val = _
+        >>> m = LinearSVC().fit(X_train, y_train)
+        >>> hy = m.predict(X_val)
+        >>> ens = RandomForestClassifier().fit(X_train, y_train)
+        >>> perf = Perf(y_val, hy, name='LinearSVC')
+        >>> perf.ci
+        (np.float64(0.9333333333333332), np.float64(1.0))
+        """
+
+        output = CI(self.statistic_samples)
+        if len(output) == 1:
+            return list(output.values())[0]
+        return output
+
     def plot(self, value_name:str=None,
              var_name:str='Performance',
              alg_legend:str='Algorithm',

CompStats/tests/test_interface.py

@@ -121,6 +121,7 @@ def test_Perf_plot_multi():
     f_grid = score.plot()
     assert f_grid is not None
 
+
 def test_Perf_statistic_one():
     """Test Perf statistic one alg"""
     from CompStats.metrics import f1_score
@@ -142,6 +143,9 @@ def test_Perf_statistic_one():
     assert isinstance(score.statistic, float)
     assert isinstance(str(score), str)
     assert isinstance(score.se, float)
+    assert isinstance(score.ci, tuple)
+    assert len(score.ci) == 2
+
 
 def test_Perf_best():
     """Test Perf best"""
@@ -191,7 +195,7 @@ def test_difference_best():
     score(svm.predict(X_val), name='svm')
     diff = score.difference()
     assert isinstance(diff.best, str)
-    
+
 
 def test_difference_str__():
     """Test f1_score"""

pyproject.toml

@@ -2,6 +2,7 @@
 name = 'CompStats'
 description = 'CompStats implements an evaluation methodology for statistically analyzing competition results and competition'
 readme = "README.rst"
+license = "Apache-2.0"
 dependencies = [
     'numpy',
     'scikit-learn>=1.3.0',
@@ -17,7 +18,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Information Technology",
     "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",

requirements.txt

@@ -0,0 +1,8 @@
+numpy
+scikit-learn
+seaborn
+jupyter
+pyyaml
+sphinx
+pytest
+statsmodels