Fix for rii encoding problems under windows + windows pytest (for 3.13 only) (#228)

* Use chardet in RII scripts

* Add windows pytest

* Fix installation of uv

* Us uv install action

* Add dispersion with encoding errors

* Cleanup diff

---------

Co-authored-by: MarJMue <[email protected]>

Author: domna

.github/workflows/pytest.yml

@@ -10,6 +10,29 @@ on:
     branches: [master]
 
 jobs:
+  test_python_win:
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Install module
+        run: |
+          uv pip install --system ".[fitting,dev]"
+          uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu
+      - name: Test with pytest
+        run: |
+          pytest --nbmake
+
   test_python:
     runs-on: ubuntu-latest
     strategy:

src/elli/database/refractive_index_info.py

@@ -28,6 +28,7 @@
     IndexDispersionSum,
 )
 from ..materials import IsotropicMaterial
+from ..utils import detect_encoding
 
 WavelengthFilterType = Union[
     None, float, int, List[Union[float, int]], Tuple[Union[float, int]]
@@ -287,18 +288,7 @@ def get_dispersion(
         Returns:
             Dispersion: A dispersion object containing the tabulated data.
         """
-
-        index = self.catalog.loc[
-            (self.catalog["book"] == book) & (self.catalog["page"] == page)
-        ].index
-
-        if len(index) != 1:
-            raise ValueError("No entry found.")
-
-        yml_file = yaml.load(
-            self.rii_path.joinpath(self.catalog.loc[index[0]]["path"]).read_text(),
-            yaml.SafeLoader,
-        )
+        yml_file = self._open_file(book, page)
 
         dispersion_list = []
         contains_index_dispersion = False
@@ -441,18 +431,7 @@ def get_reference(self, book: str, page: str) -> str:
         Returns:
             str: Reference information.
         """
-
-        index = self.catalog.loc[
-            (self.catalog["book"] == book) & (self.catalog["page"] == page)
-        ].index
-
-        if len(index) != 1:
-            raise ValueError("No entry found.")
-
-        yml_file = yaml.load(
-            self.rii_path.joinpath(self.catalog.loc[index[0]]["path"]).read_text(),
-            yaml.SafeLoader,
-        )
+        yml_file = self._open_file(book, page)
 
         return yml_file["REFERENCES"]
 
@@ -466,6 +445,20 @@ def get_comment(self, book: str, page: str) -> str:
         Returns:
             str: Dispersion information.
         """
+        yml_file = self._open_file(book, page)
+
+        return yml_file["COMMENTS"]
+
+    def _open_file(self, book: str, page: str) -> dict:
+        """Opens the selected dispersion file.
+
+        Args:
+            book (str): Name of the Material, named 'Book' on the website and the database. E.g. 'Au'
+            page (str): Name of the Source, named 'Page' on the website and the database. E.g. 'Johnson'
+
+        Returns:
+            dict: Content of the YAML file.
+        """
 
         index = self.catalog.loc[
             (self.catalog["book"] == book) & (self.catalog["page"] == page)
@@ -474,9 +467,15 @@ def get_comment(self, book: str, page: str) -> str:
         if len(index) != 1:
             raise ValueError("No entry found.")
 
-        yml_file = yaml.load(
-            self.rii_path.joinpath(self.catalog.loc[index[0]]["path"]).read_text(),
-            yaml.SafeLoader,
+        encoding = detect_encoding(
+            self.rii_path.joinpath(self.catalog.loc[index[0]]["path"])
         )
 
-        return yml_file["COMMENTS"]
+        with open(
+            self.rii_path.joinpath(self.catalog.loc[index[0]]["path"]),
+            "r",
+            encoding=encoding,
+        ) as f:
+            yml_file = yaml.load(f, yaml.SafeLoader)
+
+        return yml_file

src/elli/importer/__init__.py

@@ -1,14 +0,0 @@
-import chardet
-
-
-def detect_encoding(fname: str) -> str:
-    r"""Detects the encoding of file fname.
-    Args:
-      fname (str): Filename
-    Returns:
-      str: Encoding identifier string.
-    """
-    with open(fname, "rb") as f:
-        raw_data = f.read()
-    result = chardet.detect(raw_data)
-    return result["encoding"]

src/elli/importer/accurion.py

@@ -5,8 +5,7 @@
 import numpy as np
 import pandas as pd
 
-from ..utils import convert_delta_range
-from . import detect_encoding
+from ..utils import convert_delta_range, detect_encoding
 
 
 def read_accurion_psi_delta(fname: str) -> pd.DataFrame:

src/elli/importer/spectraray.py

@@ -8,8 +8,7 @@
 import pandas as pd
 from packaging.version import Version, parse
 
-from ..utils import calc_rho, convert_delta_range
-from . import detect_encoding
+from ..utils import calc_rho, convert_delta_range, detect_encoding
 
 
 def read_spectraray_psi_delta(

src/elli/importer/woollam.py

@@ -11,8 +11,7 @@
 from pint import DimensionalityError, UndefinedUnitError
 
 from ..units import ureg
-from ..utils import calc_rho
-from . import detect_encoding
+from ..utils import calc_rho, detect_encoding
 
 logger = logging.getLogger(__name__)
 

src/elli/utils.py

@@ -1,6 +1,7 @@
 # Encoding: utf-8
 from dataclasses import dataclass
 
+import chardet
 import numpy as np
 import numpy.typing as npt
 import pandas as pd
@@ -361,3 +362,16 @@ def rotation_v_theta(v: npt.ArrayLike, theta: float) -> npt.NDArray:
         + m_w * np.sin(np.deg2rad(theta))
         + np.linalg.matrix_power(m_w, 2) * (1 - np.cos(np.deg2rad(theta)))
     )
+
+
+def detect_encoding(fname: str) -> str:
+    r"""Detects the encoding of file fname.
+    Args:
+      fname (str): Filename
+    Returns:
+      str: Encoding identifier string.
+    """
+    with open(fname, "rb") as f:
+        raw_data = f.read()
+    result = chardet.detect(raw_data)
+    return result["encoding"]

tests/test_refractive_index_info.py

@@ -1,9 +1,10 @@
 """Tests the importing from the refractiveindex.info database."""
 
-import elli
 import numpy as np
 import pytest
 
+import elli
+
 
 class TestRefractiveIndexInfo:
     """Test if the refractive index.info parser works."""
@@ -14,6 +15,10 @@ def test_get_mat(self):
         mat = self.RII.get_mat("Au", "Johnson")
         assert isinstance(mat, elli.IsotropicMaterial)
 
+    def test_encoding(self):
+        mat = self.RII.get_mat("Ta2O5", "Franta")
+        assert isinstance(mat, elli.IsotropicMaterial)
+
     def test_dispersion_error(self):
         with pytest.raises(ValueError):
             self.RII.get_dispersion("foo", "bar")