Merge pull request #915 from davidhassell/zarr-write

Zarr write + Zarr groups

Author: davidhassell

Changelog.rst

@@ -1,8 +1,15 @@
 Version NEXTVERSION
--------------------
+--------------
 
 **2026-01-??**
 
+* Write Zarr v3 datasets with `cf.write`
+  (https://github.com/NCAS-CMS/cf-python/issues/895)
+* Read Zarr v2 and v3 datasets that contain a group hierarchy with
+  `cf.read` (https://github.com/NCAS-CMS/cf-python/issues/894)
+* Reduce the time taken to import `cf`
+  (https://github.com/NCAS-CMS/cf-python/issues/902)
+* New optional dependency: ``zarr>=3.1.3``
 * New function to control the creation of cached elements during data
   display: `cf.display_data`
   (https://github.com/NCAS-CMS/cf-python/issues/913)

README.md

@@ -87,7 +87,8 @@ of its array manipulation and can:
 * read field constructs from netCDF, CDL, Zarr, PP and UM datasets with a
   choice of netCDF backends,and in local, http, and s3 locations,
 * create new field constructs in memory,
-* write and append field and domain constructs to netCDF datasets on disk,
+* write and append field and domain constructs to netCDF and Zarr v3
+  datasets on disk,
 * read, create, and manipulate UGRID mesh topologies,
 * read, write, and create coordinates defined by geometry cells,
 * read netCDF and CDL datasets containing hierarchical groups,

cf/aggregate.py

@@ -4112,7 +4112,7 @@ def _get_hfl(
         # Record the bounds of the first and last (sorted) cells
         first, last = hfl_cache.flb.get(hash_value, (None, None))
         if first is None:
-            cached_elements = d._get_cached_elements()
+            cached_elements = d.get_cached_elements()
             x = []
             for i in (0, 1, -2, -1):
                 value = cached_elements.get(i)

cf/data/data.py

@@ -2761,7 +2761,7 @@ def Units(self, value):
             self._set_dask(dx, clear=self._ALL ^ self._CACHE ^ self._CFA)
 
             # Adjust cached values for the new units
-            cache = self._get_cached_elements()
+            cache = self.get_cached_elements()
             if cache:
                 self._set_cached_elements(
                     {index: cf_func(value) for index, value in cache.items()}

cf/dimensioncoordinate.py

@@ -165,7 +165,7 @@ def _infer_direction(self):
         if data is not None:
             # Infer the direction from the data
             if data.size > 1:
-                c = data._get_cached_elements()
+                c = data.get_cached_elements()
                 if c:
                     try:
                         return bool(c.get(0) <= c.get(1))
@@ -179,7 +179,7 @@ def _infer_direction(self):
         data = self.get_bounds_data(None, _fill_value=False)
         if data is not None:
             # Infer the direction from the bounds
-            c = data._get_cached_elements()
+            c = data.get_cached_elements()
             if c:
                 try:
                     return bool(c.get(0) <= c.get(1))

cf/field.py

@@ -6994,7 +6994,7 @@ def collapse(
                 else:
                     b = dim.data
 
-                cached_elements = b._get_cached_elements()
+                cached_elements = b.get_cached_elements()
                 try:
                     # Try to set the new bounds from cached values
                     bounds_data = Data(

cf/functions.py

@@ -3198,7 +3198,7 @@ def environment(display=True, paths=True):
     netCDF4: 1.7.2 /home/miniconda3/lib/python3.12/site-packages/netCDF4/__init__.py
     h5netcdf: 1.3.0 /home/miniconda3/lib/python3.12/site-packages/h5netcdf/__init__.py
     h5py: 3.12.1 /home/miniconda3/lib/python3.12/site-packages/h5py/__init__.py
-    zarr: 3.0.8 /home/miniconda3/lib/python3.12/site-packages/zarr/__init__.py
+    zarr: 3.1.3 /home/miniconda3/lib/python3.12/site-packages/zarr/__init__.py
     s3fs: 2024.12.0 /home/miniconda3/lib/python3.12/site-packages/s3fs/__init__.py
     scipy: 1.15.1 /home/miniconda3/lib/python3.12/site-packages/scipy/__init__.py
     dask: 2025.5.1 /home/miniconda3/lib/python3.12/site-packages/dask/__init__.py
@@ -3224,7 +3224,7 @@ def environment(display=True, paths=True):
     netCDF4: 1.7.2
     h5netcdf: 1.3.0
     h5py: 3.12.1
-    zarr: 3.0.8
+    zarr: 3.1.3
     s3fs: 2024.12.0
     scipy: 1.15.1
     dask: 2025.5.1

cf/read_write/read.py

@@ -316,6 +316,10 @@ class read(cfdm.read):
 
             .. versionadded:: 3.17.0
 
+        {{read store_dataset_shards: `bool`, optional}}
+
+            .. versionadded:: NEXTVERSION
+
         {{read cfa: `dict`, optional}}
 
             .. versionadded:: 3.15.0
@@ -328,6 +332,10 @@ class read(cfdm.read):
 
             .. versionadded:: 3.17.0
 
+        {{read group_dimension_search: `str`, optional}}
+
+            .. versionadded:: NEXTVERSION
+
         umversion: deprecated at version 3.0.0
             Use the *um* parameter instead.
 
@@ -434,6 +442,7 @@ def __new__(
         warn_valid=False,
         dask_chunks="storage-aligned",
         store_dataset_chunks=True,
+        store_dataset_shards=True,
         domain=False,
         cfa=None,
         cfa_write=None,
@@ -445,6 +454,7 @@ def __new__(
         ignore_read_error=False,
         fmt=None,
         file_type=None,
+        group_dimension_search="closest_ancestor",
     ):
         """Read field or domain constructs from a dataset."""
         kwargs = locals()

cf/test/test_Data.py

@@ -4457,28 +4457,30 @@ def test_Data__init__datetime(self):
         self.assertTrue((q == d).array.all())
         self.assertTrue((d == q).array.all())
 
-    def test_Data__str__(self):
-        """Test `Data.__str__`"""
-        elements0 = (0, -1, 1)
-        for array in ([1], [1, 2], [1, 2, 3]):
-            d = cf.Data(array)
-            d[0] = 1
-            self.assertEqual(str(d), str(array))
-            d += 0
-            self.assertEqual(str(d), str(array))
-
-        # Test when size > 3, i.e. second element is not there.
-        d = cf.Data([1, 2, 3, 4])
-
-        self.assertEqual(str(d), "[1, ..., 4]")
-        cache = d.get_cached_elements()
-        self.assertNotIn(1, cache)
-        for element in elements0[:2]:
-            self.assertIn(element, cache)
-
-        d[0] = 1
-        for element in elements0:
-            self.assertNotIn(element, d.get_cached_elements())
+    def test_Data__repr__str(self):
+        """Test all means of Data inspection."""
+        for d in [
+            cf.Data(9, units="km"),
+            cf.Data([9], units="km"),
+            cf.Data([[9]], units="km"),
+            cf.Data([8, 9], units="km"),
+            cf.Data([[8, 9]], units="km"),
+            cf.Data([7, 8, 9], units="km"),
+            cf.Data([[7, 8, 9]], units="km"),
+            cf.Data([6, 7, 8, 9], units="km"),
+            cf.Data([[6, 7, 8, 9]], units="km"),
+            cf.Data([[6, 7], [8, 9]], units="km"),
+            cf.Data([[6, 7, 8, 9], [6, 7, 8, 9]], units="km"),
+        ]:
+            _ = repr(d)
+            _ = str(d)
+
+        # Test when the data contains date-times with the first
+        # element masked
+        dt = np.ma.array([10, 20], mask=[True, False])
+        d = cf.Data(dt, units="days since 2000-01-01")
+        self.assertTrue(str(d) == "[--, 2000-01-21 00:00:00]")
+        self.assertTrue(repr(d) == "<CF Data(2): [--, 2000-01-21 00:00:00]>")
 
     def test_Data_cull_graph(self):
         """Test Data.cull_graph."""