Record alignment of class in dictionary and use in TStreamerInfo

.github/workflows/code_analysis.yml

@@ -90,8 +90,8 @@ jobs:
       run: |
         files=$(grep '\.py$' changed_files.txt || echo "")
         if [ -n "$files" ]; then
-          diff_command=""
           apply_command=""
+          failure=false
           for file in $files; do
             while IFS=- read -r start length; do
               [ -z "$start" ] && continue
@@ -101,20 +101,14 @@ jobs:
                 continue
               fi
               end=$((start + length))
-              diff_command+="ruff format --diff --range $start-$end $file && "
+              ruff format --diff --preview --range ${start}-${end} ${file} || failure=true
               apply_command+="ruff format --range $start-$end $file && "
             done < <(git diff --unified=0 HEAD~1 -- "$file" | grep '^@@' | sed -E 's/^@@ -[0-9]+(,[0-9]+)? \+([0-9]+)(,([0-9]+))? @@.*/\2-\4/')
           done
 
-          if [ -n "$diff_command" ]; then
-            diff_command=${diff_command% && }
-            if ! eval "$diff_command"; then
-              apply_command=${apply_command% && }
-              echo -e "::error::Formatting failed. To apply the changes locally, run the following command:\n$apply_command"
-              exit 123
-            fi
-          else
-            echo "No ranges detected to format."
+          if ${failure}; then
+            echo -e "::error::Formatting failed. To apply the changes locally, run the following command:\n$apply_command"
+            exit 123;
           fi
         else
           echo "No python files to format"

.github/workflows/root-ci-config/buildconfig/alma10-benchmark.txt

@@ -1,7 +1,5 @@
 CMAKE_CXX_STANDARD=20
 builtin_vdt=ON
-builtin_zlib=ON
-builtin_zstd=ON
 pythia8=ON
 r=OFF
 rootbench=ON

.github/workflows/root-ci-config/buildconfig/alma10-clang_ninja.txt

@@ -1,12 +1,13 @@
 CMAKE_C_COMPILER=clang
 CMAKE_CXX_COMPILER=clang++
 CMAKE_GENERATOR=Ninja
-builtin_vdt=ON
+builtin_freetype=ON
 builtin_gif=ON
 builtin_jpeg=ON
-builtin_png=ON
 builtin_lz4=ON
 builtin_lzma=ON
+builtin_png=ON
+builtin_vdt=ON
 builtin_zlib=ON
 builtin_zstd=ON
 pythia8=ON

.github/workflows/root-ci-config/buildconfig/alma10-minimal.txt

@@ -1,7 +1,5 @@
 ccache=ON
 builtin_vdt=ON
-builtin_zlib=ON
-builtin_zstd=ON
 fail-on-missing=ON
 minimal=ON
 roottest=ON

.github/workflows/root-ci-config/buildconfig/alma10.txt

@@ -1,5 +1,3 @@
 builtin_vdt=ON
-builtin_zlib=ON
-builtin_zstd=ON
 pythia8=ON
 r=OFF

.github/workflows/root-ci-config/buildconfig/fedora43.txt

@@ -1,6 +1,4 @@
 CMAKE_CXX_STANDARD=23
-builtin_zlib=ON
-builtin_zstd=ON
 experimental_adaptivecpp=ON
 pythia8=ON
 roofit_multiprocess=ON

.github/workflows/root-ci-config/buildconfig/fedora44.txt

@@ -1,6 +1,4 @@
 CMAKE_CXX_STANDARD=23
-builtin_zlib=ON
-builtin_zstd=ON
 experimental_adaptivecpp=ON
 pythia8=ON
 roofit_multiprocess=ON

.github/workflows/root-ci-config/buildconfig/rawhide.txt

@@ -1,5 +1,3 @@
-builtin_zlib=ON
-builtin_zstd=ON
 pythia8=ON
 test_distrdf_dask=OFF
 test_distrdf_pyspark=OFF

CMakeLists.txt

@@ -411,6 +411,9 @@ else()
   if(asimage)
     message(SEND_ERROR "minimal=ON is not compatible with asimage=ON. Consider setting instead gminimal=ON")
   endif()
+  if(asimage_tiff)
+    message(SEND_ERROR "minimal=ON is not compatible with asimage_tiff=ON. Consider setting instead gminimal=ON")
+  endif()
   if(x11)
     message(SEND_ERROR "minimal=ON is not compatible with x11=ON. Consider setting instead gminimal=ON")
   endif()

README/ReleaseNotes/v640/index.md

@@ -48,6 +48,9 @@ The following people have contributed to this new version:
   The ZeroMQ library and its C++ bindings are used by the experimental RooFit multiprocessing package, enabled by the `roofit_multiprocess` build option.
   The ZeroMQ versions it requires (>=4.3.6 or 4.3.5 with the draft API) are now available in the package managers of several platforms, for example Conda, Homebrew, Fedora and the Extra Packages for Enterprise Linux (EPEL).
   The `roofit_multiprocess` feature is only required by a small set of RooFit power uses, who are using one of these environments and therefore don't require the builtin ZeroMQ library.
+* The overloads of `RooAbsReal::createChi2()` and `RooAbsReal::chi2FitTo()` that take unbinned **RooDataSet** data objects are deprecated and will be removed in ROOT 6.42.
+  These methods implemented a specialized chi-square fit for x-y-data with errors in y and optional errors in x, which is conceptually different from the standard histogram-based chi-square in the **RooDataHist** case and can lead to ambiguous results.
+  To fit 2D data with errors in and `x` and `y`, use specialized tools like `TGraphErrors::Fit()`, or build an explicit likelihood model if you want to stay with RooFit.
 
 ## Removals
 
@@ -69,6 +72,12 @@ The following people have contributed to this new version:
 
 ## Build System
 
+### Moving from builtin dependencies to system-provided packages
+
+* The general direction of the ROOT project is to become more and more reliant on system packages. It is *recommended* to make the packages required by ROOT available on the system, e.g. via a package manager, and not with the builtin mechanism. This allows for timely updates and reduces the size of the installed binaries.
+* The previously vendored builtins `freetype`, `zlib`, `lzma`, `zstd`, `lz4`, `libpng`, `giflib`, `libjpeg`, and `openssl` should be installed in the system if possible. ROOT will not automatically fall-back to their builtin versions if these are not found: the user is informed of that with a helpful message. If installing these dependencies in the system is not possible, the CMake option `-Dbuiltin_XYZ=ON` has to be consciously chosen by the user.
+* For the builtin versions of `freetype`, `zlib`, `lzma`, `zstd`, `lz4`, `libpng`, `giflib`, `libjpeg`, the source tarballs are now fetched from [SPI](https://spi.web.cern.ch)'s [website](https://lcgpackages.web.cern.ch/), as for the vast majority of ROOT's builtins, e.g. `openssl` or `xrootd`.
+
 ## Core Libraries
 
 * ROOT now adds a RUNPATH to compiled macros. This ensures that when compiled macros are loaded, they load the libraries that belong to the ROOT installation
@@ -253,6 +262,14 @@ Now:
 Results may differ from previous ROOT versions for `n > 100` or `nSigma != 1`.
 The new implementation is statistically consistent and recommended.
 
+### Removal of global expensive object caching in RooFit
+
+The global “expensive object cache” used in RooFit to store numeric integrals and intermediate histogram results has been removed.
+
+While originally intended as a performance optimization, this mechanism could lead to **incorrect results due to cache collisions**: cached integrals or histograms created in one context (e.g. during plotting) could be reused unintentionally in a different context, even when the underlying configuration had changed.
+
+Given the risk of silently incorrect physics results, and the absence of known workflows that depend critically on this feature, this global caching mechanism has been removed. If you encounter performance regressions in workflows involving expensive integrals or convolutions, we encourage you to report your use case and performance regression as a GitHub issue, so that targeted and robust optimizations can be developed,
+
 ## RDataFrame
 
 - The message shown in ROOT 6.38 to inform users about change of default compression setting used by Snapshot (was 101 before 6.38, became 505 in 6.38) is now removed.
@@ -369,6 +386,29 @@ after importing ROOT.
 
 This option is intended **for debugging only and will be removed in ROOT 6.44**.
 
+### Drop support for calling C++ functions with non-const pointer references
+
+From now on, we disallow calling C++ functions with non-const pointer references (`T*&`) from Python.
+These allow pointer rebinding, which cannot be represented safely in Python and could previously lead to confusing behavior.
+A `TypeError` is now raised. Typical ROOT usage is unaffected.
+
+In the rare case where you want to call such a function, please change the C++ interface or - if the interface is outside your control - write a wrapper function that avoids non-const pointer references as arguments.
+
+For example, a function with the signature `bool setPtr(MyClass *&)` could be wrapped by a function that augments the return value with the updated pointer:
+```
+ROOT.gInterpreter.Declare("""
+    std::pair<bool, MyClass*> setPtrWrapper(MyClass *ptr) {
+       bool out = setPtr(ptr);
+       return {out, ptr};
+    }
+""")
+```
+Then, call it from Python as follows:
+```Python
+# Use tuple unpacking for convenience
+_, ptr = cppyy.gbl.setPtrWrapper(ptr)
+```
+
 ### UHI
 #### Backwards incompatible changes
 - `TH1.values()` now returns a **read-only** NumPy array by default. Previously it returned a writable array that allowed modifying histogram contents implicitly.
@@ -430,10 +470,18 @@ The version of the following packages has been updated:
 
  - cppzeromq: 4.10.0
  - fftw3: 3.3.10
+ - freetype: 2.14.3
  - gsl: 2.8
  - gtest: 1.17.0
+ - giflib: 5.2.2
+ - libjpeg-turbo 3.1.3 is now used as a replacement for libjpeg
+ - libpng: 1.6.4
  - libzeromq: 4.3.5
- - xrootd: 5.9.1
+ - lz4: 1.10.0
+ - lzma: 5.8.2
+ - xrootd: 5.9.2
+ - zlib: 1.3.2
+ - zstd: 1.5.7
 
 
 ## Items addressed for this release

bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/API.h

@@ -174,6 +174,10 @@ CPYCPPYY_EXTERN void* CallVoidP(Cppyy::TCppMethod_t, Cppyy::TCppObject_t, CallCo
 
 //- C++ access to cppyy objects ---------------------------------------------
 
+// Get C++ Instance (python object proxy) name.
+// Sets a TypeError and returns an empty string if the pyobject is not a CPPInstance.
+CPYCPPYY_EXTERN std::string Instance_GetScopedFinalName(PyObject* pyobject);
+
 // C++ Instance (python object proxy) to void* conversion
 CPYCPPYY_EXTERN void* Instance_AsVoidPtr(PyObject* pyobject);
 
@@ -199,6 +203,11 @@ CPYCPPYY_EXTERN bool Instance_IsLively(PyObject* pyobject);
 CPYCPPYY_EXTERN bool Overload_Check(PyObject* pyobject);
 CPYCPPYY_EXTERN bool Overload_CheckExact(PyObject* pyobject);
 
+// Sets the __reduce__ method for the CPPInstance class, which is by default not
+// implemented by cppyy but might make sense to implement by frameworks that
+// support IO of arbitrary C++ objects, like ROOT.
+CPYCPPYY_EXTERN void Instance_SetReduceMethod(PyCFunction reduceMethod);
+
 
 //- access to the python interpreter ----------------------------------------
 

bindings/pyroot/cppyy/CPyCppyy/src/API.cxx

@@ -103,6 +103,18 @@ static bool Initialize()
 
 
 //- C++ access to cppyy objects ---------------------------------------------
+std::string CPyCppyy::Instance_GetScopedFinalName(PyObject* pyobject)
+{
+   if (!Instance_Check(pyobject)) {
+       PyErr_SetString(PyExc_TypeError, "Instance_GetScopedFinalName : object is not a C++ instance");
+       return "";
+   }
+
+   Cppyy::TCppType_t pyobjectClass = ((CPPInstance *)pyobject)->ObjectIsA();
+   return Cppyy::GetScopedFinalName(pyobjectClass);
+}
+
+//-----------------------------------------------------------------------------
 void* CPyCppyy::Instance_AsVoidPtr(PyObject* pyobject)
 {
 // Extract the object pointer held by the CPPInstance pyobject.
@@ -254,6 +266,11 @@ bool CPyCppyy::Overload_CheckExact(PyObject* pyobject)
     return CPPOverload_CheckExact(pyobject);
 }
 
+//-----------------------------------------------------------------------------
+void CPyCppyy::Instance_SetReduceMethod(PyCFunction reduceMethod)
+{
+    CPPInstance::ReduceMethod() = reduceMethod;
+}
 
 //- access to the python interpreter ----------------------------------------
 bool CPyCppyy::Import(const std::string& mod_name)
@@ -338,117 +355,58 @@ void CPyCppyy::ExecScript(const std::string& name, const std::vector<std::string
     }
 
 // store a copy of the old cli for restoration
-    PyObject* oldargv = PySys_GetObject(const_cast<char*>("argv"));   // borrowed
-    if (!oldargv)                                 // e.g. apache
+    PyObject* oldargv = PySys_GetObject("argv");   // borrowed
+    if (oldargv) {
+        PyObject* copy = PyList_GetSlice(oldargv, 0, PyList_Size(oldargv));
+        oldargv = copy;  // now owned
+    } else {
         PyErr_Clear();
-    else {
-        PyObject* l = PyList_New(PyList_GET_SIZE(oldargv));
-        for (int i = 0; i < PyList_GET_SIZE(oldargv); ++i) {
-            PyObject* item = PyList_GET_ITEM(oldargv, i);
-            Py_INCREF(item);
-            PyList_SET_ITEM(l, i, item);          // steals ref
-        }
-        oldargv = l;
     }
 
-// create and set (add program name) the new command line
-    int argc = args.size() + 1;
-#if PY_VERSION_HEX < 0x03000000
-// This is a legacy implementation for Python 2
-    const char** argv = new const char*[argc];
-    for (int i = 1; i < argc; ++i) argv[i] = args[i-1].c_str();
-    argv[0] = Py_GetProgramName();
-    PySys_SetArgv(argc, const_cast<char**>(argv));
-    delete [] argv;
-#else
-// This is a common code block for Python 3. We prefer using objects to
-// automatize memory management and not introduce even more preprocessor
-// branching for deletion at the end of the method.
-//
-// FUTURE IMPROVEMENT ONCE OLD PYTHON VERSIONS ARE NOT SUPPORTED BY CPPYY:
-// Right now we use C++ objects to automatize memory management. One could use
-// RAAI and the Python memory allocation API (PEP 445) once some old Python
-// version is deprecated in CPPYY. That new feature is available since version
-// 3.4 and the preprocessor branching to also support that would be so
-// complicated to make the code unreadable.
-   std::vector<std::wstring> argv2;
-   argv2.reserve(argc);
-   argv2.emplace_back(name.c_str(), &name[name.size()]);
-
-   for (int i = 1; i < argc; ++i) {
-      auto iarg = args[i - 1].c_str();
-      argv2.emplace_back(iarg, &iarg[strlen(iarg)]);
-   }
-
-#if PY_VERSION_HEX < 0x03080000
-// Before version 3.8, the code is one simple line
-   wchar_t *argv2_arr[argc];
-   for (int i = 0; i < argc; ++i) {
-      argv2_arr[i] = const_cast<wchar_t *>(argv2[i].c_str());
-   }
-   PySys_SetArgv(argc, argv2_arr);
+// build new argv
+    const int argc = (int)args.size() + 1;
+    std::vector<wchar_t*> wargv(argc);
 
-#else
-// Here we comply to "PEP 587 – Python Initialization Configuration" to avoid
-// deprecation warnings at compile time.
-   class PyConfigHelperRAAI {
-   public:
-      PyConfigHelperRAAI(const std::vector<std::wstring> &argv2)
-      {
-         PyConfig_InitPythonConfig(&fConfig);
-         fConfig.parse_argv = 1;
-         UpdateArgv(argv2);
-         InitFromConfig();
-      }
-      ~PyConfigHelperRAAI() { PyConfig_Clear(&fConfig); }
-
-   private:
-      void InitFromConfig() { Py_InitializeFromConfig(&fConfig); };
-      void UpdateArgv(const std::vector<std::wstring> &argv2)
-      {
-         auto WideStringListAppendHelper = [](PyWideStringList *wslist, const wchar_t *wcstr) {
-            PyStatus append_status = PyWideStringList_Append(wslist, wcstr);
-            if (PyStatus_IsError(append_status)) {
-               std::wcerr << "Error: could not append element " << wcstr << " to arglist - " << append_status.err_msg
-                          << std::endl;
-            }
-         };
-
-#if PY_VERSION_HEX < 0x30d00f0
-         WideStringListAppendHelper(&fConfig.argv, Py_GetProgramName());
-#else
-         PyObject* progname = PySys_GetObject("executable");    // borrowed
-         wchar_t buf[4096];
-         Py_ssize_t sz = CPyCppyy_PyUnicode_AsWideChar(progname, buf, 4095);
-         if (0 < sz)
-             WideStringListAppendHelper(&fConfig.argv, buf);
-#endif
-         for (const auto &iarg : argv2) {
-            WideStringListAppendHelper(&fConfig.argv, iarg.c_str());
-         }
-      }
-      PyConfig fConfig;
-   };
+    wargv[0] = Py_DecodeLocale(name.c_str(), nullptr);
 
-   PyConfigHelperRAAI pych(argv2);
+    for (int i = 1; i < argc; ++i) {
+        wargv[i] = Py_DecodeLocale(args[i - 1].c_str(), nullptr);
+    }
 
-#endif // of the else branch of PY_VERSION_HEX < 0x03080000
-#endif // of the else branch of PY_VERSION_HEX < 0x03000000
+// set sys.argv
+    PyObject* sysmod = PyImport_ImportModule("sys");   // new reference
+    if (sysmod) {
+        PyObject* argv_obj = PyList_New(argc);
+        for (int i = 0; i < argc; ++i) {
+            PyList_SET_ITEM(argv_obj, i, PyUnicode_FromWideChar(wargv[i], -1));
+        }
+        PyObject_SetAttrString(sysmod, "argv", argv_obj);
+        Py_DECREF(argv_obj);
+        Py_DECREF(sysmod);
+    } else {
+        PyErr_Print();
+    }
 
 // actual script execution
     PyObject* gbl = PyDict_Copy(gMainDict);
     PyObject* result =       // PyRun_FileEx closes fp (b/c of last argument "1")
         PyRun_FileEx(fp, const_cast<char*>(name.c_str()), Py_file_input, gbl, gbl, 1);
+
     if (!result)
         PyErr_Print();
+
     Py_XDECREF(result);
     Py_DECREF(gbl);
 
 // restore original command line
     if (oldargv) {
-        PySys_SetObject(const_cast<char*>("argv"), oldargv);
+        PySys_SetObject("argv", oldargv);
         Py_DECREF(oldargv);
     }
+
+// free memory from Py_DecodeLocale
+    for (auto ptr : wargv)
+        PyMem_RawFree(ptr);
 }
 
 //-----------------------------------------------------------------------------