diff --git a/backends/openvino/CMakeLists.txt b/backends/openvino/CMakeLists.txt index 5b7a1349bf5..c3dc865bbe2 100644 --- a/backends/openvino/CMakeLists.txt +++ b/backends/openvino/CMakeLists.txt @@ -35,7 +35,11 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) add_library(openvino_backend STATIC) # Enable exceptions and RTTI for OpenVINO backend -target_compile_options(openvino_backend PRIVATE -frtti -fexceptions) +target_compile_options( + openvino_backend + PRIVATE $<$:/GR /EHsc> + $<$>:-frtti -fexceptions> +) # Add source files for OpenVINO backend target_sources( diff --git a/backends/openvino/README.md b/backends/openvino/README.md index 289cec932d8..5c6ee2a488c 100644 --- a/backends/openvino/README.md +++ b/backends/openvino/README.md @@ -14,20 +14,26 @@ For more information on the supported hardware, please refer to [OpenVINO System ## Quick Start (pip wheel) -On Linux, the OpenVINO backend is included in the ExecuTorch pip wheel. Install the OpenVINO runtime to activate it: +On Linux and Windows, the OpenVINO backend is included in the ExecuTorch pip wheel. Install the OpenVINO runtime to activate it: ```bash pip install executorch[openvino] ``` -The backend automatically discovers the OpenVINO C library from the pip-installed package — no `LD_LIBRARY_PATH` setup is needed. +The backend automatically discovers the OpenVINO C library from the pip-installed package — no `LD_LIBRARY_PATH` (Linux) or `PATH` (Windows) setup is needed. If auto-discovery fails (e.g. non-standard install), you can point to the library explicitly: +**Linux:** ```bash export OPENVINO_LIB_PATH=$(python3 -c "import openvino, os; print(os.path.join(os.path.dirname(openvino.__file__), 'libs', 'libopenvino_c.so'))") ``` +**Windows (PowerShell):** +```powershell +$env:OPENVINO_LIB_PATH = python -c "import openvino, os; print(os.path.join(os.path.dirname(openvino.__file__), 'libs', 'openvino_c.dll'))" +``` + Verify the backend is available: ```python @@ -44,18 +50,35 @@ print(_get_registered_backend_names()) executorch ├── backends │ └── openvino +│ ├── _passes +│ │ ├── __init__.py +│ │ └── decompose_floor_divide_pass.py │ ├── quantizer -│ ├── observers -│ └── nncf_observers.py -│ ├── __init__.py -│ └── quantizer.py +│ │ ├── __init__.py +│ │ ├── llm_compression.py +│ │ ├── observers.py +│ │ └── quantizer.py │ ├── runtime -│ ├── OpenvinoApi.h -│ ├── OpenvinoBackend.cpp -│ └── OpenvinoBackend.h +│ │ ├── OpenvinoApi.h +│ │ ├── OpenvinoBackend.cpp +│ │ └── OpenvinoBackend.h │ ├── scripts -│ └── openvino_build.sh +│ │ └── openvino_build.sh +│ ├── test +│ │ └── tester +│ │ ├── __init__.py +│ │ └── tester.py │ ├── tests +│ │ ├── models +│ │ │ └── test_classification.py +│ │ ├── ops +│ │ │ ├── base_openvino_op_test.py +│ │ │ └── test_*.py +│ │ ├── quantizer +│ │ │ ├── synthetic_test_models.py +│ │ │ └── test_llm_compression.py +│ │ ├── README.md +│ │ └── test_runner.py │ ├── CMakeLists.txt │ ├── README.md │ ├── __init__.py @@ -63,16 +86,109 @@ executorch │ ├── preprocess.py │ └── requirements.txt └── examples - └── openvino - ├── aot_optimize_and_infer.py - └── README.md + └── openvino # See examples/openvino/README.md ``` ## Build Instructions -### Prerequisites +### Setup + +Follow the steps below to setup your build environment: + + +1. **Create a Virtual Environment** +- Create a virtual environment and activate it by executing the commands below. + + **Linux:** + ```bash + python -m venv env + source env/bin/activate + ``` + + **Windows (PowerShell):** + ```powershell + python -m venv env + env\Scripts\Activate.ps1 + ``` +2. **Clone ExecuTorch Repository from GitHub** +- On Windows, enable symlinks before cloning. Refer to [Building from Source](https://docs.pytorch.org/executorch/main/using-executorch-building-from-source.html#environment-setup) for more details. +- Clone Executorch repository by executing the command below. + ```bash + git clone --recurse-submodules https://github.com/pytorch/executorch.git + ``` +3. **Build ExecuTorch with OpenVINO Backend** +- The following commands build and install ExecuTorch with the OpenVINO backend into `cmake-out`. + + **Linux:** + ```bash + cmake -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_OPENVINO=ON \ + -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ + -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ + -DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \ + -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \ + -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ + -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=ON \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -Bcmake-out + cmake --build cmake-out --target install --config Release -j $(nproc) + ``` + + **Windows (PowerShell):** + ```powershell + cmake -DCMAKE_INSTALL_PREFIX=cmake-out ` + -DCMAKE_BUILD_TYPE=Release ` + -DEXECUTORCH_BUILD_OPENVINO=ON ` + -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON ` + -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON ` + -DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON ` + -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON ` + -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON ` + -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON ` + -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=ON ` + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON ` + -Bcmake-out + cmake --build cmake-out --target install --config Release -j $env:NUMBER_OF_PROCESSORS + ``` + + To additionally build with LLM extension support, append `-DEXECUTORCH_BUILD_EXTENSION_LLM=ON -DEXECUTORCH_BUILD_EXTENSION_LLM_RUNNER=ON` to the configure step. + +#### Build Python Package with Pybindings + +Compiles and installs the ExecuTorch Python package with the OpenVINO backend into your Python environment, enabling python bindings required to execute OpenVINO backend tests and `aot_optimize_and_infer.py` script inside `executorch/examples/openvino` folder. + +**Linux:** +```bash +pip install -r backends/openvino/requirements.txt +CMAKE_ARGS="-DEXECUTORCH_BUILD_OPENVINO=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON" \ +CMAKE_BUILD_ARGS="--target openvino_backend" \ +./install_executorch.sh --use-pt-pinned-commit +``` +- On Linux, `backends/openvino/scripts/openvino_build.sh` can be used as a convenience wrapper with `--enable_python`, `--cpp_runtime`, and `--cpp_runtime_llm` options instead of running the above commands directly. + +**Windows (PowerShell):** +```powershell +pip install -r backends/openvino/requirements.txt +$env:CMAKE_ARGS = "-DEXECUTORCH_BUILD_OPENVINO=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON" +$env:CMAKE_BUILD_ARGS = "--target openvino_backend" +.\install_executorch.bat --use-pt-pinned-commit +``` + + + +For more information about ExecuTorch environment setup, refer to the [Environment Setup](https://pytorch.org/executorch/main/getting-started-setup#environment-setup) guide. + +## Runtime Setup + +OpenVINO is a runtime-only dependency — it is not required at build time. The backend discovers and loads the OpenVINO C library dynamically when first used. You can provide the library via pip (recommended) or a manual install. -Before you begin, ensure you have openvino installed and configured on your system. +### Install via pip (Recommended) + +```bash +pip install openvino +``` ### Use OpenVINO from Release Packages @@ -80,14 +196,23 @@ Before you begin, ensure you have openvino installed and configured on your syst 2. Extract the release package from the archive and set the environment variables. + **Linux:** ```bash tar -zxf openvino_toolkit_.tgz cd openvino_toolkit_ source setupvars.sh ``` + **Windows (PowerShell):** + ```powershell + Expand-Archive openvino_toolkit_.zip + cd openvino_toolkit_ + .\setupvars.ps1 + ``` + ### (Optional) Build OpenVINO from Source +**Linux:** ```bash git clone https://github.com/openvinotoolkit/openvino.git cd openvino @@ -103,46 +228,23 @@ cd source setupvars.sh ``` -For more information about OpenVINO build, refer to the [OpenVINO Build Instructions](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build_linux.md). - -### Setup - -Follow the steps below to setup your build environment: - - -1. **Create a Virtual Environment** -- Create a virtual environment and activate it by executing the commands below. - ```bash - python -m venv env - source env/bin/activate - ``` -2. **Clone ExecuTorch Repository from Github** -- Clone Executorch repository by executing the command below. - ```bash - git clone --recurse-submodules https://github.com/pytorch/executorch.git - ``` -3. **Build ExecuTorch with OpenVINO Backend** -- Ensure that you are inside `executorch/backends/openvino/scripts` directory. The following command builds and installs ExecuTorch with the OpenVINO backend, also compiles the C++ runtime libraries and binaries into `/cmake-out` for quick inference testing. - ```bash - openvino_build.sh - ``` -- Optionally, `openvino_build.sh` script can be used to build python package or C++ libraries/binaries seperately. +**Windows (PowerShell):** +```powershell +git clone https://github.com/openvinotoolkit/openvino.git +cd openvino +git submodule update --init --recursive +mkdir build; cd build +cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=ON +cmake --build . --config Release -j $env:NUMBER_OF_PROCESSORS - **Build OpenVINO Backend Python Package with Pybindings**: To build and install the OpenVINO backend Python package with Python bindings, run the `openvino_build.sh` script with the `--enable_python` argument as shown in the below command. This will compile and install the ExecuTorch Python package with the OpenVINO backend into your Python environment. This option will also enable python bindings required to execute OpenVINO backend tests and `aot_optimize_and_infer.py` script inside `executorch/examples/openvino` folder. - ```bash - ./openvino_build.sh --enable_python - ``` - **Build C++ Runtime Libraries for OpenVINO Backend**: Run the `openvino_build.sh` script with the `--cpp_runtime` flag to build the C++ runtime libraries as shown in the below command. The compiled libraries files and binaries can be found in the `/cmake-out` directory. The binary located at `/cmake-out/executor_runner` can be used to run inference with vision models. - ```bash - ./openvino_build.sh --cpp_runtime - ``` - **Build C++ Runtime Libraries with LLM Extension**: Run the `openvino_build.sh` script with the `--cpp_runtime_llm` flag to build the C++ runtime libraries with LLM extension as shown in the below command. Use this option instead of `--cpp_runtime` for LLM extension support which is required by LLM examples. - ```bash - ./openvino_build.sh --cpp_runtime_llm - ``` +cd .. +cmake --install build --prefix +cd +.\setupvars.ps1 +``` -For more information about ExecuTorch environment setup, refer to the [Environment Setup](https://pytorch.org/executorch/main/getting-started-setup#environment-setup) guide. +For more information about OpenVINO build, refer to the [OpenVINO Build Instructions](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build_linux.md). -### Run +### Examples -Please refer to [README.md](../../examples/openvino/README.md) for instructions on running examples of various of models with openvino backend. +Please refer to [README.md](../../examples/openvino/README.md) for instructions on running examples of various models with the OpenVINO backend. diff --git a/backends/openvino/runtime/OpenvinoApi.h b/backends/openvino/runtime/OpenvinoApi.h index 90403e24b95..fa663b1d8e9 100644 --- a/backends/openvino/runtime/OpenvinoApi.h +++ b/backends/openvino/runtime/OpenvinoApi.h @@ -8,7 +8,11 @@ #pragma once +#ifdef _WIN32 +#include +#else #include +#endif #include #include #include @@ -99,7 +103,11 @@ using ov_shape_free_fn = ov_status_e (*)(ov_shape_t*); struct DlCloser { void operator()(void* handle) { if (handle) { +#ifdef _WIN32 + FreeLibrary(static_cast(handle)); +#else dlclose(handle); +#endif } } }; diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index 3cf87e6abfa..d7b7577e16a 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -23,10 +23,26 @@ namespace openvino { namespace { +#ifdef _WIN32 +constexpr const char* kDefaultLibName = "openvino_c.dll"; +#else constexpr const char* kDefaultLibName = "libopenvino_c.so"; +#endif template FuncPtr load_symbol(void* handle, const char* name) { +#ifdef _WIN32 + void* sym = reinterpret_cast( + GetProcAddress(static_cast(handle), name)); + if (!sym) { + ET_LOG( + Error, + "OpenVINO: failed to resolve symbol '%s': error %lu", + name, + GetLastError()); + return nullptr; + } +#else dlerror(); // Clear any stale error state. void* sym = dlsym(handle, name); const char* err = dlerror(); @@ -34,6 +50,7 @@ FuncPtr load_symbol(void* handle, const char* name) { ET_LOG(Error, "OpenVINO: failed to resolve symbol '%s': %s", name, err); return nullptr; } +#endif return reinterpret_cast(sym); } @@ -47,6 +64,19 @@ bool OpenvinoBackend::ensure_loaded() const { const char* lib_path = std::getenv("OPENVINO_LIB_PATH"); const char* effective_path = lib_path ? lib_path : kDefaultLibName; +#ifdef _WIN32 + void* handle = static_cast(LoadLibrary(effective_path)); + if (!handle) { + ET_LOG( + Error, + "OpenVINO runtime not found (LoadLibrary failed: error %lu). " + "Set OPENVINO_LIB_PATH to the full path of 'openvino_c.dll', " + "or add its containing directory to PATH, or install with: " + "pip install \"openvino>=2025.1.0,<2026.0.0\"", + GetLastError()); + return; + } +#else void* handle = dlopen(effective_path, RTLD_NOW | RTLD_LOCAL); if (!handle) { ET_LOG( @@ -58,6 +88,7 @@ bool OpenvinoBackend::ensure_loaded() const { dlerror()); return; } +#endif lib_handle_.reset(handle); #define LOAD_SYM(field, symbol_name) \ diff --git a/extension/pybindings/portable_lib.py b/extension/pybindings/portable_lib.py index 4be4b8956e8..a9eb3b0b561 100644 --- a/extension/pybindings/portable_lib.py +++ b/extension/pybindings/portable_lib.py @@ -6,6 +6,7 @@ # pyre-strict + """API for loading and executing ExecuTorch PTE files using the C++ runtime. .. warning:: @@ -29,13 +30,15 @@ # the pybindings shared library extension. This will load libtorch.so and # related libs, ensuring that the pybindings lib can resolve those runtime # dependencies. + import torch as _torch logger = logging.getLogger(__name__) # Auto-discover the OpenVINO C library path from the pip-installed openvino -# package so the C++ backend's dlopen("libopenvino_c.so") works without the -# user having to set LD_LIBRARY_PATH or OPENVINO_LIB_PATH manually. +# package so the C++ backend's dlopen/LoadLibrary call works without the user +# having to set LD_LIBRARY_PATH or OPENVINO_LIB_PATH manually. + if not os.environ.get("OPENVINO_LIB_PATH"): try: import glob @@ -44,26 +47,33 @@ spec = importlib.util.find_spec("openvino") if spec is not None and spec.submodule_search_locations: _ov_dir = spec.submodule_search_locations[0] - _ov_libs = sorted( - glob.glob(os.path.join(_ov_dir, "libs", "libopenvino_c.so*")) - ) + _ov_libs_dir = os.path.join(_ov_dir, "libs") + if sys.platform == "win32": + _lib_pattern = os.path.join(_ov_libs_dir, "openvino_c.dll") + else: + _lib_pattern = os.path.join(_ov_libs_dir, "libopenvino_c.so*") + _ov_libs = sorted(glob.glob(_lib_pattern)) if _ov_libs: os.environ["OPENVINO_LIB_PATH"] = _ov_libs[0] + if sys.platform == "win32": + os.add_dll_directory(_ov_libs_dir) else: logger.warning( - "OpenVINO package found but libopenvino_c.so not in %s; " + "OpenVINO package found but %s not in %s; " "set OPENVINO_LIB_PATH manually if needed", - os.path.join(_ov_dir, "libs"), + "openvino_c.dll" if sys.platform == "win32" else "libopenvino_c.so", + _ov_libs_dir, ) - del _ov_libs, _ov_dir, spec + del _ov_libs, _ov_libs_dir, _lib_pattern, _ov_dir, spec except Exception as e: logger.debug("OpenVINO auto-discovery failed: %s", e) - # Update the DLL search path on Windows. This is the recommended way to handle native # extensions. + if sys.platform == "win32": try: # The extension DLL should be in the same directory as this file. + pybindings_dir = os.path.dirname(os.path.abspath(__file__)) os.add_dll_directory(pybindings_dir) except Exception as e: @@ -71,12 +81,12 @@ "Failed to add the pybinding extension DLL to the search path. The extension may not work.", e, ) - # Let users import everything from the C++ _portable_lib extension as if this # python file defined them. Although we could import these dynamically, it # wouldn't preserve the static type annotations. # # Note that all of these are experimental, and subject to change without notice. + from executorch.extension.pybindings._portable_lib import ( # noqa: F401 # Disable "imported but unused" (F401) checks. _create_profile_block, # noqa: F401 @@ -103,6 +113,7 @@ # Clean up so that `dir(portable_lib)` is the same as `dir(_portable_lib)` # (apart from some __dunder__ names). + del _torch del _exir_warnings del _warnings