Add OSS Buck build support and CI for Qualcomm backend (#18666)

.github/workflows/pull.yml

@@ -674,6 +674,40 @@ jobs:
       build-tool: buck2
       docker-image: ci-image:executorch-ubuntu-22.04-clang12
 
+  test-qnn-buck-build-linux:
+    name: test-qnn-buck-build-linux
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+    with:
+      runner: linux.2xlarge
+      docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        set -eux
+
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        # Download QNN SDK and get the path
+        QNN_SDK_ROOT=$(python3 backends/qualcomm/scripts/download_qnn_sdk.py --print-sdk-path)
+        echo "QNN_SDK_ROOT=${QNN_SDK_ROOT}"
+
+        # Configure Buck to find the QNN SDK
+        echo "[qualcomm]" >> .buckconfig
+        echo "  qnn_sdk_root = ${QNN_SDK_ROOT}" >> .buckconfig
+
+        # Setup buck2
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2
+
+        # Build QNN backend with Buck
+        buck2 build //backends/qualcomm/...
+
   unittest-arm-backend-with-no-deps:
     name: unittest-arm-backend-with-no-deps
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/qualcomm/aot/python/targets.bzl

@@ -3,7 +3,7 @@ load(
     "CXX",
 )
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
-load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
+load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
 
 PYTHON_MODULE_NAME = "PyQnnManagerAdaptor"
 
@@ -32,9 +32,9 @@ def define_common_targets():
             "//executorch/backends/qualcomm/runtime:logging",
             "//executorch/backends/qualcomm:schema",
             "//executorch/backends/qualcomm/runtime:runtime",
-            "fbsource//third-party/pybind11:pybind11",
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
+            qnn_third_party_dep("api"),
+            qnn_third_party_dep("app_sources"),
+            qnn_third_party_dep("pybind11"),
         ],
         external_deps = [
             "libtorch_python",
@@ -61,8 +61,8 @@ def define_common_targets():
             "//executorch/backends/qualcomm/runtime:logging",
             "//executorch/backends/qualcomm:schema",
             "//executorch/backends/qualcomm/runtime:runtime",
-            "fbsource//third-party/pybind11:pybind11",
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
+            qnn_third_party_dep("api"),
+            qnn_third_party_dep("app_sources"),
+            qnn_third_party_dep("pybind11"),
         ],
     )

backends/qualcomm/aot/wrappers/targets.bzl

@@ -3,7 +3,7 @@ load(
     "ANDROID",
 )
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
-load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
+load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
 
 def define_common_targets():
     """Defines targets that should be shared between fbcode and xplat.
@@ -23,8 +23,8 @@ def define_common_targets():
         platforms = [ANDROID],
         visibility = ["PUBLIC"],
         deps = [
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
+            qnn_third_party_dep("api"),
+            qnn_third_party_dep("app_sources"),
             "//executorch/runtime/backend:interface",
             "//executorch/runtime/core:core",
         ],

backends/qualcomm/recipes/BUCK

@@ -23,9 +23,7 @@ fbcode_target(_kind = runtime.python_library,
     ],
     visibility = ["PUBLIC"],
     deps = [
-        "//caffe2:torch",
         "//executorch/export:lib",
-        "//executorch/runtime:runtime",  # @manual
         "//executorch/backends/qualcomm/partition:partition",
         "//executorch/backends/qualcomm/serialization:serialization",
         "//executorch/backends/qualcomm/utils:utils",

backends/qualcomm/runtime/targets.bzl

@@ -3,7 +3,7 @@ load(
     "ANDROID",
 )
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
-load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
+load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
 
 def define_common_targets():
     """Defines targets that should be shared between fbcode and xplat.
@@ -24,12 +24,12 @@ def define_common_targets():
         platforms = [ANDROID],
         visibility = ["PUBLIC"],
         deps = [
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
+            qnn_third_party_dep("api"),
+            qnn_third_party_dep("app_sources"),
             "//executorch/runtime/backend:interface",
         ],
         exported_deps = [
-            "fbsource//third-party/toolchains:log",
+            qnn_third_party_dep("log"),
             "//executorch/backends/qualcomm:schema",
             "//executorch/runtime/core:core",
         ],
@@ -68,12 +68,12 @@ def define_common_targets():
             platforms = [ANDROID],
             visibility = ["PUBLIC"],
             resources = ({
-                "qnn_lib": "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()),
+                "qnn_lib": qnn_third_party_dep("qnn_offline_compile_libs"),
                 } if include_aot_qnn_lib else {
             }),
             deps = [
-                "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
-                "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
+                qnn_third_party_dep("api"),
+                qnn_third_party_dep("app_sources"),
                 ":logging",
                 "//executorch/backends/qualcomm:schema",
                 "//executorch/backends/qualcomm/aot/wrappers:wrappers",

backends/qualcomm/targets.bzl

@@ -3,12 +3,15 @@ load(
     "ANDROID",
 )
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
-load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
+load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
 
 # Construct the input and output file names. All input and output files rely on scalar_type file.
 SCHEMA_NAME = "qc_compiler_spec"
 
-INPUT_SCHEMA = "serialization/" + SCHEMA_NAME + ".fbs"
+# In OSS, genrule srcs can't use cross-package relative paths, so use
+# an export_file target. In fbcode/xplat, relative path works and avoids
+# the srcs patching gap in _patch_executorch_references.
+INPUT_SCHEMA = "//backends/qualcomm/serialization:qc_compiler_spec.fbs" if runtime.is_oss else "serialization/" + SCHEMA_NAME + ".fbs"
 
 OUTPUT_SCHEMA_HEADER = SCHEMA_NAME + "_generated.h"
 
@@ -84,7 +87,7 @@ def define_common_targets():
         define_static_target = True,
         visibility = ["PUBLIC"],
         deps = [
-            "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
+            qnn_third_party_dep("api"),
             "//executorch/runtime/backend:interface",
             "//executorch/runtime/core:core",
             "//executorch/backends/qualcomm/runtime:runtime_android_build",

backends/qualcomm/tests/BUCK

@@ -13,6 +13,8 @@ fbcode_target(_kind = runtime.python_library,
     ]
 )
 
+# Test targets have transitive deps (caffe2, torchvision, torchaudio, etc.)
+# that are not available in OSS Buck builds.
 fbcode_target(_kind = runtime.python_library,
     name = "test_qnn_delegate",
     srcs = [
@@ -28,18 +30,22 @@ fbcode_target(_kind = runtime.python_library,
         "//caffe2/functorch:functorch_src",
         "//executorch/exir/backend:partitioner",
         "//executorch/exir/dialects:lib",
-        "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
         "//executorch/backends/qualcomm/partition:partition",
         "//executorch/backends/qualcomm/quantizer:quantizer",
         "//executorch/backends/qualcomm/serialization:serialization",
         "//executorch/backends/qualcomm/utils:utils",
         "//executorch/devtools:lib",
         "//executorch/examples/qualcomm:utils",
-        "//executorch/examples/models:models",
         "//executorch/backends/qualcomm/debugger:utils",
         "//executorch/backends/qualcomm/debugger:qnn_intermediate_debugger",
-    ],
+    ] + ([] if runtime.is_oss else [
+        # These deps have pre-existing issues in OSS Buck:
+        # portable_lib: shim//build_defs/cpp_python_extension.bzl missing
+        # models: examples/models/BUCK uses raw python_library (no dep patching)
+        "//executorch/extension/pybindings:portable_lib",  # @manual
+        "//executorch/examples/models:models",
+    ]),
 )
 
 fbcode_target(_kind = runtime.python_library,
@@ -67,9 +73,11 @@ fbcode_target(_kind = runtime.python_test,
         "//executorch/backends/qualcomm/partition:partition",
         "//executorch/backends/qualcomm/serialization:serialization",
         "//executorch/backends/qualcomm/utils:utils",
+        "//executorch/backends/qualcomm/builders:builders",
+    ] + ([] if runtime.is_oss else [
+        # These deps fail in OSS: keep_gpu_sections kwarg breaks TARGETS evaluation
         "//executorch/examples/models/llama:transformer_modules",
         "//executorch/examples/qualcomm/oss_scripts/llama:masking_utils",
         "//executorch/examples/qualcomm/oss_scripts/llama:static_llama",
-        "//executorch/backends/qualcomm/builders:builders",
-    ],
+    ]),
 )

backends/qualcomm/third-party/BUCK

@@ -0,0 +1,118 @@
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+# OSS prebuilt targets for Qualcomm QNN SDK.
+# These wrap the externally-downloaded SDK (via QNN_SDK_ROOT env var or
+# backends/qualcomm/scripts/download_qnn_sdk.py) so that Buck builds
+# in OSS CI can compile the QNN backend without fbsource third-party targets.
+#
+# Set QNN_SDK_ROOT before running buck2 build, or use .buckconfig:
+#   [qualcomm]
+#   qnn_sdk_root = /path/to/qnn/sdk
+
+oncall("executorch")
+
+_QNN_SDK_ROOT = native.read_config("qualcomm", "qnn_sdk_root", "")
+_QNN_APP_SDK_FILES = [
+    ("QnnModel.cpp", "share/QNN/converter/jni/QnnModel.cpp"),
+    ("QnnWrapperUtils.cpp", "share/QNN/converter/jni/QnnWrapperUtils.cpp"),
+    ("linux_QnnModelPal.cpp", "share/QNN/converter/jni/linux/QnnModelPal.cpp"),
+    ("windows_QnnModelPal.cpp", "share/QNN/converter/jni/windows/QnnModelPal.cpp"),
+    ("QnnModel.hpp", "share/QNN/converter/jni/QnnModel.hpp"),
+    ("QnnModelPal.hpp", "share/QNN/converter/jni/QnnModelPal.hpp"),
+    ("QnnTypeMacros.hpp", "share/QNN/converter/jni/QnnTypeMacros.hpp"),
+    ("QnnWrapperUtils.hpp", "share/QNN/converter/jni/QnnWrapperUtils.hpp"),
+]
+
+_QNN_APP_SRCS = [
+    ":qnn_app_sources_files[QnnModel.cpp]",
+    ":qnn_app_sources_files[QnnWrapperUtils.cpp]",
+] + select({
+    "DEFAULT": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"],
+    "ovr_config//os:linux": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"],
+    "ovr_config//os:windows": [":qnn_app_sources_files[windows_QnnModelPal.cpp]"],
+})
+
+_QNN_APP_HEADERS = [
+    ":qnn_app_sources_files[QnnModel.hpp]",
+    ":qnn_app_sources_files[QnnModelPal.hpp]",
+    ":qnn_app_sources_files[QnnTypeMacros.hpp]",
+    ":qnn_app_sources_files[QnnWrapperUtils.hpp]",
+]
+
+_QNN_APP_EXPORTED_HEADERS = {
+    name: ":qnn_app_sources_files[{}]".format(name)
+    for name in [
+        "QnnModel.hpp",
+        "QnnModelPal.hpp",
+        "QnnTypeMacros.hpp",
+        "QnnWrapperUtils.hpp",
+    ]
+}
+
+# Mirror QNN SDK API headers into declared outputs so that Buck2's sandbox
+# can access them (raw -I to an absolute SDK path is not sandbox-safe).
+runtime.genrule(
+    name = "qnn_api_headers",
+    out = "include",
+    cmd = (
+        "cp -rLf \"{root}/include/QNN/.\" \"$OUT/\"".format(root = _QNN_SDK_ROOT)
+        if _QNN_SDK_ROOT else "mkdir -p \"$OUT\""
+    ),
+    visibility = [],
+)
+
+# QNN API headers (include/QNN/)
+runtime.cxx_library(
+    name = "qnn_api",
+    exported_preprocessor_flags = [
+        "-I$(location :qnn_api_headers)",
+    ],
+    exported_headers = {},
+    visibility = ["PUBLIC"],
+)
+
+# Mirror the fixed JNI SDK files into declared outputs instead of globbing an
+# absolute SDK path, which Buck rejects during package evaluation.
+runtime.genrule(
+    name = "qnn_app_sources_files",
+    outs = {name: [path] for name, path in _QNN_APP_SDK_FILES},
+    default_outs = ["."],
+    cmd = " && ".join(
+        [
+            "mkdir -p $OUT/share/QNN/converter/jni/linux $OUT/share/QNN/converter/jni/windows",
+        ] + (
+            [
+                "cp -f \"{root}/{path}\" \"$OUT/{path}\"".format(root = _QNN_SDK_ROOT, path = path)
+                for _, path in _QNN_APP_SDK_FILES
+            ] if _QNN_SDK_ROOT else [
+                "touch \"$OUT/{path}\"".format(path = path)
+                for _, path in _QNN_APP_SDK_FILES
+            ]
+        ),
+    ),
+)
+
+# QNN app utility sources (converter/jni/)
+runtime.cxx_library(
+    name = "qnn_app_sources",
+    srcs = _QNN_APP_SRCS if _QNN_SDK_ROOT else [],
+    headers = _QNN_APP_HEADERS if _QNN_SDK_ROOT else [],
+    header_namespace = "",
+    exported_headers = _QNN_APP_EXPORTED_HEADERS if _QNN_SDK_ROOT else {},
+    visibility = ["PUBLIC"],
+    deps = [":qnn_api"],
+)
+
+# QNN offline compile libs (placeholder — only needed for AOT resources)
+runtime.filegroup(
+    name = "qnn_offline_compile_libs",
+    srcs = [],
+    visibility = ["PUBLIC"],
+)
+
+# Android log stub (noop on non-Android)
+runtime.cxx_library(
+    name = "log",
+    exported_headers = {},
+    visibility = ["PUBLIC"],
+)

backends/qualcomm/third-party/third_party_libs.bzl

@@ -0,0 +1,33 @@
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
+
+# Dictionary mapping third-party library name to [internal_dep, oss_dep].
+# Internal deps use fbsource//third-party/qualcomm/qnn/qnn-{version}:target.
+# OSS deps use //backends/qualcomm/third-party:target (prebuilt from QNN_SDK_ROOT).
+_QNN_THIRD_PARTY_LIBS = {
+    "api": [
+        "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
+        "//backends/qualcomm/third-party:qnn_api",
+    ],
+    "app_sources": [
+        "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
+        "//backends/qualcomm/third-party:qnn_app_sources",
+    ],
+    "qnn_offline_compile_libs": [
+        "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()),
+        "//backends/qualcomm/third-party:qnn_offline_compile_libs",
+    ],
+    "log": [
+        "fbsource//third-party/toolchains:log",
+        "//backends/qualcomm/third-party:log",
+    ],
+    "pybind11": [
+        "fbsource//third-party/pybind11:pybind11",
+        "//third-party:pybind11",
+    ],
+}
+
+def qnn_third_party_dep(name):
+    if name not in _QNN_THIRD_PARTY_LIBS:
+        fail("Cannot find QNN third party library " + name + ", please register it in _QNN_THIRD_PARTY_LIBS first!")
+    return _QNN_THIRD_PARTY_LIBS[name][1] if runtime.is_oss else _QNN_THIRD_PARTY_LIBS[name][0]

devtools/inspector/TARGETS

@@ -1,4 +1,3 @@
-load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary")
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
 
 oncall("executorch")
@@ -23,7 +22,7 @@ runtime.python_library(
     ],
 )
 
-python_binary(
+runtime.python_binary(
     name = "inspector_cli",
     main_function = ".inspector_cli.main",
     main_src = "inspector_cli.py",