diff --git a/.github/workflows/enscripten.yaml b/.github/workflows/enscripten.yaml deleted file mode 100644 index 55b3f82..0000000 --- a/.github/workflows/enscripten.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: WASM - -on: - workflow_dispatch: - pull_request: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build-wasm-emscripten: - name: Pyodide - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - uses: pypa/cibuildwheel@v2.22 - env: - CIBW_PLATFORM: pyodide - - - uses: actions/upload-artifact@v4 - with: - path: dist/*.whl diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000..c792242 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,22 @@ +# This is a format job. Pre-commit has a first-party GitHub action, so we use +# that: https://github.com/pre-commit/action + +name: Format + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + pre-commit: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index c16fd69..99ba196 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: platform: [windows-latest, macos-latest, ubuntu-latest] - python-version: ["3.9", "3.13", "pypy-3.10"] + python-version: ["3.9", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index cf776f4..d2174a7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -57,6 +57,8 @@ jobs: env: CIBW_ENABLE: cpython-prerelease CIBW_ARCHS_WINDOWS: auto ARM64 + CIBW_SKIP: pp* *i686 + CIBW_TEST_SKIP: "*macosx* *win* *aarch64" - name: Verify clean directory run: git diff --exit-code @@ -97,3 +99,4 @@ jobs: - uses: pypa/gh-action-pypi-publish@release/v1 with: attestations: true + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 59bb7e4..0d0fdc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ project( VERSION ${SKBUILD_PROJECT_VERSION} LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 17) # Find the module development requirements (requires FindPython from 3.17 or # scikit-build-core's built-in backport) find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) diff --git a/README.md b/README.md index 2bdd082..059ac24 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,5 @@ # jsoncons -[![Gitter][gitter-badge]][gitter-link] +I'm using this [jsoncon](https://github.com/danielaparker/jsoncon) python binding to filter & transform json data. -| CI | status | -|----------------------|--------| -| conda.recipe | [![Conda Actions Status][actions-conda-badge]][actions-conda-link] | -| pip builds | [![Pip Actions Status][actions-pip-badge]][actions-pip-link] | - - -An example project built with [pybind11][] and [scikit-build-core][]. Python -3.9+ (see older commits for 3.7+, or even older versions of Python using [scikit-build (classic)][]). - - -[gitter-badge]: https://badges.gitter.im/pybind/Lobby.svg -[gitter-link]: https://gitter.im/pybind/Lobby -[actions-badge]: https://github.com/pybind/jsoncons/workflows/Tests/badge.svg -[actions-conda-link]: https://github.com/pybind/jsoncons/actions?query=workflow%3AConda -[actions-conda-badge]: https://github.com/pybind/jsoncons/workflows/Conda/badge.svg -[actions-pip-link]: https://github.com/pybind/jsoncons/actions?query=workflow%3APip -[actions-pip-badge]: https://github.com/pybind/jsoncons/workflows/Pip/badge.svg -[actions-wheels-link]: https://github.com/pybind/jsoncons/actions?query=workflow%3AWheels -[actions-wheels-badge]: https://github.com/pybind/jsoncons/workflows/Wheels/badge.svg - -## Installation - -- Clone this repository -- `pip install ./jsoncons` - -## Test call - -```python -import jsoncons - -jsoncons.add(1, 2) -``` - -## Files - -This example has several files that are a good idea, but aren't strictly -necessary. The necessary files are: - -* `pyproject.toml`: The Python project file -* `CMakeLists.txt`: The CMake configuration file -* `src/main.cpp`: The source file for the C++ build -* `src/jsoncons/__init__.py`: The Python portion of the module. The root of the module needs to be ``, `src/`, or `python/` to be auto-discovered. - -These files are also expected and highly recommended: - -* `.gitignore`: Git's ignore list, also used by `scikit-build-core` to select files for the SDist -* `README.md`: The source for the PyPI description -* `LICENSE`: The license file - -There are also several completely optional directories: - -* `.github`: configuration for [Dependabot][] and [GitHub Actions][] -* `conda.recipe`: Example recipe. Normally you should submit projects to conda-forge instead of building them yourself, but this is useful for testing the example. -* `docs/`: Documentation -* `tests/`: Tests go here - -And some optional files: - -* `.pre-commit-config.yaml`: Configuration for the fantastic static-check runner [pre-commit][]. -* `noxfile.py`: Configuration for the [nox][] task runner, which helps make setup easier for contributors. - -This is a simplified version of the recommendations in the [Scientific-Python -Development Guide][], which is a _highly_ recommended read for anyone -interested in Python package development (Scientific or not). The guide also -has a cookiecutter that includes scikit-build-core and pybind11 as a backend -choice. - -### CI Examples - -There are examples for CI in `.github/workflows`. A simple way to produces -binary "wheels" for all platforms is illustrated in the "wheels.yml" file, -using [cibuildwheel][]. - -## License - -pybind11 is provided under a BSD-style license that can be found in the LICENSE -file. By using, distributing, or contributing to this project, you agree to the -terms and conditions of this license. - -[cibuildwheel]: https://cibuildwheel.readthedocs.io -[scientific-python development guide]: https://learn.scientific-python.org/development -[dependabot]: https://docs.github.com/en/code-security/dependabot -[github actions]: https://docs.github.com/en/actions -[pre-commit]: https://pre-commit.com -[nox]: https://nox.thea.codes -[pybind11]: https://pybind11.readthedocs.io -[scikit-build-core]: https://scikit-build-core.readthedocs.io -[scikit-build (classic)]: https://scikit-build.readthedocs.io +See `test_json_query`, `test_json_query_json`. diff --git a/out.bin b/out.bin deleted file mode 100644 index 33661ed..0000000 Binary files a/out.bin and /dev/null differ diff --git a/out.json b/out.json deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml index 9a3e655..8a18edb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,18 @@ [build-system] -requires = ["scikit-build-core>=0.10", "pybind11"] +# https://github.com/pybind/scikit_build_example/commit/90f61def6fae75e4908b871e177a69d2b81f4548 +requires = ["scikit-build-core>=0.3.3", "pybind11"] build-backend = "scikit_build_core.build" [project] name = "jsoncons" version = "0.0.1" -description="A minimal example package (with pybind11)" +description="python binding for jsoncons (only what I needed for now)" readme = "README.md" authors = [ - { name = "My Name", email = "me@email.com" }, + { name = "zhixiong.tang", email = "me@email.com" }, ] -requires-python = ">=3.9" +requires-python = ">=3.7" classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: MIT License", @@ -35,7 +36,7 @@ minimum-version = "build-system.requires" [tool.pytest.ini_options] -minversion = "8.0" +minversion = "6.0" addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] xfail_strict = true log_cli_level = "INFO" diff --git a/src/include/README b/src/include/README new file mode 100644 index 0000000..ebfcc45 --- /dev/null +++ b/src/include/README @@ -0,0 +1 @@ +from https://github.com/danielaparker/jsoncons/tree/master/include diff --git a/src/jsoncons/__init__.py b/src/jsoncons/__init__.py index 88d2419..4de47e5 100644 --- a/src/jsoncons/__init__.py +++ b/src/jsoncons/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from ._core import ( + Json, JsonQuery, JsonQueryRepl, __doc__, @@ -14,6 +15,7 @@ "__version__", "JsonQuery", "JsonQueryRepl", + "Json", "msgpack_decode", "msgpack_encode", ] diff --git a/src/jsoncons/__init__.pyi b/src/jsoncons/__init__.pyi index 1827028..5219e88 100644 --- a/src/jsoncons/__init__.pyi +++ b/src/jsoncons/__init__.pyi @@ -1,19 +1,245 @@ """ -Pybind11 example plugin ------------------------ +Pybind11 bindings for jsoncons +------------------------------ .. currentmodule:: jsoncons .. autosummary:: :toctree: _generate - add - subtract + Json + JsonQuery + JsonQueryRepl + msgpack_decode + msgpack_encode """ -def add(i: int, j: int) -> int: +from __future__ import annotations + +from typing import overload + +__doc__: str +__version__: str + +class Json: + """ + A class for handling JSON data with conversion to/from JSON and MessagePack formats. + """ + def __init__(self) -> None: + """ + Create a new Json object. + """ + + def from_json(self, json_string: str) -> Json: + """ + Parse JSON from a string. + + Args: + json_string: JSON string to parse + + Returns: + Json: Reference to self + """ + + def to_json(self) -> str: + """ + Convert the JSON object to a string. + + Returns: + str: JSON string representation + """ + + def from_msgpack(self, msgpack_bytes: bytes) -> Json: + """ + Parse MessagePack binary data into a JSON object. + + Args: + msgpack_bytes: MessagePack binary data + + Returns: + Json: Reference to self + """ + + def to_msgpack(self) -> bytes: + """ + Convert the JSON object to MessagePack binary data. + + Returns: + bytes: MessagePack binary data + """ + +class JsonQueryRepl: + """ + A REPL (Read-Eval-Print Loop) for evaluating JMESPath expressions on JSON data. + """ + + doc: Json + debug: bool + + @overload + def __init__(self) -> None: + """ + Create a new JsonQueryRepl instance with null document. + """ + + @overload + def __init__(self, json: str, debug: bool = False) -> None: + """ + Create a new JsonQueryRepl instance. + + Args: + json: JSON text to be parsed + debug: Whether to enable debug mode (default: False) + """ + + def eval(self, expr: str) -> str: + """ + Evaluate a JMESPath expression against the JSON document. + + Args: + expr: JMESPath expression + + Returns: + str: Result of the evaluation as a string + """ + + def add_params(self, key: str, value: str) -> None: + """ + Add parameters for JMESPath evaluation. + + Args: + key: Parameter key + value: Parameter value as JSON string + """ + +class JsonQuery: + """ + A class for filtering and transforming JSON data using JMESPath expressions. + """ + + debug: bool + + def __init__(self) -> None: + """ + Create a new JsonQuery instance. + """ + + def setup_predicate(self, predicate: str) -> None: + """ + Set up the predicate expression used for filtering. + + Args: + predicate: JMESPath predicate expression + """ + + def setup_transforms(self, transforms: list[str]) -> None: + """ + Set up transform expressions used for data transformation. + + Args: + transforms: list of JMESPath transform expressions + """ + + def add_params(self, key: str, value: str) -> None: + """ + Add parameters for JMESPath evaluation. + + Args: + key: Parameter key + value: Parameter value as JSON string + """ + + def matches(self, msgpack: bytes) -> bool: + """ + Check if a MessagePack message matches the predicate. + + Args: + msgpack: MessagePack data as bytes + + Returns: + bool: True if the message matches, False otherwise + """ + + def matches_json(self, json: Json) -> bool: + """ + Check if a JSON document matches the predicate. + + Args: + json: JSON document + + Returns: + bool: True if the document matches, False otherwise + """ + + def process( + self, msgpack: bytes, *, skip_predicate: bool = False, raise_error: bool = False + ) -> bool: + """ + Process a MessagePack message with predicate matching and transformation. + + Args: + msgpack: MessagePack data as bytes + skip_predicate: Whether to skip predicate matching (default: False) + raise_error: Whether to raise errors during transformation (default: False) + + Returns: + bool: True if processing succeeded, False otherwise + """ + + def process_json( + self, json: Json, *, skip_predicate: bool = False, raise_error: bool = False + ) -> bool: + """ + Process a JSON document with predicate matching and transformation. + + Args: + json: JSON document + skip_predicate: Whether to skip predicate matching (default: False) + raise_error: Whether to raise errors during transformation (default: False) + + Returns: + bool: True if processing succeeded, False otherwise + """ + + def export(self) -> bytes: + """ + Export the processed data as MessagePack. + + Returns: + bytes: MessagePack binary data containing the processed results + """ + + def export_json(self) -> Json: + """ + Export the processed data as JSON. + + Returns: + Json: JSON array of processed data + """ + + def clear(self) -> None: + """ + Clear all processed data. + """ + +def msgpack_decode(msgpack_bytes: bytes) -> str: + """ + Convert MessagePack binary data to a JSON string. + + Args: + msgpack_bytes: MessagePack binary data + + Returns: + str: JSON string representation """ - Add two numbers - Some other explanation about the add function. +def msgpack_encode(json_string: str) -> bytes: + """ + Convert a JSON string to MessagePack binary format. + + Args: + json_string: JSON string to encode + + Returns: + bytes: MessagePack binary data """ diff --git a/src/main.cpp b/src/main.cpp index 341362d..c4a77be 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -5,6 +5,7 @@ #endif #include +#include #define STRINGIFY(x) #x #define MACRO_STRINGIFY(x) STRINGIFY(x) @@ -20,43 +21,75 @@ using jsoncons::json; namespace jmespath = jsoncons::jmespath; namespace msgpack = jsoncons::msgpack; -int add(int i, int j) { - return i + j; -} - namespace py = pybind11; +using rvp = py::return_value_policy; using namespace pybind11::literals; // https://github.com/danielaparker/jsoncons/blob/master/doc/ref/jmespath/jmespath.md - +/** + * A REPL (Read-Eval-Print Loop) for evaluating JMESPath expressions on JSON data. + */ struct JsonQueryRepl { - JsonQueryRepl(const std::string &jsontext, bool debug = false): doc_(json::parse(jsontext)), debug(debug) { } + JsonQueryRepl(): doc(json::null()), debug(false) { } + /** + * Constructor for JsonQueryRepl. + * @param jsontext JSON text to be parsed + * @param debug Whether to enable debug mode + */ + JsonQueryRepl(const std::string &jsontext, bool debug = false): doc(json::parse(jsontext)), debug(debug) { } + + /** + * Evaluate a JMESPath expression against the JSON document. + * @param expr_text JMESPath expression + * @return Result of the evaluation as a string + */ std::string eval(const std::string &expr_text) const { - // auto result = jmespath::search(doc_, expr); auto expr = jmespath::make_expression(expr_text); - auto result = expr.evaluate(doc_, params_); + auto result = expr.evaluate(doc, params_); if (debug) { std::cerr << pretty_print(result) << std::endl; } return result.to_string(); } + + /** + * Add parameters for JMESPath evaluation. + * @param key Parameter key + * @param value Parameter value as JSON string + */ void add_params(const std::string &key, const std::string &value) { params_[key] = json::parse(value); } + json doc; bool debug = false; private: - json doc_; std::map params_; }; +/** + * A class for filtering and transforming JSON data using JMESPath expressions. + */ struct JsonQuery { + /** + * Constructor for JsonQuery. + */ JsonQuery() {} + + /** + * Set up the predicate expression used for filtering. + * @param predicate JMESPath predicate expression + */ void setup_predicate(const std::string &predicate) { predicate_expr_ = std::make_unique>(jmespath::make_expression(predicate)); predicate_ = predicate; } + + /** + * Set up transform expressions used for data transformation. + * @param transforms List of JMESPath transform expressions + */ void setup_transforms(const std::vector &transforms) { transforms_expr_.clear(); transforms_expr_.reserve(transforms.size()); @@ -65,10 +98,21 @@ struct JsonQuery { } transforms_ = transforms; } + + /** + * Add parameters for JMESPath evaluation. + * @param key Parameter key + * @param value Parameter value as JSON string + */ void add_params(const std::string &key, const std::string &value) { params_[key] = json::parse(value); } + /** + * Check if a MessagePack message matches the predicate. + * @param msg MessagePack data as string + * @return True if the message matches, false otherwise + */ bool matches(const std::string &msg) const { if (!predicate_expr_) { return false; @@ -77,21 +121,62 @@ struct JsonQuery { return __matches(doc); } - bool process(const std::string &msg, bool skip_predicate = false) { + /** + * Check if a JSON document matches the predicate. + * @param doc JSON document + * @return True if the document matches, false otherwise + */ + bool matches_json(const json &doc) const { + if (!predicate_expr_) { + return false; + } + return __matches(doc); + } + + /** + * Process a MessagePack message with predicate matching and transformation. + * @param msg MessagePack data as string + * @param skip_predicate Whether to skip predicate matching + * @param raise_error Whether to raise errors during transformation + * @return True if processing succeeded, false otherwise + */ + bool process(const std::string &msg, bool skip_predicate = false, bool raise_error = false) { auto doc = msgpack::decode_msgpack(msg); + return process_json(doc, skip_predicate, raise_error); + } + + /** + * Process a JSON document with predicate matching and transformation. + * @param doc JSON document + * @param skip_predicate Whether to skip predicate matching + * @param raise_error Whether to raise errors during transformation + * @return True if processing succeeded, false otherwise + */ + bool process_json(const json &doc, bool skip_predicate = false, bool raise_error = false) { if (!skip_predicate && !__matches(doc)) { return false; } std::vector row; row.reserve(transforms_expr_.size()); for (auto &expr: transforms_expr_) { - row.push_back(expr->evaluate(doc, params_)); + try { + row.push_back(expr->evaluate(doc, params_)); + } catch (const std::exception &e) { + if (raise_error) { + throw e; + } + row.push_back(json::null()); + } } outputs_.emplace_back(std::move(row)); return true; } - std::vector export_() const { + /** + * Export the processed data as JSON. + * @return JSON array of processed data + */ + json export_json() const { json result = json::make_array(); result.reserve(outputs_.size()); for (const auto& row : outputs_) { @@ -102,11 +187,22 @@ struct JsonQuery { } result.push_back(json_row); } + return result; + } + + /** + * Export the processed data as MessagePack. + * @return Binary data containing the MessagePack representation + */ + std::vector export_() const { std::vector output; - msgpack::encode_msgpack(result, output); + msgpack::encode_msgpack(export_json(), output); return output; } + /** + * Clear all processed data. + */ void clear() { outputs_.clear(); } @@ -131,53 +227,217 @@ struct JsonQuery { PYBIND11_MODULE(_core, m) { m.doc() = R"pbdoc( - Pybind11 example plugin - ----------------------- + Python bindings for jsoncons library - .. currentmodule:: jsoncons + This module provides Python bindings for the jsoncons C++ library, allowing for + efficient JSON processing, filtering, and transformation using JMESPath expressions. - .. autosummary:: - :toctree: _generate + Classes: + Json: A class for handling JSON data with conversion to/from JSON and MessagePack formats. + JsonQueryRepl: A REPL (Read-Eval-Print Loop) for evaluating JMESPath expressions on JSON data. + JsonQuery: A class for filtering and transforming JSON data using JMESPath expressions. - add - subtract + Functions: + msgpack_encode: Convert a JSON string to MessagePack binary format. + msgpack_decode: Convert MessagePack binary data to a JSON string. )pbdoc"; - m.def("msgpack_encode", [](const std::string &input) { + py::class_(m, "Json", py::module_local(), py::dynamic_attr()) // + .def(py::init<>(), R"pbdoc( + Create a new Json object. + )pbdoc") + // from/to_json + .def("from_json", [](json &self, const std::string &input) -> json & { + self = json::parse(input); + return self; + }, "json_string"_a, rvp::reference_internal, R"pbdoc( + Parse JSON from a string. + + Args: + json_string: JSON string to parse + + Returns: + Json: Reference to self + )pbdoc") + .def("to_json", [](const json &self) { + return self.to_string(); + }, R"pbdoc( + Convert the JSON object to a string. + + Returns: + str: JSON string representation + )pbdoc") + // from/to_msgpack + .def("from_msgpack", [](json &self, const std::string &input) -> json & { + self = msgpack::decode_msgpack(input); + return self; + }, "msgpack_bytes"_a, rvp::reference_internal, R"pbdoc( + Parse MessagePack binary data into a JSON object. + + Args: + msgpack_bytes: MessagePack binary data + + Returns: + Json: Reference to self + )pbdoc") + .def("to_msgpack", [](const json &self) { std::vector output; - msgpack::encode_msgpack(json::parse(input), output); + msgpack::encode_msgpack(self, output); return py::bytes(reinterpret_cast(output.data()), output.size()); - }, "json_string"_a); - m.def("msgpack_decode", [](const std::string &input) { - auto doc = msgpack::decode_msgpack(input); - return doc.to_string(); - }, "msgpack_bytes"_a); + }, R"pbdoc( + Convert the JSON object to MessagePack binary data. + + Returns: + bytes: MessagePack binary data + )pbdoc") + // + ; py::class_(m, "JsonQueryRepl", py::module_local(), py::dynamic_attr()) // - .def(py::init(), "json"_a, "debug"_a = false) - .def("eval", &JsonQueryRepl::eval, "expr"_a) - .def("add_params", &JsonQueryRepl::add_params, "key"_a, "value"_a) - .def_readwrite("debug", &JsonQueryRepl::debug) + .def(py::init<>()) + .def(py::init(), "json"_a, "debug"_a = false, R"pbdoc( + Create a new JsonQueryRepl instance. + + Args: + json: JSON text to be parsed + debug: Whether to enable debug mode (default: False) + )pbdoc") + .def("eval", &JsonQueryRepl::eval, "expr"_a, R"pbdoc( + Evaluate a JMESPath expression against the JSON document. + + Args: + expr: JMESPath expression + + Returns: + str: Result of the evaluation as a string + )pbdoc") + .def("add_params", &JsonQueryRepl::add_params, "key"_a, "value"_a, R"pbdoc( + Add parameters for JMESPath evaluation. + + Args: + key: Parameter key + value: Parameter value as JSON string + )pbdoc") + .def_readwrite("doc", &JsonQueryRepl::doc, R"pbdoc( + The JSON document being queried. This is the data that JMESPath expressions will be evaluated against. + )pbdoc") + .def_readwrite("debug", &JsonQueryRepl::debug, R"pbdoc( + Debug mode flag. When True, evaluation results will be printed to stderr. + )pbdoc") // ; py::class_(m, "JsonQuery", py::module_local(), py::dynamic_attr()) // - .def(py::init<>()) - .def("setup_predicate", &JsonQuery::setup_predicate) - .def("setup_transforms", &JsonQuery::setup_transforms) - .def("add_params", &JsonQuery::add_params, "key"_a, "value"_a) - .def("matches", &JsonQuery::matches) - .def("process", &JsonQuery::process) + .def(py::init<>(), R"pbdoc( + Create a new JsonQuery instance. + )pbdoc") + .def("setup_predicate", &JsonQuery::setup_predicate, R"pbdoc( + Set up the predicate expression used for filtering. + + Args: + predicate: JMESPath predicate expression + )pbdoc") + .def("setup_transforms", &JsonQuery::setup_transforms, R"pbdoc( + Set up transform expressions used for data transformation. + + Args: + transforms: List of JMESPath transform expressions + )pbdoc") + .def("add_params", &JsonQuery::add_params, "key"_a, "value"_a, R"pbdoc( + Add parameters for JMESPath evaluation. + + Args: + key: Parameter key + value: Parameter value as JSON string + )pbdoc") + .def("matches", &JsonQuery::matches, "msgpack"_a, R"pbdoc( + Check if a MessagePack message matches the predicate. + + Args: + msgpack: MessagePack data as bytes + + Returns: + bool: True if the message matches, False otherwise + )pbdoc") + .def("matches_json", &JsonQuery::matches_json, "json"_a, R"pbdoc( + Check if a JSON document matches the predicate. + + Args: + json: JSON document + + Returns: + bool: True if the document matches, False otherwise + )pbdoc") + .def("process", &JsonQuery::process, "msgpack"_a, py::kw_only(), "skip_predicate"_a = false, "raise_error"_a = false, R"pbdoc( + Process a MessagePack message with predicate matching and transformation. + + Args: + msgpack: MessagePack data as bytes + skip_predicate: Whether to skip predicate matching (default: False) + raise_error: Whether to raise errors during transformation (default: False) + + Returns: + bool: True if processing succeeded, False otherwise + )pbdoc") + .def("process_json", &JsonQuery::process_json, "msgpack"_a, py::kw_only(), "skip_predicate"_a = false, "raise_error"_a = false, R"pbdoc( + Process a JSON document with predicate matching and transformation. + + Args: + json: JSON document + skip_predicate: Whether to skip predicate matching (default: False) + raise_error: Whether to raise errors during transformation (default: False) + + Returns: + bool: True if processing succeeded, False otherwise + )pbdoc") .def("export", [](const JsonQuery& self) { auto output = self.export_(); return py::bytes(reinterpret_cast(output.data()), output.size()); - }, "Export as bytes") - .def("export", &JsonQuery::export_) - .def_readwrite("debug", &JsonQuery::debug) + }, R"pbdoc( + Export the processed data as MessagePack. + + Returns: + bytes: MessagePack binary data containing the processed results + )pbdoc") + .def("export_json", &JsonQuery::export_json, R"pbdoc( + Export the processed data as JSON. + + Returns: + Json: JSON array of processed data + )pbdoc") + .def_readwrite("debug", &JsonQuery::debug, R"pbdoc( + Debug mode flag. + )pbdoc") // ; + m.def("msgpack_encode", [](const std::string &input) { + std::vector output; + msgpack::encode_msgpack(json::parse(input), output); + return py::bytes(reinterpret_cast(output.data()), output.size()); + }, "json_string"_a, R"pbdoc( + Convert a JSON string to MessagePack binary format. + + Args: + json_string: JSON string to encode + + Returns: + bytes: MessagePack binary data + )pbdoc"); + + m.def("msgpack_decode", [](const std::string &input) { + auto doc = msgpack::decode_msgpack(input); + return doc.to_string(); + }, "msgpack_bytes"_a, R"pbdoc( + Convert MessagePack binary data to a JSON string. + + Args: + msgpack_bytes: MessagePack binary data + + Returns: + str: JSON string representation + )pbdoc"); #ifdef VERSION_INFO m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); diff --git a/tests/test_basic.py b/tests/test_basic.py index 1cfa44c..b37232a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -79,4 +79,59 @@ def test_msgpack(): assert data == '{"compact":"true","schema":0}' +def test_json_query(): + """ + https://jmespath.org/tutorial.html + """ + people = [ + {"age": 5, "other": "too young", "name": "Baby"}, + {"age": 20, "other": "foo", "name": "Bob"}, + {"age": 25, "other": "bar", "name": "Fred"}, + {"age": 30, "other": "baz", "name": "George"}, + ] + + repl = m.JsonQueryRepl(json.dumps(people[0]), debug=True) + assert json.loads(repl.eval("age")) == 5 + assert json.loads(repl.eval("name")) == "Baby" + assert not json.loads(repl.eval("age >= `18`")) + + jql = m.JsonQuery() + with pytest.raises(RuntimeError) as excinfo: + jql.setup_predicate("[*].[") + assert "Syntax error" in repr(excinfo) + + jql.setup_predicate("age >= `18`") + jql.setup_transforms(["name", "age"]) + for p in people: + print(p, jql.process(m.msgpack_encode(json.dumps(p)))) + export = jql.export() + data = m.msgpack_decode(export) + assert json.loads(data) == [["Bob", 20], ["Fred", 25], ["George", 30]] + + +def test_json_type(): + obj = m.Json().from_json('{"compact":"true", "schema":0}') + assert obj.to_json() == '{"compact":"true","schema":0}' + obj2 = m.Json().from_msgpack(obj.to_msgpack()) + assert obj.to_msgpack() == obj2.to_msgpack() + + +def test_json_query_json(): + people = [ + {"age": 5, "other": "too young", "name": "Baby"}, + {"age": 20, "other": "foo", "name": "Bob"}, + {"age": 25, "other": "bar", "name": "Fred"}, + {"age": 30, "other": "baz", "name": "George"}, + ] + + jql = m.JsonQuery() + jql.setup_predicate("age >= `18`") + jql.setup_transforms(["name", "age"]) + for p in people: + j = m.Json().from_json(json.dumps(p)) + print(p, jql.process_json(j)) + export = jql.export_json() + assert json.loads(export.to_json()) == [["Bob", 20], ["Fred", 25], ["George", 30]] + + # pytest -vs tests/test_basic.py