From 8d015a4865f3bc71064d061c2488a7730b604820 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Wed, 16 Jul 2025 23:45:08 -0700 Subject: [PATCH 01/33] chore(rename): rename project from template. --- .github/workflows/build-docs.yml | 2 +- .github/workflows/python-app.yml | 6 +++--- README.md | 18 +++++++++--------- docs/source/_templates/redirect.html | 2 +- docs/source/api/index.rst | 4 ++-- docs/source/conf.py | 4 ++-- docs/source/index.rst | 2 +- pyproject.toml | 14 +++++++------- rename.py | 6 +++--- src/{PyTemplate => BenchMatcha}/__init__.py | 2 +- src/{PyTemplate => BenchMatcha}/py.typed | 0 tests/unit/test_version.py | 2 +- 12 files changed, 31 insertions(+), 31 deletions(-) rename src/{PyTemplate => BenchMatcha}/__init__.py (98%) rename src/{PyTemplate => BenchMatcha}/py.typed (100%) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index d822feb..422ded0 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false steps: - - name: Checkout PyTemplate Project + - name: Checkout BenchMatcha Project uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 9d4d723..441cc7d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: PyTemplate CI +name: BenchMatcha CI on: push: @@ -16,7 +16,7 @@ jobs: fail-fast: false steps: - - name: Checkout PyTemplate Project + - name: Checkout BenchMatcha Project uses: actions/checkout@v4 - name: Set up Python @@ -48,7 +48,7 @@ jobs: python-version: [ "3.11", "3.12" ] steps: - - name: Checkout PyTemplate Project + - name: Checkout BenchMatcha Project uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/README.md b/README.md index d9281e5..be3132a 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -# PyTemplate +# BenchMatcha [![build status][buildstatus-image]][buildstatus-url] -[buildstatus-image]: https://github.com/Spill-Tea/PyTemplate/actions/workflows/python-app.yml/badge.svg?branch=main -[buildstatus-url]: https://github.com/Spill-Tea/PyTemplate/actions?query=branch%3Amain +[buildstatus-image]: https://github.com/Spill-Tea/BenchMatcha/actions/workflows/python-app.yml/badge.svg?branch=main +[buildstatus-url]: https://github.com/Spill-Tea/BenchMatcha/actions?query=branch%3Amain Python Project Template. Be sure to create a template directly from github. ## Table of Contents -- [PyTemplate](#pytemplate) +- [BenchMatcha](#pytemplate) - [Using this template](#using-this-template) - [Manual Editing of Project Template](#manual-editing-of-project-template) - [Installation](#installation) @@ -23,7 +23,7 @@ Clone that new repository (e.g. `mynewproject`), and run the helper script `rena ```bash git clone https://github.com//.git cd -python rename.py --old-name PyTemplate --new-name +python rename.py --old-name BenchMatcha --new-name ``` We provide a simple helper script `rename.py` in the root directory to help rename a few @@ -51,7 +51,7 @@ PRO-TIP: you could theoretically run the helper script several times to replace project name, author name, email, and (github) username. Something like: ```bash -python rename.py --old-name PyTemplate --new-name +python rename.py --old-name BenchMatcha --new-name python rename.py --old-name 'Jason C Del Rio' --new-name python rename.py --old-name spillthetea917@gmail.com --new-name python rename.py --old-name Spill-Tea --new-name @@ -61,14 +61,14 @@ python rename.py --old-name Spill-Tea --new-name Clone the repository and pip install. ```bash -git clone https://github.com/Spill-Tea/PyTemplate.git -cd PyTemplate +git clone https://github.com/Spill-Tea/BenchMatcha.git +cd BenchMatcha pip install . ``` Alternatively, you may install directly from github. ```bash -pip install git+https://github.com/Spill-Tea/PyTemplate@main +pip install git+https://github.com/Spill-Tea/BenchMatcha@main ``` diff --git a/docs/source/_templates/redirect.html b/docs/source/_templates/redirect.html index 6bfc771..04be7dc 100644 --- a/docs/source/_templates/redirect.html +++ b/docs/source/_templates/redirect.html @@ -4,6 +4,6 @@ Redirecting to main development branch - + diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 135137e..c87043c 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -1,7 +1,7 @@ -PyTemplate API Documentation +BenchMatcha API Documentation ============================ -PyTemplate API documentation. +BenchMatcha API documentation. Here are python and cython code snippets to demonstrate the use of respective customized lexers with custom syntax highlighting style. These examples are not necessarily meant diff --git a/docs/source/conf.py b/docs/source/conf.py index 4668339..d722c10 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,7 +42,7 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = "PyTemplate" +project = "BenchMatcha" copyright = "2025, Jason C Del Rio (Spill-Tea)" author = "Jason C Del Rio (Spill-Tea)" release = "v0.0.1" @@ -85,7 +85,7 @@ html_static_path = ["_static"] html_css_files = ["custom.css"] # html_logo = "_static/logo.svg" -github_url = "https://github.com/Spill-Tea/PyTemplate" +github_url = "https://github.com/Spill-Tea/BenchMatcha" # Theme options html_theme_options = { diff --git a/docs/source/index.rst b/docs/source/index.rst index d86f4f0..eff0383 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,4 +1,4 @@ -PyTemplate documentation +BenchMatcha documentation ======================== Include text here. diff --git a/pyproject.toml b/pyproject.toml index dfa32f5..19ae792 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=67.6.1"] build-backend = "setuptools.build_meta" [project] -name = "PyTemplate" +name = "BenchMatcha" authors = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] maintainers = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] description = "Project description here." @@ -14,11 +14,11 @@ classifiers = ["Programming Language :: Python :: 3"] dynamic = ["version", "readme", "dependencies"] [project.urls] -homepage = "https://github.com/Spill-Tea/PyTemplate" -issues = "https://github.com/Spill-Tea/PyTemplate/issues" +homepage = "https://github.com/Spill-Tea/BenchMatcha" +issues = "https://github.com/Spill-Tea/BenchMatcha/issues" [tool.setuptools.dynamic] -version = { attr = "PyTemplate.__version__" } +version = { attr = "BenchMatcha.__version__" } readme = { file = ["README.md"], content-type = "text/markdown" } dependencies = { file = ["requirements.txt"] } @@ -33,7 +33,7 @@ exclude = ["benchmarks", "docs", "tests"] "*" = ["py.typed", "*.pyi"] [project.optional-dependencies] -dev = ["PyTemplate[doc,test,lint,type]", "tox", "pre-commit"] +dev = ["BenchMatcha[doc,test,lint,type]", "tox", "pre-commit"] doc = ["sphinx", "furo", "sphinx_multiversion"] test = ["pytest", "coverage", "pytest-xdist"] lint = ["pylint", "ruff"] @@ -46,7 +46,7 @@ addopts = "-n auto -rA" [tool.coverage.run] parallel = true branch = true -source = ["PyTemplate"] +source = ["BenchMatcha"] disable_warnings = ["no-data-collected", "module-not-imported"] [tool.coverage.paths] @@ -61,7 +61,7 @@ skip_empty = true exclude_also = ["def __repr__", 'if __name__ == "__main__"'] [tool.mypy] -mypy_path = "PyTemplate" +mypy_path = "BenchMatcha" warn_unused_ignores = true allow_redefinition = false force_uppercase_builtins = true diff --git a/rename.py b/rename.py index d48a1c3..24c2353 100644 --- a/rename.py +++ b/rename.py @@ -31,7 +31,7 @@ Arguments: new-name (str): new project name (defaults to root directory name) - old-name (str): old project name (defaults to PyTemplate) + old-name (str): old project name (defaults to BenchMatcha) path (str): root path of project (defaults to cwd) dry-run (bool): print out what files / directories would be modified timeout (int): Time in seconds to allow a subprocess to run. @@ -197,8 +197,8 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--old-name", - default="PyTemplate", - help="Old project name to replace (optional, defaults to PyTemplate)", + default="BenchMatcha", + help="Old project name to replace (optional, defaults to BenchMatcha)", ) parser.add_argument( "--path", diff --git a/src/PyTemplate/__init__.py b/src/BenchMatcha/__init__.py similarity index 98% rename from src/PyTemplate/__init__.py rename to src/BenchMatcha/__init__.py index 4a61609..1d37a94 100644 --- a/src/PyTemplate/__init__.py +++ b/src/BenchMatcha/__init__.py @@ -27,6 +27,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""PyTemplate Project.""" +"""BenchMatcha Project.""" __version__: str = "v0.0.1" diff --git a/src/PyTemplate/py.typed b/src/BenchMatcha/py.typed similarity index 100% rename from src/PyTemplate/py.typed rename to src/BenchMatcha/py.typed diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index 7c7bf28..d6a8220 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -29,7 +29,7 @@ """Example unit tests.""" -from PyTemplate import __version__ +from BenchMatcha import __version__ def test_version_type(): From 24c0a6ffe514204081b4ce823f458b3146beffa0 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Fri, 18 Jul 2025 16:36:47 -0700 Subject: [PATCH 02/33] chore(pyproject): Update project metadata. --- pyproject.toml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 19ae792..06df715 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,21 @@ build-backend = "setuptools.build_meta" name = "BenchMatcha" authors = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] maintainers = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] -description = "Project description here." +description = "Google Benchmark Suite Runner and Regression Analyzer." license = { file = "LICENSE" } -requires-python = ">=3.7" -keywords = ["keyword1", "keyword2"] -classifiers = ["Programming Language :: Python :: 3"] +requires-python = ">=3.10" +keywords = ["benchmark", "regression", "analysis"] +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 1 - Planning", + "License :: OSI Approved :: BSD License", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Utilities", + "Topic :: System :: Benchmark", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Libraries", + ] dynamic = ["version", "readme", "dependencies"] [project.urls] From 1cf53113ea299c5c41244f09b75ef33d94a136c3 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Fri, 18 Jul 2025 16:37:43 -0700 Subject: [PATCH 03/33] docs(readme): Update readme. --- README.md | 66 ++++++++++++++++++------------------------------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index be3132a..3438044 100644 --- a/README.md +++ b/README.md @@ -4,73 +4,49 @@ [buildstatus-image]: https://github.com/Spill-Tea/BenchMatcha/actions/workflows/python-app.yml/badge.svg?branch=main [buildstatus-url]: https://github.com/Spill-Tea/BenchMatcha/actions?query=branch%3Amain -Python Project Template. Be sure to create a template directly -from github. +![logo](docs/source/_static/logo.svg) + +BenchMatcha is your companion pytest-like runner to google benchmarks. +Analyze, plot, and save your results over time to evaluate regression +over the lifetime of a project. ## Table of Contents -- [BenchMatcha](#pytemplate) - - [Using this template](#using-this-template) - - [Manual Editing of Project Template](#manual-editing-of-project-template) +- [BenchMatcha](#benchmatcha) - [Installation](#installation) + - [Install from pypi](#install-from-pypi) + - [Clone the repository](#clone-the-repository) + - [Pip install directly from github.](#pip-install-directly-from-github) + - [Development](#development) - [For Developers](#for-developers) - [License](#license) -## Using this template -Create a new repository using the `Use this template` option available on github. -Clone that new repository (e.g. `mynewproject`), and run the helper script `rename.py`. - -```bash -git clone https://github.com//.git -cd -python rename.py --old-name BenchMatcha --new-name - -``` -We provide a simple helper script `rename.py` in the root directory to help rename a few -files and directory names to make your life easier. Please note that you will still need -to manually adjust the `pyproject.toml` file, specifically the `[project]` and -`[project.urls]` keys, to reflect your new project metadata. - -Also manually update the `docs/source/conf.py` file to reflect correct `author`, -`copyright`, and `release` key metadata for documentation builds. - -Finally update this `README.md` document to reflect new project urls. - -### Manual Editing of Project Template -To summarize, after running the `rename.py` script, there are three files you may need -to manually adjust for your new project: -1. `pyproject.toml` --> update metadata -1. `docs/source/conf.py` --> update metadata -1. `README.md` --> update project urls (and license type if different) - -Note: If you need to update the `LICENSE` file, you will also need to edit the license -header from files throughout the `src/` and `tests/` directories. - -PRO-TIP: you could theoretically run the helper script several times to replace the -project name, author name, email, and (github) username. Something like: +## Installation +You have options. +### Install from pypi ```bash -python rename.py --old-name BenchMatcha --new-name -python rename.py --old-name 'Jason C Del Rio' --new-name -python rename.py --old-name spillthetea917@gmail.com --new-name -python rename.py --old-name Spill-Tea --new-name +pip install BenchMatcha ``` -## Installation -Clone the repository and pip install. - +### Clone the repository ```bash git clone https://github.com/Spill-Tea/BenchMatcha.git cd BenchMatcha pip install . ``` -Alternatively, you may install directly from github. +### Pip install directly from github. ```bash pip install git+https://github.com/Spill-Tea/BenchMatcha@main ``` +## Development + +BenchMatcha is currently in the planning stages of development. This means the project +is not ready for production use, and may be prone to change api without much notice. + ## For Developers After cloning the repository, create a new virtual environment and run the following From 52cb915044b93a17afbfc3e3f6f0d717760bab3f Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Tue, 22 Jul 2025 15:49:07 -0700 Subject: [PATCH 04/33] docs(source): Fix docs post renaming. Remove irrelevant sample documentation. --- docs/source/api/index.rst | 144 +------------------------------------- docs/source/index.rst | 2 +- 2 files changed, 2 insertions(+), 144 deletions(-) diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index c87043c..3cb0d7c 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -1,150 +1,8 @@ BenchMatcha API Documentation -============================ +============================= BenchMatcha API documentation. -Here are python and cython code snippets to demonstrate the use of respective customized -lexers with custom syntax highlighting style. These examples are not necessarily meant -to be fully valid code, but to demonstrate key features not available through standard -pygments syntax highlighting styles. - -Python Example Snippet ----------------------- - -.. code-block:: python - :caption: example.py - - #!/usr/bin/env python3 - """Module level docstring.""" - from typing import ClassVar - import numpy as np - - CONSTANT_A: int = 0xFF - CONSTANT_B: float = np.pi - - # NOTE: this is an example class - class Example(object): - """Example docstring. - - Args: - arg1 (str): argument 1 - arg2 (int): argument 2 - - Attributes: - data (dict): data - - """ - arg1: str - arg2: int - data: dict - seventeen: ClassVar[list[int]] = [17, 0x11, 0o21, 0b10001] - other: ClassVar[list[int]] = [1e-5, 1.0e+3, 2j, 2l, 2.7E4J] - - def __init__(self, arg1: str, arg2: int) -> None: - self.arg1 = arg1 - self.arg2 = arg2 - self.data = { - "a": [(1, 2, (3, 4, 5)), (6, 7, (8, 9 , 10))], - "b": {"c": (7, 4, 3), "d": {"e", "f", "g"}}, - } - - def __getattr__(self, value): - return self.method(value) - - def method(self, value): - return self.data[value] - - def write(self, text): - print(f"{text:<5}\n") - - def do_something(self, value): - if value > CONSTANT_A: - return value - CONSTANT_B - else: - return value + 0b10011 - - -Cython Example Snippet ----------------------- - -.. code-block:: cython - :caption: example.pyx - - """Module level docstring.""" - import cython - from libc.stdlib cimport free, malloc - - cdef extern from "" namespace "std": - cdef cppclass vector[T]: - vector() - T& operator[](int) - - ctypedef fused StringTypeObject: - str - bytes - - ctypedef struct CustomStruct: - int y - str z - - cdef packed struct Breakfast: - int[4] spam - signed char[5] eggs - - cdef enum CheeseType: - manchego = 1 - gouda = 2 - camembert = 3 - - cdef union MyUnion: - int i - float f - char c - - cdef inline unsigned char* function(bint flag) noexcept: - cdef: - Py_ssize_t j - unsigned char* k = NULL - - k = malloc(5 * sizeof(unsigned char)) - - for j in range(5): - k[j] = "A" - - return k - - # XXX: this is an example class - cdef class Example: - """The little example class that couldn't. - - Args: - arg1 (unsigned long long): ... - arg2 (double): ... - - """ - cdef public unsigned long long v - cdef readonly double k - cdef char* mem - - def __cinit__(self, unsigned long long arg1, double arg2): - self.v = arg1 - self.k = arg2 - self.mem = malloc(5 * sizeof(char)) - - def __dealloc__(self): - free(self.mem) - - @cython.boundscheck(False) - cdef char index(self, size_t idx): - return self.mem[idx] - - # just an example of nested parenthesis to demonstrate rainbow coloring - cdef dict obj = { - "a": [(1, 2, (3, 4, 5)), (6, 7, (8, 9 , 10))], - "b": {"c": (7, 4, 3), "d": {"e", "f", "g"}}, - } - cdef tuple builtin_constants = (True, False, NULL, None,) - .. toctree:: :caption: Submodules diff --git a/docs/source/index.rst b/docs/source/index.rst index eff0383..894d4cd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ BenchMatcha documentation -======================== +========================= Include text here. From 04389f2fa8c27d6d793a29c39bd8432e5ac5bd02 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Wed, 23 Jul 2025 00:25:10 -0700 Subject: [PATCH 05/33] chore(configs): Update project config and requirements. --- pyproject.toml | 10 +- rename.py | 274 ----------------------------------------------- requirements.txt | 7 ++ tox.ini | 6 +- 4 files changed, 16 insertions(+), 281 deletions(-) delete mode 100644 rename.py diff --git a/pyproject.toml b/pyproject.toml index 06df715..47a9790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,9 @@ exclude = ["benchmarks", "docs", "tests"] [tool.setuptools.package-data] "*" = ["py.typed", "*.pyi"] +[project.scripts] +benchmatcha = "BenchMatcha.runner:main" + [project.optional-dependencies] dev = ["BenchMatcha[doc,test,lint,type]", "tox", "pre-commit"] doc = ["sphinx", "furo", "sphinx_multiversion"] @@ -67,17 +70,14 @@ fail_under = 95.0 precision = 1 show_missing = true skip_empty = true -# skip_covered = true exclude_also = ["def __repr__", 'if __name__ == "__main__"'] [tool.mypy] mypy_path = "BenchMatcha" -warn_unused_ignores = true allow_redefinition = false -force_uppercase_builtins = true +warn_unused_ignores = true [tool.pylint.main] -# extension-pkg-whitelist = [] ignore = ["tests", "dist", "build"] fail-under = 9.0 jobs = 0 @@ -99,6 +99,8 @@ max-line-length = 88 [tool.pylint."messages control"] disable = [ "R1731", # consider-using-max-builtin + "R0903", # too-few-public-methods + "R1735", # use-dict-literal ] [tool.pylint."*.pyi"] diff --git a/rename.py b/rename.py deleted file mode 100644 index 24c2353..0000000 --- a/rename.py +++ /dev/null @@ -1,274 +0,0 @@ -# BSD 3-Clause License -# -# Copyright (c) 2025, Spill-Tea -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Primitive script to update repository (project) name throughout template. - -Arguments: - new-name (str): new project name (defaults to root directory name) - old-name (str): old project name (defaults to BenchMatcha) - path (str): root path of project (defaults to cwd) - dry-run (bool): print out what files / directories would be modified - timeout (int): Time in seconds to allow a subprocess to run. - -Notes: - * client must have git installed. - -""" - -import argparse -import os -import subprocess -from collections.abc import Iterator - - -def bypass(path: str, git_root: str, timeout: int = 1) -> bool: - """Use git to identify if a path is ignored as specified by a .gitignore file.""" - # NOTE: Do not capture errors to avoid assuming a path is included or not. - result = subprocess.run( - ["git", "-C", git_root, "check-ignore", path], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - timeout=timeout, - ) - - return result.returncode == 0 - - -def find_git_root(start_path: str, timeout: int = 1) -> str: - """Confirm we are in a git repository.""" - try: - result = subprocess.run( - ["git", "-C", start_path, "rev-parse", "--show-toplevel"], - check=True, - capture_output=True, - text=True, - timeout=timeout, - ) - return result.stdout.strip() - - except subprocess.CalledProcessError as e: - raise RuntimeError("This script must be run inside a Git repository.") from e - - -def safe_scandir(path: str) -> Iterator[os.DirEntry]: - """Wrapper around os.scandir.""" - try: - with os.scandir(path) as it: - yield from it - except PermissionError: - return - - -def replace_in_file( - filepath: str, - old: str, - new: str, - dry_run: bool = False, -) -> int: - """Replace an old keyword found within a file.""" - try: - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - except (UnicodeDecodeError, FileNotFoundError) as e: - print(f"[Warning] ({e.__class__.__name__}) {filepath}") - return 0 - - if old not in content: - return 0 - - if dry_run: - print(f"[DRY RUN] Would update content within file: {filepath}") - - else: - new_content: str = content.replace(old, new) - with open(filepath, "w", encoding="utf-8") as f: - f.write(new_content) - print(f"Updated content within file: {filepath}") - - return 1 - - -def update_project_name( - path: str, - old_name: str, - new_name: str, - dry_run: bool, - git_root: str, - timeout: int = 1, -) -> int: - """Recursively search, and modify files in place to update project name if used.""" - count = 0 - for entry in safe_scandir(path): - full_path = os.path.join(path, entry.name) - - if bypass(full_path, git_root, timeout): - continue - - if entry.is_dir(follow_symlinks=False): - count += update_project_name( - full_path, old_name, new_name, dry_run, git_root - ) - - elif entry.is_file(follow_symlinks=False): - count += replace_in_file(full_path, old_name, new_name, dry_run) - - return count - - -def _filetype(entry: os.DirEntry) -> str: - key: str = "" - if entry.is_file(follow_symlinks=False): - key = " filepath" - elif entry.is_dir(follow_symlinks=False): - key = " directory" - - return key - - -def rename_directories_and_files( - path: str, - old_name: str, - new_name: str, - dry_run: bool, - git_root: str, - timeout: int = 1, -) -> int: - """Rename both directories and filenames alike if old keyword present.""" - count: int = 0 - for entry in safe_scandir(path): - full_path = os.path.join(path, entry.name) - if bypass(full_path, git_root, timeout): - continue - - # NOTE: Depth First Search. Handle all children before renaming a directory. - if entry.is_dir(follow_symlinks=False): - count += rename_directories_and_files( - full_path, old_name, new_name, dry_run, git_root - ) - - if old_name not in entry.name: - continue - - new_path = os.path.join(path, entry.name.replace(old_name, new_name)) - key: str = _filetype(entry) - if dry_run: - print(f"[DRY RUN] Would rename{key}: {full_path} -> {new_path}") - else: - os.rename(full_path, new_path) - print(f"Renamed{key}: {full_path} -> {new_path}") - count += 1 - - return count - - -def parse_args() -> argparse.Namespace: - """Define and return parsed arguments.""" - parser = argparse.ArgumentParser(description="Rename a Python project template.") - parser.add_argument( - "--new-name", - help="New project name (e.g. my_project)", - ) - parser.add_argument( - "--old-name", - default="BenchMatcha", - help="Old project name to replace (optional, defaults to BenchMatcha)", - ) - parser.add_argument( - "--path", - default=".", - help="Root path of the project (default: current directory)", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would change, but don't modify anything", - ) - parser.add_argument( - "--timeout", - default=1, - help="Time in seconds to allow a subprocess to run.", - type=int, - ) - - return parser.parse_args() - - -def main() -> None: - """Main script Entry point.""" - args: argparse.Namespace = parse_args() - git_root: str = find_git_root(args.path) - - # NOTE: When this template is forked, the project should be renamed. So we can - # reasonably assume the name of the new project. Report assumption to client. - if args.new_name is None: - args.new_name = os.path.basename(git_root) - print(f"Assuming new project name: {args.new_name}") - - if args.new_name == args.old_name: - print("Exiting. Both New and old names are identical.") - return - - print( - f"Project Found at: '{args.path}'", - f"Replacing '{args.old_name}' --> '{args.new_name}'", - sep="\n", - ) - if args.dry_run: - print("[DRY RUN] Confirming Dry Run Mode. No changes will be made.") - - # NOTE: this script may also be updated to reflect the new project name. - total: int = 0 - print("\nStep I: Update File contents.") - total += update_project_name( - args.path, - args.old_name, - args.new_name, - dry_run=args.dry_run, - git_root=git_root, - timeout=args.timeout, - ) - print("\nStep II: Update Filepath Names.") - total += rename_directories_and_files( - args.path, - args.old_name, - args.new_name, - dry_run=args.dry_run, - git_root=git_root, - timeout=args.timeout, - ) - - if args.dry_run: - print(f"\n[DRY RUN] Complete. Would modify {total} file(s).") - else: - print(f"Success. Modified {total} file(s) in total.") - - -if __name__ == "__main__": - main() diff --git a/requirements.txt b/requirements.txt index e69de29..2326fc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,7 @@ +google-benchmark +numpy +orjson +plotly +pytest +scipy +wurlitzer diff --git a/tox.ini b/tox.ini index 93f3f78..931e2ce 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -envlist = type, lint, coverage, docs, py{38,39,310,311,312}-tests +envlist = type, lint, coverage, docs, py{310,311,312}-tests [testenv] description = Base Environment @@ -12,7 +12,7 @@ commands_pre = commands = coverage run --rcfile pyproject.toml -m pytest {posargs} -[testenv:py{38,39,310,311,312}-tests] +[testenv:py{310,311,312}-tests] description = Run Unit Tests commands_pre = {envpython} --version @@ -23,7 +23,7 @@ description = Report Code Coverage skip_install = true deps = coverage parallel_show_output = true -depends = py{38,39,310,311,312}-tests +depends = py{310,311,312}-tests commands = coverage combine --quiet --rcfile pyproject.toml coverage report --rcfile pyproject.toml {posargs} From 1c77c588315d52fc50e38bf1d31f3e2df4b54fbb Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Wed, 13 Aug 2025 21:38:18 -0700 Subject: [PATCH 06/33] feat(handlers): Implement json and file stream handlers, and custom error organization. --- src/BenchMatcha/errors.py | 60 +++++++++++++++++ src/BenchMatcha/handlers.py | 125 ++++++++++++++++++++++++++++++++++++ tests/unit/conftest.py | 121 ++++++++++++++++++++++++++++++++++ tests/unit/test_errors.py | 68 ++++++++++++++++++++ tests/unit/test_handlers.py | 71 ++++++++++++++++++++ 5 files changed, 445 insertions(+) create mode 100644 src/BenchMatcha/errors.py create mode 100644 src/BenchMatcha/handlers.py create mode 100644 tests/unit/test_errors.py create mode 100644 tests/unit/test_handlers.py diff --git a/src/BenchMatcha/errors.py b/src/BenchMatcha/errors.py new file mode 100644 index 0000000..66cec7c --- /dev/null +++ b/src/BenchMatcha/errors.py @@ -0,0 +1,60 @@ +"""Custom BenchMatcha exception definitions. + +Considerations: + * All custom exceptions should be defined within this module for better project + organization. This also prevents proliferation of custom errors as project scales. + * Custom exceptions should define a class method named `response` to construct and + standardize verbiage of output message. It has the nice side effect of providing + example context for usage. + * Before creating a custom error, determine if available exceptions can be used + instead. + +""" + +from __future__ import annotations + +from json import JSONDecodeError +from typing import Self, TypeVar + + +E = TypeVar("E", bound=Exception) + +_exception_register: set[type[Exception]] = { + TypeError, + ValueError, + RuntimeError, + JSONDecodeError, + FileNotFoundError, +} + + +def register_custom_exception(cls: type[E]) -> type[E]: + """Register custom exceptions.""" + _exception_register.add(cls) + + return cls + + +@register_custom_exception +class SchemaError(Exception): + """Unsupported json schema.""" + + @classmethod + def response(cls, version: str) -> Self: + """Define standard response message.""" + msg: str = f"Unsupported json schema version: {version}" + + return cls(msg) + + +@register_custom_exception +class ParsingError(Exception): + """Failed to parse json output (from Google Benchmark).""" + + @classmethod + def response(cls) -> Self: + """Define standard response message.""" + return cls( + "Failed to parse json data. Please confirm benchmarks do not contain " + "print statements or write to stdout, which can interfere with output." + ) diff --git a/src/BenchMatcha/handlers.py b/src/BenchMatcha/handlers.py new file mode 100644 index 0000000..d7c6f8c --- /dev/null +++ b/src/BenchMatcha/handlers.py @@ -0,0 +1,125 @@ +"""IO handlers to transform using json library.""" + +from __future__ import annotations + +import json +import os +from abc import ABC, abstractmethod +from io import IOBase +from typing import IO, Any + + +def is_readable_io_protocol(obj: object) -> bool: + """Determine if object implements readable IO interface.""" + methods: set[str] = { + "read", + "readable", + "write", + # "writeable", # temporary file handler does not have this method. + "seek", + "seekable", + "tell", + "close", + "closed", # property + "flush", + } + + return ( + isinstance(obj, IO | IOBase) or all(map(lambda x: hasattr(obj, x), methods)) + ) and obj.readable() + + +class Handler(ABC): + """Abstract Handler protocol.""" + + @abstractmethod + def handle(self) -> dict[str, Any]: + """Handle parsing of object.""" + raise NotImplementedError("Must implement.") + + +class HandleText(Handler): + """Handle loading text (string) to json object.""" + + text: str + + def __init__(self, text: str): + self.text = text + + def handle(self) -> dict[str, Any]: + return json.loads(self.text) + + +class HandleBytes(Handler): + """Handle loading bytes to json object.""" + + text: bytes + encoding: str + + def __init__(self, text: bytes, encoding: str = "utf8"): + self.text = text + self.encoding = encoding + + def handle(self) -> dict[str, Any]: + return HandleText(self.text.decode(self.encoding)).handle() + + +class HandleIO(Handler): + """Handle loading io data to json object.""" + + stream: IOBase + encoding: str + + def __init__(self, stream: IOBase, encoding: str = "utf8"): + self.stream = stream + if not self.stream.readable(): + raise TypeError("Unreadable stream.") + + self.encoding = encoding + + def handle(self) -> dict[str, Any]: + text = self.stream.read() + if isinstance(text, bytes): + handler: Handler = HandleBytes(text, self.encoding) + + else: + handler = HandleText(text) + + return handler.handle() + + +class HandlePath(Handler): + """Handle loading data from file to json object.""" + + path: str + encoding: str + + def __init__(self, path: str, encoding: str = "utf8"): + self.path = path + self.encoding = encoding + + def handle(self) -> dict[str, Any]: + if not os.path.exists(self.path): + return HandleText(self.path).handle() + + with open(self.path, "r", encoding=self.encoding) as f: + return HandleIO(f).handle() + + +def dispatch(obj: object, encoding: str = "utf8") -> Handler: + """Dispatch appropriate handler in response to input object type.""" + if isinstance(obj, str): + return HandlePath(obj) + + elif isinstance(obj, bytes): + return HandleBytes(obj, encoding) + + elif is_readable_io_protocol(obj): + return HandleIO(obj, encoding) + + raise TypeError(f"Unsupported object type: {type(obj)}") + + +def load(obj: object, encoding: str = "utf8") -> dict[str, Any]: + """Load json data.""" + return dispatch(obj, encoding).handle() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 30b4382..558ab48 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -26,3 +26,124 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import tempfile +from collections.abc import Iterator + +import pytest + + +@pytest.fixture +def mock_context() -> str: + return """ +"context": { + "date": "2025-07-13T12:09:31-07:00", + "host_name": "host", + "executable": "file.py", + "num_cpus": 12, + "mhz_per_cpu": 24, + "cpu_scaling_enabled": false, + "caches": [ + { + "type": "Data", + "level": 1, + "size": 65536, + "num_sharing": 0 + }, + { + "type": "Instruction", + "level": 1, + "size": 131072, + "num_sharing": 0 + }, + { + "type": "Unified", + "level": 2, + "size": 4194304, + "num_sharing": 1 + } + ], + "load_avg": [4.69092,4.60693,4.47949], + "library_version": "1.9.4", + "library_build_type": "release", + "json_schema_version": 1 + }, +""" + + +@pytest.fixture +def mock_bench() -> str: + return """ +"benchmarks": [ + { + "name": "function/8/repeats:3", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/8/repeats:3", + "run_type": "iteration", + "repetitions": 3, + "repetition_index": 0, + "threads": 1, + "iterations": 1686012, + "real_time": 4.2350600595475760e+02, + "cpu_time": 4.2322355950016964e+02, + "time_unit": "ns" + }, + { + "name": "function/8/repeats:3", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/8/repeats:3", + "run_type": "iteration", + "repetitions": 3, + "repetition_index": 1, + "threads": 1, + "iterations": 1686012, + "real_time": 4.2872337197514543e+02, + "cpu_time": 4.2800703672334481e+02, + "time_unit": "ns" + }, + { + "name": "function/repeats:3_BigO", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/repeats:3", + "run_type": "aggregate", + "repetitions": 3, + "threads": 1, + "aggregate_name": "BigO", + "aggregate_unit": "time", + "cpu_coefficient": 1.2549524739108346e+01, + "real_coefficient": 1.2555517635286144e+01, + "big_o": "N", + "time_unit": "ns" + }, + { + "name": "function/repeats:3_RMS", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/repeats:3", + "run_type": "aggregate", + "repetitions": 3, + "threads": 1, + "aggregate_name": "RMS", + "aggregate_unit": "percentage", + "rms": 5.4107447739157266e-02 + } +] +""" + + +@pytest.fixture +def mock_data(mock_context: str, mock_bench: str) -> str: + """mock benchmark data.""" + return "{" + f"{mock_context}{mock_bench}" + "}" + + +@pytest.fixture +def mock_file(mock_data: str) -> Iterator[tempfile._TemporaryFileWrapper]: + with tempfile.NamedTemporaryFile("w+") as f: + f.write(mock_data) + f.seek(0) + + yield f diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py new file mode 100644 index 0000000..99b81e8 --- /dev/null +++ b/tests/unit/test_errors.py @@ -0,0 +1,68 @@ +import builtins +from collections.abc import Callable, Iterator +from json import JSONDecodeError + +import pytest + +from BenchMatcha import errors + + +@pytest.fixture +def register_error() -> Iterator[Callable[[type[Exception]], type[Exception]]]: + e: type[Exception] | None = None + + def inner(exc: type[Exception]) -> type[Exception]: + nonlocal e + e = errors.register_custom_exception(exc) + + return e + + yield inner + + if e is not None: + assert e in errors._exception_register, "Expected error to be registered" + errors._exception_register.remove(e) + assert e not in errors._exception_register, "Expected error to be removed." + + +def test_registering_custom_error( + register_error: Callable[[type[Exception]], type[Exception]], +): + """Confirm we can correctly register custom exception classes.""" + + class CustomMagicalUnknownUnicornError(Exception): ... + + res = register_error(CustomMagicalUnknownUnicornError) + assert res is CustomMagicalUnknownUnicornError, "Expected same exception." + + +# https://pytest-xdist.readthedocs.io/en/stable/known-limitations.html#order-and-amount-of-test-must-be-consistent +@pytest.mark.parametrize( + "custom", + sorted( # limitation of pytest-xdist -> cannot handle unordered (sets) iterables + filter( + lambda x: not (hasattr(builtins, x.__name__) or x == JSONDecodeError), + errors._exception_register, + ), + key=lambda x: x.__name__, + ), +) +def test_custom_exception_api(custom: type[Exception]) -> None: + """Test we conform to Custom exception API.""" + assert issubclass(custom, Exception), "Expected to subclass Exception." + assert hasattr(custom, "response"), "Expected to have response method." + + +@pytest.mark.parametrize( + ["cls", "args"], + [ + [errors.ParsingError, ()], + [errors.SchemaError, (1,)], + ], +) +def test_custom_exception_response(cls: type[Exception], args: tuple | None) -> None: + """Confirm response method returns an instance of the class.""" + result = cls.response(*args) + assert isinstance(result, cls), ( + f"Expected to return an instance of exception: {cls.__name__}." + ) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py new file mode 100644 index 0000000..ab8ddef --- /dev/null +++ b/tests/unit/test_handlers.py @@ -0,0 +1,71 @@ +"""Test json data handlers module.""" + +from collections.abc import Callable +from io import BytesIO, StringIO +from typing import IO, Any + +import pytest + +from BenchMatcha import handlers + + +def test_load(mock_data: str) -> None: + """Test load interface.""" + result = handlers.load(mock_data) + assert isinstance(result, dict), "Expected to load a dict." + + +@pytest.mark.parametrize( + ["handler", "transformer"], + [ + (handlers.HandlePath, lambda x: x), + (handlers.HandleBytes, lambda x: x.encode()), + (handlers.HandleIO, lambda x: StringIO(x)), + (handlers.HandleIO, lambda x: BytesIO(x.encode())), + ], +) +def test_handlers( + handler: type[handlers.Handler], + transformer: Callable[[str], Any], + mock_data: str, +) -> None: + """""" + mock = transformer(mock_data) + result = handler(mock).handle() + assert isinstance(result, dict), "Expected a dictionary loaded." + assert isinstance(handlers.dispatch(mock), handler), ( + "Expected sample handler object." + ) + + if isinstance(mock, (StringIO, BytesIO)): + mock.close() + + +@pytest.mark.parametrize( + ["handler", "transformer", "expected"], + [ + (handlers.dispatch, lambda x: x.name, handlers.HandlePath), + (handlers.dispatch, lambda x: x.read(), handlers.HandlePath), + (handlers.dispatch, lambda x: x.read().encode(), handlers.HandleBytes), + (handlers.dispatch, lambda x: x, handlers.HandleIO), + ], +) +def test_file_path_handlers( + handler: Callable[[Any], handlers.Handler], + transformer: Callable[[IO], Any], + expected: handlers.Handler, + mock_file: IO, +) -> None: + """Test dispatch handlers.""" + result: handlers.Handler = handler(transformer(mock_file)) + assert isinstance(result, expected), "Unexpected Handler dispatched." + data = result.handle() + assert isinstance(data, dict), "Expected dictionary object." + + +def test_dispatch_unsupported_object() -> None: + """Test we raise TypeError when providing unsupported object.""" + with pytest.raises(TypeError) as e: + handlers.dispatch(1) + + assert e.type is TypeError, "Expected a type error." From 7d97ce6c31256dd67bea8dee9e08195275695d05 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Wed, 13 Aug 2025 21:39:11 -0700 Subject: [PATCH 07/33] chore(precommit): Update addlicense precommit hook. --- .pre-commit-config.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da84716..5f9ea74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,11 +15,10 @@ repos: args: [ --maxkb=1024 ] - id: requirements-txt-fixer - - repo: https://github.com/Spill-Tea/addlicense-pre-commit - rev: v1.1.2 + - repo: https://github.com/google/addlicense + rev: v1.2.0 hooks: - id: addlicense - language: golang args: [ -f, LICENSE, ] From ffc4b8bd0d3bdd96d67a5bbad56b610f2c34adf7 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Wed, 13 Aug 2025 21:40:31 -0700 Subject: [PATCH 08/33] chore(license): add license to new files. --- src/BenchMatcha/errors.py | 29 +++++++++++++++++++++++++++++ src/BenchMatcha/handlers.py | 29 +++++++++++++++++++++++++++++ tests/unit/test_errors.py | 29 +++++++++++++++++++++++++++++ tests/unit/test_handlers.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/src/BenchMatcha/errors.py b/src/BenchMatcha/errors.py index 66cec7c..909492f 100644 --- a/src/BenchMatcha/errors.py +++ b/src/BenchMatcha/errors.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """Custom BenchMatcha exception definitions. Considerations: diff --git a/src/BenchMatcha/handlers.py b/src/BenchMatcha/handlers.py index d7c6f8c..fbaeb6f 100644 --- a/src/BenchMatcha/handlers.py +++ b/src/BenchMatcha/handlers.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """IO handlers to transform using json library.""" from __future__ import annotations diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index 99b81e8..8d8b546 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import builtins from collections.abc import Callable, Iterator from json import JSONDecodeError diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index ab8ddef..50f46da 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """Test json data handlers module.""" from collections.abc import Callable From 9914395f4a37ab0ed75078f2cafcf82bc1134e97 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Fri, 15 Aug 2025 02:50:20 -0700 Subject: [PATCH 09/33] feat(sifter): Include module of methods to collect and load new benchmarks. --- src/BenchMatcha/sifter.py | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/BenchMatcha/sifter.py diff --git a/src/BenchMatcha/sifter.py b/src/BenchMatcha/sifter.py new file mode 100644 index 0000000..24c6f2d --- /dev/null +++ b/src/BenchMatcha/sifter.py @@ -0,0 +1,40 @@ +"""Discovery of benchmark tests to register.""" + +import glob +import os +from collections.abc import Iterator +from pathlib import Path +from types import ModuleType + +from _pytest.pathlib import import_path + + +def scandir(filepath: str) -> Iterator[os.DirEntry[str]]: + """Simple wrapper around os.scandir to use more simply as an iterator.""" + with os.scandir(os.path.abspath(filepath)) as scanner: + yield from scanner + + +def collect(root: str, pattern: str = "bench*.py") -> Iterator[str]: + """Collect relevant filepaths recursively stemming from root directory.""" + yield from glob.iglob(os.path.join(root, pattern), root_dir=root) + + for candidate in scandir(root): + if candidate.is_dir(follow_symlinks=False): + yield from collect(candidate.path, pattern) + + +def load_benchmark(path: str, root: str) -> ModuleType: + """Load a benchmark suite.""" + return import_path( + os.path.abspath(path), + root=Path(root).absolute().resolve(), + consider_namespace_packages=False, + ) + + +def collect_benchmarks(root: str) -> None: + """Collect all benchmarks from a variety of benchmark suites.""" + root = os.path.abspath(root) + for j in collect(root): + load_benchmark(j, root=root) From befd5d67a992c99c06405677e5aa80d4d272e816 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Fri, 15 Aug 2025 02:51:07 -0700 Subject: [PATCH 10/33] feat(sifter): Include module of methods to collect and load new benchmarks. --- src/BenchMatcha/sifter.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/BenchMatcha/sifter.py b/src/BenchMatcha/sifter.py index 24c6f2d..f5910a9 100644 --- a/src/BenchMatcha/sifter.py +++ b/src/BenchMatcha/sifter.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """Discovery of benchmark tests to register.""" import glob From 9492950fd589e9feeb11bdd5f7256244467045fe Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Fri, 15 Aug 2025 19:18:42 -0700 Subject: [PATCH 11/33] feat(utils): common or miscellaneous utility functions. --- src/BenchMatcha/utils.py | 51 ++++++++++++++++++++++++++++++ tests/unit/test_utils.py | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 src/BenchMatcha/utils.py create mode 100644 tests/unit/test_utils.py diff --git a/src/BenchMatcha/utils.py b/src/BenchMatcha/utils.py new file mode 100644 index 0000000..afed66d --- /dev/null +++ b/src/BenchMatcha/utils.py @@ -0,0 +1,51 @@ +"""Miscellaneous utilities.""" + +from __future__ import annotations + +import enum +import sys + +import numpy as np + + +def power_of_2(x: int) -> int: + """Retrieve the next power of 2, if value is not already one.""" + x -= 1 + mod: int = 1 + size: int = sys.getsizeof(x) + while mod < size: + x |= x >> mod + mod *= 2 + + return x + 1 + + +def _simple_stats(x: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Compute mean and standard deviation.""" + mean: np.ndarray = np.nanmean(x, axis=1) + std: np.ndarray = np.nanstd(x, axis=1, ddof=1) + + return mean, std + + +# https://github.com/google/benchmark/blob/main/src/complexity.cc#L52-L69 +class BigO(str, enum.ReprEnum): + """Big o notation string identifiers.""" + + o1 = "(1)" + oN = "N" + oNSquared = "N^2" + oNCubed = "N^3" + oLogN = "lgN" + oNLogN = "NlgN" + oLambda = "f(N)" + + @classmethod + def get(cls, value: str) -> str: + # e.g. "o1" -> "(1)" + return cls[value].value + + @classmethod + def back(cls, value: str) -> str: + # e.g. "(1)" -> "o1" + return cls(value).name diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..0d5770e --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,67 @@ +import numpy as np +import pytest + +from BenchMatcha import utils + + +_power2: list[tuple[int, int]] = [ + (0, 0), + (1, 1), +] +for j in range(2, 24): + value = 1 << j + _power2.append((value - 1, value)) + _power2.append((value, value)) + _power2.append((value + 1, 1 << (j + 1))) + + +@pytest.mark.parametrize(["value", "expected"], _power2) +def test_power_of_2(value: int, expected: int): + """Test returns next power of two.""" + result = utils.power_of_2(value) + assert result == expected, f"Unexpected result: {result}" + + +def test_simple_stats(): + """Test mean and std.""" + x = np.asarray([[1, 2, 3], [2, 3, 1], [3, 1, 2]]) + result = utils._simple_stats(x) + assert isinstance(result, tuple), "Expected a tuple." + assert np.all(result[0] == np.asarray([2, 2, 2])) + assert np.all(result[1] == np.asarray([1, 1, 1])) + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + ("o1", "(1)"), + ("oN", "N"), + ("oNSquared", "N^2"), + ("oNCubed", "N^3"), + ("oLogN", "lgN"), + ("oNLogN", "NlgN"), + ("oLambda", "f(N)"), + ], +) +def test_bigo_enum_get(value: str, expected: str): + """Test conversion of big o notation identifier get classmethod.""" + result: str = utils.BigO.get(value) + assert result == expected, "Unexpected result." + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + ("(1)", "o1"), + ("N", "oN"), + ("N^2", "oNSquared"), + ("N^3", "oNCubed"), + ("lgN", "oLogN"), + ("NlgN", "oNLogN"), + ("f(N)", "oLambda"), + ], +) +def test_bigo_enum_back(value: str, expected: str): + """Test conversion of big o notation identifier back classmethod.""" + result: str = utils.BigO.back(value) + assert result == expected, "Unexpected result." From fa3fff2c770cd3f7849bd22fe84c2c9cc19b6bda Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Fri, 15 Aug 2025 19:19:43 -0700 Subject: [PATCH 12/33] fix(utils): add licensen. --- src/BenchMatcha/utils.py | 29 +++++++++++++++++++++++++++++ tests/unit/test_utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/BenchMatcha/utils.py b/src/BenchMatcha/utils.py index afed66d..ae80488 100644 --- a/src/BenchMatcha/utils.py +++ b/src/BenchMatcha/utils.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """Miscellaneous utilities.""" from __future__ import annotations diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0d5770e..4ef1d61 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,3 +1,32 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import numpy as np import pytest From 3396015b963d5a8d19044e8bb2840a5354ffdd2c Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sat, 16 Aug 2025 00:42:00 -0700 Subject: [PATCH 13/33] feat(complexity): Implement methods to calculate algorithmic complexity. --- src/BenchMatcha/complexity.py | 208 ++++++++++++++++++++++++++++++++++ src/BenchMatcha/handlers.py | 4 +- tests/unit/test_complexity.py | 118 +++++++++++++++++++ 3 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 src/BenchMatcha/complexity.py create mode 100644 tests/unit/test_complexity.py diff --git a/src/BenchMatcha/complexity.py b/src/BenchMatcha/complexity.py new file mode 100644 index 0000000..95e8f5a --- /dev/null +++ b/src/BenchMatcha/complexity.py @@ -0,0 +1,208 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Complexity calculations.""" + +from collections.abc import Callable +from dataclasses import dataclass + +import google_benchmark as gbench +import numpy as np +from scipy.optimize import curve_fit # type: ignore[import-untyped] + +from .utils import _simple_stats + + +@dataclass +class FitResult: + """Curve fit result. + + Args: + bigo (str): Big O notation string identifier. + params (np.ndarray): coefficient value(s). + cov (np.ndarray): covariance std of coefficients + rms (float): root mean square error of fit. + + """ + + bigo: str + params: np.ndarray + cov: np.ndarray + rms: float + + @staticmethod + def _handle(x: np.ndarray) -> str: + a = " ".join([f"{j:.3E}" for j in x.tolist()]) + + return f"[{a}]" + + def __repr__(self) -> str: + return ( + f"FitResult(bigo={self.bigo},params={self._handle(self.params)}" + f",cov={self._handle(self.cov)},rms={self.rms:.3f})" + ) + + +# Define common complexity functions with all coefficients and intercept +Equation = ( + Callable[[np.ndarray, float, float], np.ndarray] + | Callable[[np.ndarray, float, float, float], np.ndarray] + | Callable[[np.ndarray, float, float, float, float], np.ndarray] +) + + +def constant(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Constant O(1) equation.""" + return a * np.ones_like(n) + b + + +def logn(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Log O(logN) equation.""" + return a * np.log2(n) + b + + +def linear(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Linear O(N) equation.""" + return a * n + b + + +def nlogn(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Log linear O(NlogN) equation.""" + return a * n * np.log2(n) + b + + +def quadratic(n: np.ndarray, a: float, b: float, c: float) -> np.ndarray: + """Quadratic O(N^2) equation.""" + return a * np.power(n, 2) + linear(n, b, c) + + +def cubic(n: np.ndarray, a: float, b: float, c: float, d: float) -> np.ndarray: + """Cubic O(N^3) equation.""" + return a * np.power(n, 3) + quadratic(n, b, c, d) + + +complexity_functions: dict[str, Equation] = { + # gbench.oNone.name: "", + gbench.o1.name: constant, + gbench.oLogN.name: logn, + gbench.oN.name: linear, + gbench.oNLogN.name: nlogn, + gbench.oNSquared.name: quadratic, + gbench.oNCubed.name: cubic, +} + + +def compute_rmsd(y_true: np.ndarray, y_pred: np.ndarray, k: int) -> float: + r"""Mean normalized root mean square deviation (RMSD). + + Args: + y_true (np.ndarray): observed y values. + y_pred (np.ndarray): predicted y values. + k (int): number of parameters used to estimate predicted values. + + Returns: + (float): normalized RMSD + + Equations: + $\frac{1}{\bar{y}} \sqrt{\frac{\sum_{i=0}^{N} (y_i - \hat{y}_i)^2}{N - k}}$ + + """ + residuals: np.ndarray = y_true - y_pred + sum_square_error: np.float64 = (residuals * residuals).sum() + dof: np.int64 = np.prod(y_pred.size) - k + + return float(np.sqrt(sum_square_error / dof) / y_true.mean()) + + +def fit( + func: Callable, + label: str, + x: np.ndarray, + y: np.ndarray, + sigma: np.ndarray, +) -> FitResult | None: + """Fit observed data to an equation. + + Args: + func (Callable): equation to fit. + label (str): complexity label + x (np.ndarray): x input values + y (np.ndarray): observed y values + sigma (np.ndarray): observed error in y values + + Returns: + (FitResult | None) returns fit result if converged. + + """ + popt: np.ndarray + pcov: np.ndarray + try: + popt, pcov, *_ = curve_fit( + func, + x, + y, + sigma=sigma, + absolute_sigma=True, + ) + pred = func(x, *popt) + cov = np.sqrt(pcov.diagonal()) + rms = compute_rmsd(y, pred, len(popt)) + + return FitResult( + bigo=label, + params=popt, + cov=cov, + rms=rms, + ) + + except RuntimeError: + return None + + +def fit_complexity(x: np.ndarray, y: np.ndarray, sigma: np.ndarray) -> list[FitResult]: + """Perform curve fitting to available complexity algorithms.""" + results: list[FitResult] = [] + + for label, func in complexity_functions.items(): + if (res := fit(func, label, x, y, sigma)) is not None: + results.append(res) + + return results + + +def analyze_complexity(x: np.ndarray, y: np.ndarray) -> list[FitResult]: + """Analyze algorithmic complexity.""" + mean, std = _simple_stats(y) + + return sorted(fit_complexity(x, mean, std), key=lambda x: x.rms) + + +def get_best_fit(fits: list[FitResult]) -> FitResult: + """Return best fit by minimizing RMSD.""" + return min(fits, key=lambda x: x.rms) diff --git a/src/BenchMatcha/handlers.py b/src/BenchMatcha/handlers.py index fbaeb6f..24fc683 100644 --- a/src/BenchMatcha/handlers.py +++ b/src/BenchMatcha/handlers.py @@ -55,7 +55,7 @@ def is_readable_io_protocol(obj: object) -> bool: return ( isinstance(obj, IO | IOBase) or all(map(lambda x: hasattr(obj, x), methods)) - ) and obj.readable() + ) and obj.readable() # type: ignore[attr-defined] class Handler(ABC): @@ -144,7 +144,7 @@ def dispatch(obj: object, encoding: str = "utf8") -> Handler: return HandleBytes(obj, encoding) elif is_readable_io_protocol(obj): - return HandleIO(obj, encoding) + return HandleIO(obj, encoding) # type: ignore[arg-type] raise TypeError(f"Unsupported object type: {type(obj)}") diff --git a/tests/unit/test_complexity.py b/tests/unit/test_complexity.py new file mode 100644 index 0000000..450edff --- /dev/null +++ b/tests/unit/test_complexity.py @@ -0,0 +1,118 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test algorithmic complexity module.""" + +import numpy as np +import pytest + +from BenchMatcha import complexity as comp +from BenchMatcha.utils import _simple_stats + + +@pytest.fixture +def fit_result() -> comp.FitResult: + return comp.FitResult( + bigo="N", + params=np.asarray([2.73]), + cov=np.asarray([1.5]), + rms=0.479, + ) + + +@pytest.fixture +def coords() -> tuple[np.ndarray, np.ndarray]: + x = np.arange(3) + y = np.arange(9).reshape((3, 3)) + + return x, y + + +def test_fit_result_repr(fit_result: comp.FitResult) -> None: + """minor test of fit result repr dunder method.""" + result: str = repr(fit_result) + assert result == "FitResult(bigo=N,params=[2.730E+00],cov=[1.500E+00],rms=0.479)", ( + "Unexpected FitResult repr." + ) + + +@pytest.mark.parametrize( + ["x", "y", "k", "expected"], + [ + (np.arange(3), np.arange(5, 8), 1, 6.12372), + (np.arange(3), np.arange(5, 8), 2, 8.66025), + ], +) +def test_rmsd_computation( + x: np.ndarray, + y: np.ndarray, + k: int, + expected: float, +) -> None: + """Confirm computation of mean normalized rmsd.""" + result: float = comp.compute_rmsd(x, y, k) + assert isinstance(result, float), "Expected float return type." + assert np.isclose(result, expected), "Unexpected rmsd computation." + + +def test_fit(coords: tuple[np.ndarray, np.ndarray]) -> None: + """Test (linear) curve fitting.""" + x, y = coords + mean, std = _simple_stats(y) + result = comp.fit(comp.linear, "N", x, mean, std) + + assert isinstance(result, comp.FitResult), "Expected a fit result return type." + assert result.bigo == "N", "Expected correct label." + + assert len(result.params) == 2, "Expected 2 parameters." + assert np.allclose(result.params, np.asarray([3.0, 1.0])), ( + "Unexpected param values." + ) + + assert len(result.cov) == 2, "Expected 2 parameter errors." + assert np.allclose(result.cov, np.asarray([0.7071, 0.91287])), ( + "Unexpected param error values." + ) + + assert np.isclose(result.rms, 0.0), "Unexpected error." + + +def test_analyze_complexity() -> None: + """Test batch analysis of algorithmic complexity.""" + x = np.arange(10, 20) + y = np.arange(1, 31).reshape(10, 3) + result = comp.analyze_complexity(x, y) + assert isinstance(result, list), "Expected list return type." + assert len(result) == 6, "Expected 6 elements" + assert all(isinstance(x, comp.FitResult) for x in result), ( + "Expected all elements to be a FitResult type." + ) + + # Results should be sorted by best performing fit, by minimizing rmsd. + assert comp.get_best_fit(result) == result[0], "Expected same FitResult." From c88f5a8b7aad6cfeb1a829dc69a7c54399341155 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sat, 16 Aug 2025 18:46:44 -0700 Subject: [PATCH 14/33] feat(3.11): Update python requirement to minimum of 3.11. --- pyproject.toml | 2 +- src/BenchMatcha/utils.py | 2 +- tox.ini | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 47a9790..7d1cdfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] maintainers = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] description = "Google Benchmark Suite Runner and Regression Analyzer." license = { file = "LICENSE" } -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = ["benchmark", "regression", "analysis"] classifiers = [ "Programming Language :: Python :: 3", diff --git a/src/BenchMatcha/utils.py b/src/BenchMatcha/utils.py index ae80488..9aaee4f 100644 --- a/src/BenchMatcha/utils.py +++ b/src/BenchMatcha/utils.py @@ -58,7 +58,7 @@ def _simple_stats(x: np.ndarray) -> tuple[np.ndarray, np.ndarray]: # https://github.com/google/benchmark/blob/main/src/complexity.cc#L52-L69 -class BigO(str, enum.ReprEnum): +class BigO(enum.StrEnum): """Big o notation string identifiers.""" o1 = "(1)" diff --git a/tox.ini b/tox.ini index 931e2ce..8280b6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -envlist = type, lint, coverage, docs, py{310,311,312}-tests +envlist = type, lint, coverage, docs, py{311,312,313}-tests [testenv] description = Base Environment @@ -12,7 +12,7 @@ commands_pre = commands = coverage run --rcfile pyproject.toml -m pytest {posargs} -[testenv:py{310,311,312}-tests] +[testenv:py{311,312,313}-tests] description = Run Unit Tests commands_pre = {envpython} --version @@ -23,7 +23,7 @@ description = Report Code Coverage skip_install = true deps = coverage parallel_show_output = true -depends = py{310,311,312}-tests +depends = py{311,312,313}-tests commands = coverage combine --quiet --rcfile pyproject.toml coverage report --rcfile pyproject.toml {posargs} From 9a67354b11547e943fb8cfbc0070f8bb4e8bfe99 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sat, 16 Aug 2025 18:47:54 -0700 Subject: [PATCH 15/33] ci(workflow): Include python 3.13 in workflow matrix unit testing. --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 441cc7d..b586ae6 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.11", "3.12" ] + python-version: [ "3.11", "3.12", "3.13" ] steps: - name: Checkout BenchMatcha Project From 4668cdf9cd35307f6f65b4a3040762443e069eea Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sat, 16 Aug 2025 18:49:12 -0700 Subject: [PATCH 16/33] ci(workflow): Upgrade addlicense version to match pre commit version. --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index b586ae6..0e933ac 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -120,7 +120,7 @@ jobs: - name: Install addlicense run: | - go install github.com/google/addlicense@v1.1.1 + go install github.com/google/addlicense@v1.2.0 - name: Check for License Headers run: | From 5497977d89c8c251111340d0e383d332c2fe2eb0 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sat, 16 Aug 2025 18:56:25 -0700 Subject: [PATCH 17/33] feat(structure): Include module to organize json output of google benchmarks into dataclasses. --- src/BenchMatcha/structure.py | 300 +++++++++++++++++++++++++++++++++++ tests/unit/test_structure.py | 72 +++++++++ 2 files changed, 372 insertions(+) create mode 100644 src/BenchMatcha/structure.py create mode 100644 tests/unit/test_structure.py diff --git a/src/BenchMatcha/structure.py b/src/BenchMatcha/structure.py new file mode 100644 index 0000000..ae01821 --- /dev/null +++ b/src/BenchMatcha/structure.py @@ -0,0 +1,300 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Data structures defining json output of google benchmark suite.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Literal, Self + +import numpy as np + + +BuildType = Literal["release", "debug"] + + +def parse_datetime(x: str) -> datetime: + """Parse ISO 8601 datetime string.""" + return datetime.fromisoformat(x).astimezone(UTC) + + +@dataclass +class Cache: + """System cache information. + + Args: + type (str): cache type + level (int): + size (int): cache size + num_sharing (int): + + """ + + type: str + level: int + size: int + num_sharing: int + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to Cache.""" + return cls( + type=record["type"], + level=record["level"], + size=record["size"], + num_sharing=record["num_sharing"], + ) + + +@dataclass +class BenchmarkRecord: + """Benchmark result record. + + Args: + function (str): function name or alias + size (int): Input size + threads (int): thread id + iterations (int): number of iterations performed per measurement + real_time (float): total real time per measurement + cpu_time (float): total cpu time per measurement + time_unit (str): unit of time + + """ + + function: str + size: int + threads: int + iterations: int + real_time: float + cpu_time: float + time_unit: str + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to BenchmarkRecord.""" + parts: list[str] = record["name"].split("/") + function: str = parts[0] + size: int = int(parts[1]) + + return cls( + function=function, + size=size, + threads=record["threads"], + iterations=record["iterations"], + real_time=record["real_time"], + cpu_time=record["cpu_time"], + time_unit=record["time_unit"], + ) + + +@dataclass +class ComplexityInfo: + """Algorithmic time complexity result. + + Args: + function (str): function name or alias. + big_o (str): BigO notation. + real_coefficient (float): real time coefficient. + cpu_coefficient (float): cpu time coefficient. + rms (float): root mean square error of fit. + + """ + + function: str + big_o: str + real_coefficient: float + cpu_coefficient: float + rms: float = 0.0 + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to ComplexityInfo.""" + function: str = record["name"].split("/")[0] + + return cls( + function=function, + big_o=record["big_o"], + real_coefficient=record["real_coefficient"], + cpu_coefficient=record["cpu_coefficient"], + ) + + +@dataclass +class BenchmarkArray: + """Reformatted Benchmark result as an array. + + Args: + function (str): function name or alias + unit (str): unit of time + size (np.ndarray): Input size + iterations (np.ndarray): number of iterations performed per measurement + real_time (np.ndarray): total real time per measurement + cpu_time (np.ndarray): total cpu time per measurement + complexity (ComplexityInfo): algorithmic time complexity information + + """ + + function: str + unit: str + size: np.ndarray # 1D array + iterations: np.ndarray # 2D array (n_sizes x repetitions) + real_time: np.ndarray # 2D array (n_sizes x repetitions) + cpu_time: np.ndarray # 2D array (n_sizes x repetitions) + complexity: ComplexityInfo + + +def get_benchmark_records( + data: list[dict[str, Any]], +) -> dict[str, list[BenchmarkRecord]]: + """Group and parse benchmark results.""" + grouped_records = defaultdict(list) + for record in data: + if record["run_type"] != "iteration": + continue + br = BenchmarkRecord.from_json(record) + grouped_records[br.function].append(br) + + return grouped_records + + +def get_complexity_info(data: list[dict[str, Any]]) -> dict[str, ComplexityInfo]: + """Capture and parse complexity information from benchmarks.""" + complexity_info: dict[str, ComplexityInfo] = {} + # First pass: grab complexity information + for record in data: + if ( + record.get("run_type") != "aggregate" + or record.get("aggregate_name") != "BigO" + ): + continue + ci = ComplexityInfo.from_json(record) + complexity_info[ci.function] = ci + + # Second pass: grab rms fit error information + for record in data: + if ( + record.get("run_type") != "aggregate" + or record.get("aggregate_name") != "RMS" + ): + continue + function: str = record["name"].split("/")[0] + if function in complexity_info: + complexity_info[function].rms = record["rms"] + + return complexity_info + + +def convert_to_arrays( + grouped_records: dict[str, list[BenchmarkRecord]], + complexity_data: dict[str, ComplexityInfo], +) -> list[BenchmarkArray]: + """Reorganize benchmark data into numpy arrays.""" + grouped_arrays: list[BenchmarkArray] = [] + + for function, records in grouped_records.items(): + size_to_times: defaultdict[int, list[tuple[int, float, float]]] = defaultdict( + list + ) + for record in records: + size_to_times[record.size].append( + ( + record.iterations, + record.real_time, + record.cpu_time, + ) + ) + + sorted_sizes: list[int] = sorted(size_to_times) + iter_arr: list[list[int]] = [] + real_arr: list[list[float]] = [] + cpu_arr: list[list[float]] = [] + container: list[list[int] | list[float]] + idx: int + + for size in sorted_sizes: + times: list[tuple[int, float, float]] = size_to_times[size] + for idx, container in zip( # type: ignore[assignment] + range(3), (iter_arr, real_arr, cpu_arr), strict=True + ): + container.append([t[idx] for t in times]) + + # TODO: validate time unit is consistent. Adjust if assumption not true. + grouped_arrays.append( + BenchmarkArray( + function=function, + unit=records[0].time_unit, + size=np.asarray(sorted_sizes, dtype=np.int64), + iterations=np.asarray(iter_arr, dtype=np.int64), + real_time=np.asarray(real_arr, dtype=np.float64), + cpu_time=np.asarray(cpu_arr, dtype=np.float64), + complexity=complexity_data[function], + ) + ) + + return grouped_arrays + + +# TODO: Consider how we would capture custom data (Counters). +# TODO: Capture additional information not included in google_benchmark output +# (e.g. git sha) +@dataclass +class BenchmarkContext: + """Google benchmark context.""" + + # pylint: disable=R0902 + date: datetime + host_name: str + executable: str + num_cpus: int + mhz_per_cpu: int + caches: list[Cache] + cpu_scaling_enabled: bool + load_avg: list[float] + library_version: str + library_build_type: BuildType + json_schema_version: int + benchmarks: list[BenchmarkArray] + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to BenchmarkContext.""" + context: dict = record.get("context", {}).copy() + caches: list[Cache] = [Cache.from_json(i) for i in context.pop("caches", [])] + date: datetime = parse_datetime( + context.pop("date", datetime.now(UTC).isoformat()) + ) + benchmarks: list[BenchmarkArray] = convert_to_arrays( + get_benchmark_records(record["benchmarks"]), + get_complexity_info(record["benchmarks"]), + ) + + return cls(**context, caches=caches, date=date, benchmarks=benchmarks) diff --git a/tests/unit/test_structure.py b/tests/unit/test_structure.py new file mode 100644 index 0000000..d6cfe78 --- /dev/null +++ b/tests/unit/test_structure.py @@ -0,0 +1,72 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit test structure module.""" + +from datetime import UTC, datetime + +import pytest + +from BenchMatcha import structure +from BenchMatcha.handlers import load + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + ("2025-07-13T12:09:31-07:00", datetime(2025, 7, 13, 19, 9, 31, tzinfo=UTC)), + ("2025-07-12T14:01:22+07:00", datetime(2025, 7, 12, 7, 1, 22, tzinfo=UTC)), + ], +) +def test_parse_datetime(value: str, expected: datetime) -> None: + """Test parsing of ISO8601 format datetime string.""" + result = structure.parse_datetime(value) + assert isinstance(result, datetime), "Expected datetime object." + assert result == expected, "Unexpected date time." + + +def test_parse_json_data(mock_data: str) -> None: + """Test we correctly parse json data into a dataclass.""" + data = load(mock_data) + result = structure.BenchmarkContext.from_json(data) + assert isinstance(result, structure.BenchmarkContext) + assert result.date == datetime(2025, 7, 13, 19, 9, 31, tzinfo=UTC) + assert result.host_name == "host" + assert result.executable == "file.py" + assert result.num_cpus == 12 + assert result.mhz_per_cpu == 24 + + assert result.cpu_scaling_enabled is False + assert len(result.caches) == 3 + assert result.load_avg == [4.69092, 4.60693, 4.47949] + assert result.library_version == "1.9.4" + assert result.library_build_type == "release" + assert result.json_schema_version == 1 + + assert len(result.benchmarks) == 1 From f794f8a641bf46f662ee76f5a64ff62748feb02e Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 17 Aug 2025 16:21:20 -0700 Subject: [PATCH 18/33] feat(plotting): Implement functions to build plotly figures. --- src/BenchMatcha/plotting.py | 247 ++++++++++++++++++++++++++++++++++++ tests/unit/test_plotting.py | 96 ++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 src/BenchMatcha/plotting.py create mode 100644 tests/unit/test_plotting.py diff --git a/src/BenchMatcha/plotting.py b/src/BenchMatcha/plotting.py new file mode 100644 index 0000000..db0d854 --- /dev/null +++ b/src/BenchMatcha/plotting.py @@ -0,0 +1,247 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Plotting utilities.""" + +from collections.abc import Callable + +import numpy as np +import plotly.graph_objs as go # type: ignore[import-untyped] +from plotly.express import colors # type: ignore[import-untyped] +from plotly.io import to_json as _to_json # type: ignore[import-untyped] + +from .utils import _simple_stats, power_of_2 + + +Prism: list[str] = colors.qualitative.Prism[:] + + +def to_html(figure: go.Figure, path: str, mode: str = "w") -> None: + """Saves a plotly figure in HTML Format to a file. + + Args: + figure (go.Figure): Plotly figure. + path (str): Filepath to save plotly figure. + mode (str): Writing mode ("a" | "w") + + Returns: + (None) Appends/writes figure to html filepath. + + """ + with open(path, mode) as f: + f.write(figure.to_html(full_html=False, include_plotlyjs="cdn")) + + +def to_json(figure: go.Figure, path: str) -> None: + """Serialize a plotly figure to a json file.""" + with open(path, "w") as f: + f.write(_to_json(figure, False, False, True, engine="orjson")) + + +def construct_log2_axis(x: np.ndarray) -> tuple[list[int], list[str]]: + """Build a log2 power axis for plotly.""" + labels: list[str] = [] + values: list[int] = [] + + minimum = int(x.min()) + maximum = power_of_2(int(x.max())) + 1 + current = power_of_2(minimum) + if 1 < current >= minimum: + current //= 2 + power = int(np.log2(current)) + + while current < maximum: + values.append(int(current)) + labels.append(f"2{power}") + current *= 2 + power += 1 + + return values, labels + + +def create_scatter_trace( + x: np.ndarray, + y: np.ndarray, + name: str, + color: str, +) -> go.Scatter: + """Create scatter trace of mean and std. + + Args: + x (np.ndarray): x values + y (np.ndarray): y values + name (str): name to give plot + color (str): trace color + + Returns: + (go.Scatter) scatter plot trace of data. + + """ + mean, std = _simple_stats(y) + + return go.Scatter( + mode="lines+markers", + x=x, + y=mean, + name=name, + line=dict(color=color, dash="dot"), + error_y=dict(type="data", array=std, visible=True), + ) + + +def box_plot( + x: np.ndarray, + y: np.ndarray, + name: str, + color: str, + line_color: str, +) -> go.Box: + """Construct a box plot. + + Args: + x (np.ndarray): x values + y (np.ndarray): y values + name (str): name to give plot + color (str): marker color + line_color (str): line color + + Returns: + (go.Box) boxplot trace of data. + + """ + q1, med, q3 = np.nanquantile(y, [0.25, 0.5, 0.75], 1) + mean, std = _simple_stats(y) + + return go.Box( + name=name, + x=x, + mean=mean, + sd=std, + q1=q1, + median=med, + q3=q3, + marker_color=color, + marker_line_color=line_color, + ) + + +def create_annotation_text( + label: str, + error: float, +) -> dict: + """Build a simple annotation data of complexity fit information. + + Args: + label (str): Complexity label + error (float): error to fit (e.g. RMSD) + + Example: + + .. code-block:: python + + benchmark: BenchmarkArray + figure = go.Figure() + figure.add_annotation( + **create_annotation_text( + benchmark.complexity.bigo, + benchmark.complexity.rms, + ) + ) + + """ + a = f"{error:.2f}% " + b = f"O({label}) " + length = max(len(a), len(b)) + c = f" Complexity: {b: >{length}}" + d = f" RMS: {a: >{length}}" + + return dict( + xref="paper", + yref="paper", + x=0.01, + y=0.99, + showarrow=False, + text=f"{c}
{d}", + align="left", + bgcolor="rgba(255,255,255,0.6)", + bordercolor="black", + borderwidth=1, + ) + + +_benchmark_map: dict[str, Callable[[np.ndarray, float], np.ndarray]] = { + "(1)": lambda n, c: c * np.ones_like(n), + "N": lambda n, c: c * n, + "lgN": lambda n, c: c * np.log2(n), + "NlgN": lambda n, c: c * n * np.log2(n), + "N^2": lambda n, c: c * n**2, + "N^3": lambda n, c: c * n**3, +} + + +def get_big_o_function(label: str) -> Callable[[np.ndarray, float], np.ndarray]: + """Map the big-O notation label (from google benchmark) to function.""" + return _benchmark_map.get(label, lambda n, c: c * n) + + +def draw_complexity_line( + x: np.ndarray, + coefficient: float, + big_o: str, + name: str, + color: str, +) -> go.Scatter: + """Create a scatter plot to describe google benchmark complexity information. + + Args: + x (np.ndarray): x axis data (n). + coefficient (float): multiplier + big_o (str): label describing algorithmic complexity. + name (str): name (label) to give trace on plot. + color (str): color of line. + + Returns: + (go.Scatter): scatter plot trace of complexity information. + + """ + func: Callable[[np.ndarray, float], np.ndarray] = get_big_o_function(big_o) + y: np.ndarray = func(x, coefficient) + + return go.Scatter( + x=x, + y=y, + name=name, + mode="lines", + line=dict( + color=color, + dash="dash", + shape="spline", + ), + opacity=0.7, + ) diff --git a/tests/unit/test_plotting.py b/tests/unit/test_plotting.py new file mode 100644 index 0000000..35199fb --- /dev/null +++ b/tests/unit/test_plotting.py @@ -0,0 +1,96 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""unit test plotting module.""" + +import numpy as np +import plotly.graph_objs as go +import pytest + +from BenchMatcha import plotting + + +def test_construct_log2_axis() -> None: + """Confirm a log 2 axis is constructed correctly""" + x = np.asarray([2, 256]) + a, b = plotting.construct_log2_axis(x) + + assert len(a) == len(b) == 9 + assert a == [1, 2, 4, 8, 16, 32, 64, 128, 256], "Incorrect values" + assert b == [ + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + ], "Incorrect plotly labels." + + +def test_create_scatter_trace() -> None: + """Confirm function constructs a plotly scatter trace.""" + x = np.arange(5) + y = np.arange(25).reshape(5, 5) + result = plotting.create_scatter_trace(x, y, "test", "black") + assert isinstance(result, go.Scatter) + + +def test_create_box_plot() -> None: + """Confirm function constructs a plotly box plot trace.""" + x = np.arange(5) + y = np.arange(25).reshape(5, 5) + result = plotting.box_plot(x, y, "test", "black", "red") + assert isinstance(result, go.Box) + + +def test_create_annotation_text() -> None: + """Test construction of plot annotation data.""" + result = plotting.create_annotation_text("test", 1.0) + assert isinstance(result, dict), "expected a dictionary return type." + assert "text" in result, "Expected text annotation key." + + +@pytest.mark.parametrize( + ["key"], + [["(1)"], ["N"], ["lgN"], ["NlgN"], ["N^2"], ["N^3"], ["invalid"]], +) +def test_get_big_o_function(key: str) -> None: + """Test retrieval of complexity function.""" + result = plotting.get_big_o_function(key) + assert callable(result), "Expected a callable." + + +def test_draw_complexity_line() -> None: + """Confirm function produces a scatter trace.""" + x = np.arange(20) + result = plotting.draw_complexity_line(x, 1.2, "N", "test", "red") + assert isinstance(result, go.Scatter) From 5a8996dc6be9bfd8317a076c30439fa9f40dbe69 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 16:15:17 -0700 Subject: [PATCH 19/33] feat(structure): Implement methods to convert structured dataclass to dictionary objects. Include function to handle parsing differences of json output by versioning. --- pyproject.toml | 2 +- src/BenchMatcha/structure.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7d1cdfc..8f7194c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Topic :: System :: Benchmark", "Topic :: Software Development :: Testing", "Topic :: Software Development :: Libraries", - ] +] dynamic = ["version", "readme", "dependencies"] [project.urls] diff --git a/src/BenchMatcha/structure.py b/src/BenchMatcha/structure.py index ae01821..3b81f70 100644 --- a/src/BenchMatcha/structure.py +++ b/src/BenchMatcha/structure.py @@ -38,8 +38,11 @@ import numpy as np +from .errors import SchemaError + BuildType = Literal["release", "debug"] +SUPPORTED_VERSIONS: tuple[int, ...] = (1,) def parse_datetime(x: str) -> datetime: @@ -74,6 +77,9 @@ def from_json(cls, record: dict[str, Any]) -> Self: num_sharing=record["num_sharing"], ) + def to_json(self) -> dict: + return self.__dict__.copy() + @dataclass class BenchmarkRecord: @@ -115,6 +121,9 @@ def from_json(cls, record: dict[str, Any]) -> Self: time_unit=record["time_unit"], ) + def to_json(self) -> dict: + return self.__dict__.copy() + @dataclass class ComplexityInfo: @@ -147,6 +156,9 @@ def from_json(cls, record: dict[str, Any]) -> Self: cpu_coefficient=record["cpu_coefficient"], ) + def to_json(self) -> dict: + return self.__dict__.copy() + @dataclass class BenchmarkArray: @@ -171,6 +183,12 @@ class BenchmarkArray: cpu_time: np.ndarray # 2D array (n_sizes x repetitions) complexity: ComplexityInfo + def to_json(self) -> dict: + d = self.__dict__.copy() + d["complexity"] = self.complexity.to_json() + + return d + def get_benchmark_records( data: list[dict[str, Any]], @@ -298,3 +316,23 @@ def from_json(cls, record: dict[str, Any]) -> Self: ) return cls(**context, caches=caches, date=date, benchmarks=benchmarks) + + def to_json(self) -> dict: + data = self.__dict__.copy() + data["caches"] = [i.to_json() for i in self.caches] + data["benchmarks"] = [j.to_json() for j in self.benchmarks] + + return data + + +def parse_version(record: dict[str, Any]) -> BenchmarkContext: + """Map schema version to correct parsing engine.""" + schema_version = int(record.get("context", {}).get("json_schema_version", -1)) + if schema_version not in SUPPORTED_VERSIONS: + raise SchemaError.response(str(schema_version)) + + match schema_version: + case 1: + return BenchmarkContext.from_json(record) + case _: + raise SchemaError.response(str(schema_version)) From 48f5f692bf8ef112265bea7c3d85772473f108b8 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 17:02:43 -0700 Subject: [PATCH 20/33] feat(config): Implement a basic config with toml tool support. --- src/BenchMatcha/config.py | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/BenchMatcha/config.py diff --git a/src/BenchMatcha/config.py b/src/BenchMatcha/config.py new file mode 100644 index 0000000..b46b08f --- /dev/null +++ b/src/BenchMatcha/config.py @@ -0,0 +1,59 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Default runner configuration.""" + +import logging + +import toml + +from . import plotting + + +log: logging.Logger = logging.getLogger(__name__) + + +class Config: + """default configuration.""" + + color: str = plotting.Prism[3] + line_color: str = plotting.Prism[4] + font: str = "Space Grotesk Light, Courier New, monospace" + + +def update_config_from_pyproject(path: str) -> None: + """Update default config from pyproject toml file.""" + data = toml.load(path) + + for key, value in data.get("tool", {}).get("BenchMatcha", {}).items(): + if not hasattr(Config, key): + log.info(f"Unsupported tool key: {key}") + continue + + setattr(Config, key, value) From 77a7a4854bf83c5cf99f807be9b93f0128fb97a2 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 17:09:53 -0700 Subject: [PATCH 21/33] feat(runner): Implement initial prototype of benchmark runner and cli entry point. --- src/BenchMatcha/runner.py | 220 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 src/BenchMatcha/runner.py diff --git a/src/BenchMatcha/runner.py b/src/BenchMatcha/runner.py new file mode 100644 index 0000000..5a9697a --- /dev/null +++ b/src/BenchMatcha/runner.py @@ -0,0 +1,220 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Primary Benchmark Runner.""" + +import logging +import os +import sys +from json import JSONDecodeError + +import google_benchmark as gbench +import orjson +import plotly.graph_objs as go # type: ignore[import-untyped] +from wurlitzer import pipes # type: ignore[import-untyped] + +from . import plotting +from .config import Config, update_config_from_pyproject +from .errors import ParsingError +from .handlers import HandleText +from .sifter import collect_benchmarks, load_benchmark +from .structure import BenchmarkArray, BenchmarkContext, parse_version + + +log: logging.Logger = logging.getLogger(__name__) + + +def manage_registration(path: str) -> None: + """Manage import, depending on whether path is a directory or file.""" + abspath: str = os.path.abspath(path) + if not os.path.exists(abspath): + raise FileNotFoundError("Invalid filepath") + + if os.path.isdir(abspath): + collect_benchmarks(abspath) + + elif os.path.isfile(abspath) and abspath.endswith(".py"): + load_benchmark(abspath, os.path.abspath(os.path.dirname(abspath))) + + else: + log.warning( + "Unsupported path provided. While the path does exist, it is neither a" + " file nor a directory." + ) + + +def plot_benchmark_array(benchmark: BenchmarkArray) -> go.Figure: + """Plot benchmark array.""" + fig = go.Figure() + fig.add_trace( + plotting.create_scatter_trace( + benchmark.size, + benchmark.cpu_time, + "CPU Time", + Config.color, + ) + ) + + fig.add_trace( + plotting.draw_complexity_line( + benchmark.size, + benchmark.complexity.cpu_coefficient, + benchmark.complexity.big_o, + f"CPU Time Fit ({benchmark.complexity.big_o})", + Config.line_color, + ) + ) + + fig.add_annotation( + **plotting.create_annotation_text( + benchmark.complexity.big_o, + benchmark.complexity.rms, + ) + ) + + vals, labels = plotting.construct_log2_axis(benchmark.size) + if (p := len(vals) // 13) > 0: + vals = vals[:: p + 1] + labels = labels[:: p + 1] + + fig.update_layout( + title=f"Benchmark Results
{benchmark.function}", + xaxis=dict( + type="log", + tickvals=vals, + ticktext=labels, + tickmode="array", + title="Input Size (n)", + ), + yaxis=dict( + title=f"Time ({benchmark.unit})", + type="log", + dtick=1, + exponentformat="power", + ), + legend_title="Timing", + font=dict( + family=Config.font, + size=12, + ), + ) + + return fig + + +def _run() -> BenchmarkContext: + # TODO: Improve logic here (e.g. --benchmark_format=csv) + if "--benchmark_format=json" not in sys.argv: + sys.argv.append("--benchmark_format=json") + + # TODO: create python bindings of library to call and collect json data without + # serializing and capturing stdout. + # Read this excellent blog regarding redirecting stdout from c libraries: + # https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ + with pipes() as (stdout, stderr): + try: + gbench.main() + + # NOTE: bypass sys.exit(0) call from main + except SystemExit: + ... + + text: str = stdout.read() + stdout.close(), stderr.close() # pylint: disable=W0106 + + handler = HandleText(text) + try: + obj: dict = handler.handle() + except JSONDecodeError as e: + raise ParsingError.response() from e + + context: BenchmarkContext = parse_version(obj) + + return context + + +def save(context: BenchmarkContext, cache_dir: str) -> None: + """Save benchmark data.""" + for j in context.benchmarks: + figure: go.Figure = plot_benchmark_array(j) + plotting.to_html(figure, os.path.join(cache_dir, "out.html"), "a") + + # TODO: Save data to database. Serialize to json in the interim. + database: str = os.path.join(cache_dir, "benchmark.json") + data: list[dict] = [] + if os.path.exists(database): + with open(database, "br") as f: + data = orjson.loads(f.read()) + + data.append(context.to_json()) + serialized: bytes = orjson.dumps( + data, + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_SERIALIZE_DATACLASS, + ) + + with open(database, "bw") as f: + f.write(serialized) + + +def run(path: str, cache_dir: str) -> None: + """BenchMatcha Runner.""" + manage_registration(path) + + # TODO: remove arguments specific to BenchMatch to prevent failure on google + # benchmark interface. + + context: BenchmarkContext = _run() + save(context, cache_dir) + + +# TODO: Handle a list of separated filepaths. +def run_paths(paths: list[str]) -> None: + """Run benchmarks against a list of paths.""" + for path in paths: + manage_registration(path) + + context: BenchmarkContext = _run() + + +def main() -> None: + """Primary CLI Entry Point.""" + # TODO: support specification of config file path from CLI, to overwrite default + # TODO: Support command line args to overwrite default config. + cwd: str = os.getcwd() + p = os.path.join(cwd, ".pyproject.toml") + if os.path.exists(p): + update_config_from_pyproject(p) + + # Create cache if does not exist + cache = os.path.join(cwd, ".benchmatcha") + if not os.path.exists(cache): + os.mkdir(cache) + + # TODO: Determine if a list of paths have been provided instead, and handle + run(sys.argv.pop(), cache) From 9cdd3c64381fc448a026f7a9eeac05a598355cce Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 17:20:14 -0700 Subject: [PATCH 22/33] test(structure): Implement unit tests for new to_json methods for dataclass structure. --- tests/unit/test_structure.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unit/test_structure.py b/tests/unit/test_structure.py index d6cfe78..4cf7fbd 100644 --- a/tests/unit/test_structure.py +++ b/tests/unit/test_structure.py @@ -70,3 +70,26 @@ def test_parse_json_data(mock_data: str) -> None: assert result.json_schema_version == 1 assert len(result.benchmarks) == 1 + + +def test_convert_BenchmarkContext_to_json(mock_data: str) -> None: + """Test we convert dataclass into dictionary json like objects.""" + data = load(mock_data) + obj = structure.BenchmarkContext.from_json(data) + result = obj.to_json() + assert isinstance(result, dict), "Expected a dictionary object." + + caches = result["caches"] + assert isinstance(caches, list) + assert all(isinstance(i, dict) for i in caches), ( + "Expected cache instances to be dictionary objects." + ) + + benchmarks = result["benchmarks"] + assert isinstance(benchmarks, list) + assert all(isinstance(i, dict) for i in benchmarks), ( + "Expected benchmark instances to be dictionary objects." + ) + assert all(isinstance(k["complexity"], dict) for k in benchmarks), ( + "Expected benchmark complexity instances to be dictionary objects." + ) From 438860799b0404499e151a475c2914a2639bc4c5 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 18:36:29 -0700 Subject: [PATCH 23/33] chore(lint): Lint project appropriately. --- pyproject.toml | 2 ++ requirements.txt | 1 + src/BenchMatcha/config.py | 2 +- src/BenchMatcha/runner.py | 12 ++++++------ src/BenchMatcha/utils.py | 1 + 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8f7194c..c602d0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ allow_redefinition = false warn_unused_ignores = true [tool.pylint.main] +extension-pkg-allow-list = ["orjson"] ignore = ["tests", "dist", "build"] fail-under = 9.0 jobs = 0 @@ -101,6 +102,7 @@ disable = [ "R1731", # consider-using-max-builtin "R0903", # too-few-public-methods "R1735", # use-dict-literal + "W1514", # unspecified-encoding ] [tool.pylint."*.pyi"] diff --git a/requirements.txt b/requirements.txt index 2326fc3..49652d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ orjson plotly pytest scipy +toml wurlitzer diff --git a/src/BenchMatcha/config.py b/src/BenchMatcha/config.py index b46b08f..f1903f7 100644 --- a/src/BenchMatcha/config.py +++ b/src/BenchMatcha/config.py @@ -53,7 +53,7 @@ def update_config_from_pyproject(path: str) -> None: for key, value in data.get("tool", {}).get("BenchMatcha", {}).items(): if not hasattr(Config, key): - log.info(f"Unsupported tool key: {key}") + log.info("Unsupported tool key: %s", key) continue setattr(Config, key, value) diff --git a/src/BenchMatcha/runner.py b/src/BenchMatcha/runner.py index 5a9697a..c54a1e7 100644 --- a/src/BenchMatcha/runner.py +++ b/src/BenchMatcha/runner.py @@ -129,7 +129,7 @@ def plot_benchmark_array(benchmark: BenchmarkArray) -> go.Figure: def _run() -> BenchmarkContext: - # TODO: Improve logic here (e.g. --benchmark_format=csv) + # TODO: Improve logic here if "--benchmark_format=json" not in sys.argv: sys.argv.append("--benchmark_format=json") @@ -194,12 +194,12 @@ def run(path: str, cache_dir: str) -> None: # TODO: Handle a list of separated filepaths. -def run_paths(paths: list[str]) -> None: - """Run benchmarks against a list of paths.""" - for path in paths: - manage_registration(path) +# def run_paths(paths: list[str]) -> None: +# """Run benchmarks against a list of paths.""" +# for path in paths: +# manage_registration(path) - context: BenchmarkContext = _run() +# context: BenchmarkContext = _run() def main() -> None: diff --git a/src/BenchMatcha/utils.py b/src/BenchMatcha/utils.py index 9aaee4f..517cda3 100644 --- a/src/BenchMatcha/utils.py +++ b/src/BenchMatcha/utils.py @@ -57,6 +57,7 @@ def _simple_stats(x: np.ndarray) -> tuple[np.ndarray, np.ndarray]: return mean, std +# pylint: disable=invalid-name # https://github.com/google/benchmark/blob/main/src/complexity.cc#L52-L69 class BigO(enum.StrEnum): """Big o notation string identifiers.""" From 72f0563639f72ceaba317e0cea1584eecbb84e16 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 18:42:13 -0700 Subject: [PATCH 24/33] chore(config): static typing. --- src/BenchMatcha/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchMatcha/config.py b/src/BenchMatcha/config.py index f1903f7..6038b61 100644 --- a/src/BenchMatcha/config.py +++ b/src/BenchMatcha/config.py @@ -31,7 +31,7 @@ import logging -import toml +import toml # type: ignore[import-untyped] from . import plotting From 50deece3657a2b9782f39377713ddb24b9943c26 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 19:15:59 -0700 Subject: [PATCH 25/33] test(config): Improve config api to make unit testing more straightforward. --- src/BenchMatcha/config.py | 53 +++++++++++++++--- tests/unit/test_config.py | 109 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_config.py diff --git a/src/BenchMatcha/config.py b/src/BenchMatcha/config.py index 6038b61..e76913e 100644 --- a/src/BenchMatcha/config.py +++ b/src/BenchMatcha/config.py @@ -47,13 +47,52 @@ class Config: font: str = "Space Grotesk Light, Courier New, monospace" +class ConfigUpdater: + """Configuration updater through pyproject config file. + + Args: + path (str): path to valid configuration file. + config (Config): configuration class to update. + + """ + + path: str + config: type[Config] + + def __init__(self, path: str, config: type[Config] = Config) -> None: + self.path = path + self.config = config + + def load(self) -> dict: + """Load toml data from path.""" + return toml.load(self.path) + + def _update(self, data: dict) -> None: + for key, value in data.get("tool", {}).get("BenchMatcha", {}).items(): + if not hasattr(self.config, key): + log.info("Unsupported tool key: %s", key) + continue + + setattr(self.config, key, value) + + def update(self) -> None: + """Parse toml path and update default configuration.""" + data: dict = self.load() + self._update(data) + + def update_config_from_pyproject(path: str) -> None: - """Update default config from pyproject toml file.""" - data = toml.load(path) + """Update default config from pyproject toml file. + + Example: + + .. code-block: toml - for key, value in data.get("tool", {}).get("BenchMatcha", {}).items(): - if not hasattr(Config, key): - log.info("Unsupported tool key: %s", key) - continue + [tool.BenchMatcha] + color="#FFF" + line_color="#333" + font="Courier" - setattr(Config, key, value) + """ + cu = ConfigUpdater(path) + cu.update() diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..e512d71 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,109 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test config module.""" + +from io import StringIO +from unittest.mock import MagicMock, patch + +import pytest + +from BenchMatcha import config + + +@pytest.fixture +def toml_str() -> str: + return """ +[tool.BenchMatcha] +color="#FFF" +line_color="#333" +font="Courier" +upsupported_key="test" +""" + + +@pytest.fixture +def toml_data() -> dict: + return { + "tool": { + "BenchMatcha": { + "color": "#FFF", + "line_color": "#333", + "font": "Courier", + "upsupported_key": "test", + } + } + } + + +def test_config_load(toml_str: str, toml_data: dict) -> None: + """Confirm toml data loads correctly.""" + with StringIO(toml_str) as stream: + instance = config.ConfigUpdater(stream) + result = instance.load() + + assert result == toml_data, "Expected same object" + + +def _assert_config_is_updated(): + assert config.Config.color == "#FFF", "Expected color to be updated." + assert config.Config.line_color == "#333", "Expected line color to be updated." + assert config.Config.font == "Courier", "Expected font to be updated." + + +def test_config_private_update(toml_data: dict) -> None: + """Confirm config data is updated correctly.""" + instance = config.ConfigUpdater("") + instance._update(toml_data) + _assert_config_is_updated() + + +def test_config_update(toml_str: str) -> None: + """Confirm toml data loads correctly.""" + with StringIO(toml_str) as stream: + instance = config.ConfigUpdater(stream) + instance.update() + _assert_config_is_updated() + + +@patch.object(config.ConfigUpdater, "load") +def test_config_update_mock(mock: MagicMock, toml_data: dict) -> None: + """Confirm config is updated by mocking load method of ConfigUpdater.""" + mock.return_value = toml_data + instance = config.ConfigUpdater("") + instance.update() + _assert_config_is_updated() + + +@patch.object(config.ConfigUpdater, "load") +def test_config_update_function(mock: MagicMock, toml_data: dict) -> None: + """Confirm config is updated from available function api.""" + mock.return_value = toml_data + config.update_config_from_pyproject("") + _assert_config_is_updated() From 1dcc71f9e97b7f39f1da1a83f0ac0219bae2ba47 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 19:26:08 -0700 Subject: [PATCH 26/33] test(config): Use a context manager to reset config class across unit tests to prevent silent failures. --- tests/unit/test_config.py | 41 +++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index e512d71..ccb17a9 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -29,6 +29,7 @@ """Test config module.""" +from contextlib import contextmanager from io import StringIO from unittest.mock import MagicMock, patch @@ -62,48 +63,64 @@ def toml_data() -> dict: } +@contextmanager +def reset(c: type[config.Config]): + """Simple contextmanager to reset config to original default values.""" + default: dict = {key: getattr(c, key) for key in c.__annotations__.keys()} + yield + + for k, v in default.items(): + setattr(c, k, v) + + def test_config_load(toml_str: str, toml_data: dict) -> None: """Confirm toml data loads correctly.""" - with StringIO(toml_str) as stream: + with reset(config.Config), StringIO(toml_str) as stream: instance = config.ConfigUpdater(stream) result = instance.load() assert result == toml_data, "Expected same object" -def _assert_config_is_updated(): +def _assert_config_is_updated() -> None: assert config.Config.color == "#FFF", "Expected color to be updated." assert config.Config.line_color == "#333", "Expected line color to be updated." assert config.Config.font == "Courier", "Expected font to be updated." + assert not hasattr(config.Config, "upsupported_key"), ( + "Expected unsupported key to be bypassed." + ) def test_config_private_update(toml_data: dict) -> None: """Confirm config data is updated correctly.""" - instance = config.ConfigUpdater("") - instance._update(toml_data) - _assert_config_is_updated() + with reset(config.Config): + instance = config.ConfigUpdater("") + instance._update(toml_data) + _assert_config_is_updated() def test_config_update(toml_str: str) -> None: """Confirm toml data loads correctly.""" - with StringIO(toml_str) as stream: + with reset(config.Config), StringIO(toml_str) as stream: instance = config.ConfigUpdater(stream) instance.update() - _assert_config_is_updated() + _assert_config_is_updated() @patch.object(config.ConfigUpdater, "load") def test_config_update_mock(mock: MagicMock, toml_data: dict) -> None: """Confirm config is updated by mocking load method of ConfigUpdater.""" mock.return_value = toml_data - instance = config.ConfigUpdater("") - instance.update() - _assert_config_is_updated() + with reset(config.Config): + instance = config.ConfigUpdater("") + instance.update() + _assert_config_is_updated() @patch.object(config.ConfigUpdater, "load") def test_config_update_function(mock: MagicMock, toml_data: dict) -> None: """Confirm config is updated from available function api.""" mock.return_value = toml_data - config.update_config_from_pyproject("") - _assert_config_is_updated() + with reset(config.Config): + config.update_config_from_pyproject("") + _assert_config_is_updated() From 46332073dec837a263b0a4943f62ea45967ba740 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Sun, 24 Aug 2025 22:21:07 -0700 Subject: [PATCH 27/33] test(sifter): Improve sifter collection interface to facilitate facile unit testing suites. --- src/BenchMatcha/sifter.py | 28 ++++++++++-- tests/unit/test_sifter.py | 95 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_sifter.py diff --git a/src/BenchMatcha/sifter.py b/src/BenchMatcha/sifter.py index f5910a9..9415ef8 100644 --- a/src/BenchMatcha/sifter.py +++ b/src/BenchMatcha/sifter.py @@ -44,13 +44,33 @@ def scandir(filepath: str) -> Iterator[os.DirEntry[str]]: yield from scanner +class Collector: + """Collection interface.""" + + root: str + pattern: str + + def __init__(self, root: str, pattern: str = "bench*.py") -> None: + self.root = root + self.pattern = pattern + + def get(self, path: str) -> Iterator[str]: + """Get paths of a given pattern.""" + yield from glob.iglob(path, root_dir=self.root) + + def collect(self, path: str) -> Iterator[str]: + """Recursive collection of pattern matching filepaths.""" + yield from self.get(os.path.join(path, self.pattern)) + for candidate in scandir(path): + if candidate.is_dir(follow_symlinks=False): + yield from self.collect(candidate.path) + + def collect(root: str, pattern: str = "bench*.py") -> Iterator[str]: """Collect relevant filepaths recursively stemming from root directory.""" - yield from glob.iglob(os.path.join(root, pattern), root_dir=root) + col = Collector(root, pattern) - for candidate in scandir(root): - if candidate.is_dir(follow_symlinks=False): - yield from collect(candidate.path, pattern) + yield from col.collect(root) def load_benchmark(path: str, root: str) -> ModuleType: diff --git a/tests/unit/test_sifter.py b/tests/unit/test_sifter.py new file mode 100644 index 0000000..f1ba107 --- /dev/null +++ b/tests/unit/test_sifter.py @@ -0,0 +1,95 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit test BenchMatcha.sifter module.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from BenchMatcha import sifter + + +class MockDirEntry: + """mock os.DirEntry object.""" + + path: str + directory: bool + + def __init__(self, path: str, directory: bool = False): + self.path = path + self.directory = directory + + def is_dir(self, **kwargs) -> bool: + return self.directory + + def __str__(self): + return self.path + + def __eq__(self, value: MockDirEntry) -> bool: + return value.path == self.path and value.directory == self.directory + + +@patch.object(sifter.Collector, "get") +@patch.object(sifter, "scandir") +def test_collect(mock_scandir: MagicMock, mock_get: MagicMock): + """Unit test sifter.collect method.""" + mock_scandir.side_effect = [ + iter( + ( + MockDirEntry("root/bench_this.py"), + MockDirEntry("root/other.py"), + MockDirEntry("root/sub", True), + ) + ), + iter( + ( + MockDirEntry("root/sub/bench_sub.py"), + MockDirEntry("root/sub/bench_sub2.py"), + ) + ), + ] + mock_get.side_effect = [ + iter(("root/bench_this.py",)), + iter(("root/sub/bench_sub.py", "root/sub/bench_sub2.py")), + ] + + result: list[str] = list(sifter.collect("root")) + assert len(result) == 3, "Expected 3 results." + + expected = [ + "root/bench_this.py", + "root/sub/bench_sub.py", + "root/sub/bench_sub2.py", + ] + + for a, b in zip(result, expected, strict=False): + assert a == b, "Unexpected DirEntry." + + assert len(set(result).difference(set(expected))) == 0, "expected identical output." From 8477930d12ec6a90eec43082134f1928e21176ee Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Mon, 25 Aug 2025 18:42:34 -0700 Subject: [PATCH 28/33] tests(integration): Include Integration tests. --- pyproject.toml | 6 +- tests/integration/conftest.py | 37 ++++++++++ .../data/handle_imports/__init__.py | 29 ++++++++ .../data/handle_imports/bench_a.py | 44 ++++++++++++ .../integration/data/handle_imports/utils.py | 57 +++++++++++++++ tests/integration/data/single/bench_a.py | 48 +++++++++++++ tests/integration/test_runner.py | 71 +++++++++++++++++++ tests/unit/test_config.py | 1 + 8 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 tests/integration/data/handle_imports/__init__.py create mode 100644 tests/integration/data/handle_imports/bench_a.py create mode 100644 tests/integration/data/handle_imports/utils.py create mode 100644 tests/integration/data/single/bench_a.py create mode 100644 tests/integration/test_runner.py diff --git a/pyproject.toml b/pyproject.toml index c602d0d..ddc2758 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,10 @@ exclude = ["benchmarks", "docs", "tests"] benchmatcha = "BenchMatcha.runner:main" [project.optional-dependencies] -dev = ["BenchMatcha[doc,test,lint,type]", "tox", "pre-commit"] +dev = ["BenchMatcha[commit,doc,test,lint,type]"] doc = ["sphinx", "furo", "sphinx_multiversion"] -test = ["pytest", "coverage", "pytest-xdist"] +test = ["pytest", "coverage>=7.10.3", "pytest-xdist", "tox"] +commit = ["pre-commit"] lint = ["pylint", "ruff"] type = ["mypy"] @@ -59,6 +60,7 @@ addopts = "-n auto -rA" [tool.coverage.run] parallel = true branch = true +patch = ["subprocess"] source = ["BenchMatcha"] disable_warnings = ["no-data-collected", "module-not-imported"] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 30b4382..306b5ce 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -26,3 +26,40 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import subprocess +import tempfile +from collections.abc import Callable, Iterator + +import pytest + + +HERE: str = os.path.abspath(os.path.dirname(__file__)) + + +# NOTE: Coverage cannot be captured of a subprocess while changing the CWD, without +# use of the tool.coverage.run.patch argument setup to use `subprocess`. This was +# not introduced until V7.10.3. See the following for details: +# https://github.com/nedbat/coveragepy/issues/1499 +@pytest.fixture +def benchmark() -> Iterator[Callable[[list[str]], tuple[int, str, str, str]]]: + """Benchmark entry point subprocess.""" + + with tempfile.TemporaryDirectory(dir=os.getcwd()) as cursor: + + def inner(args: list[str]) -> tuple[int, str, str, str]: + response: subprocess.CompletedProcess[bytes] = subprocess.run( + ["benchmatcha", *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + cwd=cursor, + env=os.environ, + ) + output: str = response.stdout.decode() + errors: str = response.stderr.decode() + + return response.returncode, output, errors, cursor + + yield inner diff --git a/tests/integration/data/handle_imports/__init__.py b/tests/integration/data/handle_imports/__init__.py new file mode 100644 index 0000000..e62dc04 --- /dev/null +++ b/tests/integration/data/handle_imports/__init__.py @@ -0,0 +1,29 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/tests/integration/data/handle_imports/bench_a.py b/tests/integration/data/handle_imports/bench_a.py new file mode 100644 index 0000000..21337cb --- /dev/null +++ b/tests/integration/data/handle_imports/bench_a.py @@ -0,0 +1,44 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .utils import register_complexity_analysis + + +def multiply(a: float) -> float: + return a * 25.0 + + +register_complexity_analysis( + lambda x: x, + multiply, + 2, + 8, + 2, + 2, +) diff --git a/tests/integration/data/handle_imports/utils.py b/tests/integration/data/handle_imports/utils.py new file mode 100644 index 0000000..06da109 --- /dev/null +++ b/tests/integration/data/handle_imports/utils.py @@ -0,0 +1,57 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from collections.abc import Callable + +import google_benchmark as gbench + + +def register_complexity_analysis( + setup: Callable, + function: Callable, + minimum: int, + maximum: int, + step: int, + repeats: int, + *args, +): + """Dynamically build complexity google benchmark wrapper.""" + + @gbench.register(name=f"{function.__module__}.{function.__qualname__}") + @gbench.option.repetitions(repeats) + @gbench.option.range_multiplier(step) + @gbench.option.range(minimum, maximum) + @gbench.option.complexity(gbench.oAuto) + def inner(state: gbench.State): + random_arg = setup(state.range(0)) + while state: + function(random_arg, *args) + state.complexity_n = state.range(0) + + return inner diff --git a/tests/integration/data/single/bench_a.py b/tests/integration/data/single/bench_a.py new file mode 100644 index 0000000..cd308d0 --- /dev/null +++ b/tests/integration/data/single/bench_a.py @@ -0,0 +1,48 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Example simple benchmark.""" + +import google_benchmark as gbench + + +def multiply(a: float) -> float: + return a * 25.0 + + +@gbench.register +@gbench.option.repetitions(2) +@gbench.option.range_multiplier(2) +@gbench.option.range(2, 8) +@gbench.option.complexity(gbench.oAuto) +def bench_multiply(state: gbench.State) -> None: + arg: int = state.range(0) + while state: + multiply(arg) + state.complexity_n = state.range(0) diff --git a/tests/integration/test_runner.py b/tests/integration/test_runner.py new file mode 100644 index 0000000..80c8570 --- /dev/null +++ b/tests/integration/test_runner.py @@ -0,0 +1,71 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Integration test suite for cli runner entry point.""" + +import os +from collections.abc import Callable + +import pytest + + +HERE: str = os.path.abspath(os.path.dirname(__file__)) +DATA: str = os.path.join(HERE, "data") + + +@pytest.mark.parametrize( + ["path"], + [ + (os.path.join(DATA, "single"),), + (os.path.join(DATA, "handle_imports"),), + ], +) +def test_bench_directory( + path: str, + benchmark: Callable[[list[str]], tuple[int, str, str, str]], +) -> None: + """Test benchmarking a directory of benchmark suites.""" + status, out, error, tmpath = benchmark([path]) + if len(error): + print(error) + + cache: str = os.path.join(tmpath, ".benchmatcha") + + assert os.path.exists(cache), "Expected cache directory to be created." + assert os.path.isdir(cache), "expected path to be a directory." + assert os.path.exists(os.path.join(cache, "out.html")), ( + "expected figures to be generated." + ) + + # NOTE: this output is temporary. + assert os.path.exists(os.path.join(cache, "benchmark.json")), ( + "expected data to be saved." + ) + assert status == 0, "Expected no errors." + assert len(error) == 0, "Expected no errors" diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ccb17a9..d7fda90 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -71,6 +71,7 @@ def reset(c: type[config.Config]): for k, v in default.items(): setattr(c, k, v) + assert getattr(c, k) == default[k], f"Expected Value to be reset: {k}" def test_config_load(toml_str: str, toml_data: dict) -> None: From 8939e39ddb79c8a6b5962c2c5e994325c3dab668 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Mon, 25 Aug 2025 19:15:19 -0700 Subject: [PATCH 29/33] tests(plotting): Complete unit testing of plotting module. --- tests/unit/test_plotting.py | 80 ++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_plotting.py b/tests/unit/test_plotting.py index 35199fb..6e520a4 100644 --- a/tests/unit/test_plotting.py +++ b/tests/unit/test_plotting.py @@ -29,6 +29,8 @@ """unit test plotting module.""" +import tempfile + import numpy as np import plotly.graph_objs as go import pytest @@ -36,24 +38,72 @@ from BenchMatcha import plotting -def test_construct_log2_axis() -> None: +def test_serialization_to_html(): + """Confirm a plotly figure is saved to a html file.""" + figure = go.Figure() + + with tempfile.NamedTemporaryFile("w+") as file: + plotting.to_html(figure, file.name) + file.seek(0) + data: str = file.read(1024) + + assert data.startswith("
"), "Expected html serialization." + + +def test_serialization_to_json(): + """Confirm a plotly figure is saved to a json file.""" + figure = go.Figure() + + with tempfile.NamedTemporaryFile("w+") as file: + plotting.to_json(figure, file.name) + file.seek(0) + data: str = file.read(1024) + + assert data.startswith("{"), "Expected json serialization." + + +@pytest.mark.parametrize( + ["x", "length", "vals", "labels"], + [ + ( + np.asarray([2, 256]), + 9, + [1, 2, 4, 8, 16, 32, 64, 128, 256], + [ + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + ], + ), + ( + np.asarray([4, 4]), + 2, + [2, 4], + [ + "21", + "22", + ], + ), + ], +) +def test_construct_log2_axis( + x: np.ndarray, + length: int, + vals: list[int], + labels: list[str], +) -> None: """Confirm a log 2 axis is constructed correctly""" - x = np.asarray([2, 256]) a, b = plotting.construct_log2_axis(x) - assert len(a) == len(b) == 9 - assert a == [1, 2, 4, 8, 16, 32, 64, 128, 256], "Incorrect values" - assert b == [ - "20", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - ], "Incorrect plotly labels." + assert len(a) == len(b) == length + assert a == vals, "Incorrect values" + assert b == labels, "Incorrect plotly labels." def test_create_scatter_trace() -> None: From aa6d78b46d2f37df4adbcd555a382a97762bb8f8 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Mon, 25 Aug 2025 19:35:29 -0700 Subject: [PATCH 30/33] fix(runner): Fix pyproject toml file path name. --- src/BenchMatcha/runner.py | 6 +++--- tests/unit/test_handlers.py | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/BenchMatcha/runner.py b/src/BenchMatcha/runner.py index c54a1e7..9614012 100644 --- a/src/BenchMatcha/runner.py +++ b/src/BenchMatcha/runner.py @@ -133,8 +133,8 @@ def _run() -> BenchmarkContext: if "--benchmark_format=json" not in sys.argv: sys.argv.append("--benchmark_format=json") - # TODO: create python bindings of library to call and collect json data without - # serializing and capturing stdout. + # TODO: create python bindings of google_benchmark library to call and collect json + # data without serializing and capturing stdout. # Read this excellent blog regarding redirecting stdout from c libraries: # https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ with pipes() as (stdout, stderr): @@ -207,7 +207,7 @@ def main() -> None: # TODO: support specification of config file path from CLI, to overwrite default # TODO: Support command line args to overwrite default config. cwd: str = os.getcwd() - p = os.path.join(cwd, ".pyproject.toml") + p = os.path.join(cwd, "pyproject.toml") if os.path.exists(p): update_config_from_pyproject(p) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 50f46da..6e68831 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -29,6 +29,7 @@ """Test json data handlers module.""" +import tempfile from collections.abc import Callable from io import BytesIO, StringIO from typing import IO, Any @@ -70,6 +71,14 @@ def test_handlers( mock.close() +def test_stream_type_error(): + """Confirm a type error is raised when an unreadable stream is provided.""" + with pytest.raises(TypeError) as err, tempfile.NamedTemporaryFile("w") as file: + handlers.HandleIO(file) + + assert err.type is TypeError, "Expected a TypeError to be raised." + + @pytest.mark.parametrize( ["handler", "transformer", "expected"], [ From 26285b48e5ae1ea00e83f664fc12a78d3913b101 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Mon, 25 Aug 2025 19:54:22 -0700 Subject: [PATCH 31/33] test(structure): Include additional testing coverage of structure module. --- src/BenchMatcha/structure.py | 26 +++++++++++++++++-------- tests/unit/test_structure.py | 37 +++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/BenchMatcha/structure.py b/src/BenchMatcha/structure.py index 3b81f70..18238b6 100644 --- a/src/BenchMatcha/structure.py +++ b/src/BenchMatcha/structure.py @@ -50,6 +50,20 @@ def parse_datetime(x: str) -> datetime: return datetime.fromisoformat(x).astimezone(UTC) +def _get_function(record: dict[str, str]) -> list[str]: + return record["name"].split("/") + + +def get_function_name(record: dict[str, str]) -> str: + """Retrieve and parse function name.""" + return _get_function(record)[0] + + +def get_size(record: dict[str, str]) -> int: + """Get complexity size, n.""" + return int(_get_function(record)[1]) + + @dataclass class Cache: """System cache information. @@ -107,9 +121,8 @@ class BenchmarkRecord: @classmethod def from_json(cls, record: dict[str, Any]) -> Self: """Convert dictionary object to BenchmarkRecord.""" - parts: list[str] = record["name"].split("/") - function: str = parts[0] - size: int = int(parts[1]) + function: str = get_function_name(record) + size: int = get_size(record) return cls( function=function, @@ -121,9 +134,6 @@ def from_json(cls, record: dict[str, Any]) -> Self: time_unit=record["time_unit"], ) - def to_json(self) -> dict: - return self.__dict__.copy() - @dataclass class ComplexityInfo: @@ -147,7 +157,7 @@ class ComplexityInfo: @classmethod def from_json(cls, record: dict[str, Any]) -> Self: """Convert dictionary object to ComplexityInfo.""" - function: str = record["name"].split("/")[0] + function: str = get_function_name(record) return cls( function=function, @@ -224,7 +234,7 @@ def get_complexity_info(data: list[dict[str, Any]]) -> dict[str, ComplexityInfo] or record.get("aggregate_name") != "RMS" ): continue - function: str = record["name"].split("/")[0] + function: str = get_function_name(record) if function in complexity_info: complexity_info[function].rms = record["rms"] diff --git a/tests/unit/test_structure.py b/tests/unit/test_structure.py index 4cf7fbd..ffbe0c0 100644 --- a/tests/unit/test_structure.py +++ b/tests/unit/test_structure.py @@ -29,14 +29,29 @@ """Unit test structure module.""" +from collections.abc import Callable, Iterator from datetime import UTC, datetime import pytest -from BenchMatcha import structure +from BenchMatcha import errors, structure from BenchMatcha.handlers import load +@pytest.fixture +def shunt_version() -> Iterator[Callable[[tuple[int, ...]], tuple[int, ...]]]: + previous = structure.SUPPORTED_VERSIONS + + def modify(value: tuple[int, ...]) -> tuple[int, ...]: + structure.SUPPORTED_VERSIONS = value + return value + + yield modify + + structure.SUPPORTED_VERSIONS = previous + assert structure.SUPPORTED_VERSIONS == previous + + @pytest.mark.parametrize( ["value", "expected"], [ @@ -93,3 +108,23 @@ def test_convert_BenchmarkContext_to_json(mock_data: str) -> None: assert all(isinstance(k["complexity"], dict) for k in benchmarks), ( "Expected benchmark complexity instances to be dictionary objects." ) + + +def test_unavailable_version() -> None: + """Confirm SchemaError is raised when unavailable version is passed.""" + record: dict = {"context": {"json_schema_version": -10}} + with pytest.raises(errors.SchemaError) as err: + structure.parse_version(record) + + assert err.type is errors.SchemaError, "Expected to raise a SchemaError." + + +def test_unsupported_version(shunt_version) -> None: + """Confirm SchemaError is raised when unsupported version is passed.""" + shunt_version((-10,)) + + record: dict = {"context": {"json_schema_version": -10}} + with pytest.raises(errors.SchemaError) as err: + structure.parse_version(record) + + assert err.type is errors.SchemaError, "Expected to raise a SchemaError." From 08e6984b5a92cde607f0435bd35da09a0e165dc0 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Mon, 25 Aug 2025 20:11:15 -0700 Subject: [PATCH 32/33] test(runner): Implement unit and additional integration tests of runner module. --- tests/integration/test_runner.py | 43 ++++++++++++++++++++++---------- tests/unit/test_runner.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_runner.py diff --git a/tests/integration/test_runner.py b/tests/integration/test_runner.py index 80c8570..6e495b8 100644 --- a/tests/integration/test_runner.py +++ b/tests/integration/test_runner.py @@ -39,11 +39,29 @@ DATA: str = os.path.join(HERE, "data") +def _assert_cache_created(cache: str, status: int, error: str) -> None: + assert os.path.exists(cache), "Expected cache directory to be created." + assert os.path.isdir(cache), "expected path to be a directory." + assert os.path.exists(os.path.join(cache, "out.html")), ( + "expected figures to be generated." + ) + + # NOTE: this output is temporary. + assert os.path.exists(os.path.join(cache, "benchmark.json")), ( + "expected data to be saved." + ) + assert status == 0, "Expected no errors." + assert len(error) == 0, "Expected no errors" + + @pytest.mark.parametrize( ["path"], [ - (os.path.join(DATA, "single"),), - (os.path.join(DATA, "handle_imports"),), + (os.path.join(DATA, "single"),), # Directory with single file + (os.path.join(DATA, "single", "bench_a.py"),), # single file + ( + os.path.join(DATA, "handle_imports"), + ), # Directory with file that imports locally ], ) def test_bench_directory( @@ -56,16 +74,15 @@ def test_bench_directory( print(error) cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status, error) - assert os.path.exists(cache), "Expected cache directory to be created." - assert os.path.isdir(cache), "expected path to be a directory." - assert os.path.exists(os.path.join(cache, "out.html")), ( - "expected figures to be generated." - ) - # NOTE: this output is temporary. - assert os.path.exists(os.path.join(cache, "benchmark.json")), ( - "expected data to be saved." - ) - assert status == 0, "Expected no errors." - assert len(error) == 0, "Expected no errors" +def test_json_key_val(benchmark) -> None: + """Confirm including benchmark format proceeds normally.""" + path: str = os.path.join(DATA, "single") + status, out, error, tmpath = benchmark(["--benchmark_format=json", path]) + if len(error): + print(error) + + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status, error) diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py new file mode 100644 index 0000000..ca61211 --- /dev/null +++ b/tests/unit/test_runner.py @@ -0,0 +1,42 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""unit test runner module.""" + +import pytest + +from BenchMatcha import runner + + +def test_manage_registration_file_not_found_error(): + """Confirm invalid files raise correct error.""" + with pytest.raises(FileNotFoundError) as err: + runner.manage_registration("cthulu") + + assert err.type is FileNotFoundError From 54e50d7fcbf4aefcb30604be5296e5cd6c750194 Mon Sep 17 00:00:00 2001 From: Spill-Tea Date: Mon, 25 Aug 2025 20:33:20 -0700 Subject: [PATCH 33/33] fix(structure): Include additional key aslr_enabled in BenchmarkContext dataclass found on remote linux machines. Further prune context to future proof and make more robust across os'. --- src/BenchMatcha/structure.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/BenchMatcha/structure.py b/src/BenchMatcha/structure.py index 18238b6..e108402 100644 --- a/src/BenchMatcha/structure.py +++ b/src/BenchMatcha/structure.py @@ -311,6 +311,7 @@ class BenchmarkContext: library_build_type: BuildType json_schema_version: int benchmarks: list[BenchmarkArray] + aslr_enabled: bool @classmethod def from_json(cls, record: dict[str, Any]) -> Self: @@ -325,7 +326,16 @@ def from_json(cls, record: dict[str, Any]) -> Self: get_complexity_info(record["benchmarks"]), ) - return cls(**context, caches=caches, date=date, benchmarks=benchmarks) + # NOTE: key found on linux machines (remote testing), but not encountered on mac + aslr = bool(context.pop("aslr_enabled", False)) + + return cls( + **{k: v for k, v in context.items() if k in cls.__annotations__}, + caches=caches, + date=date, + benchmarks=benchmarks, + aslr_enabled=aslr, + ) def to_json(self) -> dict: data = self.__dict__.copy()