diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index d822feb..422ded0 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false steps: - - name: Checkout PyTemplate Project + - name: Checkout BenchMatcha Project uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 9d4d723..0e933ac 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python -name: PyTemplate CI +name: BenchMatcha CI on: push: @@ -16,7 +16,7 @@ jobs: fail-fast: false steps: - - name: Checkout PyTemplate Project + - name: Checkout BenchMatcha Project uses: actions/checkout@v4 - name: Set up Python @@ -45,10 +45,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.11", "3.12" ] + python-version: [ "3.11", "3.12", "3.13" ] steps: - - name: Checkout PyTemplate Project + - name: Checkout BenchMatcha Project uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -120,7 +120,7 @@ jobs: - name: Install addlicense run: | - go install github.com/google/addlicense@v1.1.1 + go install github.com/google/addlicense@v1.2.0 - name: Check for License Headers run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da84716..5f9ea74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,11 +15,10 @@ repos: args: [ --maxkb=1024 ] - id: requirements-txt-fixer - - repo: https://github.com/Spill-Tea/addlicense-pre-commit - rev: v1.1.2 + - repo: https://github.com/google/addlicense + rev: v1.2.0 hooks: - id: addlicense - language: golang args: [ -f, LICENSE, ] diff --git a/README.md b/README.md index d9281e5..3438044 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,52 @@ -# PyTemplate +# BenchMatcha [![build status][buildstatus-image]][buildstatus-url] -[buildstatus-image]: https://github.com/Spill-Tea/PyTemplate/actions/workflows/python-app.yml/badge.svg?branch=main -[buildstatus-url]: https://github.com/Spill-Tea/PyTemplate/actions?query=branch%3Amain +[buildstatus-image]: https://github.com/Spill-Tea/BenchMatcha/actions/workflows/python-app.yml/badge.svg?branch=main +[buildstatus-url]: https://github.com/Spill-Tea/BenchMatcha/actions?query=branch%3Amain -Python Project Template. Be sure to create a template directly -from github. +![logo](docs/source/_static/logo.svg) + +BenchMatcha is your companion pytest-like runner to google benchmarks. +Analyze, plot, and save your results over time to evaluate regression +over the lifetime of a project. ## Table of Contents -- [PyTemplate](#pytemplate) - - [Using this template](#using-this-template) - - [Manual Editing of Project Template](#manual-editing-of-project-template) +- [BenchMatcha](#benchmatcha) - [Installation](#installation) + - [Install from pypi](#install-from-pypi) + - [Clone the repository](#clone-the-repository) + - [Pip install directly from github.](#pip-install-directly-from-github) + - [Development](#development) - [For Developers](#for-developers) - [License](#license) -## Using this template -Create a new repository using the `Use this template` option available on github. -Clone that new repository (e.g. `mynewproject`), and run the helper script `rename.py`. - -```bash -git clone https://github.com//.git -cd -python rename.py --old-name PyTemplate --new-name - -``` -We provide a simple helper script `rename.py` in the root directory to help rename a few -files and directory names to make your life easier. Please note that you will still need -to manually adjust the `pyproject.toml` file, specifically the `[project]` and -`[project.urls]` keys, to reflect your new project metadata. - -Also manually update the `docs/source/conf.py` file to reflect correct `author`, -`copyright`, and `release` key metadata for documentation builds. - -Finally update this `README.md` document to reflect new project urls. - -### Manual Editing of Project Template -To summarize, after running the `rename.py` script, there are three files you may need -to manually adjust for your new project: -1. `pyproject.toml` --> update metadata -1. `docs/source/conf.py` --> update metadata -1. `README.md` --> update project urls (and license type if different) - -Note: If you need to update the `LICENSE` file, you will also need to edit the license -header from files throughout the `src/` and `tests/` directories. - -PRO-TIP: you could theoretically run the helper script several times to replace the -project name, author name, email, and (github) username. Something like: +## Installation +You have options. +### Install from pypi ```bash -python rename.py --old-name PyTemplate --new-name -python rename.py --old-name 'Jason C Del Rio' --new-name -python rename.py --old-name spillthetea917@gmail.com --new-name -python rename.py --old-name Spill-Tea --new-name +pip install BenchMatcha ``` -## Installation -Clone the repository and pip install. - +### Clone the repository ```bash -git clone https://github.com/Spill-Tea/PyTemplate.git -cd PyTemplate +git clone https://github.com/Spill-Tea/BenchMatcha.git +cd BenchMatcha pip install . ``` -Alternatively, you may install directly from github. +### Pip install directly from github. ```bash -pip install git+https://github.com/Spill-Tea/PyTemplate@main +pip install git+https://github.com/Spill-Tea/BenchMatcha@main ``` +## Development + +BenchMatcha is currently in the planning stages of development. This means the project +is not ready for production use, and may be prone to change api without much notice. + ## For Developers After cloning the repository, create a new virtual environment and run the following diff --git a/docs/source/_templates/redirect.html b/docs/source/_templates/redirect.html index 6bfc771..04be7dc 100644 --- a/docs/source/_templates/redirect.html +++ b/docs/source/_templates/redirect.html @@ -4,6 +4,6 @@ Redirecting to main development branch - + diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 135137e..3cb0d7c 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -1,149 +1,7 @@ -PyTemplate API Documentation -============================ +BenchMatcha API Documentation +============================= -PyTemplate API documentation. - -Here are python and cython code snippets to demonstrate the use of respective customized -lexers with custom syntax highlighting style. These examples are not necessarily meant -to be fully valid code, but to demonstrate key features not available through standard -pygments syntax highlighting styles. - -Python Example Snippet ----------------------- - -.. code-block:: python - :caption: example.py - - #!/usr/bin/env python3 - """Module level docstring.""" - from typing import ClassVar - import numpy as np - - CONSTANT_A: int = 0xFF - CONSTANT_B: float = np.pi - - # NOTE: this is an example class - class Example(object): - """Example docstring. - - Args: - arg1 (str): argument 1 - arg2 (int): argument 2 - - Attributes: - data (dict): data - - """ - arg1: str - arg2: int - data: dict - seventeen: ClassVar[list[int]] = [17, 0x11, 0o21, 0b10001] - other: ClassVar[list[int]] = [1e-5, 1.0e+3, 2j, 2l, 2.7E4J] - - def __init__(self, arg1: str, arg2: int) -> None: - self.arg1 = arg1 - self.arg2 = arg2 - self.data = { - "a": [(1, 2, (3, 4, 5)), (6, 7, (8, 9 , 10))], - "b": {"c": (7, 4, 3), "d": {"e", "f", "g"}}, - } - - def __getattr__(self, value): - return self.method(value) - - def method(self, value): - return self.data[value] - - def write(self, text): - print(f"{text:<5}\n") - - def do_something(self, value): - if value > CONSTANT_A: - return value - CONSTANT_B - else: - return value + 0b10011 - - -Cython Example Snippet ----------------------- - -.. code-block:: cython - :caption: example.pyx - - """Module level docstring.""" - import cython - from libc.stdlib cimport free, malloc - - cdef extern from "" namespace "std": - cdef cppclass vector[T]: - vector() - T& operator[](int) - - ctypedef fused StringTypeObject: - str - bytes - - ctypedef struct CustomStruct: - int y - str z - - cdef packed struct Breakfast: - int[4] spam - signed char[5] eggs - - cdef enum CheeseType: - manchego = 1 - gouda = 2 - camembert = 3 - - cdef union MyUnion: - int i - float f - char c - - cdef inline unsigned char* function(bint flag) noexcept: - cdef: - Py_ssize_t j - unsigned char* k = NULL - - k = malloc(5 * sizeof(unsigned char)) - - for j in range(5): - k[j] = "A" - - return k - - # XXX: this is an example class - cdef class Example: - """The little example class that couldn't. - - Args: - arg1 (unsigned long long): ... - arg2 (double): ... - - """ - cdef public unsigned long long v - cdef readonly double k - cdef char* mem - - def __cinit__(self, unsigned long long arg1, double arg2): - self.v = arg1 - self.k = arg2 - self.mem = malloc(5 * sizeof(char)) - - def __dealloc__(self): - free(self.mem) - - @cython.boundscheck(False) - cdef char index(self, size_t idx): - return self.mem[idx] - - # just an example of nested parenthesis to demonstrate rainbow coloring - cdef dict obj = { - "a": [(1, 2, (3, 4, 5)), (6, 7, (8, 9 , 10))], - "b": {"c": (7, 4, 3), "d": {"e", "f", "g"}}, - } - cdef tuple builtin_constants = (True, False, NULL, None,) +BenchMatcha API documentation. .. toctree:: diff --git a/docs/source/conf.py b/docs/source/conf.py index 4668339..d722c10 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,7 +42,7 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = "PyTemplate" +project = "BenchMatcha" copyright = "2025, Jason C Del Rio (Spill-Tea)" author = "Jason C Del Rio (Spill-Tea)" release = "v0.0.1" @@ -85,7 +85,7 @@ html_static_path = ["_static"] html_css_files = ["custom.css"] # html_logo = "_static/logo.svg" -github_url = "https://github.com/Spill-Tea/PyTemplate" +github_url = "https://github.com/Spill-Tea/BenchMatcha" # Theme options html_theme_options = { diff --git a/docs/source/index.rst b/docs/source/index.rst index d86f4f0..894d4cd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ -PyTemplate documentation -======================== +BenchMatcha documentation +========================= Include text here. diff --git a/pyproject.toml b/pyproject.toml index dfa32f5..ddc2758 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,22 +3,32 @@ requires = ["setuptools>=67.6.1"] build-backend = "setuptools.build_meta" [project] -name = "PyTemplate" +name = "BenchMatcha" authors = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] maintainers = [{ name = "Jason C Del Rio", email = "spillthetea917@gmail.com" }] -description = "Project description here." +description = "Google Benchmark Suite Runner and Regression Analyzer." license = { file = "LICENSE" } -requires-python = ">=3.7" -keywords = ["keyword1", "keyword2"] -classifiers = ["Programming Language :: Python :: 3"] +requires-python = ">=3.11" +keywords = ["benchmark", "regression", "analysis"] +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 1 - Planning", + "License :: OSI Approved :: BSD License", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Utilities", + "Topic :: System :: Benchmark", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Libraries", +] dynamic = ["version", "readme", "dependencies"] [project.urls] -homepage = "https://github.com/Spill-Tea/PyTemplate" -issues = "https://github.com/Spill-Tea/PyTemplate/issues" +homepage = "https://github.com/Spill-Tea/BenchMatcha" +issues = "https://github.com/Spill-Tea/BenchMatcha/issues" [tool.setuptools.dynamic] -version = { attr = "PyTemplate.__version__" } +version = { attr = "BenchMatcha.__version__" } readme = { file = ["README.md"], content-type = "text/markdown" } dependencies = { file = ["requirements.txt"] } @@ -32,10 +42,14 @@ exclude = ["benchmarks", "docs", "tests"] [tool.setuptools.package-data] "*" = ["py.typed", "*.pyi"] +[project.scripts] +benchmatcha = "BenchMatcha.runner:main" + [project.optional-dependencies] -dev = ["PyTemplate[doc,test,lint,type]", "tox", "pre-commit"] +dev = ["BenchMatcha[commit,doc,test,lint,type]"] doc = ["sphinx", "furo", "sphinx_multiversion"] -test = ["pytest", "coverage", "pytest-xdist"] +test = ["pytest", "coverage>=7.10.3", "pytest-xdist", "tox"] +commit = ["pre-commit"] lint = ["pylint", "ruff"] type = ["mypy"] @@ -46,7 +60,8 @@ addopts = "-n auto -rA" [tool.coverage.run] parallel = true branch = true -source = ["PyTemplate"] +patch = ["subprocess"] +source = ["BenchMatcha"] disable_warnings = ["no-data-collected", "module-not-imported"] [tool.coverage.paths] @@ -57,17 +72,15 @@ fail_under = 95.0 precision = 1 show_missing = true skip_empty = true -# skip_covered = true exclude_also = ["def __repr__", 'if __name__ == "__main__"'] [tool.mypy] -mypy_path = "PyTemplate" -warn_unused_ignores = true +mypy_path = "BenchMatcha" allow_redefinition = false -force_uppercase_builtins = true +warn_unused_ignores = true [tool.pylint.main] -# extension-pkg-whitelist = [] +extension-pkg-allow-list = ["orjson"] ignore = ["tests", "dist", "build"] fail-under = 9.0 jobs = 0 @@ -89,6 +102,9 @@ max-line-length = 88 [tool.pylint."messages control"] disable = [ "R1731", # consider-using-max-builtin + "R0903", # too-few-public-methods + "R1735", # use-dict-literal + "W1514", # unspecified-encoding ] [tool.pylint."*.pyi"] diff --git a/rename.py b/rename.py deleted file mode 100644 index d48a1c3..0000000 --- a/rename.py +++ /dev/null @@ -1,274 +0,0 @@ -# BSD 3-Clause License -# -# Copyright (c) 2025, Spill-Tea -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Primitive script to update repository (project) name throughout template. - -Arguments: - new-name (str): new project name (defaults to root directory name) - old-name (str): old project name (defaults to PyTemplate) - path (str): root path of project (defaults to cwd) - dry-run (bool): print out what files / directories would be modified - timeout (int): Time in seconds to allow a subprocess to run. - -Notes: - * client must have git installed. - -""" - -import argparse -import os -import subprocess -from collections.abc import Iterator - - -def bypass(path: str, git_root: str, timeout: int = 1) -> bool: - """Use git to identify if a path is ignored as specified by a .gitignore file.""" - # NOTE: Do not capture errors to avoid assuming a path is included or not. - result = subprocess.run( - ["git", "-C", git_root, "check-ignore", path], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - timeout=timeout, - ) - - return result.returncode == 0 - - -def find_git_root(start_path: str, timeout: int = 1) -> str: - """Confirm we are in a git repository.""" - try: - result = subprocess.run( - ["git", "-C", start_path, "rev-parse", "--show-toplevel"], - check=True, - capture_output=True, - text=True, - timeout=timeout, - ) - return result.stdout.strip() - - except subprocess.CalledProcessError as e: - raise RuntimeError("This script must be run inside a Git repository.") from e - - -def safe_scandir(path: str) -> Iterator[os.DirEntry]: - """Wrapper around os.scandir.""" - try: - with os.scandir(path) as it: - yield from it - except PermissionError: - return - - -def replace_in_file( - filepath: str, - old: str, - new: str, - dry_run: bool = False, -) -> int: - """Replace an old keyword found within a file.""" - try: - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - except (UnicodeDecodeError, FileNotFoundError) as e: - print(f"[Warning] ({e.__class__.__name__}) {filepath}") - return 0 - - if old not in content: - return 0 - - if dry_run: - print(f"[DRY RUN] Would update content within file: {filepath}") - - else: - new_content: str = content.replace(old, new) - with open(filepath, "w", encoding="utf-8") as f: - f.write(new_content) - print(f"Updated content within file: {filepath}") - - return 1 - - -def update_project_name( - path: str, - old_name: str, - new_name: str, - dry_run: bool, - git_root: str, - timeout: int = 1, -) -> int: - """Recursively search, and modify files in place to update project name if used.""" - count = 0 - for entry in safe_scandir(path): - full_path = os.path.join(path, entry.name) - - if bypass(full_path, git_root, timeout): - continue - - if entry.is_dir(follow_symlinks=False): - count += update_project_name( - full_path, old_name, new_name, dry_run, git_root - ) - - elif entry.is_file(follow_symlinks=False): - count += replace_in_file(full_path, old_name, new_name, dry_run) - - return count - - -def _filetype(entry: os.DirEntry) -> str: - key: str = "" - if entry.is_file(follow_symlinks=False): - key = " filepath" - elif entry.is_dir(follow_symlinks=False): - key = " directory" - - return key - - -def rename_directories_and_files( - path: str, - old_name: str, - new_name: str, - dry_run: bool, - git_root: str, - timeout: int = 1, -) -> int: - """Rename both directories and filenames alike if old keyword present.""" - count: int = 0 - for entry in safe_scandir(path): - full_path = os.path.join(path, entry.name) - if bypass(full_path, git_root, timeout): - continue - - # NOTE: Depth First Search. Handle all children before renaming a directory. - if entry.is_dir(follow_symlinks=False): - count += rename_directories_and_files( - full_path, old_name, new_name, dry_run, git_root - ) - - if old_name not in entry.name: - continue - - new_path = os.path.join(path, entry.name.replace(old_name, new_name)) - key: str = _filetype(entry) - if dry_run: - print(f"[DRY RUN] Would rename{key}: {full_path} -> {new_path}") - else: - os.rename(full_path, new_path) - print(f"Renamed{key}: {full_path} -> {new_path}") - count += 1 - - return count - - -def parse_args() -> argparse.Namespace: - """Define and return parsed arguments.""" - parser = argparse.ArgumentParser(description="Rename a Python project template.") - parser.add_argument( - "--new-name", - help="New project name (e.g. my_project)", - ) - parser.add_argument( - "--old-name", - default="PyTemplate", - help="Old project name to replace (optional, defaults to PyTemplate)", - ) - parser.add_argument( - "--path", - default=".", - help="Root path of the project (default: current directory)", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would change, but don't modify anything", - ) - parser.add_argument( - "--timeout", - default=1, - help="Time in seconds to allow a subprocess to run.", - type=int, - ) - - return parser.parse_args() - - -def main() -> None: - """Main script Entry point.""" - args: argparse.Namespace = parse_args() - git_root: str = find_git_root(args.path) - - # NOTE: When this template is forked, the project should be renamed. So we can - # reasonably assume the name of the new project. Report assumption to client. - if args.new_name is None: - args.new_name = os.path.basename(git_root) - print(f"Assuming new project name: {args.new_name}") - - if args.new_name == args.old_name: - print("Exiting. Both New and old names are identical.") - return - - print( - f"Project Found at: '{args.path}'", - f"Replacing '{args.old_name}' --> '{args.new_name}'", - sep="\n", - ) - if args.dry_run: - print("[DRY RUN] Confirming Dry Run Mode. No changes will be made.") - - # NOTE: this script may also be updated to reflect the new project name. - total: int = 0 - print("\nStep I: Update File contents.") - total += update_project_name( - args.path, - args.old_name, - args.new_name, - dry_run=args.dry_run, - git_root=git_root, - timeout=args.timeout, - ) - print("\nStep II: Update Filepath Names.") - total += rename_directories_and_files( - args.path, - args.old_name, - args.new_name, - dry_run=args.dry_run, - git_root=git_root, - timeout=args.timeout, - ) - - if args.dry_run: - print(f"\n[DRY RUN] Complete. Would modify {total} file(s).") - else: - print(f"Success. Modified {total} file(s) in total.") - - -if __name__ == "__main__": - main() diff --git a/requirements.txt b/requirements.txt index e69de29..49652d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,8 @@ +google-benchmark +numpy +orjson +plotly +pytest +scipy +toml +wurlitzer diff --git a/src/PyTemplate/__init__.py b/src/BenchMatcha/__init__.py similarity index 98% rename from src/PyTemplate/__init__.py rename to src/BenchMatcha/__init__.py index 4a61609..1d37a94 100644 --- a/src/PyTemplate/__init__.py +++ b/src/BenchMatcha/__init__.py @@ -27,6 +27,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""PyTemplate Project.""" +"""BenchMatcha Project.""" __version__: str = "v0.0.1" diff --git a/src/BenchMatcha/complexity.py b/src/BenchMatcha/complexity.py new file mode 100644 index 0000000..95e8f5a --- /dev/null +++ b/src/BenchMatcha/complexity.py @@ -0,0 +1,208 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Complexity calculations.""" + +from collections.abc import Callable +from dataclasses import dataclass + +import google_benchmark as gbench +import numpy as np +from scipy.optimize import curve_fit # type: ignore[import-untyped] + +from .utils import _simple_stats + + +@dataclass +class FitResult: + """Curve fit result. + + Args: + bigo (str): Big O notation string identifier. + params (np.ndarray): coefficient value(s). + cov (np.ndarray): covariance std of coefficients + rms (float): root mean square error of fit. + + """ + + bigo: str + params: np.ndarray + cov: np.ndarray + rms: float + + @staticmethod + def _handle(x: np.ndarray) -> str: + a = " ".join([f"{j:.3E}" for j in x.tolist()]) + + return f"[{a}]" + + def __repr__(self) -> str: + return ( + f"FitResult(bigo={self.bigo},params={self._handle(self.params)}" + f",cov={self._handle(self.cov)},rms={self.rms:.3f})" + ) + + +# Define common complexity functions with all coefficients and intercept +Equation = ( + Callable[[np.ndarray, float, float], np.ndarray] + | Callable[[np.ndarray, float, float, float], np.ndarray] + | Callable[[np.ndarray, float, float, float, float], np.ndarray] +) + + +def constant(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Constant O(1) equation.""" + return a * np.ones_like(n) + b + + +def logn(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Log O(logN) equation.""" + return a * np.log2(n) + b + + +def linear(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Linear O(N) equation.""" + return a * n + b + + +def nlogn(n: np.ndarray, a: float, b: float) -> np.ndarray: + """Log linear O(NlogN) equation.""" + return a * n * np.log2(n) + b + + +def quadratic(n: np.ndarray, a: float, b: float, c: float) -> np.ndarray: + """Quadratic O(N^2) equation.""" + return a * np.power(n, 2) + linear(n, b, c) + + +def cubic(n: np.ndarray, a: float, b: float, c: float, d: float) -> np.ndarray: + """Cubic O(N^3) equation.""" + return a * np.power(n, 3) + quadratic(n, b, c, d) + + +complexity_functions: dict[str, Equation] = { + # gbench.oNone.name: "", + gbench.o1.name: constant, + gbench.oLogN.name: logn, + gbench.oN.name: linear, + gbench.oNLogN.name: nlogn, + gbench.oNSquared.name: quadratic, + gbench.oNCubed.name: cubic, +} + + +def compute_rmsd(y_true: np.ndarray, y_pred: np.ndarray, k: int) -> float: + r"""Mean normalized root mean square deviation (RMSD). + + Args: + y_true (np.ndarray): observed y values. + y_pred (np.ndarray): predicted y values. + k (int): number of parameters used to estimate predicted values. + + Returns: + (float): normalized RMSD + + Equations: + $\frac{1}{\bar{y}} \sqrt{\frac{\sum_{i=0}^{N} (y_i - \hat{y}_i)^2}{N - k}}$ + + """ + residuals: np.ndarray = y_true - y_pred + sum_square_error: np.float64 = (residuals * residuals).sum() + dof: np.int64 = np.prod(y_pred.size) - k + + return float(np.sqrt(sum_square_error / dof) / y_true.mean()) + + +def fit( + func: Callable, + label: str, + x: np.ndarray, + y: np.ndarray, + sigma: np.ndarray, +) -> FitResult | None: + """Fit observed data to an equation. + + Args: + func (Callable): equation to fit. + label (str): complexity label + x (np.ndarray): x input values + y (np.ndarray): observed y values + sigma (np.ndarray): observed error in y values + + Returns: + (FitResult | None) returns fit result if converged. + + """ + popt: np.ndarray + pcov: np.ndarray + try: + popt, pcov, *_ = curve_fit( + func, + x, + y, + sigma=sigma, + absolute_sigma=True, + ) + pred = func(x, *popt) + cov = np.sqrt(pcov.diagonal()) + rms = compute_rmsd(y, pred, len(popt)) + + return FitResult( + bigo=label, + params=popt, + cov=cov, + rms=rms, + ) + + except RuntimeError: + return None + + +def fit_complexity(x: np.ndarray, y: np.ndarray, sigma: np.ndarray) -> list[FitResult]: + """Perform curve fitting to available complexity algorithms.""" + results: list[FitResult] = [] + + for label, func in complexity_functions.items(): + if (res := fit(func, label, x, y, sigma)) is not None: + results.append(res) + + return results + + +def analyze_complexity(x: np.ndarray, y: np.ndarray) -> list[FitResult]: + """Analyze algorithmic complexity.""" + mean, std = _simple_stats(y) + + return sorted(fit_complexity(x, mean, std), key=lambda x: x.rms) + + +def get_best_fit(fits: list[FitResult]) -> FitResult: + """Return best fit by minimizing RMSD.""" + return min(fits, key=lambda x: x.rms) diff --git a/src/BenchMatcha/config.py b/src/BenchMatcha/config.py new file mode 100644 index 0000000..e76913e --- /dev/null +++ b/src/BenchMatcha/config.py @@ -0,0 +1,98 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Default runner configuration.""" + +import logging + +import toml # type: ignore[import-untyped] + +from . import plotting + + +log: logging.Logger = logging.getLogger(__name__) + + +class Config: + """default configuration.""" + + color: str = plotting.Prism[3] + line_color: str = plotting.Prism[4] + font: str = "Space Grotesk Light, Courier New, monospace" + + +class ConfigUpdater: + """Configuration updater through pyproject config file. + + Args: + path (str): path to valid configuration file. + config (Config): configuration class to update. + + """ + + path: str + config: type[Config] + + def __init__(self, path: str, config: type[Config] = Config) -> None: + self.path = path + self.config = config + + def load(self) -> dict: + """Load toml data from path.""" + return toml.load(self.path) + + def _update(self, data: dict) -> None: + for key, value in data.get("tool", {}).get("BenchMatcha", {}).items(): + if not hasattr(self.config, key): + log.info("Unsupported tool key: %s", key) + continue + + setattr(self.config, key, value) + + def update(self) -> None: + """Parse toml path and update default configuration.""" + data: dict = self.load() + self._update(data) + + +def update_config_from_pyproject(path: str) -> None: + """Update default config from pyproject toml file. + + Example: + + .. code-block: toml + + [tool.BenchMatcha] + color="#FFF" + line_color="#333" + font="Courier" + + """ + cu = ConfigUpdater(path) + cu.update() diff --git a/src/BenchMatcha/errors.py b/src/BenchMatcha/errors.py new file mode 100644 index 0000000..909492f --- /dev/null +++ b/src/BenchMatcha/errors.py @@ -0,0 +1,89 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Custom BenchMatcha exception definitions. + +Considerations: + * All custom exceptions should be defined within this module for better project + organization. This also prevents proliferation of custom errors as project scales. + * Custom exceptions should define a class method named `response` to construct and + standardize verbiage of output message. It has the nice side effect of providing + example context for usage. + * Before creating a custom error, determine if available exceptions can be used + instead. + +""" + +from __future__ import annotations + +from json import JSONDecodeError +from typing import Self, TypeVar + + +E = TypeVar("E", bound=Exception) + +_exception_register: set[type[Exception]] = { + TypeError, + ValueError, + RuntimeError, + JSONDecodeError, + FileNotFoundError, +} + + +def register_custom_exception(cls: type[E]) -> type[E]: + """Register custom exceptions.""" + _exception_register.add(cls) + + return cls + + +@register_custom_exception +class SchemaError(Exception): + """Unsupported json schema.""" + + @classmethod + def response(cls, version: str) -> Self: + """Define standard response message.""" + msg: str = f"Unsupported json schema version: {version}" + + return cls(msg) + + +@register_custom_exception +class ParsingError(Exception): + """Failed to parse json output (from Google Benchmark).""" + + @classmethod + def response(cls) -> Self: + """Define standard response message.""" + return cls( + "Failed to parse json data. Please confirm benchmarks do not contain " + "print statements or write to stdout, which can interfere with output." + ) diff --git a/src/BenchMatcha/handlers.py b/src/BenchMatcha/handlers.py new file mode 100644 index 0000000..24fc683 --- /dev/null +++ b/src/BenchMatcha/handlers.py @@ -0,0 +1,154 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""IO handlers to transform using json library.""" + +from __future__ import annotations + +import json +import os +from abc import ABC, abstractmethod +from io import IOBase +from typing import IO, Any + + +def is_readable_io_protocol(obj: object) -> bool: + """Determine if object implements readable IO interface.""" + methods: set[str] = { + "read", + "readable", + "write", + # "writeable", # temporary file handler does not have this method. + "seek", + "seekable", + "tell", + "close", + "closed", # property + "flush", + } + + return ( + isinstance(obj, IO | IOBase) or all(map(lambda x: hasattr(obj, x), methods)) + ) and obj.readable() # type: ignore[attr-defined] + + +class Handler(ABC): + """Abstract Handler protocol.""" + + @abstractmethod + def handle(self) -> dict[str, Any]: + """Handle parsing of object.""" + raise NotImplementedError("Must implement.") + + +class HandleText(Handler): + """Handle loading text (string) to json object.""" + + text: str + + def __init__(self, text: str): + self.text = text + + def handle(self) -> dict[str, Any]: + return json.loads(self.text) + + +class HandleBytes(Handler): + """Handle loading bytes to json object.""" + + text: bytes + encoding: str + + def __init__(self, text: bytes, encoding: str = "utf8"): + self.text = text + self.encoding = encoding + + def handle(self) -> dict[str, Any]: + return HandleText(self.text.decode(self.encoding)).handle() + + +class HandleIO(Handler): + """Handle loading io data to json object.""" + + stream: IOBase + encoding: str + + def __init__(self, stream: IOBase, encoding: str = "utf8"): + self.stream = stream + if not self.stream.readable(): + raise TypeError("Unreadable stream.") + + self.encoding = encoding + + def handle(self) -> dict[str, Any]: + text = self.stream.read() + if isinstance(text, bytes): + handler: Handler = HandleBytes(text, self.encoding) + + else: + handler = HandleText(text) + + return handler.handle() + + +class HandlePath(Handler): + """Handle loading data from file to json object.""" + + path: str + encoding: str + + def __init__(self, path: str, encoding: str = "utf8"): + self.path = path + self.encoding = encoding + + def handle(self) -> dict[str, Any]: + if not os.path.exists(self.path): + return HandleText(self.path).handle() + + with open(self.path, "r", encoding=self.encoding) as f: + return HandleIO(f).handle() + + +def dispatch(obj: object, encoding: str = "utf8") -> Handler: + """Dispatch appropriate handler in response to input object type.""" + if isinstance(obj, str): + return HandlePath(obj) + + elif isinstance(obj, bytes): + return HandleBytes(obj, encoding) + + elif is_readable_io_protocol(obj): + return HandleIO(obj, encoding) # type: ignore[arg-type] + + raise TypeError(f"Unsupported object type: {type(obj)}") + + +def load(obj: object, encoding: str = "utf8") -> dict[str, Any]: + """Load json data.""" + return dispatch(obj, encoding).handle() diff --git a/src/BenchMatcha/plotting.py b/src/BenchMatcha/plotting.py new file mode 100644 index 0000000..db0d854 --- /dev/null +++ b/src/BenchMatcha/plotting.py @@ -0,0 +1,247 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Plotting utilities.""" + +from collections.abc import Callable + +import numpy as np +import plotly.graph_objs as go # type: ignore[import-untyped] +from plotly.express import colors # type: ignore[import-untyped] +from plotly.io import to_json as _to_json # type: ignore[import-untyped] + +from .utils import _simple_stats, power_of_2 + + +Prism: list[str] = colors.qualitative.Prism[:] + + +def to_html(figure: go.Figure, path: str, mode: str = "w") -> None: + """Saves a plotly figure in HTML Format to a file. + + Args: + figure (go.Figure): Plotly figure. + path (str): Filepath to save plotly figure. + mode (str): Writing mode ("a" | "w") + + Returns: + (None) Appends/writes figure to html filepath. + + """ + with open(path, mode) as f: + f.write(figure.to_html(full_html=False, include_plotlyjs="cdn")) + + +def to_json(figure: go.Figure, path: str) -> None: + """Serialize a plotly figure to a json file.""" + with open(path, "w") as f: + f.write(_to_json(figure, False, False, True, engine="orjson")) + + +def construct_log2_axis(x: np.ndarray) -> tuple[list[int], list[str]]: + """Build a log2 power axis for plotly.""" + labels: list[str] = [] + values: list[int] = [] + + minimum = int(x.min()) + maximum = power_of_2(int(x.max())) + 1 + current = power_of_2(minimum) + if 1 < current >= minimum: + current //= 2 + power = int(np.log2(current)) + + while current < maximum: + values.append(int(current)) + labels.append(f"2{power}") + current *= 2 + power += 1 + + return values, labels + + +def create_scatter_trace( + x: np.ndarray, + y: np.ndarray, + name: str, + color: str, +) -> go.Scatter: + """Create scatter trace of mean and std. + + Args: + x (np.ndarray): x values + y (np.ndarray): y values + name (str): name to give plot + color (str): trace color + + Returns: + (go.Scatter) scatter plot trace of data. + + """ + mean, std = _simple_stats(y) + + return go.Scatter( + mode="lines+markers", + x=x, + y=mean, + name=name, + line=dict(color=color, dash="dot"), + error_y=dict(type="data", array=std, visible=True), + ) + + +def box_plot( + x: np.ndarray, + y: np.ndarray, + name: str, + color: str, + line_color: str, +) -> go.Box: + """Construct a box plot. + + Args: + x (np.ndarray): x values + y (np.ndarray): y values + name (str): name to give plot + color (str): marker color + line_color (str): line color + + Returns: + (go.Box) boxplot trace of data. + + """ + q1, med, q3 = np.nanquantile(y, [0.25, 0.5, 0.75], 1) + mean, std = _simple_stats(y) + + return go.Box( + name=name, + x=x, + mean=mean, + sd=std, + q1=q1, + median=med, + q3=q3, + marker_color=color, + marker_line_color=line_color, + ) + + +def create_annotation_text( + label: str, + error: float, +) -> dict: + """Build a simple annotation data of complexity fit information. + + Args: + label (str): Complexity label + error (float): error to fit (e.g. RMSD) + + Example: + + .. code-block:: python + + benchmark: BenchmarkArray + figure = go.Figure() + figure.add_annotation( + **create_annotation_text( + benchmark.complexity.bigo, + benchmark.complexity.rms, + ) + ) + + """ + a = f"{error:.2f}% " + b = f"O({label}) " + length = max(len(a), len(b)) + c = f" Complexity: {b: >{length}}" + d = f" RMS: {a: >{length}}" + + return dict( + xref="paper", + yref="paper", + x=0.01, + y=0.99, + showarrow=False, + text=f"{c}
{d}", + align="left", + bgcolor="rgba(255,255,255,0.6)", + bordercolor="black", + borderwidth=1, + ) + + +_benchmark_map: dict[str, Callable[[np.ndarray, float], np.ndarray]] = { + "(1)": lambda n, c: c * np.ones_like(n), + "N": lambda n, c: c * n, + "lgN": lambda n, c: c * np.log2(n), + "NlgN": lambda n, c: c * n * np.log2(n), + "N^2": lambda n, c: c * n**2, + "N^3": lambda n, c: c * n**3, +} + + +def get_big_o_function(label: str) -> Callable[[np.ndarray, float], np.ndarray]: + """Map the big-O notation label (from google benchmark) to function.""" + return _benchmark_map.get(label, lambda n, c: c * n) + + +def draw_complexity_line( + x: np.ndarray, + coefficient: float, + big_o: str, + name: str, + color: str, +) -> go.Scatter: + """Create a scatter plot to describe google benchmark complexity information. + + Args: + x (np.ndarray): x axis data (n). + coefficient (float): multiplier + big_o (str): label describing algorithmic complexity. + name (str): name (label) to give trace on plot. + color (str): color of line. + + Returns: + (go.Scatter): scatter plot trace of complexity information. + + """ + func: Callable[[np.ndarray, float], np.ndarray] = get_big_o_function(big_o) + y: np.ndarray = func(x, coefficient) + + return go.Scatter( + x=x, + y=y, + name=name, + mode="lines", + line=dict( + color=color, + dash="dash", + shape="spline", + ), + opacity=0.7, + ) diff --git a/src/PyTemplate/py.typed b/src/BenchMatcha/py.typed similarity index 100% rename from src/PyTemplate/py.typed rename to src/BenchMatcha/py.typed diff --git a/src/BenchMatcha/runner.py b/src/BenchMatcha/runner.py new file mode 100644 index 0000000..9614012 --- /dev/null +++ b/src/BenchMatcha/runner.py @@ -0,0 +1,220 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Primary Benchmark Runner.""" + +import logging +import os +import sys +from json import JSONDecodeError + +import google_benchmark as gbench +import orjson +import plotly.graph_objs as go # type: ignore[import-untyped] +from wurlitzer import pipes # type: ignore[import-untyped] + +from . import plotting +from .config import Config, update_config_from_pyproject +from .errors import ParsingError +from .handlers import HandleText +from .sifter import collect_benchmarks, load_benchmark +from .structure import BenchmarkArray, BenchmarkContext, parse_version + + +log: logging.Logger = logging.getLogger(__name__) + + +def manage_registration(path: str) -> None: + """Manage import, depending on whether path is a directory or file.""" + abspath: str = os.path.abspath(path) + if not os.path.exists(abspath): + raise FileNotFoundError("Invalid filepath") + + if os.path.isdir(abspath): + collect_benchmarks(abspath) + + elif os.path.isfile(abspath) and abspath.endswith(".py"): + load_benchmark(abspath, os.path.abspath(os.path.dirname(abspath))) + + else: + log.warning( + "Unsupported path provided. While the path does exist, it is neither a" + " file nor a directory." + ) + + +def plot_benchmark_array(benchmark: BenchmarkArray) -> go.Figure: + """Plot benchmark array.""" + fig = go.Figure() + fig.add_trace( + plotting.create_scatter_trace( + benchmark.size, + benchmark.cpu_time, + "CPU Time", + Config.color, + ) + ) + + fig.add_trace( + plotting.draw_complexity_line( + benchmark.size, + benchmark.complexity.cpu_coefficient, + benchmark.complexity.big_o, + f"CPU Time Fit ({benchmark.complexity.big_o})", + Config.line_color, + ) + ) + + fig.add_annotation( + **plotting.create_annotation_text( + benchmark.complexity.big_o, + benchmark.complexity.rms, + ) + ) + + vals, labels = plotting.construct_log2_axis(benchmark.size) + if (p := len(vals) // 13) > 0: + vals = vals[:: p + 1] + labels = labels[:: p + 1] + + fig.update_layout( + title=f"Benchmark Results
{benchmark.function}", + xaxis=dict( + type="log", + tickvals=vals, + ticktext=labels, + tickmode="array", + title="Input Size (n)", + ), + yaxis=dict( + title=f"Time ({benchmark.unit})", + type="log", + dtick=1, + exponentformat="power", + ), + legend_title="Timing", + font=dict( + family=Config.font, + size=12, + ), + ) + + return fig + + +def _run() -> BenchmarkContext: + # TODO: Improve logic here + if "--benchmark_format=json" not in sys.argv: + sys.argv.append("--benchmark_format=json") + + # TODO: create python bindings of google_benchmark library to call and collect json + # data without serializing and capturing stdout. + # Read this excellent blog regarding redirecting stdout from c libraries: + # https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/ + with pipes() as (stdout, stderr): + try: + gbench.main() + + # NOTE: bypass sys.exit(0) call from main + except SystemExit: + ... + + text: str = stdout.read() + stdout.close(), stderr.close() # pylint: disable=W0106 + + handler = HandleText(text) + try: + obj: dict = handler.handle() + except JSONDecodeError as e: + raise ParsingError.response() from e + + context: BenchmarkContext = parse_version(obj) + + return context + + +def save(context: BenchmarkContext, cache_dir: str) -> None: + """Save benchmark data.""" + for j in context.benchmarks: + figure: go.Figure = plot_benchmark_array(j) + plotting.to_html(figure, os.path.join(cache_dir, "out.html"), "a") + + # TODO: Save data to database. Serialize to json in the interim. + database: str = os.path.join(cache_dir, "benchmark.json") + data: list[dict] = [] + if os.path.exists(database): + with open(database, "br") as f: + data = orjson.loads(f.read()) + + data.append(context.to_json()) + serialized: bytes = orjson.dumps( + data, + option=orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_SERIALIZE_DATACLASS, + ) + + with open(database, "bw") as f: + f.write(serialized) + + +def run(path: str, cache_dir: str) -> None: + """BenchMatcha Runner.""" + manage_registration(path) + + # TODO: remove arguments specific to BenchMatch to prevent failure on google + # benchmark interface. + + context: BenchmarkContext = _run() + save(context, cache_dir) + + +# TODO: Handle a list of separated filepaths. +# def run_paths(paths: list[str]) -> None: +# """Run benchmarks against a list of paths.""" +# for path in paths: +# manage_registration(path) + +# context: BenchmarkContext = _run() + + +def main() -> None: + """Primary CLI Entry Point.""" + # TODO: support specification of config file path from CLI, to overwrite default + # TODO: Support command line args to overwrite default config. + cwd: str = os.getcwd() + p = os.path.join(cwd, "pyproject.toml") + if os.path.exists(p): + update_config_from_pyproject(p) + + # Create cache if does not exist + cache = os.path.join(cwd, ".benchmatcha") + if not os.path.exists(cache): + os.mkdir(cache) + + # TODO: Determine if a list of paths have been provided instead, and handle + run(sys.argv.pop(), cache) diff --git a/src/BenchMatcha/sifter.py b/src/BenchMatcha/sifter.py new file mode 100644 index 0000000..9415ef8 --- /dev/null +++ b/src/BenchMatcha/sifter.py @@ -0,0 +1,89 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Discovery of benchmark tests to register.""" + +import glob +import os +from collections.abc import Iterator +from pathlib import Path +from types import ModuleType + +from _pytest.pathlib import import_path + + +def scandir(filepath: str) -> Iterator[os.DirEntry[str]]: + """Simple wrapper around os.scandir to use more simply as an iterator.""" + with os.scandir(os.path.abspath(filepath)) as scanner: + yield from scanner + + +class Collector: + """Collection interface.""" + + root: str + pattern: str + + def __init__(self, root: str, pattern: str = "bench*.py") -> None: + self.root = root + self.pattern = pattern + + def get(self, path: str) -> Iterator[str]: + """Get paths of a given pattern.""" + yield from glob.iglob(path, root_dir=self.root) + + def collect(self, path: str) -> Iterator[str]: + """Recursive collection of pattern matching filepaths.""" + yield from self.get(os.path.join(path, self.pattern)) + for candidate in scandir(path): + if candidate.is_dir(follow_symlinks=False): + yield from self.collect(candidate.path) + + +def collect(root: str, pattern: str = "bench*.py") -> Iterator[str]: + """Collect relevant filepaths recursively stemming from root directory.""" + col = Collector(root, pattern) + + yield from col.collect(root) + + +def load_benchmark(path: str, root: str) -> ModuleType: + """Load a benchmark suite.""" + return import_path( + os.path.abspath(path), + root=Path(root).absolute().resolve(), + consider_namespace_packages=False, + ) + + +def collect_benchmarks(root: str) -> None: + """Collect all benchmarks from a variety of benchmark suites.""" + root = os.path.abspath(root) + for j in collect(root): + load_benchmark(j, root=root) diff --git a/src/BenchMatcha/structure.py b/src/BenchMatcha/structure.py new file mode 100644 index 0000000..e108402 --- /dev/null +++ b/src/BenchMatcha/structure.py @@ -0,0 +1,358 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Data structures defining json output of google benchmark suite.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Literal, Self + +import numpy as np + +from .errors import SchemaError + + +BuildType = Literal["release", "debug"] +SUPPORTED_VERSIONS: tuple[int, ...] = (1,) + + +def parse_datetime(x: str) -> datetime: + """Parse ISO 8601 datetime string.""" + return datetime.fromisoformat(x).astimezone(UTC) + + +def _get_function(record: dict[str, str]) -> list[str]: + return record["name"].split("/") + + +def get_function_name(record: dict[str, str]) -> str: + """Retrieve and parse function name.""" + return _get_function(record)[0] + + +def get_size(record: dict[str, str]) -> int: + """Get complexity size, n.""" + return int(_get_function(record)[1]) + + +@dataclass +class Cache: + """System cache information. + + Args: + type (str): cache type + level (int): + size (int): cache size + num_sharing (int): + + """ + + type: str + level: int + size: int + num_sharing: int + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to Cache.""" + return cls( + type=record["type"], + level=record["level"], + size=record["size"], + num_sharing=record["num_sharing"], + ) + + def to_json(self) -> dict: + return self.__dict__.copy() + + +@dataclass +class BenchmarkRecord: + """Benchmark result record. + + Args: + function (str): function name or alias + size (int): Input size + threads (int): thread id + iterations (int): number of iterations performed per measurement + real_time (float): total real time per measurement + cpu_time (float): total cpu time per measurement + time_unit (str): unit of time + + """ + + function: str + size: int + threads: int + iterations: int + real_time: float + cpu_time: float + time_unit: str + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to BenchmarkRecord.""" + function: str = get_function_name(record) + size: int = get_size(record) + + return cls( + function=function, + size=size, + threads=record["threads"], + iterations=record["iterations"], + real_time=record["real_time"], + cpu_time=record["cpu_time"], + time_unit=record["time_unit"], + ) + + +@dataclass +class ComplexityInfo: + """Algorithmic time complexity result. + + Args: + function (str): function name or alias. + big_o (str): BigO notation. + real_coefficient (float): real time coefficient. + cpu_coefficient (float): cpu time coefficient. + rms (float): root mean square error of fit. + + """ + + function: str + big_o: str + real_coefficient: float + cpu_coefficient: float + rms: float = 0.0 + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to ComplexityInfo.""" + function: str = get_function_name(record) + + return cls( + function=function, + big_o=record["big_o"], + real_coefficient=record["real_coefficient"], + cpu_coefficient=record["cpu_coefficient"], + ) + + def to_json(self) -> dict: + return self.__dict__.copy() + + +@dataclass +class BenchmarkArray: + """Reformatted Benchmark result as an array. + + Args: + function (str): function name or alias + unit (str): unit of time + size (np.ndarray): Input size + iterations (np.ndarray): number of iterations performed per measurement + real_time (np.ndarray): total real time per measurement + cpu_time (np.ndarray): total cpu time per measurement + complexity (ComplexityInfo): algorithmic time complexity information + + """ + + function: str + unit: str + size: np.ndarray # 1D array + iterations: np.ndarray # 2D array (n_sizes x repetitions) + real_time: np.ndarray # 2D array (n_sizes x repetitions) + cpu_time: np.ndarray # 2D array (n_sizes x repetitions) + complexity: ComplexityInfo + + def to_json(self) -> dict: + d = self.__dict__.copy() + d["complexity"] = self.complexity.to_json() + + return d + + +def get_benchmark_records( + data: list[dict[str, Any]], +) -> dict[str, list[BenchmarkRecord]]: + """Group and parse benchmark results.""" + grouped_records = defaultdict(list) + for record in data: + if record["run_type"] != "iteration": + continue + br = BenchmarkRecord.from_json(record) + grouped_records[br.function].append(br) + + return grouped_records + + +def get_complexity_info(data: list[dict[str, Any]]) -> dict[str, ComplexityInfo]: + """Capture and parse complexity information from benchmarks.""" + complexity_info: dict[str, ComplexityInfo] = {} + # First pass: grab complexity information + for record in data: + if ( + record.get("run_type") != "aggregate" + or record.get("aggregate_name") != "BigO" + ): + continue + ci = ComplexityInfo.from_json(record) + complexity_info[ci.function] = ci + + # Second pass: grab rms fit error information + for record in data: + if ( + record.get("run_type") != "aggregate" + or record.get("aggregate_name") != "RMS" + ): + continue + function: str = get_function_name(record) + if function in complexity_info: + complexity_info[function].rms = record["rms"] + + return complexity_info + + +def convert_to_arrays( + grouped_records: dict[str, list[BenchmarkRecord]], + complexity_data: dict[str, ComplexityInfo], +) -> list[BenchmarkArray]: + """Reorganize benchmark data into numpy arrays.""" + grouped_arrays: list[BenchmarkArray] = [] + + for function, records in grouped_records.items(): + size_to_times: defaultdict[int, list[tuple[int, float, float]]] = defaultdict( + list + ) + for record in records: + size_to_times[record.size].append( + ( + record.iterations, + record.real_time, + record.cpu_time, + ) + ) + + sorted_sizes: list[int] = sorted(size_to_times) + iter_arr: list[list[int]] = [] + real_arr: list[list[float]] = [] + cpu_arr: list[list[float]] = [] + container: list[list[int] | list[float]] + idx: int + + for size in sorted_sizes: + times: list[tuple[int, float, float]] = size_to_times[size] + for idx, container in zip( # type: ignore[assignment] + range(3), (iter_arr, real_arr, cpu_arr), strict=True + ): + container.append([t[idx] for t in times]) + + # TODO: validate time unit is consistent. Adjust if assumption not true. + grouped_arrays.append( + BenchmarkArray( + function=function, + unit=records[0].time_unit, + size=np.asarray(sorted_sizes, dtype=np.int64), + iterations=np.asarray(iter_arr, dtype=np.int64), + real_time=np.asarray(real_arr, dtype=np.float64), + cpu_time=np.asarray(cpu_arr, dtype=np.float64), + complexity=complexity_data[function], + ) + ) + + return grouped_arrays + + +# TODO: Consider how we would capture custom data (Counters). +# TODO: Capture additional information not included in google_benchmark output +# (e.g. git sha) +@dataclass +class BenchmarkContext: + """Google benchmark context.""" + + # pylint: disable=R0902 + date: datetime + host_name: str + executable: str + num_cpus: int + mhz_per_cpu: int + caches: list[Cache] + cpu_scaling_enabled: bool + load_avg: list[float] + library_version: str + library_build_type: BuildType + json_schema_version: int + benchmarks: list[BenchmarkArray] + aslr_enabled: bool + + @classmethod + def from_json(cls, record: dict[str, Any]) -> Self: + """Convert dictionary object to BenchmarkContext.""" + context: dict = record.get("context", {}).copy() + caches: list[Cache] = [Cache.from_json(i) for i in context.pop("caches", [])] + date: datetime = parse_datetime( + context.pop("date", datetime.now(UTC).isoformat()) + ) + benchmarks: list[BenchmarkArray] = convert_to_arrays( + get_benchmark_records(record["benchmarks"]), + get_complexity_info(record["benchmarks"]), + ) + + # NOTE: key found on linux machines (remote testing), but not encountered on mac + aslr = bool(context.pop("aslr_enabled", False)) + + return cls( + **{k: v for k, v in context.items() if k in cls.__annotations__}, + caches=caches, + date=date, + benchmarks=benchmarks, + aslr_enabled=aslr, + ) + + def to_json(self) -> dict: + data = self.__dict__.copy() + data["caches"] = [i.to_json() for i in self.caches] + data["benchmarks"] = [j.to_json() for j in self.benchmarks] + + return data + + +def parse_version(record: dict[str, Any]) -> BenchmarkContext: + """Map schema version to correct parsing engine.""" + schema_version = int(record.get("context", {}).get("json_schema_version", -1)) + if schema_version not in SUPPORTED_VERSIONS: + raise SchemaError.response(str(schema_version)) + + match schema_version: + case 1: + return BenchmarkContext.from_json(record) + case _: + raise SchemaError.response(str(schema_version)) diff --git a/src/BenchMatcha/utils.py b/src/BenchMatcha/utils.py new file mode 100644 index 0000000..517cda3 --- /dev/null +++ b/src/BenchMatcha/utils.py @@ -0,0 +1,81 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Miscellaneous utilities.""" + +from __future__ import annotations + +import enum +import sys + +import numpy as np + + +def power_of_2(x: int) -> int: + """Retrieve the next power of 2, if value is not already one.""" + x -= 1 + mod: int = 1 + size: int = sys.getsizeof(x) + while mod < size: + x |= x >> mod + mod *= 2 + + return x + 1 + + +def _simple_stats(x: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Compute mean and standard deviation.""" + mean: np.ndarray = np.nanmean(x, axis=1) + std: np.ndarray = np.nanstd(x, axis=1, ddof=1) + + return mean, std + + +# pylint: disable=invalid-name +# https://github.com/google/benchmark/blob/main/src/complexity.cc#L52-L69 +class BigO(enum.StrEnum): + """Big o notation string identifiers.""" + + o1 = "(1)" + oN = "N" + oNSquared = "N^2" + oNCubed = "N^3" + oLogN = "lgN" + oNLogN = "NlgN" + oLambda = "f(N)" + + @classmethod + def get(cls, value: str) -> str: + # e.g. "o1" -> "(1)" + return cls[value].value + + @classmethod + def back(cls, value: str) -> str: + # e.g. "(1)" -> "o1" + return cls(value).name diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 30b4382..306b5ce 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -26,3 +26,40 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import subprocess +import tempfile +from collections.abc import Callable, Iterator + +import pytest + + +HERE: str = os.path.abspath(os.path.dirname(__file__)) + + +# NOTE: Coverage cannot be captured of a subprocess while changing the CWD, without +# use of the tool.coverage.run.patch argument setup to use `subprocess`. This was +# not introduced until V7.10.3. See the following for details: +# https://github.com/nedbat/coveragepy/issues/1499 +@pytest.fixture +def benchmark() -> Iterator[Callable[[list[str]], tuple[int, str, str, str]]]: + """Benchmark entry point subprocess.""" + + with tempfile.TemporaryDirectory(dir=os.getcwd()) as cursor: + + def inner(args: list[str]) -> tuple[int, str, str, str]: + response: subprocess.CompletedProcess[bytes] = subprocess.run( + ["benchmatcha", *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + cwd=cursor, + env=os.environ, + ) + output: str = response.stdout.decode() + errors: str = response.stderr.decode() + + return response.returncode, output, errors, cursor + + yield inner diff --git a/tests/integration/data/handle_imports/__init__.py b/tests/integration/data/handle_imports/__init__.py new file mode 100644 index 0000000..e62dc04 --- /dev/null +++ b/tests/integration/data/handle_imports/__init__.py @@ -0,0 +1,29 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/tests/integration/data/handle_imports/bench_a.py b/tests/integration/data/handle_imports/bench_a.py new file mode 100644 index 0000000..21337cb --- /dev/null +++ b/tests/integration/data/handle_imports/bench_a.py @@ -0,0 +1,44 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .utils import register_complexity_analysis + + +def multiply(a: float) -> float: + return a * 25.0 + + +register_complexity_analysis( + lambda x: x, + multiply, + 2, + 8, + 2, + 2, +) diff --git a/tests/integration/data/handle_imports/utils.py b/tests/integration/data/handle_imports/utils.py new file mode 100644 index 0000000..06da109 --- /dev/null +++ b/tests/integration/data/handle_imports/utils.py @@ -0,0 +1,57 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from collections.abc import Callable + +import google_benchmark as gbench + + +def register_complexity_analysis( + setup: Callable, + function: Callable, + minimum: int, + maximum: int, + step: int, + repeats: int, + *args, +): + """Dynamically build complexity google benchmark wrapper.""" + + @gbench.register(name=f"{function.__module__}.{function.__qualname__}") + @gbench.option.repetitions(repeats) + @gbench.option.range_multiplier(step) + @gbench.option.range(minimum, maximum) + @gbench.option.complexity(gbench.oAuto) + def inner(state: gbench.State): + random_arg = setup(state.range(0)) + while state: + function(random_arg, *args) + state.complexity_n = state.range(0) + + return inner diff --git a/tests/integration/data/single/bench_a.py b/tests/integration/data/single/bench_a.py new file mode 100644 index 0000000..cd308d0 --- /dev/null +++ b/tests/integration/data/single/bench_a.py @@ -0,0 +1,48 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Example simple benchmark.""" + +import google_benchmark as gbench + + +def multiply(a: float) -> float: + return a * 25.0 + + +@gbench.register +@gbench.option.repetitions(2) +@gbench.option.range_multiplier(2) +@gbench.option.range(2, 8) +@gbench.option.complexity(gbench.oAuto) +def bench_multiply(state: gbench.State) -> None: + arg: int = state.range(0) + while state: + multiply(arg) + state.complexity_n = state.range(0) diff --git a/tests/integration/test_runner.py b/tests/integration/test_runner.py new file mode 100644 index 0000000..6e495b8 --- /dev/null +++ b/tests/integration/test_runner.py @@ -0,0 +1,88 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Integration test suite for cli runner entry point.""" + +import os +from collections.abc import Callable + +import pytest + + +HERE: str = os.path.abspath(os.path.dirname(__file__)) +DATA: str = os.path.join(HERE, "data") + + +def _assert_cache_created(cache: str, status: int, error: str) -> None: + assert os.path.exists(cache), "Expected cache directory to be created." + assert os.path.isdir(cache), "expected path to be a directory." + assert os.path.exists(os.path.join(cache, "out.html")), ( + "expected figures to be generated." + ) + + # NOTE: this output is temporary. + assert os.path.exists(os.path.join(cache, "benchmark.json")), ( + "expected data to be saved." + ) + assert status == 0, "Expected no errors." + assert len(error) == 0, "Expected no errors" + + +@pytest.mark.parametrize( + ["path"], + [ + (os.path.join(DATA, "single"),), # Directory with single file + (os.path.join(DATA, "single", "bench_a.py"),), # single file + ( + os.path.join(DATA, "handle_imports"), + ), # Directory with file that imports locally + ], +) +def test_bench_directory( + path: str, + benchmark: Callable[[list[str]], tuple[int, str, str, str]], +) -> None: + """Test benchmarking a directory of benchmark suites.""" + status, out, error, tmpath = benchmark([path]) + if len(error): + print(error) + + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status, error) + + +def test_json_key_val(benchmark) -> None: + """Confirm including benchmark format proceeds normally.""" + path: str = os.path.join(DATA, "single") + status, out, error, tmpath = benchmark(["--benchmark_format=json", path]) + if len(error): + print(error) + + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status, error) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 30b4382..558ab48 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -26,3 +26,124 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import tempfile +from collections.abc import Iterator + +import pytest + + +@pytest.fixture +def mock_context() -> str: + return """ +"context": { + "date": "2025-07-13T12:09:31-07:00", + "host_name": "host", + "executable": "file.py", + "num_cpus": 12, + "mhz_per_cpu": 24, + "cpu_scaling_enabled": false, + "caches": [ + { + "type": "Data", + "level": 1, + "size": 65536, + "num_sharing": 0 + }, + { + "type": "Instruction", + "level": 1, + "size": 131072, + "num_sharing": 0 + }, + { + "type": "Unified", + "level": 2, + "size": 4194304, + "num_sharing": 1 + } + ], + "load_avg": [4.69092,4.60693,4.47949], + "library_version": "1.9.4", + "library_build_type": "release", + "json_schema_version": 1 + }, +""" + + +@pytest.fixture +def mock_bench() -> str: + return """ +"benchmarks": [ + { + "name": "function/8/repeats:3", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/8/repeats:3", + "run_type": "iteration", + "repetitions": 3, + "repetition_index": 0, + "threads": 1, + "iterations": 1686012, + "real_time": 4.2350600595475760e+02, + "cpu_time": 4.2322355950016964e+02, + "time_unit": "ns" + }, + { + "name": "function/8/repeats:3", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/8/repeats:3", + "run_type": "iteration", + "repetitions": 3, + "repetition_index": 1, + "threads": 1, + "iterations": 1686012, + "real_time": 4.2872337197514543e+02, + "cpu_time": 4.2800703672334481e+02, + "time_unit": "ns" + }, + { + "name": "function/repeats:3_BigO", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/repeats:3", + "run_type": "aggregate", + "repetitions": 3, + "threads": 1, + "aggregate_name": "BigO", + "aggregate_unit": "time", + "cpu_coefficient": 1.2549524739108346e+01, + "real_coefficient": 1.2555517635286144e+01, + "big_o": "N", + "time_unit": "ns" + }, + { + "name": "function/repeats:3_RMS", + "family_index": 0, + "per_family_instance_index": 0, + "run_name": "function/repeats:3", + "run_type": "aggregate", + "repetitions": 3, + "threads": 1, + "aggregate_name": "RMS", + "aggregate_unit": "percentage", + "rms": 5.4107447739157266e-02 + } +] +""" + + +@pytest.fixture +def mock_data(mock_context: str, mock_bench: str) -> str: + """mock benchmark data.""" + return "{" + f"{mock_context}{mock_bench}" + "}" + + +@pytest.fixture +def mock_file(mock_data: str) -> Iterator[tempfile._TemporaryFileWrapper]: + with tempfile.NamedTemporaryFile("w+") as f: + f.write(mock_data) + f.seek(0) + + yield f diff --git a/tests/unit/test_complexity.py b/tests/unit/test_complexity.py new file mode 100644 index 0000000..450edff --- /dev/null +++ b/tests/unit/test_complexity.py @@ -0,0 +1,118 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test algorithmic complexity module.""" + +import numpy as np +import pytest + +from BenchMatcha import complexity as comp +from BenchMatcha.utils import _simple_stats + + +@pytest.fixture +def fit_result() -> comp.FitResult: + return comp.FitResult( + bigo="N", + params=np.asarray([2.73]), + cov=np.asarray([1.5]), + rms=0.479, + ) + + +@pytest.fixture +def coords() -> tuple[np.ndarray, np.ndarray]: + x = np.arange(3) + y = np.arange(9).reshape((3, 3)) + + return x, y + + +def test_fit_result_repr(fit_result: comp.FitResult) -> None: + """minor test of fit result repr dunder method.""" + result: str = repr(fit_result) + assert result == "FitResult(bigo=N,params=[2.730E+00],cov=[1.500E+00],rms=0.479)", ( + "Unexpected FitResult repr." + ) + + +@pytest.mark.parametrize( + ["x", "y", "k", "expected"], + [ + (np.arange(3), np.arange(5, 8), 1, 6.12372), + (np.arange(3), np.arange(5, 8), 2, 8.66025), + ], +) +def test_rmsd_computation( + x: np.ndarray, + y: np.ndarray, + k: int, + expected: float, +) -> None: + """Confirm computation of mean normalized rmsd.""" + result: float = comp.compute_rmsd(x, y, k) + assert isinstance(result, float), "Expected float return type." + assert np.isclose(result, expected), "Unexpected rmsd computation." + + +def test_fit(coords: tuple[np.ndarray, np.ndarray]) -> None: + """Test (linear) curve fitting.""" + x, y = coords + mean, std = _simple_stats(y) + result = comp.fit(comp.linear, "N", x, mean, std) + + assert isinstance(result, comp.FitResult), "Expected a fit result return type." + assert result.bigo == "N", "Expected correct label." + + assert len(result.params) == 2, "Expected 2 parameters." + assert np.allclose(result.params, np.asarray([3.0, 1.0])), ( + "Unexpected param values." + ) + + assert len(result.cov) == 2, "Expected 2 parameter errors." + assert np.allclose(result.cov, np.asarray([0.7071, 0.91287])), ( + "Unexpected param error values." + ) + + assert np.isclose(result.rms, 0.0), "Unexpected error." + + +def test_analyze_complexity() -> None: + """Test batch analysis of algorithmic complexity.""" + x = np.arange(10, 20) + y = np.arange(1, 31).reshape(10, 3) + result = comp.analyze_complexity(x, y) + assert isinstance(result, list), "Expected list return type." + assert len(result) == 6, "Expected 6 elements" + assert all(isinstance(x, comp.FitResult) for x in result), ( + "Expected all elements to be a FitResult type." + ) + + # Results should be sorted by best performing fit, by minimizing rmsd. + assert comp.get_best_fit(result) == result[0], "Expected same FitResult." diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..d7fda90 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,127 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test config module.""" + +from contextlib import contextmanager +from io import StringIO +from unittest.mock import MagicMock, patch + +import pytest + +from BenchMatcha import config + + +@pytest.fixture +def toml_str() -> str: + return """ +[tool.BenchMatcha] +color="#FFF" +line_color="#333" +font="Courier" +upsupported_key="test" +""" + + +@pytest.fixture +def toml_data() -> dict: + return { + "tool": { + "BenchMatcha": { + "color": "#FFF", + "line_color": "#333", + "font": "Courier", + "upsupported_key": "test", + } + } + } + + +@contextmanager +def reset(c: type[config.Config]): + """Simple contextmanager to reset config to original default values.""" + default: dict = {key: getattr(c, key) for key in c.__annotations__.keys()} + yield + + for k, v in default.items(): + setattr(c, k, v) + assert getattr(c, k) == default[k], f"Expected Value to be reset: {k}" + + +def test_config_load(toml_str: str, toml_data: dict) -> None: + """Confirm toml data loads correctly.""" + with reset(config.Config), StringIO(toml_str) as stream: + instance = config.ConfigUpdater(stream) + result = instance.load() + + assert result == toml_data, "Expected same object" + + +def _assert_config_is_updated() -> None: + assert config.Config.color == "#FFF", "Expected color to be updated." + assert config.Config.line_color == "#333", "Expected line color to be updated." + assert config.Config.font == "Courier", "Expected font to be updated." + assert not hasattr(config.Config, "upsupported_key"), ( + "Expected unsupported key to be bypassed." + ) + + +def test_config_private_update(toml_data: dict) -> None: + """Confirm config data is updated correctly.""" + with reset(config.Config): + instance = config.ConfigUpdater("") + instance._update(toml_data) + _assert_config_is_updated() + + +def test_config_update(toml_str: str) -> None: + """Confirm toml data loads correctly.""" + with reset(config.Config), StringIO(toml_str) as stream: + instance = config.ConfigUpdater(stream) + instance.update() + _assert_config_is_updated() + + +@patch.object(config.ConfigUpdater, "load") +def test_config_update_mock(mock: MagicMock, toml_data: dict) -> None: + """Confirm config is updated by mocking load method of ConfigUpdater.""" + mock.return_value = toml_data + with reset(config.Config): + instance = config.ConfigUpdater("") + instance.update() + _assert_config_is_updated() + + +@patch.object(config.ConfigUpdater, "load") +def test_config_update_function(mock: MagicMock, toml_data: dict) -> None: + """Confirm config is updated from available function api.""" + mock.return_value = toml_data + with reset(config.Config): + config.update_config_from_pyproject("") + _assert_config_is_updated() diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py new file mode 100644 index 0000000..8d8b546 --- /dev/null +++ b/tests/unit/test_errors.py @@ -0,0 +1,97 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import builtins +from collections.abc import Callable, Iterator +from json import JSONDecodeError + +import pytest + +from BenchMatcha import errors + + +@pytest.fixture +def register_error() -> Iterator[Callable[[type[Exception]], type[Exception]]]: + e: type[Exception] | None = None + + def inner(exc: type[Exception]) -> type[Exception]: + nonlocal e + e = errors.register_custom_exception(exc) + + return e + + yield inner + + if e is not None: + assert e in errors._exception_register, "Expected error to be registered" + errors._exception_register.remove(e) + assert e not in errors._exception_register, "Expected error to be removed." + + +def test_registering_custom_error( + register_error: Callable[[type[Exception]], type[Exception]], +): + """Confirm we can correctly register custom exception classes.""" + + class CustomMagicalUnknownUnicornError(Exception): ... + + res = register_error(CustomMagicalUnknownUnicornError) + assert res is CustomMagicalUnknownUnicornError, "Expected same exception." + + +# https://pytest-xdist.readthedocs.io/en/stable/known-limitations.html#order-and-amount-of-test-must-be-consistent +@pytest.mark.parametrize( + "custom", + sorted( # limitation of pytest-xdist -> cannot handle unordered (sets) iterables + filter( + lambda x: not (hasattr(builtins, x.__name__) or x == JSONDecodeError), + errors._exception_register, + ), + key=lambda x: x.__name__, + ), +) +def test_custom_exception_api(custom: type[Exception]) -> None: + """Test we conform to Custom exception API.""" + assert issubclass(custom, Exception), "Expected to subclass Exception." + assert hasattr(custom, "response"), "Expected to have response method." + + +@pytest.mark.parametrize( + ["cls", "args"], + [ + [errors.ParsingError, ()], + [errors.SchemaError, (1,)], + ], +) +def test_custom_exception_response(cls: type[Exception], args: tuple | None) -> None: + """Confirm response method returns an instance of the class.""" + result = cls.response(*args) + assert isinstance(result, cls), ( + f"Expected to return an instance of exception: {cls.__name__}." + ) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py new file mode 100644 index 0000000..6e68831 --- /dev/null +++ b/tests/unit/test_handlers.py @@ -0,0 +1,109 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test json data handlers module.""" + +import tempfile +from collections.abc import Callable +from io import BytesIO, StringIO +from typing import IO, Any + +import pytest + +from BenchMatcha import handlers + + +def test_load(mock_data: str) -> None: + """Test load interface.""" + result = handlers.load(mock_data) + assert isinstance(result, dict), "Expected to load a dict." + + +@pytest.mark.parametrize( + ["handler", "transformer"], + [ + (handlers.HandlePath, lambda x: x), + (handlers.HandleBytes, lambda x: x.encode()), + (handlers.HandleIO, lambda x: StringIO(x)), + (handlers.HandleIO, lambda x: BytesIO(x.encode())), + ], +) +def test_handlers( + handler: type[handlers.Handler], + transformer: Callable[[str], Any], + mock_data: str, +) -> None: + """""" + mock = transformer(mock_data) + result = handler(mock).handle() + assert isinstance(result, dict), "Expected a dictionary loaded." + assert isinstance(handlers.dispatch(mock), handler), ( + "Expected sample handler object." + ) + + if isinstance(mock, (StringIO, BytesIO)): + mock.close() + + +def test_stream_type_error(): + """Confirm a type error is raised when an unreadable stream is provided.""" + with pytest.raises(TypeError) as err, tempfile.NamedTemporaryFile("w") as file: + handlers.HandleIO(file) + + assert err.type is TypeError, "Expected a TypeError to be raised." + + +@pytest.mark.parametrize( + ["handler", "transformer", "expected"], + [ + (handlers.dispatch, lambda x: x.name, handlers.HandlePath), + (handlers.dispatch, lambda x: x.read(), handlers.HandlePath), + (handlers.dispatch, lambda x: x.read().encode(), handlers.HandleBytes), + (handlers.dispatch, lambda x: x, handlers.HandleIO), + ], +) +def test_file_path_handlers( + handler: Callable[[Any], handlers.Handler], + transformer: Callable[[IO], Any], + expected: handlers.Handler, + mock_file: IO, +) -> None: + """Test dispatch handlers.""" + result: handlers.Handler = handler(transformer(mock_file)) + assert isinstance(result, expected), "Unexpected Handler dispatched." + data = result.handle() + assert isinstance(data, dict), "Expected dictionary object." + + +def test_dispatch_unsupported_object() -> None: + """Test we raise TypeError when providing unsupported object.""" + with pytest.raises(TypeError) as e: + handlers.dispatch(1) + + assert e.type is TypeError, "Expected a type error." diff --git a/tests/unit/test_plotting.py b/tests/unit/test_plotting.py new file mode 100644 index 0000000..6e520a4 --- /dev/null +++ b/tests/unit/test_plotting.py @@ -0,0 +1,146 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""unit test plotting module.""" + +import tempfile + +import numpy as np +import plotly.graph_objs as go +import pytest + +from BenchMatcha import plotting + + +def test_serialization_to_html(): + """Confirm a plotly figure is saved to a html file.""" + figure = go.Figure() + + with tempfile.NamedTemporaryFile("w+") as file: + plotting.to_html(figure, file.name) + file.seek(0) + data: str = file.read(1024) + + assert data.startswith("
"), "Expected html serialization." + + +def test_serialization_to_json(): + """Confirm a plotly figure is saved to a json file.""" + figure = go.Figure() + + with tempfile.NamedTemporaryFile("w+") as file: + plotting.to_json(figure, file.name) + file.seek(0) + data: str = file.read(1024) + + assert data.startswith("{"), "Expected json serialization." + + +@pytest.mark.parametrize( + ["x", "length", "vals", "labels"], + [ + ( + np.asarray([2, 256]), + 9, + [1, 2, 4, 8, 16, 32, 64, 128, 256], + [ + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + ], + ), + ( + np.asarray([4, 4]), + 2, + [2, 4], + [ + "21", + "22", + ], + ), + ], +) +def test_construct_log2_axis( + x: np.ndarray, + length: int, + vals: list[int], + labels: list[str], +) -> None: + """Confirm a log 2 axis is constructed correctly""" + a, b = plotting.construct_log2_axis(x) + + assert len(a) == len(b) == length + assert a == vals, "Incorrect values" + assert b == labels, "Incorrect plotly labels." + + +def test_create_scatter_trace() -> None: + """Confirm function constructs a plotly scatter trace.""" + x = np.arange(5) + y = np.arange(25).reshape(5, 5) + result = plotting.create_scatter_trace(x, y, "test", "black") + assert isinstance(result, go.Scatter) + + +def test_create_box_plot() -> None: + """Confirm function constructs a plotly box plot trace.""" + x = np.arange(5) + y = np.arange(25).reshape(5, 5) + result = plotting.box_plot(x, y, "test", "black", "red") + assert isinstance(result, go.Box) + + +def test_create_annotation_text() -> None: + """Test construction of plot annotation data.""" + result = plotting.create_annotation_text("test", 1.0) + assert isinstance(result, dict), "expected a dictionary return type." + assert "text" in result, "Expected text annotation key." + + +@pytest.mark.parametrize( + ["key"], + [["(1)"], ["N"], ["lgN"], ["NlgN"], ["N^2"], ["N^3"], ["invalid"]], +) +def test_get_big_o_function(key: str) -> None: + """Test retrieval of complexity function.""" + result = plotting.get_big_o_function(key) + assert callable(result), "Expected a callable." + + +def test_draw_complexity_line() -> None: + """Confirm function produces a scatter trace.""" + x = np.arange(20) + result = plotting.draw_complexity_line(x, 1.2, "N", "test", "red") + assert isinstance(result, go.Scatter) diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py new file mode 100644 index 0000000..ca61211 --- /dev/null +++ b/tests/unit/test_runner.py @@ -0,0 +1,42 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""unit test runner module.""" + +import pytest + +from BenchMatcha import runner + + +def test_manage_registration_file_not_found_error(): + """Confirm invalid files raise correct error.""" + with pytest.raises(FileNotFoundError) as err: + runner.manage_registration("cthulu") + + assert err.type is FileNotFoundError diff --git a/tests/unit/test_sifter.py b/tests/unit/test_sifter.py new file mode 100644 index 0000000..f1ba107 --- /dev/null +++ b/tests/unit/test_sifter.py @@ -0,0 +1,95 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit test BenchMatcha.sifter module.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from BenchMatcha import sifter + + +class MockDirEntry: + """mock os.DirEntry object.""" + + path: str + directory: bool + + def __init__(self, path: str, directory: bool = False): + self.path = path + self.directory = directory + + def is_dir(self, **kwargs) -> bool: + return self.directory + + def __str__(self): + return self.path + + def __eq__(self, value: MockDirEntry) -> bool: + return value.path == self.path and value.directory == self.directory + + +@patch.object(sifter.Collector, "get") +@patch.object(sifter, "scandir") +def test_collect(mock_scandir: MagicMock, mock_get: MagicMock): + """Unit test sifter.collect method.""" + mock_scandir.side_effect = [ + iter( + ( + MockDirEntry("root/bench_this.py"), + MockDirEntry("root/other.py"), + MockDirEntry("root/sub", True), + ) + ), + iter( + ( + MockDirEntry("root/sub/bench_sub.py"), + MockDirEntry("root/sub/bench_sub2.py"), + ) + ), + ] + mock_get.side_effect = [ + iter(("root/bench_this.py",)), + iter(("root/sub/bench_sub.py", "root/sub/bench_sub2.py")), + ] + + result: list[str] = list(sifter.collect("root")) + assert len(result) == 3, "Expected 3 results." + + expected = [ + "root/bench_this.py", + "root/sub/bench_sub.py", + "root/sub/bench_sub2.py", + ] + + for a, b in zip(result, expected, strict=False): + assert a == b, "Unexpected DirEntry." + + assert len(set(result).difference(set(expected))) == 0, "expected identical output." diff --git a/tests/unit/test_structure.py b/tests/unit/test_structure.py new file mode 100644 index 0000000..ffbe0c0 --- /dev/null +++ b/tests/unit/test_structure.py @@ -0,0 +1,130 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit test structure module.""" + +from collections.abc import Callable, Iterator +from datetime import UTC, datetime + +import pytest + +from BenchMatcha import errors, structure +from BenchMatcha.handlers import load + + +@pytest.fixture +def shunt_version() -> Iterator[Callable[[tuple[int, ...]], tuple[int, ...]]]: + previous = structure.SUPPORTED_VERSIONS + + def modify(value: tuple[int, ...]) -> tuple[int, ...]: + structure.SUPPORTED_VERSIONS = value + return value + + yield modify + + structure.SUPPORTED_VERSIONS = previous + assert structure.SUPPORTED_VERSIONS == previous + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + ("2025-07-13T12:09:31-07:00", datetime(2025, 7, 13, 19, 9, 31, tzinfo=UTC)), + ("2025-07-12T14:01:22+07:00", datetime(2025, 7, 12, 7, 1, 22, tzinfo=UTC)), + ], +) +def test_parse_datetime(value: str, expected: datetime) -> None: + """Test parsing of ISO8601 format datetime string.""" + result = structure.parse_datetime(value) + assert isinstance(result, datetime), "Expected datetime object." + assert result == expected, "Unexpected date time." + + +def test_parse_json_data(mock_data: str) -> None: + """Test we correctly parse json data into a dataclass.""" + data = load(mock_data) + result = structure.BenchmarkContext.from_json(data) + assert isinstance(result, structure.BenchmarkContext) + assert result.date == datetime(2025, 7, 13, 19, 9, 31, tzinfo=UTC) + assert result.host_name == "host" + assert result.executable == "file.py" + assert result.num_cpus == 12 + assert result.mhz_per_cpu == 24 + + assert result.cpu_scaling_enabled is False + assert len(result.caches) == 3 + assert result.load_avg == [4.69092, 4.60693, 4.47949] + assert result.library_version == "1.9.4" + assert result.library_build_type == "release" + assert result.json_schema_version == 1 + + assert len(result.benchmarks) == 1 + + +def test_convert_BenchmarkContext_to_json(mock_data: str) -> None: + """Test we convert dataclass into dictionary json like objects.""" + data = load(mock_data) + obj = structure.BenchmarkContext.from_json(data) + result = obj.to_json() + assert isinstance(result, dict), "Expected a dictionary object." + + caches = result["caches"] + assert isinstance(caches, list) + assert all(isinstance(i, dict) for i in caches), ( + "Expected cache instances to be dictionary objects." + ) + + benchmarks = result["benchmarks"] + assert isinstance(benchmarks, list) + assert all(isinstance(i, dict) for i in benchmarks), ( + "Expected benchmark instances to be dictionary objects." + ) + assert all(isinstance(k["complexity"], dict) for k in benchmarks), ( + "Expected benchmark complexity instances to be dictionary objects." + ) + + +def test_unavailable_version() -> None: + """Confirm SchemaError is raised when unavailable version is passed.""" + record: dict = {"context": {"json_schema_version": -10}} + with pytest.raises(errors.SchemaError) as err: + structure.parse_version(record) + + assert err.type is errors.SchemaError, "Expected to raise a SchemaError." + + +def test_unsupported_version(shunt_version) -> None: + """Confirm SchemaError is raised when unsupported version is passed.""" + shunt_version((-10,)) + + record: dict = {"context": {"json_schema_version": -10}} + with pytest.raises(errors.SchemaError) as err: + structure.parse_version(record) + + assert err.type is errors.SchemaError, "Expected to raise a SchemaError." diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..4ef1d61 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,96 @@ +# BSD 3-Clause License +# +# Copyright (c) 2025, Spill-Tea +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import numpy as np +import pytest + +from BenchMatcha import utils + + +_power2: list[tuple[int, int]] = [ + (0, 0), + (1, 1), +] +for j in range(2, 24): + value = 1 << j + _power2.append((value - 1, value)) + _power2.append((value, value)) + _power2.append((value + 1, 1 << (j + 1))) + + +@pytest.mark.parametrize(["value", "expected"], _power2) +def test_power_of_2(value: int, expected: int): + """Test returns next power of two.""" + result = utils.power_of_2(value) + assert result == expected, f"Unexpected result: {result}" + + +def test_simple_stats(): + """Test mean and std.""" + x = np.asarray([[1, 2, 3], [2, 3, 1], [3, 1, 2]]) + result = utils._simple_stats(x) + assert isinstance(result, tuple), "Expected a tuple." + assert np.all(result[0] == np.asarray([2, 2, 2])) + assert np.all(result[1] == np.asarray([1, 1, 1])) + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + ("o1", "(1)"), + ("oN", "N"), + ("oNSquared", "N^2"), + ("oNCubed", "N^3"), + ("oLogN", "lgN"), + ("oNLogN", "NlgN"), + ("oLambda", "f(N)"), + ], +) +def test_bigo_enum_get(value: str, expected: str): + """Test conversion of big o notation identifier get classmethod.""" + result: str = utils.BigO.get(value) + assert result == expected, "Unexpected result." + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + ("(1)", "o1"), + ("N", "oN"), + ("N^2", "oNSquared"), + ("N^3", "oNCubed"), + ("lgN", "oLogN"), + ("NlgN", "oNLogN"), + ("f(N)", "oLambda"), + ], +) +def test_bigo_enum_back(value: str, expected: str): + """Test conversion of big o notation identifier back classmethod.""" + result: str = utils.BigO.back(value) + assert result == expected, "Unexpected result." diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index 7c7bf28..d6a8220 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -29,7 +29,7 @@ """Example unit tests.""" -from PyTemplate import __version__ +from BenchMatcha import __version__ def test_version_type(): diff --git a/tox.ini b/tox.ini index 93f3f78..8280b6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -envlist = type, lint, coverage, docs, py{38,39,310,311,312}-tests +envlist = type, lint, coverage, docs, py{311,312,313}-tests [testenv] description = Base Environment @@ -12,7 +12,7 @@ commands_pre = commands = coverage run --rcfile pyproject.toml -m pytest {posargs} -[testenv:py{38,39,310,311,312}-tests] +[testenv:py{311,312,313}-tests] description = Run Unit Tests commands_pre = {envpython} --version @@ -23,7 +23,7 @@ description = Report Code Coverage skip_install = true deps = coverage parallel_show_output = true -depends = py{38,39,310,311,312}-tests +depends = py{311,312,313}-tests commands = coverage combine --quiet --rcfile pyproject.toml coverage report --rcfile pyproject.toml {posargs}