diff --git a/src/BenchMatcha/config.py b/src/BenchMatcha/config.py index e76913e..b3fd28d 100644 --- a/src/BenchMatcha/config.py +++ b/src/BenchMatcha/config.py @@ -40,11 +40,20 @@ class Config: - """default configuration.""" + """default configuration. + + Attributes: + color (str): plot marker color. + line_color (str): plot line color. + font (str): plot font family style. + x_axis (int): Maximum number of line ticks on x-axis. + + """ color: str = plotting.Prism[3] line_color: str = plotting.Prism[4] font: str = "Space Grotesk Light, Courier New, monospace" + x_axis: int = 13 class ConfigUpdater: @@ -92,6 +101,7 @@ def update_config_from_pyproject(path: str) -> None: color="#FFF" line_color="#333" font="Courier" + x_axis=5 """ cu = ConfigUpdater(path) diff --git a/src/BenchMatcha/plotting.py b/src/BenchMatcha/plotting.py index db0d854..042e908 100644 --- a/src/BenchMatcha/plotting.py +++ b/src/BenchMatcha/plotting.py @@ -72,8 +72,8 @@ def construct_log2_axis(x: np.ndarray) -> tuple[list[int], list[str]]: minimum = int(x.min()) maximum = power_of_2(int(x.max())) + 1 current = power_of_2(minimum) - if 1 < current >= minimum: - current //= 2 + if current >= minimum: + current = max(1, current // 2) power = int(np.log2(current)) while current < maximum: diff --git a/src/BenchMatcha/runner.py b/src/BenchMatcha/runner.py index 9614012..ac15962 100644 --- a/src/BenchMatcha/runner.py +++ b/src/BenchMatcha/runner.py @@ -29,6 +29,7 @@ """Primary Benchmark Runner.""" +import argparse import logging import os import sys @@ -40,6 +41,8 @@ from wurlitzer import pipes # type: ignore[import-untyped] from . import plotting + +# from .complexity import analyze_complexity from .config import Config, update_config_from_pyproject from .errors import ParsingError from .handlers import HandleText @@ -53,6 +56,7 @@ def manage_registration(path: str) -> None: """Manage import, depending on whether path is a directory or file.""" abspath: str = os.path.abspath(path) + log.debug("Loading path: %s", abspath) if not os.path.exists(abspath): raise FileNotFoundError("Invalid filepath") @@ -65,8 +69,10 @@ def manage_registration(path: str) -> None: else: log.warning( "Unsupported path provided. While the path does exist, it is neither a" - " file nor a directory." + " python file nor a directory: %s", + abspath, ) + raise TypeError(f"Unsupported path type: {abspath}") def plot_benchmark_array(benchmark: BenchmarkArray) -> go.Figure: @@ -99,7 +105,7 @@ def plot_benchmark_array(benchmark: BenchmarkArray) -> go.Figure: ) vals, labels = plotting.construct_log2_axis(benchmark.size) - if (p := len(vals) // 13) > 0: + if (p := len(vals) // Config.x_axis) > 0: vals = vals[:: p + 1] labels = labels[:: p + 1] @@ -128,8 +134,8 @@ def plot_benchmark_array(benchmark: BenchmarkArray) -> go.Figure: return fig +# TODO: Consider defining CLI Exit Status in an Enum def _run() -> BenchmarkContext: - # TODO: Improve logic here if "--benchmark_format=json" not in sys.argv: sys.argv.append("--benchmark_format=json") @@ -146,8 +152,13 @@ def _run() -> BenchmarkContext: ... text: str = stdout.read() + error: str = stderr.read() stdout.close(), stderr.close() # pylint: disable=W0106 + # Pass stderr from google_benchmark + if len(error): + log.error(error) + handler = HandleText(text) try: obj: dict = handler.handle() @@ -182,39 +193,147 @@ def save(context: BenchmarkContext, cache_dir: str) -> None: f.write(serialized) -def run(path: str, cache_dir: str) -> None: +def run(cache_dir: str) -> None: """BenchMatcha Runner.""" - manage_registration(path) + context: BenchmarkContext = _run() - # TODO: remove arguments specific to BenchMatch to prevent failure on google - # benchmark interface. + # TODO: Capture re-analyzed complexity information. Determine where to store, or + # how to present this information in a manner that is useful. + # for bench in context.benchmarks: + # analyze_complexity(bench.size, bench.real_time) - context: BenchmarkContext = _run() save(context, cache_dir) -# TODO: Handle a list of separated filepaths. -# def run_paths(paths: list[str]) -> None: -# """Run benchmarks against a list of paths.""" -# for path in paths: -# manage_registration(path) +def get_args() -> argparse.Namespace: + """Get BenchMatcha command line arguments and reset to support google_benchmark.""" + args = argparse.ArgumentParser("benchmatcha", conflict_handler="error") + args.add_argument( + "-v", + "--verbose", + action="store_true", + help="Set Logging Level to DEBUG.", + required=False, + ) + args.add_argument( + "-c", + "--color", + default=None, + help="Scatterplot marker color.", + required=False, + ) + args.add_argument( + "-l", + "--line-color", + default=None, + help="Scatterplot complexity fit line color.", + required=False, + ) + args.add_argument( + "-x", + "--x-axis", + default=None, + help="Maximum Number of units displayed on x-axis.", + required=False, + type=int, + ) + + cwd: str = os.getcwd() + args.add_argument( + "--config", + default=os.path.join(cwd, "pyproject.toml"), + help="Path location of pyproject.toml configuration file. " + "Defaults to Current Working Directory.", + ) + args.add_argument( + "--cache", + default=os.path.join(cwd, ".benchmatcha"), + help="Path location of cache directory. Defaults to Current Working Directory.", + ) + args.add_argument( + "--path", + action="extend", + nargs="+", + help="Valid file or directory path to benchmarks.", + ) -# context: BenchmarkContext = _run() + # Capture anything that doesn't fit (to be fed downstream to google_benchmark cli) + args.add_argument("others", nargs=argparse.REMAINDER) + + # TODO: Plotting over time (pulling from database) + # sub = args.add_subparsers() + # plot = sub.add_parser("plot") + # plot.add_argument( + # "--min-date", + # default=None, + # help="Filter data after minimum date (inclusive).", + # ) + # plot.add_argument( + # "--max-date", + # default=None, + # help="Filter data before date (inclusive).", + # ) + # plot.add_argument("--host", default=None, help="Filter data by specific host.") + # plot.add_argument("--os", default=None, help="Filter data by specific OS type.") + # plot.add_argument( + # "--function", + # default=None, + # help="Filter data to present a specific function name.", + # ) + known, unknown = args.parse_known_args() + + # NOTE: Only validate `benchmark_format` argument from google_benchmark cli, since + # we require json format to correctly work downstream. All other argument + # validations should be handled by google_benchmark cli parsing directly. + problems: list[str] = [] + for k in filter( + lambda x: isinstance(x, str) and "--benchmark_format=" in x, + unknown, + ): + if "json" not in k: + log.warning("Benchmark Format must be json: `%s`", k) + problems.append(k) + for p in problems: + unknown.remove(p) + + # Prune / Reset for google_benchmark + sys.argv = [sys.argv[0], *unknown, *known.others] + + return known def main() -> None: """Primary CLI Entry Point.""" - # TODO: support specification of config file path from CLI, to overwrite default - # TODO: Support command line args to overwrite default config. - cwd: str = os.getcwd() - p = os.path.join(cwd, "pyproject.toml") - if os.path.exists(p): - update_config_from_pyproject(p) + args: argparse.Namespace = get_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + log.setLevel(logging.DEBUG) - # Create cache if does not exist - cache = os.path.join(cwd, ".benchmatcha") - if not os.path.exists(cache): + if os.path.exists(args.config): + log.debug("Updating default configuration from file: %s", args.config) + update_config_from_pyproject(args.config) + + # NOTE: Configuration Args should overwrite values set in config file + if args.color is not None: + log.debug("Overriding color from arg: %s", args.color) + Config.color = args.color + + if args.line_color is not None: + log.debug("Overriding line_color from arg: %s", args.line_color) + Config.line_color = args.line_color + + if args.x_axis is not None: + log.debug("Overriding x_axis from arg: %s", args.x_axis) + Config.x_axis = args.x_axis + + # Create cache directory if it does not exist + if not os.path.exists(cache := args.cache): + log.debug("Creating cache directory at: %s", cache) os.mkdir(cache) - # TODO: Determine if a list of paths have been provided instead, and handle - run(sys.argv.pop(), cache) + # Natively handle multiple provided paths + for path in args.path: + manage_registration(path) + + run(cache) diff --git a/src/BenchMatcha/structure.py b/src/BenchMatcha/structure.py index e108402..3f881a9 100644 --- a/src/BenchMatcha/structure.py +++ b/src/BenchMatcha/structure.py @@ -92,6 +92,7 @@ def from_json(cls, record: dict[str, Any]) -> Self: ) def to_json(self) -> dict: + """Convert to json dictionary object.""" return self.__dict__.copy() @@ -167,6 +168,7 @@ def from_json(cls, record: dict[str, Any]) -> Self: ) def to_json(self) -> dict: + """Convert to json dictionary object.""" return self.__dict__.copy() @@ -194,6 +196,7 @@ class BenchmarkArray: complexity: ComplexityInfo def to_json(self) -> dict: + """Convert to json dictionary object.""" d = self.__dict__.copy() d["complexity"] = self.complexity.to_json() @@ -338,6 +341,7 @@ def from_json(cls, record: dict[str, Any]) -> Self: ) def to_json(self) -> dict: + """Convert to json dictionary object.""" data = self.__dict__.copy() data["caches"] = [i.to_json() for i in self.caches] data["benchmarks"] = [j.to_json() for j in self.benchmarks] diff --git a/src/BenchMatcha/utils.py b/src/BenchMatcha/utils.py index 517cda3..1bda5b9 100644 --- a/src/BenchMatcha/utils.py +++ b/src/BenchMatcha/utils.py @@ -72,10 +72,12 @@ class BigO(enum.StrEnum): @classmethod def get(cls, value: str) -> str: + """Get value from key string.""" # e.g. "o1" -> "(1)" return cls[value].value @classmethod def back(cls, value: str) -> str: + """Get key string from value.""" # e.g. "(1)" -> "o1" return cls(value).name diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 306b5ce..368b789 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -43,14 +43,21 @@ # not introduced until V7.10.3. See the following for details: # https://github.com/nedbat/coveragepy/issues/1499 @pytest.fixture -def benchmark() -> Iterator[Callable[[list[str]], tuple[int, str, str, str]]]: +def benchmark() -> Iterator[ + Callable[[list[str], Callable[[str], None] | None], tuple[int, str, str, str]] +]: """Benchmark entry point subprocess.""" + with tempfile.TemporaryDirectory(dir=HERE) as cursor: - with tempfile.TemporaryDirectory(dir=os.getcwd()) as cursor: + def inner( + args: list[str], + setup: Callable[[str], None] | None = None, + ) -> tuple[int, str, str, str]: + if setup is not None and callable(setup): + setup(cursor) - def inner(args: list[str]) -> tuple[int, str, str, str]: response: subprocess.CompletedProcess[bytes] = subprocess.run( - ["benchmatcha", *args], + ["benchmatcha", "--benchmark_dry_run", *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, diff --git a/tests/integration/test_runner.py b/tests/integration/test_runner.py index 6e495b8..c4dcd34 100644 --- a/tests/integration/test_runner.py +++ b/tests/integration/test_runner.py @@ -39,7 +39,7 @@ DATA: str = os.path.join(HERE, "data") -def _assert_cache_created(cache: str, status: int, error: str) -> None: +def _assert_cache_created(cache: str, status: int) -> None: assert os.path.exists(cache), "Expected cache directory to be created." assert os.path.isdir(cache), "expected path to be a directory." assert os.path.exists(os.path.join(cache, "out.html")), ( @@ -51,7 +51,6 @@ def _assert_cache_created(cache: str, status: int, error: str) -> None: "expected data to be saved." ) assert status == 0, "Expected no errors." - assert len(error) == 0, "Expected no errors" @pytest.mark.parametrize( @@ -69,20 +68,111 @@ def test_bench_directory( benchmark: Callable[[list[str]], tuple[int, str, str, str]], ) -> None: """Test benchmarking a directory of benchmark suites.""" - status, out, error, tmpath = benchmark([path]) - if len(error): - print(error) + status, out, error, tmpath = benchmark(["--path", path]) cache: str = os.path.join(tmpath, ".benchmatcha") - _assert_cache_created(cache, status, error) + _assert_cache_created(cache, status) -def test_json_key_val(benchmark) -> None: +@pytest.mark.parametrize( + ["form"], + [ + ("--benchmark_format=json",), # frivolously provide format + ("--benchmark_format=csv",), # providing incorrect format is overridden + ], +) +def test_json_key_val( + form: str, + benchmark: Callable[[list[str]], tuple[int, str, str, str]], +) -> None: """Confirm including benchmark format proceeds normally.""" path: str = os.path.join(DATA, "single") - status, out, error, tmpath = benchmark(["--benchmark_format=json", path]) - if len(error): - print(error) + status, out, error, tmpath = benchmark([form, "--path", path]) cache: str = os.path.join(tmpath, ".benchmatcha") - _assert_cache_created(cache, status, error) + _assert_cache_created(cache, status) + + +def _setup_pyproject(x: str) -> None: + p: str = os.path.join(x, "pyproject.toml") + with open(p, "w") as f: + ... + + +def test_empty_pyproject_config_file( + benchmark: Callable[[list[str], Callable[[str], None]], tuple[int, str, str, str]], +) -> None: + """Perform run with empty pyproject config.""" + path: str = os.path.join(DATA, "single") + + status, out, error, tmpath = benchmark(["--path", path], _setup_pyproject) + assert os.path.exists(os.path.join(tmpath, "pyproject.toml")), ( + "Expected pyproject config file to be setup." + ) + + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status) + + +def _setup_cache(x: str) -> None: + p: str = os.path.join(x, ".benchmatcha") + os.mkdir(p) + + +def test_when_cache_folder_already_exists( + benchmark: Callable[[list[str], Callable[[str], None]], tuple[int, str, str, str]], +) -> None: + """Perform run where cache already exists.""" + path: str = os.path.join(DATA, "single") + + status, out, error, tmpath = benchmark(["--path", path], _setup_cache) + + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status) + + +def _setup_db(x: str) -> None: + _setup_cache(x) + p: str = os.path.join(x, ".benchmatcha", "benchmark.json") + with open(p, "w") as f: + f.write("[]") + + +def test_previous_db( + benchmark: Callable[[list[str], Callable[[str], None]], tuple[int, str, str, str]], +) -> None: + """Perform run where database already exists.""" + path: str = os.path.join(DATA, "single") + + status, out, error, tmpath = benchmark(["--path", path], _setup_db) + + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status) + + +@pytest.mark.parametrize( + ["param", "value"], + [ + ("--color", "red"), + ("--line-color", "black"), + ("--x-axis", "2"), + ("--verbose", None), + ], +) +def test_config_parameters( + param: str, + value: str | None, + benchmark: Callable[[list[str], Callable[[str], None]], tuple[int, str, str, str]], +) -> None: + """Test available cli flags to modify configuration.""" + path: str = os.path.join(DATA, "single") + args: list[str] = ["--path", path, param] + if value is not None: + args.append(value) + + status, out, error, tmpath = benchmark(args) + cache: str = os.path.join(tmpath, ".benchmatcha") + _assert_cache_created(cache, status) + + if param == "--verbose": + assert "DEBUG" in error, "Expected debug logging in stderr." diff --git a/tests/unit/test_complexity.py b/tests/unit/test_complexity.py index 450edff..da35df9 100644 --- a/tests/unit/test_complexity.py +++ b/tests/unit/test_complexity.py @@ -29,6 +29,8 @@ """Test algorithmic complexity module.""" +from collections.abc import Iterator + import numpy as np import pytest @@ -54,6 +56,18 @@ def coords() -> tuple[np.ndarray, np.ndarray]: return x, y +@pytest.fixture +def shunt_fit() -> Iterator[None]: + def _failed_fit(x, a, b): + raise RuntimeError("Intentionally Fail.") + + comp.complexity_functions["runtime_error"] = _failed_fit + + yield + + comp.complexity_functions.pop("runtime_error") + + def test_fit_result_repr(fit_result: comp.FitResult) -> None: """minor test of fit result repr dunder method.""" result: str = repr(fit_result) @@ -103,7 +117,7 @@ def test_fit(coords: tuple[np.ndarray, np.ndarray]) -> None: assert np.isclose(result.rms, 0.0), "Unexpected error." -def test_analyze_complexity() -> None: +def test_analyze_complexity(shunt_fit) -> None: """Test batch analysis of algorithmic complexity.""" x = np.arange(10, 20) y = np.arange(1, 31).reshape(10, 3)