diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index b11a517e..106dce5c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -19,6 +19,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint + pip install pyyaml - name: Analysing the code with pylint run: | diff --git a/requirements.txt b/requirements.txt index d3dfdb3d..d4057cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pytest pylint codechecker >= 6.26 +pyyaml diff --git a/test/foss/README.md b/test/foss/README.md new file mode 100644 index 00000000..a66c1317 --- /dev/null +++ b/test/foss/README.md @@ -0,0 +1,15 @@ +# Add new FOSS test project: + +To add a new FOSS test project, add a yaml file to this directory. +This yaml file should specify: +- **\[name\]** The name of the project. +- **\[url\]** The git url of the project. +- **\[targets\]** A list of targets to be tested. + - \[name\] Name of the target. +- **\[version_tags\]** A list of bazel versions and project hashes to test on. + - \[bazel_version\] The major bazel version to be tested on as a string. + - \[hash\] The tag or commit hash to be checked out for this bazel version. + - \[bzlmod\] Boolean wether to use the bzlmod (`MODULE.bazel`) or the legacy (`WORKSPACE`) system. (Default to False on bazel 6, 7 True on bazel 8 and onwards) + - \[patch\] Optional cmd to run after successfully cloned and checked out the repository, and have set up targets. (Should be a list, starting with the full path of an executable, e.g. `["/usr/bin/env bash", ...]`) + +For template use any of the existing project configurations. diff --git a/test/foss/test_foss.py b/test/foss/test_foss.py index 6dd78d3b..325f7464 100644 --- a/test/foss/test_foss.py +++ b/test/foss/test_foss.py @@ -17,30 +17,39 @@ """ import logging +import shutil +import subprocess import unittest import os import tempfile from pathlib import Path from types import FunctionType +import yaml from common.base import TestBase ROOT_DIR = f"{os.path.dirname(os.path.abspath(__file__))}/" -NOT_PROJECT_FOLDERS = ["templates", "__pycache__", ".pytest_cache"] -def get_test_dirs() -> list[str]: +def get_test_config() -> list[Path]: """ - Collect directories containing a test project + Collect config files for test projects """ - dirs = [] - for entry in os.listdir(ROOT_DIR): - full_path = os.path.join(ROOT_DIR, entry) - if os.path.isdir(full_path) and entry not in NOT_PROJECT_FOLDERS: - dirs.append(entry) - return dirs + path = Path(ROOT_DIR) + yaml_files = list(path.rglob("*.yaml")) + list(path.rglob("*.yml")) + return yaml_files -PROJECT_DIRS = get_test_dirs() +def get_bazel_version(): + """ + Return the installed Bazel version string + """ + out = subprocess.check_output(["bazel", "--version"], text=True).strip() + version = out.split(" ")[1] + return version + + +BAZEL_VERSION = get_bazel_version() +BAZEL_MAJOR_VERSION = BAZEL_VERSION.split(".")[0] # This will contain the generated tests. @@ -58,66 +67,154 @@ class FOSSTestCollector(TestBase): # Creates test functions with the parameter: directory_name. Based on: # https://eli.thegreenplace.net/2014/04/02/dynamically-generating-python-test-cases -def create_test_method(directory_name: str) -> FunctionType: +def create_test_method( + project_name: str, + url: str, + targets: list[dict[str, str]], + context, + bzlmod, +) -> FunctionType: """ Returns a function pointer that points to a function for the given directory """ + git_hash = context["hash"] + patch = context.get("patch", "") def test_runner(self) -> None: - project_root = os.path.join(ROOT_DIR, directory_name) - - self.assertTrue( - os.path.exists(os.path.join(project_root, "init.sh")), - f"Missing 'init.sh' in {directory_name}\n" - + "Please consult with the README on how to add a new FOSS project", - ) with tempfile.TemporaryDirectory() as test_dir: self.assertTrue(os.path.exists(test_dir)) logging.info("Initializing project...") - ret, _, _ = self.run_command( - f"sh init.sh {test_dir}", project_root + subprocess.run(["git", "clone", url, test_dir], check=True) + subprocess.run( + [ + "git", + "-C", + test_dir, + "checkout", + git_hash, # pyright: ignore + ], + check=True, ) - skip_test = Path(os.path.join(test_dir, ".skipfosstest")) - if os.path.exists(skip_test): - self.skipTest( - "This project is not compatible with this bazel version" + + bazelversion = Path("../../.bazelversion") + if bazelversion.is_file(): + shutil.copy( + bazelversion, os.path.join(test_dir, ".bazelversion") + ) + + build_file = Path(os.path.join(test_dir), "BUILD") + if not build_file.is_file(): + build_file = Path(os.path.join(test_dir), "BUILD.bazel") + if not build_file.is_file(): + self.fail( + f"No build file found for project {project_name}", + ) + + with open(build_file, "a", encoding="utf-8") as f: + f.write( + "#-------------------------------------------------------\n" + "# codechecker rules\n" + "load(\n" + '"@rules_codechecker//src:codechecker.bzl",\n' + '"codechecker_test",\n' + ")\n" + ) + for target in targets: + target_name = target["name"] + f.write( + "codechecker_test(\n" + f'name = "codechecker_test_{target_name}",\n' + "targets = [\n" + f'":{target_name}",\n' + "],\n" + ")\n" + ) + f.write( + "codechecker_test(\n" + f'name = "per_file_test_{target_name}",\n' + "targets = [\n" + f'":{target_name}",\n' + "],\n" + "per_file = True,\n" + ")\n" + ) + f.write( + "#-------------------------------------------------------\n" + ) + if bzlmod: + module_template = Path("templates/MODULE.template").read_text( + "utf-8" ) - module_file = Path(os.path.join(test_dir, "MODULE.bazel")) - if os.path.exists(module_file): - content = module_file.read_text("utf-8").replace( + module_final = module_template.replace( "{rule_path}", f"{os.path.dirname(os.path.abspath(__file__))}/../../", ) - module_file.write_text(content, "utf-8") - workspace_file = Path( - os.path.join(test_dir, "WORKSPACE") - ) - if os.path.exists(workspace_file): - content = workspace_file.read_text("utf-8").replace( + module_file = Path(os.path.join(test_dir, "MODULE.bazel")) + with open(module_file, "a", encoding="utf-8") as f: + f.write(module_final) + if BAZEL_MAJOR_VERSION == "6": + with open( + os.path.join(test_dir, ".bazelrc"), + "a", + encoding="utf-8", + ) as f: + f.write("common --enable_bzlmod") + Path(os.path.join(test_dir, "WORKSPACE")).touch() + else: + workspace_template = Path( + "templates/WORKSPACE.template" + ).read_text("utf-8") + workspace_final = workspace_template.replace( "{rule_path}", f"{os.path.dirname(os.path.abspath(__file__))}/../../", ) - workspace_file.write_text(content, "utf-8") + workspace_file = Path(os.path.join(test_dir, "WORKSPACE")) + with open(workspace_file, "a", encoding="utf-8") as f: + f.write(workspace_final) + if patch: + subprocess.run(patch, cwd=test_dir, check=True) logging.info("Running monolithic rule...") - ret, _, stderr = self.run_command( - "bazel build :codechecker_test", test_dir - ) - self.assertEqual(ret, 0, stderr) + for target in targets: + ret, _, stderr = self.run_command( + f"bazel build :codechecker_test_{target['name']}", + test_dir, + ) + self.assertEqual(ret, 0, stderr) logging.info("Running per_file rule...") - ret, _, stderr = self.run_command( - "bazel build :per_file_test", test_dir - ) - self.assertEqual(ret, 0, stderr) + for target in targets: + ret, _, stderr = self.run_command( + f"bazel build :per_file_test_{target['name']}", test_dir + ) + self.assertEqual(ret, 0, stderr) return test_runner # Dynamically add a test method for each project -# For each project directory it adds a new test function to the class -# This must be outside of the __main__ if, pytest doesn't run it that way -for dir_name in PROJECT_DIRS: - test_name = f"test_{dir_name}" - setattr(FOSSTestCollector, test_name, create_test_method(dir_name)) +# For each project config it adds new test functions to the class +# This must be outside of the __main__, to work well with pytest +for config_file in get_test_config(): + CONTENT = None + with open(config_file, "r", encoding="utf-8") as conf: + CONTENT = yaml.safe_load(conf) + assert CONTENT is not None + test_name: str = CONTENT["name"] + + for tag in CONTENT["version_tags"]: + bazel_version: str = tag["bazel_version"] + bzlmod_on: bool = tag.get("bzlmod", (int(bazel_version) >= 8)) + if bazel_version == BAZEL_MAJOR_VERSION: + setattr( + FOSSTestCollector, + f"test_{test_name}_{'bzlmod' if bzlmod_on else 'workspace'}", + create_test_method( + test_name, + CONTENT["url"], + CONTENT["targets"], + tag, + bzlmod_on, + ), + ) if __name__ == "__main__": unittest.main() diff --git a/test/foss/yaml-cpp.yaml b/test/foss/yaml-cpp.yaml new file mode 100644 index 00000000..e9cc82d4 --- /dev/null +++ b/test/foss/yaml-cpp.yaml @@ -0,0 +1,17 @@ +name: "yaml-cpp" +url: "https://github.com/jbeder/yaml-cpp.git" +targets: + - name: "yaml-cpp" +version_tags: + - bazel_version: "6" + hash: "yaml-cpp-0.7.0" + bzlmod: False + - bazel_version: "7" + hash: "yaml-cpp-0.7.0" + bzlmod: False + - bazel_version: "7" + hash: "yaml-cpp-0.9.0" + bzlmod: True + - bazel_version: "8" + hash: "yaml-cpp-0.9.0" + bzlmod: True diff --git a/test/foss/yaml-cpp/init.sh b/test/foss/yaml-cpp/init.sh deleted file mode 100755 index d6c901b1..00000000 --- a/test/foss/yaml-cpp/init.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Copyright 2023 Ericsson AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ -z "$1" ]; then - echo "[Error]: Missing parameter." - echo "Usage: $0 [folder_name]" - printf "%s %s %s\n" \ - "[WARNING]: This script was meant to be used in automated testing." \ - "To use it manually, provide a folder name where the project" \ - "should be initialized." - exit 1 -fi - -# Skip this test on bazel 8 -MAJOR_VERSION=$(bazel --version | cut -d' ' -f2 | cut -d'.' -f1) -if [ "$MAJOR_VERSION" -ge 8 ]; then - echo "" >> $1/.skipfosstest - exit 0 -fi - -git clone https://github.com/jbeder/yaml-cpp.git "$1" -git -C "$1" checkout yaml-cpp-0.7.0 - -# This file must be in the root of the project to be analyzed for bazelisk to work -bazelversion="../../../.bazelversion" -[ -f $bazelversion ] && cp $bazelversion "$1" - -# Add codechecker to the project -cat <> "$1/BUILD.bazel" -#------------------------------------------------------- - -# codechecker rules -load( - "@rules_codechecker//src:codechecker.bzl", - "codechecker_test", -) - - -codechecker_test( - name = "codechecker_test", - targets = [ - ":yaml-cpp", - ], -) - -codechecker_test( - name = "per_file_test", - targets = [ - ":yaml-cpp", - ], - per_file = True, -) - -#------------------------------------------------------- -EOF - -# Add rules_codechecker repo to WORKSPACE -cat ../templates/WORKSPACE.template >> "$1/WORKSPACE" diff --git a/test/foss/zlib-module/init.sh b/test/foss/zlib-module/init.sh deleted file mode 100644 index eb818e63..00000000 --- a/test/foss/zlib-module/init.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Copyright 2023 Ericsson AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ -z "$1" ]; then - echo "[Error]: Missing parameter." - echo "Usage: $0 [folder_name]" - printf "%s %s %s\n" \ - "[WARNING]: This script was meant to be used in automated testing." \ - "To use it manually, provide a folder name where the project" \ - "should be initialized." - exit 1 -fi - -git clone --recurse https://github.com/madler/zlib.git "$1" -git -C "$1" checkout 5a82f71ed1dfc0bec044d9702463dbdf84ea3b71 - -# This file must be in the root of the project to be analyzed for bazelisk to work -bazelversion="../../../.bazelversion" -[ -f $bazelversion ] && cp $bazelversion "$1" - -# Add codechecker to the project -cat <> "$1/BUILD.bazel" -#------------------------------------------------------- - -# codechecker rules -load( - "@rules_codechecker//src:codechecker.bzl", - "codechecker_test", -) -codechecker_test( - name = "codechecker_test", - targets = [ - ":z", - ], -) - -codechecker_test( - name = "per_file_test", - targets = [ - ":z", - ], - per_file = True, -) - -#------------------------------------------------------- -EOF - -# Enable MODULE.bazel (in Bazel 6) -echo "common --enable_bzlmod" > "$1/.bazelrc" -# Add rules_codechecker repo MODULE.bazel -cat ../templates/MODULE.template >> "$1/MODULE.bazel" -# An empty workspace file is required to keep Bazel versions older than 6.5.0 -# in the project directory -touch "$1/WORKSPACE" diff --git a/test/foss/zlib.yaml b/test/foss/zlib.yaml new file mode 100644 index 00000000..e0572b22 --- /dev/null +++ b/test/foss/zlib.yaml @@ -0,0 +1,20 @@ +name: "zlib" +url: "https://github.com/madler/zlib.git" +targets: + - name: "z" +version_tags: + - bazel_version: "6" + hash: "5a82f71ed1dfc0bec044d9702463dbdf84ea3b71" + bzlmod: False + - bazel_version: "6" + hash: "5a82f71ed1dfc0bec044d9702463dbdf84ea3b71" + bzlmod: True + - bazel_version: "7" + hash: "5a82f71ed1dfc0bec044d9702463dbdf84ea3b71" + bzlmod: False + - bazel_version: "7" + hash: "5a82f71ed1dfc0bec044d9702463dbdf84ea3b71" + bzlmod: True + - bazel_version: "8" + hash: "5a82f71ed1dfc0bec044d9702463dbdf84ea3b71" + bzlmod: True diff --git a/test/foss/zlib/init.sh b/test/foss/zlib/init.sh deleted file mode 100644 index f24b979a..00000000 --- a/test/foss/zlib/init.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Copyright 2023 Ericsson AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ -z "$1" ]; then - echo "[Error]: Missing parameter." - echo "Usage: $0 [folder_name]" - printf "%s %s %s\n" \ - "[WARNING]: This script was meant to be used in automated testing." \ - "To use it manually, provide a folder name where the project" \ - "should be initialized." - exit 1 -fi - -# Skip this test on bazel 8 -MAJOR_VERSION=$(bazel --version | cut -d' ' -f2 | cut -d'.' -f1) -if [ "$MAJOR_VERSION" -ge 8 ]; then - echo "" >> $1/.skipfosstest - exit 0 -fi - -git clone https://github.com/madler/zlib.git "$1" -git -C "$1" checkout 5a82f71ed1dfc0bec044d9702463dbdf84ea3b71 - -# This file must be in the root of the project to be analyzed for bazelisk to work -bazelversion="../../../.bazelversion" -[ -f $bazelversion ] && cp $bazelversion "$1" - -# Add codechecker to the project -cat <> "$1/BUILD.bazel" -#------------------------------------------------------- - -# codechecker rules -load( - "@rules_codechecker//src:codechecker.bzl", - "codechecker_test", -) - - -codechecker_test( - name = "codechecker_test", - targets = [ - ":z", - ], -) - -codechecker_test( - name = "per_file_test", - targets = [ - ":z", - ], - per_file = True, -) - -#------------------------------------------------------- -EOF - -# Add rules_codechecker repo to WORKSPACE -cat ../templates/WORKSPACE.template >> "$1/WORKSPACE"