diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8d2ac3b1c8..f59b16e2cd1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -125,17 +125,19 @@ jobs: # folders that is commented below requires credentials, no need to spare time to run them tests_config: - name: "integ-buildcmd-arm64" - params: "-n 2 --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py" + params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/buildcmd/test_build_cmd_arm64.py" + - name: "integ-buildcmd-python" + params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/buildcmd/test_build_cmd_python.py" - name: "integ-buildcmd-main" - params: "-n 2 --reruns 3 tests/integration/buildcmd/test_build_cmd_dotnet.py tests/integration/buildcmd/test_build_cmd_provided.py tests/integration/buildcmd/test_build_cmd_rust.py tests/integration/buildcmd/test_build_cmd_python.py tests/integration/buildcmd/test_build_cmd_node.py" + params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/buildcmd/test_build_cmd_dotnet.py tests/integration/buildcmd/test_build_cmd_provided.py tests/integration/buildcmd/test_build_cmd_rust.py tests/integration/buildcmd/test_build_cmd_node.py" - name: "integ-buildcmd-java" - params: "-n 2 --reruns 5 tests/integration/buildcmd/test_build_cmd_java.py" + params: "-n 3 --reruns 5 tests/integration/buildcmd/test_build_cmd_java.py" - name: "integ-buildcmd-other" params: "-n 2 --reruns 3 tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_cmd_dotnet.py --ignore=tests/integration/buildcmd/test_build_cmd_provided.py --ignore=tests/integration/buildcmd/test_build_cmd_rust.py --ignore=tests/integration/buildcmd/test_build_cmd_python.py --ignore=tests/integration/buildcmd/test_build_cmd_node.py --ignore=tests/integration/buildcmd/test_build_cmd_java.py" #- "delete" #- "deploy" - name: "integ-all-other" - params: "-n 2 --reruns 3 tests/integration/init tests/integration/local tests/integration/pipeline tests/integration/root tests/integration/scripts tests/integration/telemetry tests/integration/docs --ignore=tests/integration/local/invoke/test_invoke_durable.py --ignore=tests/integration/local/start_api/test_start_api_durable.py --ignore=tests/integration/local/start_lambda/test_start_lambda_durable.py" + params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/init tests/integration/local tests/integration/pipeline tests/integration/root tests/integration/scripts tests/integration/telemetry tests/integration/docs --ignore=tests/integration/local/invoke/test_invoke_durable.py --ignore=tests/integration/local/start_api/test_start_api_durable.py --ignore=tests/integration/local/start_lambda/test_start_lambda_durable.py" - name: "durable-functions" params: "--reruns 3 tests/integration/local/invoke/test_invoke_durable.py tests/integration/local/start_api/test_start_api_durable.py tests/integration/local/start_lambda/test_start_lambda_durable.py" #- "list" @@ -214,22 +216,30 @@ jobs: - "3.10" - "3.11" tests_config: - - name: "Smoke & Functional Tests - All" + - name: "Smoke Tests - First Half" + os: ubuntu-latest + params: "-n 4 tests/smoke" + env_vars: "first-half" + - name: "Smoke Tests - Second Half & Functional Tests" os: ubuntu-latest params: "-n 4 tests/smoke tests/functional" - env_vars: "all" - - name: "Smoke Tests - First Third" + env_vars: "second-half" + - name: "Smoke Tests - First Quarter" + os: windows-latest + params: "-n 4 tests/smoke" + env_vars: "first-quarter" + - name: "Smoke Tests - Second Quarter" os: windows-latest params: "-n 4 tests/smoke" - env_vars: "first-third" - - name: "Smoke Tests - Second Third" + env_vars: "second-quarter" + - name: "Smoke Tests - Third Quarter" os: windows-latest params: "-n 4 tests/smoke" - env_vars: "second-third" - - name: "Smoke Tests - Third Third & Functional Tests" + env_vars: "third-quarter" + - name: "Smoke Tests - Fourth Quarter & Functional Tests" os: windows-latest params: "-n 4 tests/smoke tests/functional" - env_vars: "third-third" + env_vars: "fourth-quarter" steps: - uses: actions/checkout@v6 - name: Set TEMP to D:/Temp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 67ae218d505..4dca73c0237 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,6 +123,20 @@ def test_tier1_my_feature_in_container(self): 4. Each runtime should have one non-container and one container tier 1 test. +### Skipping tests in PR workflow (`pr_skip`) + +Tests marked with `@pytest.mark.pr_skip` are excluded from the PR workflow (`.github/workflows/build.yml`) but still run in the integration workflow (`.github/workflows/integration-tests.yml`). Use this marker for tests that are redundant with other tests in the same file or low-risk for PR validation. + +```python +@pytest.mark.pr_skip +class TestMyRedundantFeature(TestBase): + ... +``` + +The marker is registered in `pytest.ini`. PR workflow jobs filter with `-m "not pr_skip"`. + +Before adding `pr_skip`, verify the test is covered by at least one job in `integration-tests.yml`. + ## Finding contributions to work on Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws/aws-sam-cli/labels/help%20wanted) issues is a great place to start. diff --git a/pytest.ini b/pytest.ini index c0873997738..ab3f535e8f3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -27,3 +27,4 @@ markers = provided dotnet al2023: Marker to note tests that require a newer Docker version on Windows + pr_skip: Tests skipped in PR workflow but run in nightly/integration tests diff --git a/tests/integration/buildcmd/test_build_cmd_arm64.py b/tests/integration/buildcmd/test_build_cmd_arm64.py index 92184408b36..361ff5c6d1b 100644 --- a/tests/integration/buildcmd/test_build_cmd_arm64.py +++ b/tests/integration/buildcmd/test_build_cmd_arm64.py @@ -117,6 +117,7 @@ def test_tier1_node_arm64_build_in_container(self, runtime, code_uri, expected_f @pytest.mark.nodejs +@pytest.mark.pr_skip class TestBuildCommand_EsbuildFunctions_With_External_Manifest_arm64(BuildIntegEsbuildBase): template = "template_with_metadata_esbuild.yaml" MANIFEST_PATH = "Esbuild/npm_manifest/package.json" @@ -672,6 +673,7 @@ def test_tier1_provided_arm64_build(self): self._test_with_Makefile("provided.al2023", False, None, ARM64) +@pytest.mark.pr_skip @skipIf( ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), "Skip build tests on windows when running in CI unless overridden", diff --git a/tests/integration/buildcmd/test_build_cmd_node.py b/tests/integration/buildcmd/test_build_cmd_node.py index 4f132d68be2..20ec6a09d27 100644 --- a/tests/integration/buildcmd/test_build_cmd_node.py +++ b/tests/integration/buildcmd/test_build_cmd_node.py @@ -77,6 +77,7 @@ def test_tier1_node_build_in_container(self): ) +@pytest.mark.pr_skip class TestBuildCommand_EsbuildFunctions_With_External_Manifest(BuildIntegEsbuildBase): template = "template_with_metadata_esbuild.yaml" MANIFEST_PATH = "Esbuild/npm_manifest/package.json" @@ -145,6 +146,7 @@ def test_building_default_package_json( ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), "Skip build tests on windows when running in CI unless overridden", ) +@pytest.mark.pr_skip @parameterized_class( ("template",), [ diff --git a/tests/integration/buildcmd/test_build_cmd_python.py b/tests/integration/buildcmd/test_build_cmd_python.py index bca9aa0b446..139063354e3 100644 --- a/tests/integration/buildcmd/test_build_cmd_python.py +++ b/tests/integration/buildcmd/test_build_cmd_python.py @@ -543,6 +543,7 @@ def test_cdk_app_with_default_requirements(self): self._verify_invoke_built_function(self.built_template, self.FUNCTION_LOGICAL_ID, {}, expected) +@pytest.mark.pr_skip class TestBuildCommand_PythonFunctions_With_Specified_Architecture(BuildIntegPythonBase): template = "template_with_architecture.yaml" diff --git a/tests/integration/docs/test_docs_command.py b/tests/integration/docs/test_docs_command.py index 8eb1b1648a0..1a1a09eccdd 100644 --- a/tests/integration/docs/test_docs_command.py +++ b/tests/integration/docs/test_docs_command.py @@ -1,5 +1,7 @@ import os +import pytest + from parameterized import parameterized from samcli.commands.docs.command_context import SUCCESS_MESSAGE, ERROR_MESSAGE, DocsCommandContext @@ -10,6 +12,7 @@ COMMAND_URL_PAIR = [(command, url) for command, url in Documentation.load().items()] +@pytest.mark.pr_skip class TestDocsCommand(DocsIntegBase): @parameterized.expand(COMMAND_URL_PAIR) def test_docs_command(self, command, url): diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index 8a81afb980e..2ddf6ff86e9 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -568,6 +568,7 @@ def _assert_template_with_cfn_lint(self, cwd): @pytest.mark.xdist_group(name="sam_init") +@pytest.mark.pr_skip class TestInitForParametersCompatibility(TestCase): def test_init_command_no_interactive_missing_name(self): stderr = None diff --git a/tests/integration/scripts/test_copy_terraform_built_artifacts.py b/tests/integration/scripts/test_copy_terraform_built_artifacts.py index ac66df79cfb..d1b8b645a06 100644 --- a/tests/integration/scripts/test_copy_terraform_built_artifacts.py +++ b/tests/integration/scripts/test_copy_terraform_built_artifacts.py @@ -10,6 +10,10 @@ TIMEOUT = 3 +import pytest + + +@pytest.mark.pr_skip class TestCopyTerraformBuiltArtifacts(TestCase): def setUp(self) -> None: self.samcli_root_path = pathlib.Path(__file__).parents[3].joinpath("samcli") diff --git a/tests/smoke/test_all_commands.py b/tests/smoke/test_all_commands.py index ae689dc7e36..00fb0cf75b2 100644 --- a/tests/smoke/test_all_commands.py +++ b/tests/smoke/test_all_commands.py @@ -13,28 +13,26 @@ # Check environment variable to determine which subset to use SMOKE_TEST_SUBSET = os.environ.get("SMOKE_TEST_SUBSET", "").lower() -# Calculate split points for dividing into thirds +# Calculate split points for dividing into quarters total_count = len(ALL_TEMPLATE_FILE_NAMES) -first_third = total_count // 3 -second_third = (total_count * 2) // 3 - -# Adjust split points to balance load (since the third part also runs functional tests) -if second_third > 2: - # third part will also run functional tests, distribute more tests to earlier parts - first_third = first_third + 1 - second_third = second_third + 2 +q1 = total_count // 4 +q2 = total_count // 2 +q3 = (total_count * 3) // 4 TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES # Select appropriate subset based on environment variable -if SMOKE_TEST_SUBSET == "first-third": - # Select first third of templates - TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[:first_third] -elif SMOKE_TEST_SUBSET == "second-third": - # Select second third of templates - TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[first_third:second_third] -elif SMOKE_TEST_SUBSET == "third-third": - # Select last third of templates - TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[second_third:] +if SMOKE_TEST_SUBSET == "first-quarter": + TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[:q1] +elif SMOKE_TEST_SUBSET == "second-quarter": + TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q1:q2] +elif SMOKE_TEST_SUBSET == "third-quarter": + TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q2:q3] +elif SMOKE_TEST_SUBSET == "fourth-quarter": + TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q3:] +elif SMOKE_TEST_SUBSET == "first-half": + TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[:q2] +elif SMOKE_TEST_SUBSET == "second-half": + TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q2:] class TestAllCommands(TestCase):