Split long-running integ tests to stay under 10 min

.github/workflows/build.yml

@@ -125,17 +125,19 @@ jobs:
         # folders that is commented below requires credentials, no need to spare time to run them
         tests_config:
           - name: "integ-buildcmd-arm64"
-            params: "-n 2 --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py"
+            params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/buildcmd/test_build_cmd_arm64.py"
+          - name: "integ-buildcmd-python"
+            params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/buildcmd/test_build_cmd_python.py"
           - name: "integ-buildcmd-main"
-            params: "-n 2 --reruns 3 tests/integration/buildcmd/test_build_cmd_dotnet.py tests/integration/buildcmd/test_build_cmd_provided.py tests/integration/buildcmd/test_build_cmd_rust.py tests/integration/buildcmd/test_build_cmd_python.py tests/integration/buildcmd/test_build_cmd_node.py"
+            params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/buildcmd/test_build_cmd_dotnet.py tests/integration/buildcmd/test_build_cmd_provided.py tests/integration/buildcmd/test_build_cmd_rust.py tests/integration/buildcmd/test_build_cmd_node.py"
           - name: "integ-buildcmd-java"
-            params: "-n 2 --reruns 5 tests/integration/buildcmd/test_build_cmd_java.py"
+            params: "-n 3 --reruns 5 tests/integration/buildcmd/test_build_cmd_java.py"
           - name: "integ-buildcmd-other"
             params: "-n 2 --reruns 3 tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_cmd_dotnet.py --ignore=tests/integration/buildcmd/test_build_cmd_provided.py --ignore=tests/integration/buildcmd/test_build_cmd_rust.py --ignore=tests/integration/buildcmd/test_build_cmd_python.py --ignore=tests/integration/buildcmd/test_build_cmd_node.py --ignore=tests/integration/buildcmd/test_build_cmd_java.py"
           #- "delete"
           #- "deploy"
           - name: "integ-all-other"
-            params: "-n 2 --reruns 3 tests/integration/init tests/integration/local tests/integration/pipeline tests/integration/root tests/integration/scripts tests/integration/telemetry tests/integration/docs --ignore=tests/integration/local/invoke/test_invoke_durable.py --ignore=tests/integration/local/start_api/test_start_api_durable.py --ignore=tests/integration/local/start_lambda/test_start_lambda_durable.py"
+            params: "-n 2 --reruns 3 -m 'not pr_skip' tests/integration/init tests/integration/local tests/integration/pipeline tests/integration/root tests/integration/scripts tests/integration/telemetry tests/integration/docs --ignore=tests/integration/local/invoke/test_invoke_durable.py --ignore=tests/integration/local/start_api/test_start_api_durable.py --ignore=tests/integration/local/start_lambda/test_start_lambda_durable.py"
           - name: "durable-functions"
             params: "--reruns 3 tests/integration/local/invoke/test_invoke_durable.py tests/integration/local/start_api/test_start_api_durable.py tests/integration/local/start_lambda/test_start_lambda_durable.py"
           #- "list"
@@ -214,22 +216,30 @@ jobs:
           - "3.10"
           - "3.11"
         tests_config:
-          - name: "Smoke & Functional Tests - All"
+          - name: "Smoke Tests - First Half"
+            os: ubuntu-latest
+            params: "-n 4 tests/smoke"
+            env_vars: "first-half"
+          - name: "Smoke Tests - Second Half & Functional Tests"
             os: ubuntu-latest
             params: "-n 4 tests/smoke tests/functional"
-            env_vars: "all"
-          - name: "Smoke Tests - First Third"
+            env_vars: "second-half"
+          - name: "Smoke Tests - First Quarter"
+            os: windows-latest
+            params: "-n 4 tests/smoke"
+            env_vars: "first-quarter"
+          - name: "Smoke Tests - Second Quarter"
             os: windows-latest
             params: "-n 4 tests/smoke"
-            env_vars: "first-third"
-          - name: "Smoke Tests - Second Third"
+            env_vars: "second-quarter"
+          - name: "Smoke Tests - Third Quarter"
             os: windows-latest
             params: "-n 4 tests/smoke"
-            env_vars: "second-third"
-          - name: "Smoke Tests - Third Third & Functional Tests"
+            env_vars: "third-quarter"
+          - name: "Smoke Tests - Fourth Quarter & Functional Tests"
             os: windows-latest
             params: "-n 4 tests/smoke tests/functional"
-            env_vars: "third-third"
+            env_vars: "fourth-quarter"
     steps:
       - uses: actions/checkout@v6
       - name: Set TEMP to D:/Temp

CONTRIBUTING.md

@@ -123,6 +123,20 @@ def test_tier1_my_feature_in_container(self):
 
 4. Each runtime should have one non-container and one container tier 1 test.
 
+### Skipping tests in PR workflow (`pr_skip`)
+
+Tests marked with `@pytest.mark.pr_skip` are excluded from the PR workflow (`.github/workflows/build.yml`) but still run in the integration workflow (`.github/workflows/integration-tests.yml`). Use this marker for tests that are redundant with other tests in the same file or low-risk for PR validation.
+
+```python
+@pytest.mark.pr_skip
+class TestMyRedundantFeature(TestBase):
+    ...
+```
+
+The marker is registered in `pytest.ini`. PR workflow jobs filter with `-m "not pr_skip"`.
+
+Before adding `pr_skip`, verify the test is covered by at least one job in `integration-tests.yml`.
+
 
 ## Finding contributions to work on
 Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws/aws-sam-cli/labels/help%20wanted) issues is a great place to start. 

pytest.ini

@@ -27,3 +27,4 @@ markers =
     provided
     dotnet
     al2023: Marker to note tests that require a newer Docker version on Windows
+    pr_skip: Tests skipped in PR workflow but run in nightly/integration tests

tests/integration/buildcmd/test_build_cmd_arm64.py

@@ -117,6 +117,7 @@ def test_tier1_node_arm64_build_in_container(self, runtime, code_uri, expected_f
 
 
 @pytest.mark.nodejs
+@pytest.mark.pr_skip
 class TestBuildCommand_EsbuildFunctions_With_External_Manifest_arm64(BuildIntegEsbuildBase):
     template = "template_with_metadata_esbuild.yaml"
     MANIFEST_PATH = "Esbuild/npm_manifest/package.json"
@@ -672,6 +673,7 @@ def test_tier1_provided_arm64_build(self):
         self._test_with_Makefile("provided.al2023", False, None, ARM64)
 
 
+@pytest.mark.pr_skip
 @skipIf(
     ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE),
     "Skip build tests on windows when running in CI unless overridden",

tests/integration/buildcmd/test_build_cmd_node.py

@@ -77,6 +77,7 @@ def test_tier1_node_build_in_container(self):
         )
 
 
+@pytest.mark.pr_skip
 class TestBuildCommand_EsbuildFunctions_With_External_Manifest(BuildIntegEsbuildBase):
     template = "template_with_metadata_esbuild.yaml"
     MANIFEST_PATH = "Esbuild/npm_manifest/package.json"
@@ -145,6 +146,7 @@ def test_building_default_package_json(
     ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE),
     "Skip build tests on windows when running in CI unless overridden",
 )
+@pytest.mark.pr_skip
 @parameterized_class(
     ("template",),
     [

tests/integration/buildcmd/test_build_cmd_python.py

@@ -543,6 +543,7 @@ def test_cdk_app_with_default_requirements(self):
         self._verify_invoke_built_function(self.built_template, self.FUNCTION_LOGICAL_ID, {}, expected)
 
 
+@pytest.mark.pr_skip
 class TestBuildCommand_PythonFunctions_With_Specified_Architecture(BuildIntegPythonBase):
     template = "template_with_architecture.yaml"
 

tests/integration/docs/test_docs_command.py

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from parameterized import parameterized
 
 from samcli.commands.docs.command_context import SUCCESS_MESSAGE, ERROR_MESSAGE, DocsCommandContext
@@ -10,6 +12,7 @@
 COMMAND_URL_PAIR = [(command, url) for command, url in Documentation.load().items()]
 
 
+@pytest.mark.pr_skip
 class TestDocsCommand(DocsIntegBase):
     @parameterized.expand(COMMAND_URL_PAIR)
     def test_docs_command(self, command, url):

tests/integration/init/test_init_command.py

@@ -568,6 +568,7 @@ def _assert_template_with_cfn_lint(self, cwd):
 
 
 @pytest.mark.xdist_group(name="sam_init")
+@pytest.mark.pr_skip
 class TestInitForParametersCompatibility(TestCase):
     def test_init_command_no_interactive_missing_name(self):
         stderr = None

tests/integration/scripts/test_copy_terraform_built_artifacts.py

@@ -10,6 +10,10 @@
 TIMEOUT = 3
 
 
+import pytest
+
+
+@pytest.mark.pr_skip
 class TestCopyTerraformBuiltArtifacts(TestCase):
     def setUp(self) -> None:
         self.samcli_root_path = pathlib.Path(__file__).parents[3].joinpath("samcli")

tests/smoke/test_all_commands.py

@@ -13,28 +13,26 @@
 # Check environment variable to determine which subset to use
 SMOKE_TEST_SUBSET = os.environ.get("SMOKE_TEST_SUBSET", "").lower()
 
-# Calculate split points for dividing into thirds
+# Calculate split points for dividing into quarters
 total_count = len(ALL_TEMPLATE_FILE_NAMES)
-first_third = total_count // 3
-second_third = (total_count * 2) // 3
-
-# Adjust split points to balance load (since the third part also runs functional tests)
-if second_third > 2:
-    # third part will also run functional tests, distribute more tests to earlier parts
-    first_third = first_third + 1
-    second_third = second_third + 2
+q1 = total_count // 4
+q2 = total_count // 2
+q3 = (total_count * 3) // 4
 
 TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES
 # Select appropriate subset based on environment variable
-if SMOKE_TEST_SUBSET == "first-third":
-    # Select first third of templates
-    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[:first_third]
-elif SMOKE_TEST_SUBSET == "second-third":
-    # Select second third of templates
-    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[first_third:second_third]
-elif SMOKE_TEST_SUBSET == "third-third":
-    # Select last third of templates
-    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[second_third:]
+if SMOKE_TEST_SUBSET == "first-quarter":
+    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[:q1]
+elif SMOKE_TEST_SUBSET == "second-quarter":
+    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q1:q2]
+elif SMOKE_TEST_SUBSET == "third-quarter":
+    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q2:q3]
+elif SMOKE_TEST_SUBSET == "fourth-quarter":
+    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q3:]
+elif SMOKE_TEST_SUBSET == "first-half":
+    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[:q2]
+elif SMOKE_TEST_SUBSET == "second-half":
+    TEMPLATE_FILE_NAMES = ALL_TEMPLATE_FILE_NAMES[q2:]
 
 
 class TestAllCommands(TestCase):