Debug PT2.6 SM Inference Test Failure (#5451)

* run test with prod image

* disable autopatch

* use ami-04864586635537982 to test

* use ami-024d21dba4813c0fb to test

* pin sagemaker version

* pin sagemaker version

* pin sagemaker version

* test protobuf

* check docker compose

* add back docker compose

* revert sagemaker version pin in test requriements

* update docker version

* change docker version

* add docker compose v1 to local test

* format

* add docker-compose before requriements

* create a requirments file only for pytorch-inference

* revert toml

---------

Co-authored-by: Yadan Wei <[email protected]>

Author: Yadan-Wei

test/sagemaker_tests/pytorch/inference/pytorch-inference-requirements.txt

@@ -0,0 +1,30 @@
+boto3
+coverage
+# SageMaker PyTorch Inference Toolkit still use docker compose v1
+# https://github.com/aws/sagemaker-pytorch-inference-toolkit/blob/v2.0.25/setup.py#L58
+# Docker v7.0.0 breaks compatibility with Docker Compose v1 (SageMaker Local)
+docker==6.1.3
+docker-compose
+Flask==1.1.1
+fabric
+flake8==3.7.7
+gitpython
+invoke
+mock
+numpy
+Pillow
+packaging
+# Preserve pytest caching behavior in earlier versions
+pytest<8.1
+pytest-cov
+pytest-rerunfailures
+pytest-xdist
+requests<2.32.0
+requests_mock
+retrying==1.3.3
+sagemaker>=2,<3
+sagemaker-inference
+tenacity
+toml
+torch<2.5.0
+torchvision

test/test_utils/__init__.py

@@ -115,6 +115,7 @@ def get_ami_id_ssm(region_name, parameter_path):
     region_name="us-west-2",
     parameter_path="/aws/service/deeplearning/ami/x86_64/base-oss-nvidia-driver-gpu-amazon-linux-2023/latest/ami-id",
 )
+
 AL2023_BASE_DLAMI_US_EAST_1 = get_ami_id_ssm(
     region_name="us-east-1",
     parameter_path="/aws/service/deeplearning/ami/x86_64/base-oss-nvidia-driver-gpu-amazon-linux-2023/latest/ami-id",

test/test_utils/sagemaker.py

@@ -345,8 +345,13 @@ def execute_local_tests(image, pytest_cache_params):
             "sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose"
         )
         ec2_conn.run("sudo chmod +x /usr/local/bin/docker-compose")
+
         with ec2_conn.cd(path):
-            ec2_conn.run(f"pip install -r requirements.txt")
+            if "pytorch-inference" in image:
+                ec2_conn.run(f"pip install -r pytorch-inference-requirements.txt")
+            else:
+                ec2_conn.run(f"pip install -r requirements.txt")
+
             pytest_cache_util.download_pytest_cache_from_s3_to_ec2(
                 ec2_conn, path, **pytest_cache_params
             )