Added integration test covering real execution with all model architectures (#205)

* Added integration test covering real execution with all model architectures

* Updated to latest artifacts action

* Test disabling macos upper memory limit for integration tests on gha runners

* Bumped version to release with fix

* Attempt to switch to ubuntu-latest for integration tests due to MPS backend OOMing

* Set up self hosted runner for integration tests

* Swap python setup

* Swap poetry before python again

* Attempt n

* Remove poetry caching for now

* Install pipx

* prefix with python

* Fixed PATH

* Restrict to one model for fast iteration right now

* Re-enable all models in integration test

Author: beveradb

.github/workflows/run-integration-tests.yaml

@@ -0,0 +1,76 @@
+name: run-integration-tests
+
+on:
+  pull_request:
+
+jobs:
+  integration-test:
+    runs-on: self-hosted
+    env:
+      AUDIO_SEPARATOR_MODEL_DIR: ${{ github.workspace }}/models
+
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install pipx
+        run: python -m pip install --user pipx && python -m pipx ensurepath
+
+      - name: Install poetry
+        run: python -m pipx install poetry
+
+      - name: Setup PATH
+        run: echo "/root/.local/bin" >> $GITHUB_PATH
+  
+      - name: Install system dependencies
+        run: |
+          apt-get update
+          apt-get install -y ffmpeg libsamplerate0 libsamplerate-dev
+  
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: poetry
+
+      - name: Create models directory
+        run: mkdir -p $AUDIO_SEPARATOR_MODEL_DIR
+
+      - name: Cache models directory
+        uses: actions/cache@v3
+        id: model-cache
+        with:
+          path: ${{ env.AUDIO_SEPARATOR_MODEL_DIR }}
+          key: model-cache-${{ hashFiles('tests/integration/test_cli_integration.py') }}
+          restore-keys: model-cache-
+
+      - name: Install Poetry dependencies
+        run: poetry install -E cpu
+
+      - name: Display model cache status
+        run: |
+          echo "Model cache hit: ${{ steps.model-cache.outputs.cache-hit == 'true' }}"
+          echo "Models directory contents:"
+          ls -la $AUDIO_SEPARATOR_MODEL_DIR || echo "Directory empty or doesn't exist"
+
+      - name: Run integration tests
+        run: poetry run pytest tests -v --cov=audio_separator --cov-report=xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-results
+          path: |
+            *.flac
+            tests/*.flac

.github/workflows/run-tests.yaml .github/workflows/run-unit-tests.yaml.github/workflows/run-tests.yaml renamed to .github/workflows/run-unit-tests.yaml

@@ -1,7 +1,6 @@
-name: run-tests
+name: run-unit-tests
 
 on:
-  push:
   pull_request:
 
 jobs:
@@ -29,12 +28,7 @@ jobs:
         run: poetry install -E cpu
 
       - name: Run unit tests with coverage
-        run: poetry run pytest tests/unit --cov=./ --cov-report=xml
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: poetry run pytest tests/unit
 
   test-macos:
     runs-on: macos-latest
@@ -61,12 +55,7 @@ jobs:
 
       - name: Run unit tests with coverage
         run: |
-          poetry run pytest tests/unit --cov=./ --cov-report=xml
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          poetry run pytest tests/unit
 
   test-windows:
     runs-on: windows-latest
@@ -92,9 +81,4 @@ jobs:
         run: poetry install -E cpu
 
       - name: Run unit tests with coverage
-        run: poetry run pytest tests/unit --cov=./ --cov-report=xml
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: poetry run pytest tests/unit

.gitignore

@@ -9,7 +9,7 @@
 *.onnx
 *.pth
 *.wav
-*.flac
+/*.flac
 *.mp3
 tests/model-metrics/results
 tests/model-metrics/datasets

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "audio-separator"
-version = "0.31.2"
+version = "0.31.3"
 description = "Easy to use audio stem separation, using various models from UVR trained primarily by @Anjok07"
 authors = ["Andrew Beveridge <[email protected]>"]
 license = "MIT"

tests/inputs/mardy20s.flac

@@ -0,0 +1,41 @@
+# Integration Tests
+
+These tests verify the end-to-end functionality of the audio-separator CLI.
+
+## Running the tests
+
+To run the integration tests, use:
+
+```bash
+pytest tests/integration
+```
+
+To run a specific model test, you can use pytest's parameter selection:
+
+```bash
+# Run only the kuielab_b_vocals.onnx test
+pytest tests/integration/test_cli_integration.py::test_model_separation[kuielab_b_vocals.onnx-expected_files0]
+
+# Run only the MGM_MAIN_v4.pth test
+pytest tests/integration/test_cli_integration.py::test_model_separation[MGM_MAIN_v4.pth-expected_files1]
+```
+
+## Adding New Model Tests
+
+To add a new model test, simply add a new entry to the `MODEL_PARAMS` list in the test file:
+
+```python
+(
+    "new_model_filename.onnx",
+    ["mardy20s_(Instrumental)_new_model_filename.flac", "mardy20s_(Vocals)_new_model_filename.flac"]
+),
+```
+
+No additional test functions are needed.
+
+## Notes
+
+- These tests use actual audio files and models, and will run the full audio separation process.
+- Tests may take longer to run than unit tests, as they perform actual audio processing.
+- The model files will be automatically downloaded if they don't exist locally.
+- The test requires the test audio file at `tests/inputs/mardy20s.flac` to exist. 

tests/integration/README.md

@@ -0,0 +1,80 @@
+import os
+import subprocess
+import pytest
+
+
+@pytest.fixture(name="input_file")
+def fixture_input_file():
+    """Fixture providing the test input audio file path."""
+    return "tests/inputs/mardy20s.flac"
+
+
+@pytest.fixture(name="cleanup_output_files")
+def fixture_cleanup_output_files():
+    """Fixture to clean up output files before and after test."""
+    # This list will be populated by the test functions
+    output_files = []
+
+    # Yield to allow the test to run and add files to the list
+    yield output_files
+
+    # Clean up output files after test
+    for file in output_files:
+        if os.path.exists(file):
+            print(f"Test output file exists: {file}")
+            # os.remove(file)
+
+
+def run_separation_test(model, audio_path, expected_files):
+    """Helper function to run a separation test with a specific model."""
+    # Clean up any existing output files before the test
+    for file in expected_files:
+        if os.path.exists(file):
+            print(f"Deleting existing test output file {file}")
+            os.remove(file)
+
+    # Run the CLI command
+    result = subprocess.run(["audio-separator", "-m", model, audio_path], capture_output=True, text=True, check=False)  # Explicitly set check to False as we handle errors manually
+
+    # Check that the command completed successfully
+    assert result.returncode == 0, f"Command failed with output: {result.stderr}"
+
+    # Check that the output files were created
+    for file in expected_files:
+        assert os.path.exists(file), f"Output file {file} was not created"
+        assert os.path.getsize(file) > 0, f"Output file {file} is empty"
+
+    return result
+
+
+# Parameterized test for multiple models
+MODEL_PARAMS = [
+    # (model_filename, expected_output_filenames)
+    ("kuielab_b_vocals.onnx", ["mardy20s_(Instrumental)_kuielab_b_vocals.flac", "mardy20s_(Vocals)_kuielab_b_vocals.flac"]),
+    ("MGM_MAIN_v4.pth", ["mardy20s_(Instrumental)_MGM_MAIN_v4.flac", "mardy20s_(Vocals)_MGM_MAIN_v4.flac"]),
+    ("UVR-MDX-NET-Inst_HQ_4.onnx", ["mardy20s_(Instrumental)_UVR-MDX-NET-Inst_HQ_4.flac", "mardy20s_(Vocals)_UVR-MDX-NET-Inst_HQ_4.flac"]),
+    ("2_HP-UVR.pth", ["mardy20s_(Instrumental)_2_HP-UVR.flac", "mardy20s_(Vocals)_2_HP-UVR.flac"]),
+    (
+        "htdemucs_6s.yaml",
+        [
+            "mardy20s_(Vocals)_htdemucs_6s.flac",
+            "mardy20s_(Drums)_htdemucs_6s.flac",
+            "mardy20s_(Bass)_htdemucs_6s.flac",
+            "mardy20s_(Other)_htdemucs_6s.flac",
+            "mardy20s_(Guitar)_htdemucs_6s.flac",
+            "mardy20s_(Piano)_htdemucs_6s.flac",
+        ],
+    ),
+    ("model_bs_roformer_ep_937_sdr_10.5309.ckpt", ["mardy20s_(Drum-Bass)_model_bs_roformer_ep_937_sdr_10.flac", "mardy20s_(No Drum-Bass)_model_bs_roformer_ep_937_sdr_10.flac"]),
+    ("model_bs_roformer_ep_317_sdr_12.9755.ckpt", ["mardy20s_(Instrumental)_model_bs_roformer_ep_317_sdr_12.flac", "mardy20s_(Vocals)_model_bs_roformer_ep_317_sdr_12.flac"]),
+]
+
+
+@pytest.mark.parametrize("model,expected_files", MODEL_PARAMS)
+def test_model_separation(model, expected_files, input_file, cleanup_output_files):
+    """Parameterized test for multiple model files."""
+    # Add files to the cleanup list
+    cleanup_output_files.extend(expected_files)
+
+    # Run the test
+    run_separation_test(model, input_file, expected_files)

tests/integration/test_cli_integration.py

@@ -240,23 +240,7 @@ def test_cli_use_autocast_argument(common_expected_args):
             expected_args["use_autocast"] = True
 
             # Assertions
-            mock_separator.assert_called_once_with(**common_expected_args)
-
-
-# Test using use_autocast argument
-def test_cli_use_autocast_argument(common_expected_args):
-    test_args = ["cli.py", "test_audio.mp3", "--use_autocast"]
-    with patch("sys.argv", test_args):
-        with patch("audio_separator.separator.Separator") as mock_separator:
-            mock_separator_instance = mock_separator.return_value
-            mock_separator_instance.separate.return_value = ["output_file.mp3"]
-            main()
-
-            # Update expected args for this specific test
-            common_expected_args["use_autocast"] = True
-
-            # Assertions
-            mock_separator.assert_called_once_with(**common_expected_args)
+            mock_separator.assert_called_once_with(**expected_args)
 
 
 # Test using custom_output_names argument