more benchmarks #96
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| pull_request: | |
| jobs: | |
| format-and-lint: | |
| name: Format & Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install formatting tools | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y clang-format | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install black ruff | |
| - name: Clang-format check | |
| run: | | |
| set -euo pipefail | |
| files=$(git ls-files '*.h' '*.hpp' '*.c' '*.cpp' '*.cc') | |
| if [ -n "$files" ]; then | |
| clang-format --dry-run --Werror $files | |
| fi | |
| - name: Black check | |
| run: black --check python t81 tests scripts | |
| - name: Ruff check | |
| run: ruff check python t81 tests scripts | |
| build-and-test: | |
| name: Matrix build & tests | |
| needs: format-and-lint | |
| runs-on: ${{ matrix.compiler.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compiler: | |
| - name: gcc | |
| c: gcc | |
| cxx: g++ | |
| os: ubuntu-latest | |
| - name: clang | |
| c: clang | |
| cxx: clang++ | |
| os: ubuntu-latest | |
| - name: appleclang | |
| c: clang | |
| cxx: clang++ | |
| os: macos-14 | |
| configuration: | |
| - name: minimal | |
| python_bindings: OFF | |
| - name: python | |
| python_bindings: ON | |
| python_version: "3.11" | |
| simd: | |
| - name: scalar | |
| flags: "-mno-avx" | |
| - name: avx2 | |
| flags: "-mavx2" | |
| build_type: | |
| - Release | |
| - Debug | |
| exclude: | |
| - compiler: | |
| name: appleclang | |
| simd: | |
| name: avx2 | |
| env: | |
| BUILD_DIR: build-${{ matrix.compiler.os }}-${{ matrix.compiler.name }}-${{ matrix.configuration.name }}-${{ matrix.simd.name }}-${{ matrix.build_type }} | |
| ARTIFACT_LABEL: ci-${{ matrix.compiler.os }}-${{ matrix.compiler.name }}-${{ matrix.configuration.name }}-${{ matrix.simd.name }}-${{ matrix.build_type }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| if: matrix.configuration.name == 'python' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.configuration.python_version }} | |
| - name: Configure | |
| run: | | |
| cmake -S . -B "$BUILD_DIR" \ | |
| -DT81LIB_BUILD_TESTS=ON \ | |
| -DT81LIB_BUILD_PYTHON_BINDINGS=${{ matrix.configuration.python_bindings }} \ | |
| -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCMAKE_C_COMPILER=${{ matrix.compiler.c }} \ | |
| -DCMAKE_CXX_COMPILER=${{ matrix.compiler.cxx }} \ | |
| ${{ runner.os == 'Linux' && format('-DCMAKE_C_FLAGS={0} -DCMAKE_CXX_FLAGS={0}', matrix.simd.flags) || '' }} | |
| - name: Build | |
| run: cmake --build "$BUILD_DIR" --parallel | |
| - name: Test | |
| run: ctest --test-dir "$BUILD_DIR" --output-on-failure | |
| - name: Python binding + GGUF regression tests | |
| if: matrix.configuration.name == 'python' && runner.os == 'Linux' | |
| run: | | |
| set -euo pipefail | |
| mkdir -p artifacts | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install torch transformers pytest | |
| PYTHONPATH="$BUILD_DIR" python3 tests/python/test_bindings.py 2>&1 | tee artifacts/${BUILD_DIR}-python-tests.log | |
| PYTHONPATH="$BUILD_DIR" python3 -m pytest tests/python/test_gguf.py 2>&1 | tee artifacts/${BUILD_DIR}-gguf-tests.log | |
| - name: Collect test logs | |
| run: | | |
| mkdir -p artifacts | |
| if [ -f "$BUILD_DIR/Testing/Temporary/LastTest.log" ]; then | |
| cp "$BUILD_DIR/Testing/Temporary/LastTest.log" "artifacts/${BUILD_DIR}-ctest.log" | |
| fi | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACT_LABEL }} | |
| path: artifacts | |
| coverage: | |
| name: Coverage report | |
| runs-on: ubuntu-latest | |
| needs: build-and-test | |
| env: | |
| BUILD_DIR: build-coverage | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install coverage tooling | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install gcovr | |
| - name: Configure | |
| run: | | |
| cmake -S . -B "$BUILD_DIR" \ | |
| -DT81LIB_BUILD_TESTS=ON \ | |
| -DT81LIB_BUILD_PYTHON_BINDINGS=OFF \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DCMAKE_C_FLAGS="--coverage -O0 -g" \ | |
| -DCMAKE_CXX_FLAGS="--coverage -O0 -g" | |
| - name: Build | |
| run: cmake --build "$BUILD_DIR" --parallel | |
| - name: Test | |
| run: ctest --test-dir "$BUILD_DIR" --output-on-failure | |
| - name: Generate coverage report | |
| run: | | |
| mkdir -p artifacts | |
| gcovr --root . \ | |
| --exclude "$BUILD_DIR/.*" \ | |
| --xml --output artifacts/coverage.xml \ | |
| --html --html-details --output artifacts/coverage.html | |
| - name: Upload coverage artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: artifacts | |
| gpu-cuda: | |
| name: CUDA GPU smoke | |
| runs-on: ubuntu-latest | |
| needs: build-and-test | |
| env: | |
| BUILD_DIR: build-gpu-cuda | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install CUDA toolchain | |
| uses: nvidia/setup-cuda@v2 | |
| with: | |
| cuda-version: "12.4" | |
| - name: Configure | |
| run: | | |
| cmake -S . -B "$BUILD_DIR" \ | |
| -DT81LIB_BUILD_TESTS=ON \ | |
| -DT81LIB_BUILD_PYTHON_BINDINGS=ON \ | |
| -DUSE_CUDA=ON \ | |
| -DUSE_ROCM=OFF | |
| - name: Build | |
| run: cmake --build "$BUILD_DIR" --parallel | |
| - name: Test | |
| run: ctest --test-dir "$BUILD_DIR" --output-on-failure | |
| - name: Run GPU-focused python tests | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| PYTHONPATH="$BUILD_DIR" python3 tests/python/test_gpu_ops.py | |
| gpu-rocm: | |
| name: ROCm GPU smoke | |
| runs-on: ubuntu-22.04 | |
| needs: build-and-test | |
| env: | |
| BUILD_DIR: build-gpu-rocm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install ROCm toolchain | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y rocm-dev hipblas | |
| echo "/opt/rocm/bin" >> $GITHUB_PATH | |
| echo "/opt/rocm/lib" >> $GITHUB_PATH | |
| echo "LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV | |
| - name: Configure | |
| run: | | |
| cmake -S . -B "$BUILD_DIR" \ | |
| -DT81LIB_BUILD_TESTS=ON \ | |
| -DT81LIB_BUILD_PYTHON_BINDINGS=ON \ | |
| -DUSE_ROCM=ON \ | |
| -DUSE_CUDA=OFF \ | |
| -DCMAKE_PREFIX_PATH=/opt/rocm | |
| - name: Build | |
| run: cmake --build "$BUILD_DIR" --parallel | |
| - name: Test | |
| run: ctest --test-dir "$BUILD_DIR" --output-on-failure | |
| - name: Run ROCm-focused python tests | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install pytest | |
| PYTHONPATH="$BUILD_DIR" python3 tests/python/test_gpu_ops.py |