Skip to content

Commit 809eb2d

Browse files
tianyuzhou668tianyuzhou668
authored
[Iluvatar] Support compilation on aarch64 platform (#2379)
* [Iluvatar] Support compilation on aarch64 platform * [Iluvatar] Remove test patch
1 parent d5f0155 commit 809eb2d

File tree

5 files changed

+53
-89
lines changed

5 files changed

+53
-89
lines changed

‎backends/iluvatar_gpu/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ if(WITH_COREX)
8686
add_definitions(-DEIGEN_USE_GPU)
8787
endif()
8888

89+
if(WITH_ARM)
90+
add_definitions(-DPADDLE_WITH_ARM)
91+
endif()
92+
8993
list(APPEND CMAKE_MODULE_PATH "${PADDLE_SOURCE_DIR}/cmake"
9094
"${PADDLE_SOURCE_DIR}/cmake/external")
9195

‎backends/iluvatar_gpu/build_inc.sh‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ STATE_FILE="${CURRENT_DIR}/.build_state"
3232
# set BUILD_WITH_FLAGCX to 1 if we want to use flagcx as communication backend
3333
BUILD_WITH_FLAGCX=0
3434
FLAGCX_ROOT="/workspace/FlagCX"
35+
PLATFORM_ID=$(uname -i)
3536

3637
if [ "$BUILD_WITH_FLAGCX" == "1" ]; then
3738
WITH_FLAGCX="ON"
@@ -136,21 +137,36 @@ pushd "$BUILD_DIR" > /dev/null
136137
rm -f compile.log
137138
if [[ ! -f "Makefile" && ! -f "build.ninja" ]]; then
138139
echo "Running cmake for first time build..."
139-
cmake -G Ninja -DPY_VERSION=${PYTHON_VERSION} -DWITH_COREX=ON -DPADDLE_SOURCE_DIR=${PADDLE_SOURCE_DIR} \
140-
-DWITH_DISTRIBUTE=ON -DWITH_NCCL=ON -DWITH_FLAGCX=${WITH_FLAGCX} -DWITH_RCCL=OFF -DCMAKE_BUILD_TYPE=Release \
141-
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DON_INFER=ON -DCOREX_VERSION=${COREX_VERSION} -DCOREX_ARCH=${COREX_ARCH} \
142-
-DFLAGCX_ROOT=${FLAGCX_ROOT} \
143-
-DCMAKE_CXX_FLAGS='-Wno-error=pessimizing-move -Wno-error=deprecated-copy -Wno-error=init-list-lifetime -pthread' \
144-
-DCMAKE_CUDA_FLAGS='-Xclang -fcuda-allow-variadic-functions -mllvm --skip-double' \
145-
-DCMAKE_C_FLAGS="-pthread" \
146-
-DWITH_ARM=OFF -DWITH_DGC=OFF .. 2>&1 | tee compile.log
140+
if [[ "${PLATFORM_ID}" == "aarch64" ]]; then
141+
cmake -G Ninja -DPY_VERSION=${PYTHON_VERSION} -DWITH_COREX=ON -DPADDLE_SOURCE_DIR=${PADDLE_SOURCE_DIR} \
142+
-DWITH_DISTRIBUTE=ON -DWITH_NCCL=ON -DWITH_FLAGCX=${WITH_FLAGCX} -DWITH_RCCL=OFF -DCMAKE_BUILD_TYPE=Release \
143+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DON_INFER=ON -DCOREX_VERSION=${COREX_VERSION} -DCOREX_ARCH=${COREX_ARCH} \
144+
-DFLAGCX_ROOT=${FLAGCX_ROOT} \
145+
-DCMAKE_CXX_FLAGS='-Wno-error=pessimizing-move -Wno-error=deprecated-copy -Wno-error=init-list-lifetime -pthread' \
146+
-DCMAKE_CUDA_FLAGS='-Xclang -fcuda-allow-variadic-functions -mllvm --skip-double' \
147+
-DCMAKE_C_FLAGS="-pthread" \
148+
-DWITH_ARM=ON -DWITH_DGC=OFF .. 2>&1 | tee compile.log
149+
else
150+
cmake -G Ninja -DPY_VERSION=${PYTHON_VERSION} -DWITH_COREX=ON -DPADDLE_SOURCE_DIR=${PADDLE_SOURCE_DIR} \
151+
-DWITH_DISTRIBUTE=ON -DWITH_NCCL=ON -DWITH_FLAGCX=${WITH_FLAGCX} -DWITH_RCCL=OFF -DCMAKE_BUILD_TYPE=Release \
152+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DON_INFER=ON -DCOREX_VERSION=${COREX_VERSION} -DCOREX_ARCH=${COREX_ARCH} \
153+
-DFLAGCX_ROOT=${FLAGCX_ROOT} \
154+
-DCMAKE_CXX_FLAGS='-Wno-error=pessimizing-move -Wno-error=deprecated-copy -Wno-error=init-list-lifetime -pthread' \
155+
-DCMAKE_CUDA_FLAGS='-Xclang -fcuda-allow-variadic-functions -mllvm --skip-double' \
156+
-DCMAKE_C_FLAGS="-pthread" \
157+
-DWITH_ARM=OFF -DWITH_DGC=OFF .. 2>&1 | tee compile.log
158+
fi
147159
else
148160
echo "Cmake configuration already exists, skipping..."
149161
fi
150162

151163
# Compile
152164
echo "Starting compilation..."
153-
ninja -j$(nproc) 2>&1
165+
if [[ "${PLATFORM_ID}" == "aarch64" ]]; then
166+
env TARGET=ARMV8 ninja -j$(nproc) 2>&1
167+
else
168+
ninja -j$(nproc) 2>&1
169+
fi
154170
FAILED_LOG="failed_files.log"
155171
grep -E "FAILED: " compile.log | tee ${FAILED_LOG}
156172
echo "Failed files are listed in ${FAILED_LOG}"

‎backends/iluvatar_gpu/build_paddle.sh‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ PATCH_FILE="${CURRENT_DIR}/patches/paddle-corex.patch"
3232
# set BUILD_WITH_FLAGCX to 1 if we want to use flagcx as communication backend
3333
BUILD_WITH_FLAGCX=0
3434
FLAGCX_ROOT="/workspace/FlagCX"
35+
PLATFORM_ID=$(uname -i)
3536

3637
if [ "$BUILD_WITH_FLAGCX" == "1" ]; then
3738
WITH_FLAGCX="ON"
@@ -61,15 +62,27 @@ if [[ ! -d "build" ]]; then
6162
fi
6263
pushd build
6364

64-
cmake -G Ninja -DPY_VERSION=${PYTHON_VERSION} -DWITH_COREX=ON -DPADDLE_SOURCE_DIR=${PADDLE_SOURCE_DIR} \
65-
-DWITH_DISTRIBUTE=ON -DWITH_NCCL=ON -DWITH_FLAGCX=${WITH_FLAGCX} -DWITH_RCCL=OFF -DCMAKE_BUILD_TYPE=Release \
66-
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DON_INFER=ON -DCOREX_VERSION=${COREX_VERSION} -DCOREX_ARCH=${COREX_ARCH} \
67-
-DFLAGCX_ROOT=${FLAGCX_ROOT} \
68-
-DCMAKE_CXX_FLAGS='-Wno-error=pessimizing-move -Wno-error=deprecated-copy -Wno-error=init-list-lifetime -pthread' \
69-
-DCMAKE_CUDA_FLAGS='-Xclang -fcuda-allow-variadic-functions -mllvm --skip-double' \
70-
-DCMAKE_C_FLAGS="-pthread" \
71-
-DWITH_ARM=OFF -DWITH_DGC=OFF .. || { echo "Error: CMake configuration failed!"; exit 1; }
72-
ninja -j$(nproc) || { echo "Error: Paddle-iluvatar-gpu build failed!"; exit 1; }
65+
if [[ "${PLATFORM_ID}" == "aarch64" ]]; then
66+
cmake -G Ninja -DPY_VERSION=${PYTHON_VERSION} -DWITH_COREX=ON -DPADDLE_SOURCE_DIR=${PADDLE_SOURCE_DIR} \
67+
-DWITH_DISTRIBUTE=ON -DWITH_NCCL=ON -DWITH_FLAGCX=${WITH_FLAGCX} -DWITH_RCCL=OFF -DCMAKE_BUILD_TYPE=Release \
68+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DON_INFER=ON -DCOREX_VERSION=${COREX_VERSION} -DCOREX_ARCH=${COREX_ARCH} \
69+
-DFLAGCX_ROOT=${FLAGCX_ROOT} \
70+
-DCMAKE_CXX_FLAGS='-Wno-error=pessimizing-move -Wno-error=deprecated-copy -Wno-error=init-list-lifetime -pthread' \
71+
-DCMAKE_CUDA_FLAGS='-Xclang -fcuda-allow-variadic-functions -mllvm --skip-double' \
72+
-DCMAKE_C_FLAGS="-pthread" \
73+
-DWITH_ARM=ON -DWITH_DGC=OFF .. || { echo "Error: CMake configuration failed!"; exit 1; }
74+
env TARGET=ARMV8 ninja -j$(nproc) || { echo "Error: Paddle-iluvatar-gpu build failed!"; exit 1; }
75+
else
76+
cmake -G Ninja -DPY_VERSION=${PYTHON_VERSION} -DWITH_COREX=ON -DPADDLE_SOURCE_DIR=${PADDLE_SOURCE_DIR} \
77+
-DWITH_DISTRIBUTE=ON -DWITH_NCCL=ON -DWITH_FLAGCX=${WITH_FLAGCX} -DWITH_RCCL=OFF -DCMAKE_BUILD_TYPE=Release \
78+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DON_INFER=ON -DCOREX_VERSION=${COREX_VERSION} -DCOREX_ARCH=${COREX_ARCH} \
79+
-DFLAGCX_ROOT=${FLAGCX_ROOT} \
80+
-DCMAKE_CXX_FLAGS='-Wno-error=pessimizing-move -Wno-error=deprecated-copy -Wno-error=init-list-lifetime -pthread' \
81+
-DCMAKE_CUDA_FLAGS='-Xclang -fcuda-allow-variadic-functions -mllvm --skip-double' \
82+
-DCMAKE_C_FLAGS="-pthread" \
83+
-DWITH_ARM=OFF -DWITH_DGC=OFF .. || { echo "Error: CMake configuration failed!"; exit 1; }
84+
ninja -j$(nproc) || { echo "Error: Paddle-iluvatar-gpu build failed!"; exit 1; }
85+
fi
7386
popd
7487

7588
if [[ ! -d "build_pip" ]]; then

‎backends/iluvatar_gpu/patches/paddle-corex-test.patch‎

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎backends/iluvatar_gpu/tests/run_test.sh‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ export LD_LIBRARY_PATH=/usr/local/corex/lib
2121
export LIBRARY_PATH=/usr/local/corex/lib
2222
export PYTHONPATH="${LEGACY_TEST_PATH}:${PYTHONPATH}"
2323

24+
python -m pip install parameterized
25+
2426
if [[ -z "${LD_LIBRARY_PATH:-}" ]]; then
2527
echo "ERROR: LD_LIBRARY_PATH is not set!" >&2
2628
exit 1
@@ -43,15 +45,6 @@ export FLAG_SKIP_FLOAT64=1
4345

4446
CURRENT_DIR=$(pwd)
4547
PADDLE_SOURCE_DIR="${CURRENT_DIR}/../../../Paddle"
46-
PATCH_FILE="${CURRENT_DIR}/../patches/paddle-corex-test.patch"
47-
48-
if ! git -C "$PADDLE_SOURCE_DIR" apply --reverse --check "$PATCH_FILE" > /dev/null 2>&1; then
49-
if ! git -C "$PADDLE_SOURCE_DIR" apply "$PATCH_FILE"; then
50-
echo "Error: Failed to apply patch!"
51-
exit 1
52-
fi
53-
echo "Patch applied successfully!"
54-
fi
5548

5649
mkdir -p build || { echo "ERROR: Failed to create build directory"; exit 1; }
5750
cd build || { echo "ERROR: Failed to enter build directory"; exit 1; }
@@ -69,11 +62,6 @@ ctest --output-on-failure -V -j8 || {
6962
exit 1
7063
}
7164

72-
if git -C "$PADDLE_SOURCE_DIR" apply --reverse --check "$PATCH_FILE" > /dev/null 2>&1; then
73-
git -C "$PADDLE_SOURCE_DIR" apply --reverse "$PATCH_FILE"
74-
echo "Patch successfully reverted!"
75-
fi
76-
7765
cd ..
7866

7967
# Run legacy tests only if primary tests passed

0 commit comments

Comments
 (0)