[CELEBORN-2239] Support Spark 4.1

Author: SteNicholas

.github/workflows/deps.yml

@@ -129,6 +129,7 @@ jobs:
       matrix:
         module:
           - 'spark-4.0'
+          - 'spark-4.1'
     steps:
       - uses: actions/checkout@v4
       - name: Setup JDK 17

README.md

@@ -46,7 +46,7 @@ Celeborn worker's slot count decreases when a partition is allocated and increme
 
 Build Celeborn via `make-distribution.sh`:
 ```shell
-./build/make-distribution.sh -Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pflink-2.1/-Pflink-2.2/-Pmr
+./build/make-distribution.sh -Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pspark-4.1/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pflink-2.1/-Pflink-2.2/-Pmr
 ```
 
 Package `apache-celeborn-${project.version}-bin.tgz` will be generated.
@@ -63,6 +63,7 @@ Package `apache-celeborn-${project.version}-bin.tgz` will be generated.
 | Spark 3.4  | ❌          | ✔          | ✔           | ✔           | ✔          | ✔           | ✔           |
 | Spark 3.5  | ❌          | ✔          | ✔           | ✔           | ✔          | ✔           | ✔           |
 | Spark 4.0  | ❌          | ❌          | ❌           | ❌           | ❌          | ❌           | ✔           |
+| Spark 4.1  | ❌          | ❌          | ❌           | ❌           | ❌          | ❌           | ✔           |
 | Flink 1.16 | ❌          | ✔          | ✔           | ❌           | ❌          | ❌           | ❌           |
 | Flink 1.17 | ❌          | ✔          | ✔           | ❌           | ❌          | ❌           | ❌           |
 | Flink 1.18 | ❌          | ✔          | ✔           | ❌           | ❌          | ❌           | ❌           |

build/make-distribution.sh

@@ -356,6 +356,7 @@ if [ "$SBT_ENABLED" == "true" ]; then
     sbt_build_client -Pspark-3.5
     export JAVA_HOME=$JAVA17_HOME
     sbt_build_client -Pspark-4.0
+    sbt_build_client -Pspark-4.1
     export JAVA_HOME=$JAVA8_HOME
     sbt_build_client -Pflink-1.16
     sbt_build_client -Pflink-1.17
@@ -399,6 +400,7 @@ else
     build_spark_client -Pspark-3.5
     export JAVA_HOME=$JAVA17_HOME
     build_spark_client -Pspark-4.0
+    build_spark_client -Pspark-4.1
     export JAVA_HOME=$JAVA8_HOME
     build_flink_client -Pflink-1.16
     build_flink_client -Pflink-1.17

build/release/release.sh

@@ -115,7 +115,7 @@ upload_nexus_staging() {
 
   export JAVA_HOME=$JAVA17_HOME
   echo "Deploying celeborn-client-spark-4-shaded_2.13"
-  ${PROJECT_DIR}/build/sbt -Pspark-4.0 "clean;celeborn-client-spark-4-shaded/publishSigned"
+  ${PROJECT_DIR}/build/sbt -Pspark-4.1 "clean;celeborn-client-spark-4-shaded/publishSigned"
   export JAVA_HOME=$JAVA8_HOME
 
   echo "Deploying celeborn-client-flink-1.16-shaded_2.12"

dev/deps/dependencies-client-spark-4.1

@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar
+commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-io/2.17.0//commons-io-2.17.0.jar
+commons-lang3/3.17.0//commons-lang3-3.17.0.jar
+commons-logging/1.1.3//commons-logging-1.1.3.jar
+failureaccess/1.0.2//failureaccess-1.0.2.jar
+guava/33.1.0-jre//guava-33.1.0-jre.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
+jackson-annotations/2.15.3//jackson-annotations-2.15.3.jar
+jackson-core/2.15.3//jackson-core-2.15.3.jar
+jackson-databind/2.15.3//jackson-databind-2.15.3.jar
+jackson-module-scala_2.13/2.15.3//jackson-module-scala_2.13-2.15.3.jar
+jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
+jsr305/1.3.9//jsr305-1.3.9.jar
+jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
+leveldbjni-all/1.8//leveldbjni-all-1.8.jar
+lz4-java/1.8.0//lz4-java-1.8.0.jar
+metrics-core/4.2.25//metrics-core-4.2.25.jar
+metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
+metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
+netty-all/4.1.118.Final//netty-all-4.1.118.Final.jar
+netty-buffer/4.1.118.Final//netty-buffer-4.1.118.Final.jar
+netty-buffer/4.2.7.Final//netty-buffer-4.2.7.Final.jar
+netty-codec-base/4.2.7.Final//netty-codec-base-4.2.7.Final.jar
+netty-codec-compression/4.2.7.Final//netty-codec-compression-4.2.7.Final.jar
+netty-codec-dns/4.1.118.Final//netty-codec-dns-4.1.118.Final.jar
+netty-codec-dns/4.2.7.Final//netty-codec-dns-4.2.7.Final.jar
+netty-codec-haproxy/4.1.118.Final//netty-codec-haproxy-4.1.118.Final.jar
+netty-codec-http/4.1.118.Final//netty-codec-http-4.1.118.Final.jar
+netty-codec-http/4.2.7.Final//netty-codec-http-4.2.7.Final.jar
+netty-codec-http2/4.1.118.Final//netty-codec-http2-4.1.118.Final.jar
+netty-codec-http2/4.2.7.Final//netty-codec-http2-4.2.7.Final.jar
+netty-codec-marshalling/4.2.7.Final//netty-codec-marshalling-4.2.7.Final.jar
+netty-codec-memcache/4.1.118.Final//netty-codec-memcache-4.1.118.Final.jar
+netty-codec-mqtt/4.1.118.Final//netty-codec-mqtt-4.1.118.Final.jar
+netty-codec-protobuf/4.2.7.Final//netty-codec-protobuf-4.2.7.Final.jar
+netty-codec-redis/4.1.118.Final//netty-codec-redis-4.1.118.Final.jar
+netty-codec-smtp/4.1.118.Final//netty-codec-smtp-4.1.118.Final.jar
+netty-codec-socks/4.1.118.Final//netty-codec-socks-4.1.118.Final.jar
+netty-codec-socks/4.2.7.Final//netty-codec-socks-4.2.7.Final.jar
+netty-codec-stomp/4.1.118.Final//netty-codec-stomp-4.1.118.Final.jar
+netty-codec-xml/4.1.118.Final//netty-codec-xml-4.1.118.Final.jar
+netty-codec/4.1.118.Final//netty-codec-4.1.118.Final.jar
+netty-codec/4.2.7.Final//netty-codec-4.2.7.Final.jar
+netty-common/4.1.118.Final//netty-common-4.1.118.Final.jar
+netty-common/4.2.7.Final//netty-common-4.2.7.Final.jar
+netty-handler-proxy/4.1.118.Final//netty-handler-proxy-4.1.118.Final.jar
+netty-handler-proxy/4.2.7.Final//netty-handler-proxy-4.2.7.Final.jar
+netty-handler/4.1.118.Final//netty-handler-4.1.118.Final.jar
+netty-handler/4.2.7.Final//netty-handler-4.2.7.Final.jar
+netty-resolver-dns-classes-macos/4.1.118.Final//netty-resolver-dns-classes-macos-4.1.118.Final.jar
+netty-resolver-dns-native-macos/4.1.118.Final/osx-aarch_64/netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar
+netty-resolver-dns-native-macos/4.1.118.Final/osx-x86_64/netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar
+netty-resolver-dns/4.1.118.Final//netty-resolver-dns-4.1.118.Final.jar
+netty-resolver-dns/4.2.7.Final//netty-resolver-dns-4.2.7.Final.jar
+netty-resolver/4.1.118.Final//netty-resolver-4.1.118.Final.jar
+netty-resolver/4.2.7.Final//netty-resolver-4.2.7.Final.jar
+netty-transport-classes-epoll/4.1.118.Final//netty-transport-classes-epoll-4.1.118.Final.jar
+netty-transport-classes-epoll/4.2.7.Final//netty-transport-classes-epoll-4.2.7.Final.jar
+netty-transport-classes-kqueue/4.1.118.Final//netty-transport-classes-kqueue-4.1.118.Final.jar
+netty-transport-classes-kqueue/4.2.7.Final//netty-transport-classes-kqueue-4.2.7.Final.jar
+netty-transport-native-epoll/4.1.118.Final/linux-aarch_64/netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.1.118.Final/linux-riscv64/netty-transport-native-epoll-4.1.118.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.1.118.Final/linux-x86_64/netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar
+netty-transport-native-epoll/4.2.7.Final/linux-aarch_64/netty-transport-native-epoll-4.2.7.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.2.7.Final/linux-riscv64/netty-transport-native-epoll-4.2.7.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.2.7.Final/linux-x86_64/netty-transport-native-epoll-4.2.7.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.1.118.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.118.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.1.118.Final/osx-x86_64/netty-transport-native-kqueue-4.1.118.Final-osx-x86_64.jar
+netty-transport-native-kqueue/4.2.7.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.7.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.2.7.Final/osx-x86_64/netty-transport-native-kqueue-4.2.7.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.1.118.Final//netty-transport-native-unix-common-4.1.118.Final.jar
+netty-transport-native-unix-common/4.2.7.Final//netty-transport-native-unix-common-4.2.7.Final.jar
+netty-transport-rxtx/4.1.118.Final//netty-transport-rxtx-4.1.118.Final.jar
+netty-transport-sctp/4.1.118.Final//netty-transport-sctp-4.1.118.Final.jar
+netty-transport-udt/4.1.118.Final//netty-transport-udt-4.1.118.Final.jar
+netty-transport/4.1.118.Final//netty-transport-4.1.118.Final.jar
+netty-transport/4.2.7.Final//netty-transport-4.2.7.Final.jar
+paranamer/2.8.3//paranamer-2.8.3.jar
+paranamer/2.8//paranamer-2.8.jar
+protobuf-java/3.25.5//protobuf-java-3.25.5.jar
+scala-library/2.13.17//scala-library-2.13.17.jar
+scala-reflect/2.13.17//scala-reflect-2.13.17.jar
+slf4j-api/1.7.36//slf4j-api-1.7.36.jar
+snakeyaml/2.2//snakeyaml-2.2.jar
+snappy-java/1.1.10.5//snappy-java-1.1.10.5.jar
+zstd-jni/1.5.7-6//zstd-jni-1.5.7-6.jar

dev/reformat

@@ -35,6 +35,7 @@ else
   ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-3.3
   ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-3.5
   ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-4.0
+  ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-4.1
   ${PROJECT_DIR}/build/mvn spotless:apply -Paws
   ${PROJECT_DIR}/build/mvn spotless:apply -Paliyun
   ${PROJECT_DIR}/build/mvn spotless:apply -Pmr

docs/developers/sbt.md

@@ -36,6 +36,7 @@ The following table indicates the compatibility of Celeborn Spark and Flink clie
 | Spark 3.4  | ❌          | ✔          | ✔           | ✔           | ✔          | ✔           | ✔           |
 | Spark 3.5  | ❌          | ✔          | ✔           | ✔           | ✔          | ✔           | ✔           |
 | Spark 4.0  | ❌          | ❌          | ❌           | ❌           | ❌          | ❌           | ✔           |
+| Spark 4.1  | ❌          | ❌          | ❌           | ❌           | ❌          | ❌           | ✔           |
 | Flink 1.16 | ❌          | ✔          | ✔           | ❌           | ❌          | ❌           | ❌           |
 | Flink 1.17 | ❌          | ✔          | ✔           | ❌           | ❌          | ❌           | ❌           |
 | Flink 1.18 | ❌          | ✔          | ✔           | ❌           | ❌          | ❌           | ❌           |

pom.xml

@@ -1581,6 +1581,25 @@
       </properties>
     </profile>
 
+    <profile>
+      <id>spark-4.1</id>
+      <modules>
+        <module>client-spark/common</module>
+        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-columnar-common</module>
+        <module>client-spark/spark-4-columnar-shuffle</module>
+        <module>client-spark/spark-4-shaded</module>
+        <module>tests/spark-it</module>
+      </modules>
+      <properties>
+        <lz4-java.version>1.8.0</lz4-java.version>
+        <scala.version>2.13.17</scala.version>
+        <scala.binary.version>2.13</scala.binary.version>
+        <spark.version>4.1.0</spark.version>
+        <zstd-jni.version>1.5.7-6</zstd-jni.version>
+      </properties>
+    </profile>
+
     <profile>
       <id>jdk-8</id>
       <activation>

project/CelebornBuild.scala

@@ -494,6 +494,7 @@ object Utils {
     case Some("spark-3.4") => Some(Spark34)
     case Some("spark-3.5") => Some(Spark35)
     case Some("spark-4.0") => Some(Spark40)
+    case Some("spark-4.1") => Some(Spark41)
     case _ => None
   }
 
@@ -942,6 +943,23 @@ object Spark40 extends SparkClientProjects {
   override val sparkColumnarShuffleVersion: String = "4"
 }
 
+object Spark41 extends SparkClientProjects {
+
+  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectName = "celeborn-client-spark-4"
+  val sparkClientShadedProjectPath = "client-spark/spark-4-shaded"
+  val sparkClientShadedProjectName = "celeborn-client-spark-4-shaded"
+
+  val lz4JavaVersion = "1.8.0"
+  val sparkProjectScalaVersion = "2.13.17"
+
+  val sparkVersion = "4.1.0"
+  val zstdJniVersion = "1.5.7-6"
+  val scalaBinaryVersion = "2.13"
+
+  override val sparkColumnarShuffleVersion: String = "4"
+}
+
 trait SparkClientProjects {
 
   val sparkClientProjectPath: String

tests/spark-it/pom.xml

@@ -328,5 +328,23 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>spark-4.1</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.celeborn</groupId>
+          <artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+          <version>${project.version}</version>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.celeborn</groupId>
+          <artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+          <version>${project.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
   </profiles>
 </project>