BIGTOP-4297: Install Tez with tarball (#120)

kevinw66 · web-flow · commit faa3f1974a56 · 2024-12-09T17:16:24.000+08:00
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/flink/metainfo.xml
@@ -52,17 +52,19 @@
             <package-specific>
                 <architectures>
                     <arch>x86_64</arch>
+                    <arch>aarch64</arch>
                 </architectures>
                 <packages>
                     <package>
-                        <name>flink_3_3_0</name>
+                        <name>flink-1.16.2-1.tgz</name>
+                        <checksum>SHA-256:8c6da7cbab14b9cf0ce39c8c873b29f16c83df0b95b7af5cf3061a468efac228</checksum>
                     </package>
                 </packages>
             </package-specific>
         </package-specifics>
 
         <required-services>
-            <service>yarn</service>
+            <service>hadoop</service>
         </required-services>
     </service>
 </metainfo>
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml
@@ -112,7 +112,7 @@
                 <packages>
                     <package>
                         <name>hadoop-3.3.6-1.tar.gz</name>
-                        <checksum>SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442</checksum>
+                        <checksum>SHA-256:0da6364126c6ea6dd0f5b545afec33957ee801d6db6c0874edb0aa5be1e89995</checksum>
                     </package>
                 </packages>
             </package-specific>
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/kafka/metainfo.xml
@@ -43,10 +43,12 @@
             <package-specific>
                 <architectures>
                     <arch>x86_64</arch>
+                    <arch>aarch64</arch>
                 </architectures>
                 <packages>
                     <package>
-                        <name>kafka_3_3_0</name>
+                        <name>kafka-2.8.2-1.tgz</name>
+                        <checksum>SHA-256:30a2d69ef081813624273d8a406c9b803c5868df998484e27207f85ea217870f</checksum>
                     </package>
                 </packages>
             </package-specific>
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/solr/metainfo.xml
@@ -47,10 +47,12 @@
             <package-specific>
                 <architectures>
                     <arch>x86_64</arch>
+                    <arch>aarch64</arch>
                 </architectures>
                 <packages>
                     <package>
-                        <name>solr_3_3_0</name>
+                        <name>solr-8.11.2-2.tgz</name>
+                        <checksum>SHA-256:fba6618e4c3a3133a4c4d57718e52cc797d10374f056825329a9e76b93de9f48</checksum>
                     </package>
                 </packages>
             </package-specific>
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/metainfo.xml
@@ -39,17 +39,19 @@
             <package-specific>
                 <architectures>
                     <arch>x86_64</arch>
+                    <arch>aarch64</arch>
                 </architectures>
                 <packages>
                     <package>
-                        <name>tez_3_3_0</name>
+                        <name>tez-0.10.2-1.tar.gz</name>
+                        <checksum>SHA-256:ebc2c195780fc76fb9b12e8987e48c301ca0b1916dca515f5c7a1122d74f397e</checksum>
                     </package>
                 </packages>
             </package-specific>
         </package-specifics>
 
         <required-services>
-            <service>hdfs</service>
+            <service>hadoop</service>
         </required-services>
     </service>
 </metainfo>
diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/metainfo.xml
@@ -54,7 +54,7 @@
                 <packages>
                     <package>
                         <name>zookeeper-3.7.2-1.tar.gz</name>
-                        <checksum>SHA-256:380f15d55c0282e33fdc7c2ec551bc5586f0ac126c243cd84347ccb775b846f3</checksum>
+                        <checksum>SHA-256:83e07f914eb3477c77245f4dc44031b82ea6ef1be3691687f469ddc4b8c720bb</checksum>
                     </package>
                 </packages>
             </package-specific>
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java
@@ -18,21 +18,49 @@
  */
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
 
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
+import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo;
 import org.apache.bigtop.manager.common.shell.ShellResult;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
 import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript;
 import org.apache.bigtop.manager.stack.core.spi.script.Script;
+import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;
 
 import com.google.auto.service.AutoService;
 import lombok.extern.slf4j.Slf4j;
 
+import java.io.File;
+import java.util.List;
+
 @Slf4j
 @AutoService(Script.class)
 public class TezClientScript extends AbstractClientScript {
 
+    /**
+     * Tez tarball file doesn't need to be extracted, so here we override {@link #add(Params)} method
+     * and do nothing after download the tarball file.
+     *
+     * @param params the parameters required for installation
+     * @return ShellResult
+     */
+    @Override
+    public ShellResult add(Params params) {
+        RepoInfo repo = params.repo();
+        List<PackageInfo> packages = params.packages();
+
+        String repoUrl = repo.getBaseUrl();
+        String stackHome = params.stackHome();
+        for (PackageInfo packageInfo : packages) {
+            String remoteUrl = repoUrl + File.separator + packageInfo.getName();
+            TarballDownloader.download(remoteUrl, stackHome, packageInfo);
+        }
+
+        return ShellResult.success();
+    }
+
     @Override
     public ShellResult configure(Params params) {
-        return TezSetup.config(params);
+        return TezSetup.configure(params);
     }
 
     @Override
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java
@@ -77,27 +77,14 @@ public Map<String, Object> tezEnv() {
         return tezEnv;
     }
 
-    @Override
-    public String confDir() {
-        return "/etc/tez/conf";
-    }
-
     public String hadoopConfDir() {
-        return "/etc/hadoop/conf";
+        return hadoopHome() + "/etc/hadoop";
     }
 
     public String hadoopHome() {
         return stackHome() + "/hadoop";
     }
 
-    public String hdfsHome() {
-        return stackHome() + "/hadoop-hdfs";
-    }
-
-    public String yarnHome() {
-        return stackHome() + "/hadoop-yarn";
-    }
-
     @Override
     public String getServiceName() {
         return "tez";
diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
@@ -19,8 +19,8 @@
 package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez;
 
 import org.apache.bigtop.manager.common.constants.Constants;
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
 import org.apache.bigtop.manager.common.shell.ShellResult;
-import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil;
 import org.apache.bigtop.manager.stack.core.enums.ConfigType;
 import org.apache.bigtop.manager.stack.core.spi.param.Params;
 import org.apache.bigtop.manager.stack.core.utils.LocalSettings;
@@ -31,23 +31,37 @@
 import lombok.extern.slf4j.Slf4j;
 
 import java.text.MessageFormat;
+import java.util.List;
 
 import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;
 
 @Slf4j
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class TezSetup {
 
-    public static ShellResult config(Params params) {
+    public static ShellResult configure(Params params) {
+        log.info("Configuring Tez");
         TezParams tezParams = (TezParams) params;
 
         String confDir = tezParams.confDir();
-        String hdfsUser = LocalSettings.users().get("hdfs");
+        String serviceHome = tezParams.serviceHome();
+        String libDir = serviceHome + "/lib";
+        String hadoopUser = LocalSettings.users().get("hadoop");
         String tezUser = tezParams.user();
         String tezGroup = tezParams.group();
 
-        // tez-site
-        log.info("Generating [{}/tez-site.xml] file", confDir);
+        LinuxFileUtils.createDirectories(serviceHome, tezUser, tezGroup, Constants.PERMISSION_755, true);
+        LinuxFileUtils.createDirectories(confDir, tezUser, tezGroup, Constants.PERMISSION_755, true);
+        LinuxFileUtils.createDirectories(libDir, tezUser, tezGroup, Constants.PERMISSION_755, true);
+
+        List<PackageInfo> packages = params.packages();
+        for (PackageInfo packageInfo : packages) {
+            String source = tezParams.stackHome() + "/" + packageInfo.getName();
+            String dest = libDir + "/tez.tar.gz";
+            LinuxFileUtils.copyFile(source, dest);
+            LinuxFileUtils.updateOwner(dest, tezUser, tezGroup, false);
+        }
+
         LinuxFileUtils.toFile(
                 ConfigType.XML,
                 MessageFormat.format("{0}/tez-site.xml", confDir),
@@ -57,8 +71,6 @@ public static ShellResult config(Params params) {
                 tezParams.tezSite(),
                 tezParams.getGlobalParamsMap());
 
-        // tez-env
-        log.info("Generating [{}/tez-env.sh] file", confDir);
         LinuxFileUtils.toFileByTemplate(
                 tezParams.getTezEnvContent(),
                 MessageFormat.format("{0}/tez-env.sh", confDir),
@@ -67,10 +79,10 @@ public static ShellResult config(Params params) {
                 PERMISSION_755,
                 tezParams.getGlobalParamsMap());
 
-        HdfsUtil.createDirectory(hdfsUser, "/apps");
-        HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");
+        //        HdfsUtil.createDirectory(hadoopUser, "/apps");
+        //        HdfsUtil.uploadFile(tezUser, tezParams.serviceHome() + "/lib/tez.tar.gz", "/apps/tez");
 
         log.info("Successfully configured Tez");
-        return ShellResult.success("Tez Configure success!");
+        return ShellResult.success();
     }
 }
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java
@@ -18,6 +18,9 @@
  */
 package org.apache.bigtop.manager.stack.core.tarball;
 
+import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
+import org.apache.bigtop.manager.stack.core.exception.StackException;
+
 import lombok.extern.slf4j.Slf4j;
 
 import java.io.File;
@@ -30,7 +33,55 @@
 public class TarballDownloader {
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
-    public static Boolean downloadFile(String fileUrl, String saveDir) {
+    public static void download(String remoteUrl, String saveDir, PackageInfo packageInfo) {
+        File localFile = new File(saveDir + File.separator + packageInfo.getName());
+        String algorithm = packageInfo.getChecksum().split(":")[0];
+        String checksum = packageInfo.getChecksum().split(":")[1];
+
+        if (localFile.exists()) {
+            log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath());
+        } else {
+            log.info("Downloading [{}] to [{}]", remoteUrl, saveDir);
+            download(remoteUrl, saveDir);
+        }
+
+        boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
+        if (!validateChecksum) {
+            log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath());
+            localFile.delete();
+            download(remoteUrl, saveDir);
+        }
+
+        validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
+        if (!validateChecksum) {
+            log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath());
+            throw new StackException("Invalid checksum for " + localFile.getAbsolutePath());
+        }
+
+        log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath());
+    }
+
+    private static void download(String remoteUrl, String saveDir) {
+        int i = 1;
+        while (true) {
+            Boolean downloaded = downloadFile(remoteUrl, saveDir);
+            if (downloaded) {
+                break;
+            } else {
+                if (i == 3) {
+                    log.error("Failed to download [{}], exiting...", remoteUrl);
+                    throw new StackException("Failed to download " + remoteUrl);
+                } else {
+                    log.error("Failed to download [{}], retrying...: {}", remoteUrl, i);
+                }
+            }
+
+            i++;
+        }
+    }
+
+    @SuppressWarnings("ResultOfMethodCallIgnored")
+    private static Boolean downloadFile(String fileUrl, String saveDir) {
         HttpURLConnection httpConn = null;
         try {
             URL url = new URL(fileUrl);
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java
@@ -126,7 +126,7 @@ private static void extractTarGz(File tarball, Function<TarArchiveInputStream, B
     }
 
     private static boolean isTarGz(String filePath) {
-        return filePath.endsWith(".tar.gz");
+        return filePath.endsWith(".tar.gz") || filePath.endsWith(".tgz");
     }
 
     private static boolean isTar(String filePath) {
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/TarballUtils.java
@@ -19,8 +19,6 @@
 package org.apache.bigtop.manager.stack.core.utils;
 
 import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
-import org.apache.bigtop.manager.stack.core.exception.StackException;
-import org.apache.bigtop.manager.stack.core.tarball.ChecksumValidator;
 import org.apache.bigtop.manager.stack.core.tarball.TarballDownloader;
 import org.apache.bigtop.manager.stack.core.tarball.TarballExtractor;
 
@@ -33,7 +31,6 @@
 @Slf4j
 public class TarballUtils {
 
-    @SuppressWarnings("ResultOfMethodCallIgnored")
     public static void installPackage(
             String repoUrl, String stackHome, String serviceHome, PackageInfo packageInfo, Integer skipLevels) {
         if (Files.exists(Path.of(serviceHome))) {
@@ -43,51 +40,10 @@ public static void installPackage(
 
         String remoteUrl = repoUrl + File.separator + packageInfo.getName();
         File localFile = new File(stackHome + File.separator + packageInfo.getName());
-        String algorithm = packageInfo.getChecksum().split(":")[0];
-        String checksum = packageInfo.getChecksum().split(":")[1];
+        TarballDownloader.download(remoteUrl, stackHome, packageInfo);
 
-        if (localFile.exists()) {
-            log.info("File [{}] exists, validating checksum", localFile.getAbsolutePath());
-        } else {
-            log.info("Downloading [{}] to [{}]", remoteUrl, stackHome);
-            downloadPackage(remoteUrl, stackHome);
-        }
-
-        boolean validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
-        if (!validateChecksum) {
-            log.warn("Invalid checksum for [{}], re-downloading...", localFile.getAbsolutePath());
-            localFile.delete();
-            downloadPackage(remoteUrl, stackHome);
-        }
-
-        validateChecksum = ChecksumValidator.validateChecksum(algorithm, checksum, localFile);
-        if (!validateChecksum) {
-            log.error("Invalid checksum for [{}], exiting...", localFile.getAbsolutePath());
-            throw new StackException("Invalid checksum for " + localFile.getAbsolutePath());
-        }
-
-        log.info("Checksum validate successfully for [{}]", localFile.getAbsolutePath());
         log.info("Extracting [{}] to [{}]", localFile.getAbsolutePath(), serviceHome);
         TarballExtractor.extractTarball(localFile.getAbsolutePath(), serviceHome, skipLevels);
         log.info("File [{}] successfully extracted to [{}]", localFile.getAbsolutePath(), serviceHome);
     }
-
-    private static void downloadPackage(String remoteUrl, String saveDir) {
-        int i = 1;
-        while (true) {
-            Boolean downloaded = TarballDownloader.downloadFile(remoteUrl, saveDir);
-            if (downloaded) {
-                break;
-            } else {
-                if (i == 3) {
-                    log.error("Failed to download [{}], exiting...", remoteUrl);
-                    throw new StackException("Failed to download " + remoteUrl);
-                } else {
-                    log.error("Failed to download [{}], retrying...: {}", remoteUrl, i);
-                }
-            }
-
-            i++;
-        }
-    }
 }
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java
