BIGTOP-4295: Install Hadoop with tarball

bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java

@@ -34,5 +34,5 @@ public interface ServiceDao extends BaseDao<ServicePO> {
 
     ServicePO findByClusterIdAndName(@Param("clusterId") Long clusterId, @Param("name") String name);
 
-    List<ServicePO> findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("name") List<String> names);
+    List<ServicePO> findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("names") List<String> names);
 }

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml

@@ -19,18 +19,6 @@
 -->
 
 <configuration>
-    <property>
-        <name>hadoop_log_dir_prefix</name>
-        <value>/var/log/hadoop</value>
-        <description>Hadoop Log Dir Prefix</description>
-        <display-name>Hadoop Log Dir Prefix</display-name>
-    </property>
-    <property>
-        <name>hadoop_pid_dir_prefix</name>
-        <value>/var/run/hadoop</value>
-        <display-name>Hadoop PID Dir Prefix</display-name>
-        <description>Hadoop PID Dir Prefix</description>
-    </property>
     <property>
         <name>hadoop_root_logger</name>
         <value>INFO,RFA</value>
@@ -49,16 +37,6 @@
         <description>Hadoop minimum Java heap size</description>
         <display-name>Hadoop minimum Java heap size</display-name>
     </property>
-    <property>
-        <name>hdfs_user_nofile_limit</name>
-        <value>128000</value>
-        <description>Max open files limit setting for HDFS user.</description>
-    </property>
-    <property>
-        <name>hdfs_user_nproc_limit</name>
-        <value>65536</value>
-        <description>Max number of processes limit setting for HDFS user.</description>
-    </property>
     <!-- hadoop-env.sh -->
     <property>
         <name>content</name>
@@ -87,9 +65,9 @@
 USER="$(whoami)"
 export JAVA_HOME=${java_home!}
 export HADOOP_HOME=${hadoop_home!}
-export HADOOP_COMMON_HOME=${hadoop_home!}
-export HADOOP_HDFS_HOME=${hadoop_hdfs_home!}
 export HADOOP_CONF_DIR=${hadoop_conf_dir!}
+export HADOOP_LOG_DIR=${hadoop_log_dir}
+export HADOOP_PID_DIR=${hadoop_pid_dir}
 export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max}
 export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
 
@@ -99,11 +77,9 @@ export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
 export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
 </#noparse>
 
-export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER
-export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER
 export HADOOP_ROOT_LOGGER=${hadoop_root_logger}
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
-
+export LD_LIBRARY_PATH=${hadoop_home!}/lib/native
 ]]>
         </value>
         <attrs>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs.conf.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop.conf.xml

@@ -19,10 +19,20 @@
 -->
 
 <configuration>
+    <property>
+        <name>hadoop_user_nofile_limit</name>
+        <value>128000</value>
+        <description>Max open files limit setting for Hadoop user.</description>
+    </property>
+    <property>
+        <name>hadoop_user_nproc_limit</name>
+        <value>65536</value>
+        <description>Max number of processes limit setting for Hadoop user.</description>
+    </property>
     <property>
         <name>content</name>
-        <display-name>hdfs.conf template</display-name>
-        <description>This is the freemarker template for hdfs file</description>
+        <display-name>hadoop.conf template</display-name>
+        <description>This is the freemarker template for hadoop file</description>
         <value><![CDATA[
 #
 # Licensed to the Apache Software Foundation (ASF) under one
@@ -42,8 +52,8 @@
 # limitations under the License.
 #
 
-${hdfs_user}    - nofile   ${hdfs_user_nofile_limit}
-${hdfs_group}   - nproc    ${hdfs_user_nproc_limit}
+${hadoop_user}    - nofile   ${hdfs_user_nofile_limit}
+${hadoop_group}   - nproc    ${hdfs_user_nproc_limit}
 
 ]]>
         </value>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-site.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml

@@ -323,7 +323,7 @@
     </property>
     <property>
         <name>dfs.domain.socket.path</name>
-        <value>/var/run/hadoop-hdfs/dn._PORT</value>
+        <value>/var/run/hadoop/dn._PORT</value>
         <description>
             This is a path to a UNIX domain socket that will be used for communication between the DataNode and local
             HDFS clients.

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml

@@ -19,34 +19,12 @@
 -->
 
 <configuration>
-    <property>
-        <name>mapred_log_dir_prefix</name>
-        <value>/var/log/hadoop-mapreduce</value>
-        <display-name>Mapreduce Log Dir Prefix</display-name>
-        <description>Mapreduce Log Dir Prefix</description>
-    </property>
-    <property>
-        <name>mapred_pid_dir_prefix</name>
-        <value>/var/run/hadoop-mapreduce</value>
-        <display-name>Mapreduce PID Dir Prefix</display-name>
-        <description>Mapreduce PID Dir Prefix</description>
-    </property>
     <property>
         <name>jobhistory_heapsize</name>
         <display-name>History Server heap size</display-name>
         <value>1024</value>
         <description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
     </property>
-    <property>
-        <name>mapred_user_nofile_limit</name>
-        <value>32768</value>
-        <description>Max open files limit setting for MAPREDUCE user.</description>
-    </property>
-    <property>
-        <name>mapred_user_nproc_limit</name>
-        <value>65536</value>
-        <description>Max number of processes limit setting for MAPREDUCE user.</description>
-    </property>
     <property>
         <name>content</name>
         <display-name>mapred-env template</display-name>
@@ -57,12 +35,8 @@ export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=${jobhistory_heapsize!}
 export HADOOP_ROOT_LOGGER=INFO,RFA
 
 export HADOOP_HOME=${hadoop_home}
-export HADOOP_YARN_HOME=${hadoop_yarn_home}
-export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
 
 # Could be enabled from deployment option if necessary
-export HADOOP_LOG_DIR=${mapred_log_dir_prefix!}/$USER
-export HADOOP_PID_DIR=${mapred_pid_dir_prefix!}/$USER
 export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
 export HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-site.xml

@@ -253,7 +253,7 @@
     </property>
     <property>
         <name>mapreduce.application.classpath</name>
-        <value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${hadoop_mapred_home!}/*,${hadoop_mapred_home!}/lib/*</value>
+        <value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*</value>
         <description>CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries.</description>
     </property>
     <property>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/workers.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/workers.xml

@@ -21,7 +21,7 @@
 <configuration>
     <property>
         <name>content</name>
-        <display-name>hdfs slaves template</display-name>
+        <display-name>hdfs workers template</display-name>
         <description>This is the freemarker template for hdfs file</description>
         <value><![CDATA[
 #

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-env.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-env.xml

@@ -18,18 +18,6 @@
   ~ under the License.
 -->
 <configuration>
-    <property>
-        <name>yarn_log_dir_prefix</name>
-        <value>/var/log/hadoop-yarn</value>
-        <display-name>YARN Log Dir Prefix</display-name>
-        <description>YARN Log Dir Prefix</description>
-    </property>
-    <property>
-        <name>yarn_pid_dir_prefix</name>
-        <value>/var/run/hadoop-yarn</value>
-        <display-name>YARN PID Dir Prefix</display-name>
-        <description>YARN PID Dir Prefix</description>
-    </property>
     <property>
         <name>yarn_heapsize</name>
         <value>1024</value>
@@ -59,43 +47,22 @@
         <value>false</value>
         <description>Set to false by default,  needs to be set to true in stacks that use Ranger Yarn Plugin</description>
     </property>
-    <property>
-        <name>yarn_user_nofile_limit</name>
-        <value>32768</value>
-        <description>Max open files limit setting for YARN user.</description>
-    </property>
-    <property>
-        <name>yarn_user_nproc_limit</name>
-        <value>65536</value>
-        <description>Max number of processes limit setting for YARN user.</description>
-    </property>
-    <property>
-        <name>hadoop_java_io_tmpdir</name>
-        <value>/tmp/hadoop-yarn</value>
-        <description>Max number of processes limit setting for YARN user.</description>
-    </property>
     <!-- yarn-env.sh -->
     <property>
         <name>content</name>
         <display-name>yarn-env template</display-name>
         <description>This is the freemarker template for yarn-env.sh file</description>
         <value><![CDATA[
 export HADOOP_HOME=${hadoop_home}
-export HADOOP_YARN_HOME=${hadoop_yarn_home}
-export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
 USER="$(whoami)"
-export HADOOP_LOG_DIR=${yarn_log_dir_prefix}/$USER
-export HADOOP_PID_DIR=${yarn_pid_dir_prefix}/$USER
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
 export JAVA_HOME=${java_home}
-export JAVA_LIBRARY_PATH="<#noparse>${JAVA_LIBRARY_PATH}</#noparse>:${hadoop_java_io_tmpdir}"
 <#noparse>
 # User for YARN daemons
-export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-
+export HADOOP_YARN_USER=${hadoop_user}
 
 # resolve links - $0 may be a softlink
-export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}"
+export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/etc/hadoop}"
 
 # some Java parameters
 # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
@@ -170,7 +137,7 @@ IFS=
 <#noparse>
 # default log directory and file
 if [ "$HADOOP_LOG_DIR" = "" ]; then
-HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs"
+HADOOP_LOG_DIR="$HADOOP_HOME/logs"
 fi
 if [ "$HADOOP_LOGFILE" = "" ]; then
 HADOOP_LOGFILE='yarn.log'
@@ -189,7 +156,6 @@ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
 HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE"
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
 HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
 HADOOP_OPTS="$HADOOP_OPTS -Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
@@ -200,7 +166,6 @@ HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
 fi
 HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
 </#noparse>
-HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir=${hadoop_java_io_tmpdir}"
 HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
 ]]>
         </value>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml → bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml

@@ -76,7 +76,7 @@
     </property>
     <property>
         <name>yarn.admin.acl</name>
-        <value />
+        <value>hadoop</value>
         <description>ACL of who can be admin of the YARN cluster.</description>
     </property>
     <!-- NodeManager -->
@@ -93,7 +93,7 @@
     </property>
     <property>
         <name>yarn.application.classpath</name>
-        <value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*</value>
+        <value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*</value>
         <description>Classpath for typical applications.</description>
     </property>
     <property>
@@ -291,16 +291,6 @@
             URI for the HistoryServer's log resource
         </description>
     </property>
-    <property>
-        <name>yarn.resourcemanager.nodes.exclude-path</name>
-        <value>/etc/hadoop/conf/yarn.exclude</value>
-        <description>
-            Names a file that contains a list of hosts that are
-            not permitted to connect to the resource manager. The full pathname of the
-            file must be specified. If the value is empty, no hosts are
-            excluded.
-        </description>
-    </property>
     <property>
         <name>manage.include.files</name>
         <value>false</value>
@@ -333,7 +323,7 @@
     </property>
     <property>
         <name>yarn.timeline-service.leveldb-timeline-store.path</name>
-        <value>/var/log/hadoop-yarn/timeline</value>
+        <value>/var/log/hadoop/timeline</value>
         <description>
             Store file name for leveldb timeline store
         </description>
@@ -361,14 +351,14 @@
         </description>
     </property>
     <property>
-        <description>Enable age off of timeline store data.</description>
         <name>yarn.timeline-service.ttl-enable</name>
         <value>true</value>
+        <description>Enable age off of timeline store data.</description>
     </property>
     <property>
-        <description>Time to live for timeline store data in milliseconds.</description>
         <name>yarn.timeline-service.ttl-ms</name>
         <value>2678400000</value>
+        <description>Time to live for timeline store data in milliseconds.</description>
     </property>
     <property>
         <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
@@ -383,22 +373,7 @@
             true, then yarn.timeline-service.state-store-class must be specified.
         </description>
     </property>
-    <property>
-        <name>yarn.acl.enable</name>
-        <value>false</value>
-        <description>Are acls enabled.</description>
-    </property>
-    <property>
-        <name>yarn.authorization-provider</name>
-        <description>Yarn authorization provider class.</description>
-    </property>
-    <property>
-        <name>yarn.admin.acl</name>
-        <value>yarn</value>
-        <description>ACL of who can be admin of the YARN cluster.</description>
-    </property>
-    <!--ats
-    v1.5 properties-->
+    <!--ats v1.5 properties-->
     <property>
         <name>yarn.timeline-service.store-class</name>
         <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>