Skip to content

Commit 10425e7

Browse files
kevinw66kevinw66
authored
BIGTOP-4295: Install Hadoop with tarball (#118)
1 parent 8a37894 commit 10425e7

File tree

80 files changed

+883
-1447
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+883
-1447
lines changed

‎bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,5 @@ public interface ServiceDao extends BaseDao<ServicePO> {
3434

3535
ServicePO findByClusterIdAndName(@Param("clusterId") Long clusterId, @Param("name") String name);
3636

37-
List<ServicePO> findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("name") List<String> names);
37+
List<ServicePO> findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("names") List<String> names);
3838
}

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/core-site.xml

File renamed without changes.

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,6 @@
1919
-->
2020

2121
<configuration>
22-
<property>
23-
<name>hadoop_log_dir_prefix</name>
24-
<value>/var/log/hadoop</value>
25-
<description>Hadoop Log Dir Prefix</description>
26-
<display-name>Hadoop Log Dir Prefix</display-name>
27-
</property>
28-
<property>
29-
<name>hadoop_pid_dir_prefix</name>
30-
<value>/var/run/hadoop</value>
31-
<display-name>Hadoop PID Dir Prefix</display-name>
32-
<description>Hadoop PID Dir Prefix</description>
33-
</property>
3422
<property>
3523
<name>hadoop_root_logger</name>
3624
<value>INFO,RFA</value>
@@ -49,16 +37,6 @@
4937
<description>Hadoop minimum Java heap size</description>
5038
<display-name>Hadoop minimum Java heap size</display-name>
5139
</property>
52-
<property>
53-
<name>hdfs_user_nofile_limit</name>
54-
<value>128000</value>
55-
<description>Max open files limit setting for HDFS user.</description>
56-
</property>
57-
<property>
58-
<name>hdfs_user_nproc_limit</name>
59-
<value>65536</value>
60-
<description>Max number of processes limit setting for HDFS user.</description>
61-
</property>
6240
<!-- hadoop-env.sh -->
6341
<property>
6442
<name>content</name>
@@ -87,9 +65,9 @@
8765
USER="$(whoami)"
8866
export JAVA_HOME=${java_home!}
8967
export HADOOP_HOME=${hadoop_home!}
90-
export HADOOP_COMMON_HOME=${hadoop_home!}
91-
export HADOOP_HDFS_HOME=${hadoop_hdfs_home!}
9268
export HADOOP_CONF_DIR=${hadoop_conf_dir!}
69+
export HADOOP_LOG_DIR=${hadoop_log_dir}
70+
export HADOOP_PID_DIR=${hadoop_pid_dir}
9371
export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max}
9472
export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
9573
@@ -99,11 +77,9 @@ export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
9977
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
10078
</#noparse>
10179
102-
export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER
103-
export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER
10480
export HADOOP_ROOT_LOGGER=${hadoop_root_logger}
10581
export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
106-
82+
export LD_LIBRARY_PATH=${hadoop_home!}/lib/native
10783
]]>
10884
</value>
10985
<attrs>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-policy.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-policy.xml

File renamed without changes.

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs.conf.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop.conf.xml

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,20 @@
1919
-->
2020

2121
<configuration>
22+
<property>
23+
<name>hadoop_user_nofile_limit</name>
24+
<value>128000</value>
25+
<description>Max open files limit setting for Hadoop user.</description>
26+
</property>
27+
<property>
28+
<name>hadoop_user_nproc_limit</name>
29+
<value>65536</value>
30+
<description>Max number of processes limit setting for Hadoop user.</description>
31+
</property>
2232
<property>
2333
<name>content</name>
24-
<display-name>hdfs.conf template</display-name>
25-
<description>This is the freemarker template for hdfs file</description>
34+
<display-name>hadoop.conf template</display-name>
35+
<description>This is the freemarker template for hadoop file</description>
2636
<value><![CDATA[
2737
#
2838
# Licensed to the Apache Software Foundation (ASF) under one
@@ -42,8 +52,8 @@
4252
# limitations under the License.
4353
#
4454
45-
${hdfs_user} - nofile ${hdfs_user_nofile_limit}
46-
${hdfs_group} - nproc ${hdfs_user_nproc_limit}
55+
${hadoop_user} - nofile ${hdfs_user_nofile_limit}
56+
${hadoop_group} - nproc ${hdfs_user_nproc_limit}
4757
4858
]]>
4959
</value>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-log4j.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-log4j.xml

File renamed without changes.

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-site.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@
323323
</property>
324324
<property>
325325
<name>dfs.domain.socket.path</name>
326-
<value>/var/run/hadoop-hdfs/dn._PORT</value>
326+
<value>/var/run/hadoop/dn._PORT</value>
327327
<description>
328328
This is a path to a UNIX domain socket that will be used for communication between the DataNode and local
329329
HDFS clients.

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -19,34 +19,12 @@
1919
-->
2020

2121
<configuration>
22-
<property>
23-
<name>mapred_log_dir_prefix</name>
24-
<value>/var/log/hadoop-mapreduce</value>
25-
<display-name>Mapreduce Log Dir Prefix</display-name>
26-
<description>Mapreduce Log Dir Prefix</description>
27-
</property>
28-
<property>
29-
<name>mapred_pid_dir_prefix</name>
30-
<value>/var/run/hadoop-mapreduce</value>
31-
<display-name>Mapreduce PID Dir Prefix</display-name>
32-
<description>Mapreduce PID Dir Prefix</description>
33-
</property>
3422
<property>
3523
<name>jobhistory_heapsize</name>
3624
<display-name>History Server heap size</display-name>
3725
<value>1024</value>
3826
<description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
3927
</property>
40-
<property>
41-
<name>mapred_user_nofile_limit</name>
42-
<value>32768</value>
43-
<description>Max open files limit setting for MAPREDUCE user.</description>
44-
</property>
45-
<property>
46-
<name>mapred_user_nproc_limit</name>
47-
<value>65536</value>
48-
<description>Max number of processes limit setting for MAPREDUCE user.</description>
49-
</property>
5028
<property>
5129
<name>content</name>
5230
<display-name>mapred-env template</display-name>
@@ -57,12 +35,8 @@ export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=${jobhistory_heapsize!}
5735
export HADOOP_ROOT_LOGGER=INFO,RFA
5836
5937
export HADOOP_HOME=${hadoop_home}
60-
export HADOOP_YARN_HOME=${hadoop_yarn_home}
61-
export HADOOP_MAPRED_HOME=${hadoop_mapred_home}
6238
6339
# Could be enabled from deployment option if necessary
64-
export HADOOP_LOG_DIR=${mapred_log_dir_prefix!}/$USER
65-
export HADOOP_PID_DIR=${mapred_pid_dir_prefix!}/$USER
6640
export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
6741
export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
6842
export HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-site.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@
253253
</property>
254254
<property>
255255
<name>mapreduce.application.classpath</name>
256-
<value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${hadoop_mapred_home!}/*,${hadoop_mapred_home!}/lib/*</value>
256+
<value>${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*</value>
257257
<description>CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries.</description>
258258
</property>
259259
<property>

bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/ssl-client.xml renamed to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/ssl-client.xml

File renamed without changes.

0 commit comments

Comments
 (0)