diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java index 7a24a3e6b..06a268087 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ServiceDao.java @@ -34,5 +34,5 @@ public interface ServiceDao extends BaseDao { ServicePO findByClusterIdAndName(@Param("clusterId") Long clusterId, @Param("name") String name); - List findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("name") List names); + List findByClusterIdAndNames(@Param("clusterId") Long clusterId, @Param("names") List names); } diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/core-site.xml similarity index 100% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/core-site.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/core-site.xml diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml similarity index 76% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml index 01ec60841..170d3a546 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-env.xml @@ -19,18 +19,6 @@ --> - - hadoop_log_dir_prefix - /var/log/hadoop - Hadoop Log Dir Prefix - Hadoop Log Dir Prefix - - - hadoop_pid_dir_prefix - /var/run/hadoop - Hadoop PID Dir Prefix - Hadoop PID Dir Prefix - hadoop_root_logger INFO,RFA @@ -49,16 +37,6 @@ Hadoop minimum Java heap size Hadoop minimum Java heap size - - hdfs_user_nofile_limit - 128000 - Max open files limit setting for HDFS user. - - - hdfs_user_nproc_limit - 65536 - Max number of processes limit setting for HDFS user. - content @@ -87,9 +65,9 @@ USER="$(whoami)" export JAVA_HOME=${java_home!} export HADOOP_HOME=${hadoop_home!} -export HADOOP_COMMON_HOME=${hadoop_home!} -export HADOOP_HDFS_HOME=${hadoop_hdfs_home!} export HADOOP_CONF_DIR=${hadoop_conf_dir!} +export HADOOP_LOG_DIR=${hadoop_log_dir} +export HADOOP_PID_DIR=${hadoop_pid_dir} export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max} export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min} @@ -99,11 +77,9 @@ export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min} export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} -export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER -export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER export HADOOP_ROOT_LOGGER=${hadoop_root_logger} export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir} - +export LD_LIBRARY_PATH=${hadoop_home!}/lib/native ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-policy.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-policy.xml similarity index 100% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-policy.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop-policy.xml diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs.conf.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop.conf.xml similarity index 74% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs.conf.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop.conf.xml index 72b7868cd..427fd926f 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs.conf.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hadoop.conf.xml @@ -19,10 +19,20 @@ --> + + hadoop_user_nofile_limit + 128000 + Max open files limit setting for Hadoop user. + + + hadoop_user_nproc_limit + 65536 + Max number of processes limit setting for Hadoop user. + content - hdfs.conf template - This is the freemarker template for hdfs file + hadoop.conf template + This is the freemarker template for hadoop file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-log4j.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-log4j.xml similarity index 100% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-log4j.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-log4j.xml diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml similarity index 99% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-site.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml index aba86039a..85fd05d4c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hdfs-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/hdfs-site.xml @@ -323,7 +323,7 @@ dfs.domain.socket.path - /var/run/hadoop-hdfs/dn._PORT + /var/run/hadoop/dn._PORT This is a path to a UNIX domain socket that will be used for communication between the DataNode and local HDFS clients. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml similarity index 63% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml index ad177ca52..99be32046 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-env.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-env.xml @@ -19,34 +19,12 @@ --> - - mapred_log_dir_prefix - /var/log/hadoop-mapreduce - Mapreduce Log Dir Prefix - Mapreduce Log Dir Prefix - - - mapred_pid_dir_prefix - /var/run/hadoop-mapreduce - Mapreduce PID Dir Prefix - Mapreduce PID Dir Prefix - jobhistory_heapsize History Server heap size 1024 Value for JobHistoryServer heap_size variable in hadoop-env.sh - - mapred_user_nofile_limit - 32768 - Max open files limit setting for MAPREDUCE user. - - - mapred_user_nproc_limit - 65536 - Max number of processes limit setting for MAPREDUCE user. - content mapred-env template @@ -57,12 +35,8 @@ export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=${jobhistory_heapsize!} export HADOOP_ROOT_LOGGER=INFO,RFA export HADOOP_HOME=${hadoop_home} -export HADOOP_YARN_HOME=${hadoop_yarn_home} -export HADOOP_MAPRED_HOME=${hadoop_mapred_home} # Could be enabled from deployment option if necessary -export HADOOP_LOG_DIR=${mapred_log_dir_prefix!}/$USER -export HADOOP_PID_DIR=${mapred_pid_dir_prefix!}/$USER export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir} export HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED" diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-site.xml similarity index 98% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-site.xml index 2cd6f6d84..37dee5991 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapred-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/mapred-site.xml @@ -253,7 +253,7 @@ mapreduce.application.classpath - ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${hadoop_mapred_home!}/*,${hadoop_mapred_home!}/lib/* + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/* CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/ssl-client.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/ssl-client.xml similarity index 100% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/ssl-client.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/ssl-client.xml diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/ssl-server.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/ssl-server.xml similarity index 100% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/ssl-server.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/ssl-server.xml diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/workers.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/workers.xml similarity index 97% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/workers.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/workers.xml index 4d5da3efa..d10cc4e0c 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/workers.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/workers.xml @@ -21,7 +21,7 @@ content - hdfs slaves template + hdfs workers template This is the freemarker template for hdfs file - - yarn_log_dir_prefix - /var/log/hadoop-yarn - YARN Log Dir Prefix - YARN Log Dir Prefix - - - yarn_pid_dir_prefix - /var/run/hadoop-yarn - YARN PID Dir Prefix - YARN PID Dir Prefix - yarn_heapsize 1024 @@ -59,21 +47,6 @@ false Set to false by default, needs to be set to true in stacks that use Ranger Yarn Plugin - - yarn_user_nofile_limit - 32768 - Max open files limit setting for YARN user. - - - yarn_user_nproc_limit - 65536 - Max number of processes limit setting for YARN user. - - - hadoop_java_io_tmpdir - /tmp/hadoop-yarn - Max number of processes limit setting for YARN user. - content @@ -81,21 +54,15 @@ This is the freemarker template for yarn-env.sh file ${JAVA_LIBRARY_PATH}:${hadoop_java_io_tmpdir}" <#noparse> # User for YARN daemons -export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} - +export HADOOP_YARN_USER=${hadoop_user} # resolve links - $0 may be a softlink -export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}" +export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/etc/hadoop}" # some Java parameters # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ @@ -170,7 +137,7 @@ IFS= <#noparse> # default log directory and file if [ "$HADOOP_LOG_DIR" = "" ]; then -HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs" +HADOOP_LOG_DIR="$HADOOP_HOME/logs" fi if [ "$HADOOP_LOGFILE" = "" ]; then HADOOP_LOGFILE='yarn.log' @@ -189,7 +156,6 @@ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE" HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE" -HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME" HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}" HADOOP_OPTS="$HADOOP_OPTS -Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}" @@ -200,7 +166,6 @@ HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" -HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir=${hadoop_java_io_tmpdir}" HADOOP_OPTS="$HADOOP_OPTS --add-opens java.base/java.lang=ALL-UNNAMED" ]]> diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-log4j.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-log4j.xml similarity index 100% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-log4j.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-log4j.xml diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml similarity index 94% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml index 8ff439501..a67158646 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/configuration/yarn-site.xml @@ -76,7 +76,7 @@ yarn.admin.acl - + hadoop ACL of who can be admin of the YARN cluster. @@ -93,7 +93,7 @@ yarn.application.classpath - ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/* + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/* Classpath for typical applications. @@ -291,16 +291,6 @@ URI for the HistoryServer's log resource - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - Names a file that contains a list of hosts that are - not permitted to connect to the resource manager. The full pathname of the - file must be specified. If the value is empty, no hosts are - excluded. - - manage.include.files false @@ -333,7 +323,7 @@ yarn.timeline-service.leveldb-timeline-store.path - /var/log/hadoop-yarn/timeline + /var/log/hadoop/timeline Store file name for leveldb timeline store @@ -361,14 +351,14 @@ - Enable age off of timeline store data. yarn.timeline-service.ttl-enable true + Enable age off of timeline store data. - Time to live for timeline store data in milliseconds. yarn.timeline-service.ttl-ms 2678400000 + Time to live for timeline store data in milliseconds. yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms @@ -383,22 +373,7 @@ true, then yarn.timeline-service.state-store-class must be specified. - - yarn.acl.enable - false - Are acls enabled. - - - yarn.authorization-provider - Yarn authorization provider class. - - - yarn.admin.acl - yarn - ACL of who can be admin of the YARN cluster. - - + yarn.timeline-service.store-class org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml similarity index 53% rename from bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml rename to bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml index 2d8215b04..ce0615be0 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/metainfo.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/metainfo.xml @@ -20,13 +20,14 @@ - hdfs - HDFS + hadoop + Hadoop Apache Hadoop Distributed File System. 3.3.6-1 - hdfs + hadoop + namenode NameNode @@ -39,9 +40,9 @@ NameNode UI dfs.namenode.http-address - 50070 + 9870 dfs.namenode.http-address - 50470 + 9870 @@ -51,14 +52,52 @@ 1+ - secondary_namenode + secondarynamenode SNameNode server 1 + + + + resourcemanager + ResourceManager + server + 1-2 + + ResourceManager UI + yarn.resourcemanager.webapp.address + 8088 + yarn.resourcemanager.webapp.https.address + 8090 + + + + nodemanager + NodeManager + server + 1+ + + + - hdfs_client - HDFS Client + history_server + History Server + server + 1 + + JobHistory UI + mapreduce.jobhistory.webapp.address + 19888 + mapreduce.jobhistory.webapp.https.address + 19890 + + + + + + hadoop_client + Hadoop Client client 1+ @@ -68,16 +107,12 @@ x86_64 + aarch64 - hadoop_3_3_0 - - - hadoop_3_3_0-client - - - hadoop_3_3_0-libhdfs + hadoop-3.3.6-1.tar.gz + SHA-256:2a9312eac69a542c6266f1689d29ded243dde2d61add333a9762829008f97442 @@ -86,6 +121,5 @@ zookeeper - diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/order.json new file mode 100644 index 000000000..41035f89e --- /dev/null +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hadoop/order.json @@ -0,0 +1,47 @@ +{ + "SECONDARYNAMENODE-START": [ + "NAMENODE-START" + ], + "SECONDARYNAMENODE-RESTART": [ + "NAMENODE-RESTART" + ], + "DATANODE-STOP": [ + "NAMENODE-START" + ], + "NAMENODE-STOP": [ + "DATANODE-STOP", + "SECONDARYNAMENODE-STOP" + ], + "NAMENODE-START": [ + "ZKFC-START", + "JOURNALNODE-START", + "ZOOKEEPER_SERVER-START" + ], + "ZKFC-START": [ + "ZOOKEEPER_SERVER-START" + ], + "ZKFC-STOP": [ + "NAMENODE-STOP" + ], + "JOURNALNODE-STOP": [ + "NAMENODE-STOP" + ], + "RESOURCEMANAGER-START": [ + "ZOOKEEPER_SERVER-START", + "NAMENODE-START", + "DATANODE-START" + ], + "NODEMANAGER-START": [ + "NAMENODE-START", + "DATANODE-START", + "RESOURCEMANAGER-START" + ], + "RESOURCEMANAGER-RESTART": [ + "NAMENODE-RESTART" + ], + "NODEMANAGER-RESTART": [ + "NAMENODE-RESTART" + ], + "HISTORY_SERVER-START": ["NAMENODE-START", "DATANODE-START"], + "HISTORY_SERVER-RESTART": ["NAMENODE-RESTART"] +} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/order.json deleted file mode 100644 index 1b0c1c384..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/order.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "SECONDARY_NAMENODE-START": [ - "NAMENODE-START" - ], - "SECONDARY_NAMENODE-RESTART": [ - "NAMENODE-RESTART" - ], - "DATANODE-STOP": [ - "NAMENODE-START" - ], - "NAMENODE-STOP": [ - "DATANODE-STOP", - "SECONDARY_NAMENODE-STOP" - ], - "NAMENODE-START": [ - "ZKFC-START", - "JOURNALNODE-START", - "ZOOKEEPER_SERVER-START" - ], - "ZKFC-START": [ - "ZOOKEEPER_SERVER-START" - ], - "ZKFC-STOP": [ - "NAMENODE-STOP" - ], - "JOURNALNODE-STOP": [ - "NAMENODE-STOP" - ] -} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml deleted file mode 100644 index 52807898f..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/configuration/mapreduce.conf.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - content - mapreduce.conf template - This is the freemarker template for mapreduce file - - - - longtext - - - \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml deleted file mode 100644 index 698fdcd74..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/metainfo.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - mapreduce2 - MapReduce2 - Apache Hadoop NextGen MapReduce. - 3.3.6-1 - mapred - - - - history_server - History Server - server - 1 - - JobHistory UI - mapreduce.jobhistory.webapp.address - 19888 - mapreduce.jobhistory.webapp.https.address - 19890 - - - - mapreduce2_client - MapReduce2 Client - client - 1+ - - - - - - - x86_64 - - - - hadoop_3_3_0-yarn - - - hadoop_3_3_0-hdfs - - - hadoop_3_3_0-mapreduce - - - - - - - yarn - - - diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json deleted file mode 100644 index 883fc62b8..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/mapred/order.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "HISTORY_SERVER-START": ["NAMENODE-START", "DATANODE-START"], - "HISTORY_SERVER-RESTART": ["NAMENODE-RESTART"] -} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml index 98d5f2912..130ac59cb 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml @@ -22,7 +22,7 @@ tez.lib.uris.classpath TEZ Lib URIs Classpath - ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${hadoop_hdfs_home!}/*,${hadoop_hdfs_home!}/lib/*,${hadoop_yarn_home!}/*,${hadoop_yarn_home!}/lib/*,${tez_home!}/*,${tez_home!}/lib/*,${tez_conf_dir!} + ${hadoop_conf_dir!},${hadoop_home!}/*,${hadoop_home!}/lib/*,${tez_home!}/*,${tez_home!}/lib/*,${tez_conf_dir!} Comma-delimited list of the location of the Tez libraries Classpath which will be localized for DAGs. diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml deleted file mode 100644 index aaf803580..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/exclude-nodes.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - content - hdfs slaves template - This is the freemarker template for hdfs file - -<#list exclude_hosts as host> -${host} - - - -]]> - - - longtext - - - \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml deleted file mode 100644 index facf11fff..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/configuration/yarn.conf.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - content - yarn.conf template - This is the freemarker template for yarn file - - - - longtext - - - \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml deleted file mode 100644 index ffc8eb337..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/metainfo.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - yarn - YARN - Apache Hadoop NextGen YARN. - 3.3.6-1 - yarn - - - - resourcemanager - ResourceManager - server - 1-2 - - ResourceManager UI - yarn.resourcemanager.webapp.address - 8088 - yarn.resourcemanager.webapp.https.address - 8090 - - - - nodemanager - NodeManager - server - 1+ - - - yarn_client - YARN Client - client - 1+ - - - - - - - x86_64 - - - - hadoop_3_3_0-yarn - - - hadoop_3_3_0-hdfs - - - hadoop_3_3_0-mapreduce - - - - - - - zookeeper - hdfs - - - - diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json deleted file mode 100644 index ea235ec14..000000000 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/yarn/order.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "RESOURCEMANAGER-START": [ - "ZOOKEEPER_SERVER-START", - "NAMENODE-START", - "DATANODE-START" - ], - "NODEMANAGER-START": [ - "NAMENODE-START", - "DATANODE-START", - "RESOURCEMANAGER-START" - ], - "RESOURCEMANAGER-RESTART": [ - "NAMENODE-RESTART" - ], - "NODEMANAGER-RESTART": [ - "NAMENODE-RESTART" - ] -} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml index 35ef34907..bd09086c9 100644 --- a/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml +++ b/bigtop-manager-server/src/main/resources/stacks/bigtop/3.3.0/services/zookeeper/configuration/zoo.cfg.xml @@ -45,6 +45,12 @@ Port for running ZK Server Port for running ZK Server. + + dataDir + /hadoop/zookeeper + ZooKeeper directory + Data directory for ZooKeeper. + autopurge.snapRetainCount 30 diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java index b30efc93d..c3a229c99 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/param/BigtopParams.java @@ -19,8 +19,11 @@ package org.apache.bigtop.manager.stack.bigtop.param; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; -import org.apache.bigtop.manager.stack.core.param.BaseParams; +import org.apache.bigtop.manager.stack.core.spi.param.BaseParams; +import lombok.NoArgsConstructor; + +@NoArgsConstructor public abstract class BigtopParams extends BaseParams { protected BigtopParams(CommandPayload commandPayload) { diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java index fdfaccd57..38a56f843 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkClientScript.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.flink; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; @@ -34,4 +34,9 @@ public class FlinkClientScript extends AbstractClientScript { public ShellResult configure(Params params) { return FlinkSetup.config(params); } + + @Override + public String getComponentName() { + return "flink_client"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java index 0c4a41033..923231a51 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkHistoryServerScript.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -65,4 +65,9 @@ public ShellResult status(Params params) { FlinkParams flinkParams = (FlinkParams) params; return LinuxOSUtils.checkProcess(flinkParams.getHistoryServerPidFile()); } + + @Override + public String getComponentName() { + return "flink_history_server"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java index 6d7d4bfbf..70db4fada 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkParams.java @@ -22,9 +22,12 @@ import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; @@ -32,6 +35,8 @@ @Getter @Slf4j +@AutoService(Params.class) +@NoArgsConstructor public class FlinkParams extends BigtopParams { private String flinkLogDir; @@ -64,7 +69,7 @@ public FlinkParams(CommandPayload commandPayload) { @GlobalParams public Map flinkConf() { - Map configurations = LocalSettings.configurations(serviceName(), "flink-conf"); + Map configurations = LocalSettings.configurations(getServiceName(), "flink-conf"); flinkConfContent = (String) configurations.get("content"); jobManagerArchiveFsDir = (String) configurations.get("jobmanager.archive.fs.dir"); @@ -77,7 +82,7 @@ public Map flinkConf() { @GlobalParams public Map flinkEnv() { - Map configurations = LocalSettings.configurations(serviceName(), "flink-env"); + Map configurations = LocalSettings.configurations(getServiceName(), "flink-env"); flinkLogDir = (String) configurations.get("flink_log_dir"); flinkPidDir = (String) configurations.get("flink_pid_dir"); historyServerPidFile = MessageFormat.format("{0}/flink-{1}-historyserver.pid", flinkPidDir, user()); @@ -86,14 +91,15 @@ public Map flinkEnv() { @GlobalParams public Map flinkLog4jProperties() { - Map configurations = LocalSettings.configurations(serviceName(), "flink-log4j-properties"); + Map configurations = LocalSettings.configurations(getServiceName(), "flink-log4j-properties"); flinkLog4jPropertiesContent = (String) configurations.get("content"); return configurations; } @GlobalParams public Map flinkLog4jCLiProperties() { - Map configurations = LocalSettings.configurations(serviceName(), "flink-log4j-cli-properties"); + Map configurations = + LocalSettings.configurations(getServiceName(), "flink-log4j-cli-properties"); flinkLog4jCLiPropertiesContent = (String) configurations.get("content"); return configurations; } @@ -101,7 +107,7 @@ public Map flinkLog4jCLiProperties() { @GlobalParams public Map flinkLog4jConsoleProperties() { Map configurations = - LocalSettings.configurations(serviceName(), "flink-log4j-console-properties"); + LocalSettings.configurations(getServiceName(), "flink-log4j-console-properties"); flinkLog4jConsolePropertiesContent = (String) configurations.get("content"); return configurations; } @@ -109,7 +115,7 @@ public Map flinkLog4jConsoleProperties() { @GlobalParams public Map flinkLog4jSessionProperties() { Map configurations = - LocalSettings.configurations(serviceName(), "flink-log4j-session-properties"); + LocalSettings.configurations(getServiceName(), "flink-log4j-session-properties"); flinkLog4jSessionPropertiesContent = (String) configurations.get("content"); return configurations; } @@ -126,4 +132,9 @@ public String hadoopConfDir() { public String hadoopHome() { return stackHome() + "/hadoop"; } + + @Override + public String getServiceName() { + return "flink"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkSetup.java index c47319e25..9256b176c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/flink/FlinkSetup.java @@ -21,7 +21,7 @@ import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; import lombok.AccessLevel; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java similarity index 62% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java index b48090f8d..2883ebc99 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/DataNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/DataNodeScript.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -29,24 +29,33 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.Properties; @Slf4j @AutoService(Script.class) public class DataNodeScript extends AbstractServerScript { + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + @Override public ShellResult configure(Params params) { - return HdfsSetup.config(params, "datanode"); + return HadoopSetup.configure(params, getComponentName()); } @Override public ShellResult start(Params params) { configure(params); - HdfsParams hdfsParams = (HdfsParams) params; + HadoopParams hadoopParams = (HadoopParams) params; - String cmd = MessageFormat.format("{0} --daemon start datanode", hdfsParams.hdfsExec()); + String cmd = MessageFormat.format("{0}/hdfs --daemon start datanode", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -54,10 +63,10 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - String cmd = MessageFormat.format("{0} --daemon stop datanode", hdfsParams.hdfsExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/hdfs --daemon stop datanode", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -65,7 +74,12 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - return LinuxOSUtils.checkProcess(hdfsParams.getDataNodePidFile()); + HadoopParams hadoopParams = (HadoopParams) params; + return LinuxOSUtils.checkProcess(hadoopParams.getDataNodePidFile()); + } + + @Override + public String getComponentName() { + return "datanode"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java similarity index 67% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java index 1aea95774..4f33aff02 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopClientScript.java @@ -16,22 +16,37 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; +import java.util.Properties; + @Slf4j @AutoService(Script.class) -public class MapredClientScript extends AbstractClientScript { +public class HadoopClientScript extends AbstractClientScript { + + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } @Override public ShellResult configure(Params params) { - return MapredSetup.config(params); + return HadoopSetup.configure(params); + } + + @Override + public String getComponentName() { + return "hadoop_client"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopParams.java similarity index 53% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopParams.java index 4d5c33716..142d28af3 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopParams.java @@ -16,73 +16,87 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.commons.lang3.StringUtils; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @Getter @Slf4j -public class HdfsParams extends BigtopParams { - - private String hadoopLogDir = "/var/log/hadoop"; - private String hadoopPidDir = "/var/run/hadoop"; - private String dfsDataDir = "/hadoop/dfs/data"; - private String dfsNameNodeDir = "/hadoop/dfs/name"; - private String dfsNameNodeCheckPointDir = "/hadoop/dfs/namesecondary"; - private String dfsDomainSocketPathPrefix = "/var/run/hadoop-hdfs"; - /* pid file */ - private String nameNodePidFile = hadoopPidDir + "/hdfs/hadoop-hdfs-namenode.pid"; - private String dataNodePidFile = hadoopPidDir + "/hdfs/hadoop-hdfs-datanode.pid"; - private String sNameNodePidFile = hadoopPidDir + "/hdfs/hadoop-hdfs-secondarynamenode.pid"; - private String journalNodePidFile = hadoopPidDir + "/hdfs/hadoop-hdfs-journalnode.pid"; - private String zkfcPidFile = hadoopPidDir + "/hdfs/hadoop-hdfs-zkfc.pid"; - /* pid file */ +@AutoService(Params.class) +@NoArgsConstructor +public class HadoopParams extends BigtopParams { + + private final String hadoopLogDir = "/var/log/hadoop"; + private final String hadoopPidDir = "/var/run/hadoop"; + + // hadoop-${user}-${component}.pid + private final String nameNodePidFile = hadoopPidDir + "/hadoop-hadoop-namenode.pid"; + private final String dataNodePidFile = hadoopPidDir + "/hadoop-hadoop-datanode.pid"; + private final String sNameNodePidFile = hadoopPidDir + "/hadoop-hadoop-secondarynamenode.pid"; + private final String journalNodePidFile = hadoopPidDir + "/hadoop-hadoop-journalnode.pid"; + private final String zkfcPidFile = hadoopPidDir + "/hadoop-hadoop-zkfc.pid"; + private final String resourceManagerPidFile = hadoopPidDir + "/hadoop-hadoop-resourcemanager.pid"; + private final String nodeManagerPidFile = hadoopPidDir + "/hadoop-hadoop-nodemanager.pid"; + private final String historyServerPidFile = hadoopPidDir + "/hadoop-hadoop-historyserver.pid"; + + private String dfsDataDir; + private String dfsNameNodeDir; + private String dfsNameNodeCheckPointDir; + private String dfsDomainSocketPathPrefix; + + private String nodeManagerLogDir = "/hadoop/yarn/log"; + private String nodeManagerLocalDir = "/hadoop/yarn/local"; + private List nameNodeFormattedDirs; - public HdfsParams(CommandPayload commandPayload) { + public HadoopParams(CommandPayload commandPayload) { super(commandPayload); globalParamsMap.put("hdfs_user", user()); globalParamsMap.put("hdfs_group", group()); globalParamsMap.put("datanode_hosts", LocalSettings.hosts("datanode")); globalParamsMap.put("java_home", Environments.getJavaHome()); globalParamsMap.put("hadoop_home", serviceHome()); - globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); globalParamsMap.put("hadoop_conf_dir", confDir()); globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec"); + globalParamsMap.put("exclude_hosts", new ArrayList<>()); } - public String hdfsLimits() { - Map hdfsConf = LocalSettings.configurations(serviceName(), "hdfs.conf"); - return (String) hdfsConf.get("content"); + public String hadoopLimits() { + Map hadoopConf = LocalSettings.configurations(getServiceName(), "hadoop.conf"); + return (String) hadoopConf.get("content"); } public String workers() { - Map hdfsConf = LocalSettings.configurations(serviceName(), "workers"); + Map hdfsConf = LocalSettings.configurations(getServiceName(), "workers"); return (String) hdfsConf.get("content"); } @GlobalParams public Map hdfsLog4j() { - return LocalSettings.configurations(serviceName(), "hdfs-log4j"); + return LocalSettings.configurations(getServiceName(), "hdfs-log4j"); } @GlobalParams public Map coreSite() { - Map coreSite = LocalSettings.configurations(serviceName(), "core-site"); + Map coreSite = LocalSettings.configurations(getServiceName(), "core-site"); List namenodeList = LocalSettings.hosts("namenode"); if (!namenodeList.isEmpty()) { coreSite.put( @@ -93,12 +107,12 @@ public Map coreSite() { @GlobalParams public Map hadoopPolicy() { - return LocalSettings.configurations(serviceName(), "hadoop-policy"); + return LocalSettings.configurations(getServiceName(), "hadoop-policy"); } @GlobalParams public Map hdfsSite() { - Map hdfsSite = LocalSettings.configurations(serviceName(), "hdfs-site"); + Map hdfsSite = LocalSettings.configurations(getServiceName(), "hdfs-site"); List namenodeList = LocalSettings.hosts("namenode"); if (!namenodeList.isEmpty()) { hdfsSite.put( @@ -126,30 +140,58 @@ public Map hdfsSite() { return hdfsSite; } + @GlobalParams + public Map yarnLog4j() { + return LocalSettings.configurations(getServiceName(), "yarn-log4j"); + } + + @GlobalParams + public Map yarnSite() { + Map yarnSite = LocalSettings.configurations(getServiceName(), "yarn-site"); + List resourcemanagerList = LocalSettings.hosts("resourcemanager"); + if (!resourcemanagerList.isEmpty()) { + yarnSite.put("yarn.resourcemanager.hostname", MessageFormat.format("{0}", resourcemanagerList.get(0))); + } + + nodeManagerLogDir = (String) yarnSite.get("yarn.nodemanager.log-dirs"); + nodeManagerLocalDir = (String) yarnSite.get("yarn.nodemanager.local-dirs"); + return yarnSite; + } + + @GlobalParams + public Map mapredSite() { + return LocalSettings.configurations(getServiceName(), "mapred-site"); + } + @GlobalParams public Map hadoopEnv() { - Map hadoopEnv = LocalSettings.configurations(serviceName(), "hadoop-env"); + Map configurations = LocalSettings.configurations(getServiceName(), "hadoop-env"); + configurations.put("hadoop_log_dir", hadoopLogDir); + configurations.put("hadoop_pid_dir", hadoopPidDir); + return configurations; + } - hadoopLogDir = (String) hadoopEnv.get("hadoop_log_dir_prefix"); - hadoopPidDir = (String) hadoopEnv.get("hadoop_pid_dir_prefix"); - nameNodePidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-namenode.pid", hadoopPidDir, user()); - dataNodePidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-datanode.pid", hadoopPidDir, user()); - sNameNodePidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-secondarynamenode.pid", hadoopPidDir, user()); - journalNodePidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-journalnode.pid", hadoopPidDir, user()); - zkfcPidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-zkfc.pid", hadoopPidDir, user()); - return hadoopEnv; + @GlobalParams + public Map yarnEnv() { + return LocalSettings.configurations(getServiceName(), "yarn-env"); + } + + @GlobalParams + public Map mapredEnv() { + return LocalSettings.configurations(getServiceName(), "mapred-env"); } @Override public String confDir() { - return "/etc/hadoop/conf"; + return serviceHome() + "/etc/hadoop"; } - public String hdfsHome() { - return stackHome() + "/hadoop-hdfs"; + public String binDir() { + return serviceHome() + "/bin"; } - public String hdfsExec() { - return serviceHome() + "/bin/hdfs"; + @Override + public String getServiceName() { + return "hadoop"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopSetup.java new file mode 100644 index 000000000..c96b185d6 --- /dev/null +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HadoopSetup.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; + +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.shell.ShellResult; +import org.apache.bigtop.manager.stack.core.enums.ConfigType; +import org.apache.bigtop.manager.stack.core.exception.StackException; +import org.apache.bigtop.manager.stack.core.spi.param.Params; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; +import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; + +import org.apache.commons.lang3.StringUtils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.text.MessageFormat; +import java.util.Map; + +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class HadoopSetup { + + public static ShellResult configure(Params params) { + return configure(params, null); + } + + public static ShellResult configure(Params params, String componentName) { + log.info("Configuring Hadoop"); + HadoopParams hadoopParams = (HadoopParams) params; + + String confDir = hadoopParams.confDir(); + String hadoopUser = hadoopParams.user(); + String hadoopGroup = hadoopParams.group(); + Map hadoopEnv = hadoopParams.hadoopEnv(); + Map yarnEnv = hadoopParams.yarnEnv(); + Map mapredEnv = hadoopParams.mapredEnv(); + + if (StringUtils.isNotBlank(componentName)) { + switch (componentName) { + case "namenode": { + LinuxFileUtils.createDirectories( + hadoopParams.getDfsNameNodeDir(), hadoopUser, hadoopGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.createDirectories( + hadoopParams.getDfsNameNodeCheckPointDir(), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_755, + true); + } + case "secondarynamenode": { + LinuxFileUtils.createDirectories( + hadoopParams.getDfsNameNodeCheckPointDir(), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_755, + true); + } + case "datanode": { + LinuxFileUtils.createDirectories( + hadoopParams.getDfsDomainSocketPathPrefix(), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_755, + true); + LinuxFileUtils.createDirectories( + hadoopParams.getDfsDataDir(), hadoopUser, hadoopGroup, Constants.PERMISSION_755, true); + } + case "nodemanager": { + LinuxFileUtils.createDirectories( + hadoopParams.getNodeManagerLogDir(), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_755, + true); + LinuxFileUtils.createDirectories( + hadoopParams.getNodeManagerLocalDir(), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_755, + true); + } + } + } + + // mkdir directories + LinuxFileUtils.createDirectories( + hadoopParams.getHadoopLogDir(), hadoopUser, hadoopGroup, Constants.PERMISSION_755, true); + LinuxFileUtils.createDirectories( + hadoopParams.getHadoopPidDir(), hadoopUser, hadoopGroup, Constants.PERMISSION_755, true); + + LinuxFileUtils.toFileByTemplate( + hadoopParams.hadoopLimits(), + MessageFormat.format("{0}/hadoop.conf", HadoopParams.LIMITS_CONF_DIR), + Constants.ROOT_USER, + Constants.ROOT_USER, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + hadoopEnv.get("content").toString(), + MessageFormat.format("{0}/hadoop-env.sh", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/core-site.xml", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.coreSite()); + + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/hdfs-site.xml", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.hdfsSite()); + + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/hadoop-policy.xml", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.hadoopPolicy()); + + LinuxFileUtils.toFileByTemplate( + hadoopParams.workers(), + MessageFormat.format("{0}/workers", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + hadoopParams.hdfsLog4j().get("content").toString(), + MessageFormat.format("{0}/log4j.properties", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + yarnEnv.get("content").toString(), + MessageFormat.format("{0}/yarn-env.sh", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/yarn-site.xml", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.yarnSite(), + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + hadoopParams.yarnLog4j().get("content").toString(), + MessageFormat.format("{0}/yarnservice-log4j.properties", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFileByTemplate( + mapredEnv.get("content").toString(), + MessageFormat.format("{0}/mapred-env.sh", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.getGlobalParamsMap()); + + LinuxFileUtils.toFile( + ConfigType.XML, + MessageFormat.format("{0}/mapred-site.xml", confDir), + hadoopUser, + hadoopGroup, + Constants.PERMISSION_644, + hadoopParams.mapredSite(), + hadoopParams.getGlobalParamsMap()); + + // HdfsUtil.createDirectory(hadoopUser, "/apps"); + // HdfsUtil.createDirectory(hadoopUser, "/app-logs"); + // HdfsUtil.createDirectory(hadoopUser, "/apps/mapred"); + // HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/staging"); + // HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history"); + // HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history/tmp"); + // HdfsUtil.createDirectory(hadoopUser, "/apps/mapred/history/done"); + + log.info("Successfully configured Hadoop"); + return ShellResult.success(); + } + + public static void formatNameNode(HadoopParams hadoopParams) { + if (!isNameNodeFormatted(hadoopParams)) { + String formatCmd = MessageFormat.format( + "{0}/hdfs --config {1} namenode -format -nonInteractive", + hadoopParams.binDir(), hadoopParams.confDir()); + try { + LinuxOSUtils.sudoExecCmd(formatCmd, hadoopParams.user()); + } catch (Exception e) { + throw new StackException(e); + } + + for (String nameNodeFormattedDir : hadoopParams.getNameNodeFormattedDirs()) { + LinuxFileUtils.createDirectories( + nameNodeFormattedDir, + hadoopParams.user(), + hadoopParams.group(), + Constants.PERMISSION_755, + true); + } + } + } + + public static boolean isNameNodeFormatted(HadoopParams hadoopParams) { + + boolean isFormatted = false; + for (String nameNodeFormattedDir : hadoopParams.getNameNodeFormattedDirs()) { + File file = new File(nameNodeFormattedDir); + if (file.exists() && file.isDirectory()) { + log.info("{} exists. Namenode DFS already formatted", nameNodeFormattedDir); + isFormatted = true; + } + } + + if (isFormatted) { + for (String nameNodeFormattedDir : hadoopParams.getNameNodeFormattedDirs()) { + LinuxFileUtils.createDirectories( + nameNodeFormattedDir, + hadoopParams.user(), + hadoopParams.group(), + Constants.PERMISSION_755, + true); + } + return true; + } + + // Check if name dirs are not empty + String[] nameNodeDirs = hadoopParams.getDfsNameNodeDir().split(","); + + for (String nameNodeDir : nameNodeDirs) { + File file = new File(nameNodeDir); + if (!file.exists()) { + log.info( + "NameNode will not be formatted because the directory {} is missing or cannot be checked for content.", + nameNodeDir); + return true; + } else { + File[] files = file.listFiles(); + if (files != null && files.length > 0) { + log.info("NameNode will not be formatted since {} exists and contains content", nameNodeDir); + return true; + } + } + } + + return false; + } +} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java similarity index 61% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java index b07d4b9fc..f4ae58c08 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/HistoryServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/HistoryServerScript.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -28,22 +28,31 @@ import com.google.auto.service.AutoService; import java.text.MessageFormat; +import java.util.Properties; @AutoService(Script.class) public class HistoryServerScript extends AbstractServerScript { + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + @Override public ShellResult configure(Params params) { - return MapredSetup.config(params); + return HadoopSetup.configure(params, getComponentName()); } @Override public ShellResult start(Params params) { configure(params); - Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; - String cmd = MessageFormat.format("{0} --daemon start historyserver", mapreduce2Params.mapredExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/mapred --daemon start historyserver", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, mapreduce2Params.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -51,10 +60,10 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { - Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; - String cmd = MessageFormat.format("{0} --daemon stop historyserver", mapreduce2Params.mapredExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/mapred --daemon stop historyserver", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, mapreduce2Params.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -62,7 +71,12 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; - return LinuxOSUtils.checkProcess(mapreduce2Params.getHistoryServerPidFile()); + HadoopParams hadoopParams = (HadoopParams) params; + return LinuxOSUtils.checkProcess(hadoopParams.getHistoryServerPidFile()); + } + + @Override + public String getComponentName() { + return "history_server"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java similarity index 57% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java index 2172414ae..b6f89c939 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/NameNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NameNodeScript.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -29,26 +29,35 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.Properties; @Slf4j @AutoService(Script.class) public class NameNodeScript extends AbstractServerScript { + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + @Override public ShellResult configure(Params params) { - return HdfsSetup.config(params, "namenode"); + return HadoopSetup.configure(params, getComponentName()); } @Override public ShellResult start(Params params) { configure(params); - HdfsParams hdfsParams = (HdfsParams) params; + HadoopParams hadoopParams = (HadoopParams) params; - HdfsSetup.formatNameNode(hdfsParams); + HadoopSetup.formatNameNode(hadoopParams); - String cmd = MessageFormat.format("{0} --daemon start namenode", hdfsParams.hdfsExec()); + String cmd = MessageFormat.format("{0}/hdfs --daemon start namenode", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -56,10 +65,10 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - String cmd = MessageFormat.format("{0} --daemon stop namenode", hdfsParams.hdfsExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/hdfs --daemon stop namenode", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -67,27 +76,32 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - return LinuxOSUtils.checkProcess(hdfsParams.getNameNodePidFile()); + HadoopParams hadoopParams = (HadoopParams) params; + return LinuxOSUtils.checkProcess(hadoopParams.getNameNodePidFile()); } public ShellResult rebalanceHdfs(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - String cmd = MessageFormat.format("{0} balancer", hdfsParams.hdfsExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/hdfs balancer", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } } public ShellResult printTopology(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - String cmd = MessageFormat.format("{0} dfsadmin -printTopology", hdfsParams.hdfsExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/hdfs dfsadmin -printTopology", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } } + + @Override + public String getComponentName() { + return "namenode"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java similarity index 61% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java index ac90d6b99..b92f64b69 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/NodeManagerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/NodeManagerScript.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -29,24 +29,33 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.Properties; @Slf4j @AutoService(Script.class) public class NodeManagerScript extends AbstractServerScript { + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + @Override public ShellResult configure(Params params) { - return YarnSetup.config(params, "nodemanager"); + return HadoopSetup.configure(params, getComponentName()); } @Override public ShellResult start(Params params) { configure(params); - YarnParams yarnParams = (YarnParams) params; + HadoopParams hadoopParams = (HadoopParams) params; - String cmd = MessageFormat.format("{0} --daemon start nodemanager", yarnParams.yarnExec()); + String cmd = MessageFormat.format("{0}/yarn --daemon start nodemanager", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -54,10 +63,10 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { - YarnParams yarnParams = (YarnParams) params; - String cmd = MessageFormat.format("{0} --daemon stop nodemanager", yarnParams.yarnExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/yarn --daemon stop nodemanager", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -65,7 +74,12 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - YarnParams yarnParams = (YarnParams) params; - return LinuxOSUtils.checkProcess(yarnParams.getNodeManagerPidFile()); + HadoopParams hadoopParams = (HadoopParams) params; + return LinuxOSUtils.checkProcess(hadoopParams.getNodeManagerPidFile()); + } + + @Override + public String getComponentName() { + return "nodemanager"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java similarity index 61% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java index 416b69fef..2f7bd857d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/ResourceManagerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/ResourceManagerScript.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -29,24 +29,33 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.Properties; @Slf4j @AutoService(Script.class) public class ResourceManagerScript extends AbstractServerScript { + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + @Override public ShellResult configure(Params params) { - return YarnSetup.config(params, "resourcemanager"); + return HadoopSetup.configure(params, getComponentName()); } @Override public ShellResult start(Params params) { configure(params); - YarnParams yarnParams = (YarnParams) params; + HadoopParams hadoopParams = (HadoopParams) params; - String cmd = MessageFormat.format("{0} --daemon start resourcemanager", yarnParams.yarnExec()); + String cmd = MessageFormat.format("{0}/yarn --daemon start resourcemanager", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -54,10 +63,10 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { - YarnParams yarnParams = (YarnParams) params; - String cmd = MessageFormat.format("{0} --daemon stop resourcemanager", yarnParams.yarnExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/yarn --daemon stop resourcemanager", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, yarnParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -65,7 +74,12 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - YarnParams yarnParams = (YarnParams) params; - return LinuxOSUtils.checkProcess(yarnParams.getResourceManagerPidFile()); + HadoopParams hadoopParams = (HadoopParams) params; + return LinuxOSUtils.checkProcess(hadoopParams.getResourceManagerPidFile()); + } + + @Override + public String getComponentName() { + return "resourcemanager"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java similarity index 61% rename from bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java rename to bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java index c92a28311..dc8ba46f7 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/SNameNodeScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hadoop/SNameNodeScript.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; +package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hadoop; import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -29,24 +29,33 @@ import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; +import java.util.Properties; @Slf4j @AutoService(Script.class) public class SNameNodeScript extends AbstractServerScript { + @Override + public ShellResult add(Params params) { + Properties properties = new Properties(); + properties.setProperty(PROPERTY_KEY_SKIP_LEVELS, "1"); + + return super.add(params, properties); + } + @Override public ShellResult configure(Params params) { - return HdfsSetup.config(params, "secondarynamenode"); + return HadoopSetup.configure(params, getComponentName()); } @Override public ShellResult start(Params params) { configure(params); - HdfsParams hdfsParams = (HdfsParams) params; + HadoopParams hadoopParams = (HadoopParams) params; - String cmd = MessageFormat.format("{0} --daemon start secondarynamenode", hdfsParams.hdfsExec()); + String cmd = MessageFormat.format("{0}/hdfs --daemon start secondarynamenode", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -54,10 +63,10 @@ public ShellResult start(Params params) { @Override public ShellResult stop(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - String cmd = MessageFormat.format("{0} --daemon stop secondarynamenode", hdfsParams.hdfsExec()); + HadoopParams hadoopParams = (HadoopParams) params; + String cmd = MessageFormat.format("{0}/hdfs --daemon stop secondarynamenode", hadoopParams.binDir()); try { - return LinuxOSUtils.sudoExecCmd(cmd, hdfsParams.user()); + return LinuxOSUtils.sudoExecCmd(cmd, hadoopParams.user()); } catch (Exception e) { throw new StackException(e); } @@ -65,7 +74,12 @@ public ShellResult stop(Params params) { @Override public ShellResult status(Params params) { - HdfsParams hdfsParams = (HdfsParams) params; - return LinuxOSUtils.checkProcess(hdfsParams.getSNameNodePidFile()); + HadoopParams hadoopParams = (HadoopParams) params; + return LinuxOSUtils.checkProcess(hadoopParams.getSNameNodePidFile()); + } + + @Override + public String getComponentName() { + return "secondarynamenode"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java deleted file mode 100644 index 455d9400a..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsClientScript.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; - -import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; -import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; -import org.apache.bigtop.manager.stack.core.spi.script.Script; - -import com.google.auto.service.AutoService; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@AutoService(Script.class) -public class HdfsClientScript extends AbstractClientScript { - - @Override - public ShellResult configure(Params params) { - return HdfsSetup.config(params); - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java deleted file mode 100644 index db7d0979e..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.hdfs; - -import org.apache.bigtop.manager.common.constants.Constants; -import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.bigtop.v3_3_0.kafka.KafkaParams; -import org.apache.bigtop.manager.stack.core.enums.ConfigType; -import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; -import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; -import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; - -import org.apache.commons.lang3.StringUtils; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.io.File; -import java.text.MessageFormat; -import java.util.Map; - -@Slf4j -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class HdfsSetup { - - public static ShellResult config(Params params) { - return config(params, null); - } - - public static ShellResult config(Params params, String componentName) { - log.info("Setting hdfs config"); - HdfsParams hdfsParams = (HdfsParams) params; - - String confDir = hdfsParams.confDir(); - String hdfsUser = hdfsParams.user(); - String hdfsGroup = hdfsParams.group(); - Map hadoopEnv = hdfsParams.hadoopEnv(); - - if (StringUtils.isNotBlank(componentName)) { - switch (componentName) { - case "namenode": { - LinuxFileUtils.createDirectories( - hdfsParams.getDfsNameNodeDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); - } - case "secondarynamenode": { - LinuxFileUtils.createDirectories( - hdfsParams.getDfsNameNodeCheckPointDir(), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_755, - true); - } - case "datanode": { - LinuxFileUtils.createDirectories( - hdfsParams.getDfsDomainSocketPathPrefix(), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_755, - true); - } - } - } - - // mkdir directories - LinuxFileUtils.createDirectories( - hdfsParams.getDfsDataDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); - LinuxFileUtils.createDirectories( - hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); - LinuxFileUtils.createDirectories( - hdfsParams.getHadoopPidDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true); - - // hdfs.limits - LinuxFileUtils.toFileByTemplate( - hdfsParams.hdfsLimits(), - MessageFormat.format("{0}/hdfs.conf", KafkaParams.LIMITS_CONF_DIR), - Constants.ROOT_USER, - Constants.ROOT_USER, - Constants.PERMISSION_644, - hdfsParams.getGlobalParamsMap()); - - // hadoop-env.sh - LinuxFileUtils.toFileByTemplate( - hadoopEnv.get("content").toString(), - MessageFormat.format("{0}/hadoop-env.sh", confDir), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_644, - hdfsParams.getGlobalParamsMap()); - - // core-site.xml - LinuxFileUtils.toFile( - ConfigType.XML, - MessageFormat.format("{0}/core-site.xml", confDir), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_644, - hdfsParams.coreSite()); - - // hdfs-site.xml - LinuxFileUtils.toFile( - ConfigType.XML, - MessageFormat.format("{0}/hdfs-site.xml", confDir), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_644, - hdfsParams.hdfsSite()); - - // hdfs-policy.xml - LinuxFileUtils.toFile( - ConfigType.XML, - MessageFormat.format("{0}/hadoop-policy.xml", confDir), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_644, - hdfsParams.hadoopPolicy()); - - // hdfs-policy.xml - LinuxFileUtils.toFileByTemplate( - hdfsParams.workers(), - MessageFormat.format("{0}/workers", confDir), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_644, - hdfsParams.getGlobalParamsMap()); - - // log4j - LinuxFileUtils.toFileByTemplate( - hdfsParams.hdfsLog4j().get("content").toString(), - MessageFormat.format("{0}/log4j.properties", confDir), - hdfsUser, - hdfsGroup, - Constants.PERMISSION_644, - hdfsParams.getGlobalParamsMap()); - - log.info("Successfully configured HDFS"); - return ShellResult.success("HDFS Configure success!"); - } - - public static void formatNameNode(HdfsParams hdfsParams) { - if (!isNameNodeFormatted(hdfsParams)) { - String formatCmd = MessageFormat.format( - "{0} --config {1} namenode -format -nonInteractive", hdfsParams.hdfsExec(), hdfsParams.confDir()); - try { - LinuxOSUtils.sudoExecCmd(formatCmd, hdfsParams.user()); - } catch (Exception e) { - throw new StackException(e); - } - - for (String nameNodeFormattedDir : hdfsParams.getNameNodeFormattedDirs()) { - LinuxFileUtils.createDirectories( - nameNodeFormattedDir, hdfsParams.user(), hdfsParams.group(), Constants.PERMISSION_755, true); - } - } - } - - public static boolean isNameNodeFormatted(HdfsParams hdfsParams) { - - boolean isFormatted = false; - for (String nameNodeFormattedDir : hdfsParams.getNameNodeFormattedDirs()) { - File file = new File(nameNodeFormattedDir); - if (file.exists() && file.isDirectory()) { - log.info("{} exists. Namenode DFS already formatted", nameNodeFormattedDir); - isFormatted = true; - } - } - - if (isFormatted) { - for (String nameNodeFormattedDir : hdfsParams.getNameNodeFormattedDirs()) { - LinuxFileUtils.createDirectories( - nameNodeFormattedDir, hdfsParams.user(), hdfsParams.group(), Constants.PERMISSION_755, true); - } - return true; - } - - // Check if name dirs are not empty - String[] nameNodeDirs = hdfsParams.getDfsNameNodeDir().split(","); - - for (String nameNodeDir : nameNodeDirs) { - File file = new File(nameNodeDir); - if (!file.exists()) { - log.info( - "NameNode will not be formatted because the directory {} is missing or cannot be checked for content.", - nameNodeDir); - return true; - } else { - File[] files = file.listFiles(); - if (files != null && files.length > 0) { - log.info("NameNode will not be formatted since {} exists and contains content", nameNodeDir); - return true; - } - } - } - - return false; - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java index 308b102b4..75d26758c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaBrokerScript.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -80,4 +80,9 @@ public ShellResult test(Params params) { throw new StackException(e); } } + + @Override + public String getComponentName() { + return "kafka_broker"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java index 12f401bde..d1d2ea9ad 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaParams.java @@ -22,13 +22,18 @@ import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import java.util.Map; @Getter +@AutoService(Params.class) +@NoArgsConstructor public class KafkaParams extends BigtopParams { private String kafkaLogDir = "/var/log/kafka"; @@ -49,14 +54,14 @@ public KafkaParams(CommandPayload commandPayload) { @GlobalParams public Map kafkaBroker() { - Map kafkaBroker = LocalSettings.configurations(serviceName(), "kafka-broker"); + Map kafkaBroker = LocalSettings.configurations(getServiceName(), "kafka-broker"); kafkaDataDir = (String) kafkaBroker.get("log.dirs"); return kafkaBroker; } @GlobalParams public Map kafkaEnv() { - Map kafkaEnv = LocalSettings.configurations(serviceName(), "kafka-env"); + Map kafkaEnv = LocalSettings.configurations(getServiceName(), "kafka-env"); kafkaPidDir = (String) kafkaEnv.get("kafka_pid_dir"); kafkaPidFile = kafkaPidDir + "/kafka_broker.pid"; kafkaLogDir = (String) kafkaEnv.get("kafka_log_dir"); @@ -66,13 +71,18 @@ public Map kafkaEnv() { @GlobalParams public Map kafkaLog4j() { - Map kafkaLog4j = LocalSettings.configurations(serviceName(), "kafka-log4j"); + Map kafkaLog4j = LocalSettings.configurations(getServiceName(), "kafka-log4j"); kafkaLog4jContent = (String) kafkaLog4j.get("content"); return kafkaLog4j; } public String kafkaLimits() { - Map kafkaLimits = LocalSettings.configurations(serviceName(), "kafka.conf"); + Map kafkaLimits = LocalSettings.configurations(getServiceName(), "kafka.conf"); return (String) kafkaLimits.get("content"); } + + @Override + public String getServiceName() { + return "kafka"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java index 02bd702f3..d21a4072c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/kafka/KafkaSetup.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.enums.ConfigType; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java deleted file mode 100644 index 79ba53f79..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/MapredSetup.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; - -import org.apache.bigtop.manager.common.constants.Constants; -import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil; -import org.apache.bigtop.manager.stack.core.enums.ConfigType; -import org.apache.bigtop.manager.stack.core.param.BaseParams; -import org.apache.bigtop.manager.stack.core.param.Params; -import org.apache.bigtop.manager.stack.core.utils.LocalSettings; -import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.text.MessageFormat; - -@Slf4j -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class MapredSetup { - - public static ShellResult config(Params params) { - Mapreduce2Params mapreduce2Params = (Mapreduce2Params) params; - String hdfsUser = LocalSettings.users().get("hdfs"); - String mapredUser = params.user(); - String mapredGroup = params.group(); - String confDir = mapreduce2Params.confDir(); - - LinuxFileUtils.createDirectories( - mapreduce2Params.getMapredLogDir(), mapredUser, mapredGroup, Constants.PERMISSION_755, true); - LinuxFileUtils.createDirectories( - mapreduce2Params.getMapredPidDir(), mapredUser, mapredGroup, Constants.PERMISSION_755, true); - - // mapreduce.conf - log.info("Generating [{}/mapreduce.conf] file", BaseParams.LIMITS_CONF_DIR); - LinuxFileUtils.toFileByTemplate( - mapreduce2Params.mapredLimits(), - MessageFormat.format("{0}/mapreduce.conf", BaseParams.LIMITS_CONF_DIR), - Constants.ROOT_USER, - Constants.ROOT_USER, - Constants.PERMISSION_644, - mapreduce2Params.getGlobalParamsMap()); - - // mapred-env.sh - log.info("Generating [{}/mapred-env.sh] file", confDir); - LinuxFileUtils.toFileByTemplate( - mapreduce2Params.mapredEnv().get("content").toString(), - MessageFormat.format("{0}/mapred-env.sh", confDir), - mapredUser, - mapredGroup, - Constants.PERMISSION_644, - mapreduce2Params.getGlobalParamsMap()); - - // mapred-site.xml - log.info("Generating [{}/mapred-site.xml] file", confDir); - LinuxFileUtils.toFile( - ConfigType.XML, - MessageFormat.format("{0}/mapred-site.xml", confDir), - mapredUser, - mapredGroup, - Constants.PERMISSION_644, - mapreduce2Params.mapredSite(), - mapreduce2Params.getGlobalParamsMap()); - - HdfsUtil.createDirectory(hdfsUser, "/apps"); - HdfsUtil.createDirectory(hdfsUser, "/app-logs"); - HdfsUtil.createDirectory(mapredUser, "/apps/mapred"); - HdfsUtil.createDirectory(mapredUser, "/apps/mapred/staging"); - HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history"); - HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history/tmp"); - HdfsUtil.createDirectory(mapredUser, "/apps/mapred/history/done"); - - log.info("Successfully configured MapReduce2"); - return ShellResult.success("MapReduce2 Configure success!"); - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java deleted file mode 100644 index e5c23e1c8..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/mapred/Mapreduce2Params.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.mapred; - -import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; -import org.apache.bigtop.manager.common.utils.Environments; -import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; -import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.utils.LocalSettings; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import java.text.MessageFormat; -import java.util.Map; - -@Getter -@Slf4j -public class Mapreduce2Params extends BigtopParams { - - private String mapredEnvContent; - private String mapredLogDir; - private String mapredPidDir; - private String historyServerPidFile; - - public Mapreduce2Params(CommandPayload commandPayload) { - super(commandPayload); - globalParamsMap.put("mapred_user", user()); - globalParamsMap.put("mapred_group", group()); - globalParamsMap.put("java_home", Environments.getJavaHome()); - globalParamsMap.put("hadoop_home", serviceHome()); - globalParamsMap.put("hadoop_conf_dir", confDir()); - globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); - globalParamsMap.put("hadoop_yarn_home", yarnHome()); - globalParamsMap.put("hadoop_mapred_home", mapredHome()); - globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec"); - } - - public String mapredLimits() { - Map yarnConf = LocalSettings.configurations(serviceName(), "mapreduce.conf"); - return (String) yarnConf.get("content"); - } - - @GlobalParams - public Map mapredSite() { - return LocalSettings.configurations(serviceName(), "mapred-site"); - } - - @GlobalParams - public Map mapredEnv() { - Map mapredEnv = LocalSettings.configurations(serviceName(), "mapred-env"); - mapredEnvContent = (String) mapredEnv.get("content"); - mapredLogDir = (String) mapredEnv.get("mapred_log_dir_prefix"); - mapredPidDir = (String) mapredEnv.get("mapred_pid_dir_prefix"); - historyServerPidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-historyserver.pid", mapredPidDir, user()); - return mapredEnv; - } - - @Override - public String confDir() { - return "/etc/hadoop/conf"; - } - - public String hdfsHome() { - return stackHome() + "/hadoop-hdfs"; - } - - public String yarnHome() { - return stackHome() + "/hadoop-yarn"; - } - - public String mapredHome() { - return stackHome() + "/hadoop-mapreduce"; - } - - public String mapredExec() { - return serviceHome() + "/bin/mapred"; - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java index e7a2b6ac8..e8997d229 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrClientScript.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.solr; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; @@ -34,4 +34,9 @@ public class SolrClientScript extends AbstractClientScript { public ShellResult configure(Params params) { return SolrSetup.config(params); } + + @Override + public String getComponentName() { + return "solr_client"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java index 13aea30fb..75613a80a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrParams.java @@ -22,9 +22,12 @@ import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; @@ -33,6 +36,8 @@ @Getter @Slf4j +@AutoService(Params.class) +@NoArgsConstructor public class SolrParams extends BigtopParams { private String solrPort = "8983"; @@ -54,16 +59,16 @@ public SolrParams(CommandPayload commandPayload) { @GlobalParams public Map solrXml() { - return LocalSettings.configurations(serviceName(), "solr-xml"); + return LocalSettings.configurations(getServiceName(), "solr-xml"); } @GlobalParams public Map solrLog4j() { - return LocalSettings.configurations(serviceName(), "solr-log4j"); + return LocalSettings.configurations(getServiceName(), "solr-log4j"); } public String getZnode() { - Map solrEnv = LocalSettings.configurations(serviceName(), "solr-env"); + Map solrEnv = LocalSettings.configurations(getServiceName(), "solr-env"); return (String) solrEnv.get("solr_znode"); } @@ -82,7 +87,7 @@ public String zkHost() { @GlobalParams public Map solrEnv() { - Map solrEnv = LocalSettings.configurations(serviceName(), "solr-env"); + Map solrEnv = LocalSettings.configurations(getServiceName(), "solr-env"); solrLogDir = (String) solrEnv.get("solr_log_dir"); solrPidDir = (String) solrEnv.get("solr_pid_dir"); solrDataDir = (String) solrEnv.get("solr_datadir"); @@ -90,4 +95,9 @@ public Map solrEnv() { solrPidFile = solrPidDir + "/solr-" + solrPort + ".pid"; return solrEnv; } + + @Override + public String getServiceName() { + return "solr"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java index dd93fa79c..2e74824f3 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrServerScript.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -86,4 +86,9 @@ public ShellResult create_solr_znode(Params params) { throw new StackException(e); } } + + @Override + public String getComponentName() { + return "solr_server"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java index de48181f6..9bc57742d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/solr/SolrSetup.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java index 832ada59a..e65478b7f 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezClientScript.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.tez; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; @@ -34,4 +34,9 @@ public class TezClientScript extends AbstractClientScript { public ShellResult configure(Params params) { return TezSetup.config(params); } + + @Override + public String getComponentName() { + return "tez_client"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java index c91934477..25995737c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezParams.java @@ -22,11 +22,14 @@ import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.commons.lang3.StringUtils; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; @@ -34,6 +37,8 @@ @Getter @Slf4j +@AutoService(Params.class) +@NoArgsConstructor public class TezParams extends BigtopParams { private String headDumpOpts; @@ -46,8 +51,6 @@ public TezParams(CommandPayload commandPayload) { globalParamsMap.put("java_home", Environments.getJavaHome()); globalParamsMap.put("hadoop_home", hadoopHome()); globalParamsMap.put("hadoop_conf_dir", hadoopConfDir()); - globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); - globalParamsMap.put("hadoop_yarn_home", yarnHome()); globalParamsMap.put("tez_home", serviceHome()); globalParamsMap.put("tez_conf_dir", confDir()); globalParamsMap.put("head_dump_opts", headDumpOpts); @@ -55,12 +58,12 @@ public TezParams(CommandPayload commandPayload) { @GlobalParams public Map tezSite() { - return LocalSettings.configurations(serviceName(), "tez-site"); + return LocalSettings.configurations(getServiceName(), "tez-site"); } @GlobalParams public Map tezEnv() { - Map tezEnv = LocalSettings.configurations(serviceName(), "tez-env"); + Map tezEnv = LocalSettings.configurations(getServiceName(), "tez-env"); String heapDumpEnabled = (String) tezEnv.get("enable_heap_dump"); if (StringUtils.isNotBlank(heapDumpEnabled) && Boolean.parseBoolean(heapDumpEnabled)) { @@ -94,4 +97,9 @@ public String hdfsHome() { public String yarnHome() { return stackHome() + "/hadoop-yarn"; } + + @Override + public String getServiceName() { + return "tez"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java index 739bd3042..cd995251e 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java @@ -22,7 +22,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.bigtop.utils.HdfsUtil; import org.apache.bigtop.manager.stack.core.enums.ConfigType; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java deleted file mode 100644 index cbd29d0ee..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnClientScript.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn; - -import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; -import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; -import org.apache.bigtop.manager.stack.core.spi.script.Script; - -import com.google.auto.service.AutoService; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@AutoService(Script.class) -public class YarnClientScript extends AbstractClientScript { - - @Override - public ShellResult configure(Params params) { - return YarnSetup.config(params); - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java deleted file mode 100644 index 16f2095da..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnParams.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn; - -import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; -import org.apache.bigtop.manager.common.utils.Environments; -import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; -import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; -import org.apache.bigtop.manager.stack.core.utils.LocalSettings; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import java.text.MessageFormat; -import java.util.List; -import java.util.Map; - -@Getter -@Slf4j -public class YarnParams extends BigtopParams { - - private String yarnLogDir = "/var/log/hadoop-yarn"; - private String yarnPidDir = "/var/run/hadoop-yarn"; - private String rmNodesExcludeDir = "/etc/hadoop/conf/yarn.exclude"; - private String tmpDir = "/tmp/hadoop-yarn"; - private String nodemanagerLogDir = "/hadoop/yarn/log"; - private String nodemanagerLocalDir = "/hadoop/yarn/local"; - /* pid file */ - private String resourceManagerPidFile = yarnPidDir + "/yarn/hadoop-yarn-resourcemanager.pid"; - private String nodeManagerPidFile = yarnPidDir + "/yarn/hadoop-yarn-nodemanager.pid"; - /* pid file */ - private List excludeHosts = List.of(); - - public YarnParams(CommandPayload commandPayload) { - super(commandPayload); - globalParamsMap.put("yarn_user", user()); - globalParamsMap.put("yarn_group", group()); - globalParamsMap.put("java_home", Environments.getJavaHome()); - globalParamsMap.put("hadoop_home", serviceHome()); - globalParamsMap.put("hadoop_hdfs_home", hdfsHome()); - globalParamsMap.put("hadoop_yarn_home", yarnHome()); - globalParamsMap.put("hadoop_mapred_home", mapredHome()); - globalParamsMap.put("hadoop_conf_dir", confDir()); - globalParamsMap.put("hadoop_libexec_dir", serviceHome() + "/libexec"); - globalParamsMap.put("exclude_hosts", excludeHosts); - } - - public String yarnLimits() { - Map yarnConf = LocalSettings.configurations(serviceName(), "yarn.conf"); - return (String) yarnConf.get("content"); - } - - public String excludeNodesContent() { - Map excludeNodesContent = LocalSettings.configurations(serviceName(), "exclude-nodes"); - return (String) excludeNodesContent.get("content"); - } - - @GlobalParams - public Map yarnLog4j() { - return LocalSettings.configurations(serviceName(), "yarn-log4j"); - } - - @GlobalParams - public Map yarnSite() { - Map yarnSite = LocalSettings.configurations(serviceName(), "yarn-site"); - List resourcemanagerList = LocalSettings.hosts("resourcemanager"); - if (!resourcemanagerList.isEmpty()) { - yarnSite.put("yarn.resourcemanager.hostname", MessageFormat.format("{0}", resourcemanagerList.get(0))); - } - - rmNodesExcludeDir = (String) yarnSite.get("yarn.resourcemanager.nodes.exclude-path"); - nodemanagerLogDir = (String) yarnSite.get("yarn.nodemanager.log-dirs"); - nodemanagerLocalDir = (String) yarnSite.get("yarn.nodemanager.local-dirs"); - return yarnSite; - } - - @GlobalParams - public Map yarnEnv() { - Map yarnEnv = LocalSettings.configurations(serviceName(), "yarn-env"); - - tmpDir = (String) yarnEnv.get("hadoop_java_io_tmpdir"); - yarnLogDir = (String) yarnEnv.get("yarn_log_dir_prefix"); - yarnPidDir = (String) yarnEnv.get("yarn_pid_dir_prefix"); - resourceManagerPidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-resourcemanager.pid", yarnPidDir, user()); - nodeManagerPidFile = MessageFormat.format("{0}/{1}/hadoop-{1}-nodemanager.pid", yarnPidDir, user()); - return yarnEnv; - } - - @Override - public String confDir() { - return "/etc/hadoop/conf"; - } - - public String hdfsHome() { - return stackHome() + "/hadoop-hdfs"; - } - - public String yarnExec() { - return stackHome() + "/yarn"; - } - - public String yarnHome() { - return stackHome() + "/hadoop-yarn"; - } - - public String mapredHome() { - return stackHome() + "/hadoop-mapreduce"; - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java deleted file mode 100644 index ccce82037..000000000 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/yarn/YarnSetup.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bigtop.manager.stack.bigtop.v3_3_0.yarn; - -import org.apache.bigtop.manager.common.constants.Constants; -import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.enums.ConfigType; -import org.apache.bigtop.manager.stack.core.param.BaseParams; -import org.apache.bigtop.manager.stack.core.param.Params; -import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; - -import org.apache.commons.lang3.StringUtils; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.text.MessageFormat; -import java.util.Map; - -@Slf4j -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class YarnSetup { - - public static ShellResult config(Params params) { - return config(params, null); - } - - public static ShellResult config(Params params, String componentName) { - log.info("Setting yarn config"); - YarnParams yarnParams = (YarnParams) params; - - String confDir = yarnParams.confDir(); - String yarnUser = yarnParams.user(); - String yarnGroup = yarnParams.group(); - Map yarnEnv = yarnParams.yarnEnv(); - - if (StringUtils.isNotBlank(componentName)) { - switch (componentName) { - case "resourcemanager": { - LinuxFileUtils.toFileByTemplate( - yarnParams.excludeNodesContent(), - yarnParams.getRmNodesExcludeDir(), - yarnUser, - yarnGroup, - Constants.PERMISSION_644, - yarnParams.getGlobalParamsMap()); - } - case "nodemanager": { - LinuxFileUtils.createDirectories( - yarnParams.getNodemanagerLogDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); - LinuxFileUtils.createDirectories( - yarnParams.getNodemanagerLocalDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); - } - } - } - - // mkdir directories - LinuxFileUtils.createDirectories( - yarnParams.getYarnLogDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); - LinuxFileUtils.createDirectories( - yarnParams.getYarnPidDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); - LinuxFileUtils.createDirectories(yarnParams.getTmpDir(), yarnUser, yarnGroup, Constants.PERMISSION_755, true); - - // yarn.conf - LinuxFileUtils.toFileByTemplate( - yarnParams.yarnLimits(), - MessageFormat.format("{0}/yarn.conf", BaseParams.LIMITS_CONF_DIR), - Constants.ROOT_USER, - Constants.ROOT_USER, - Constants.PERMISSION_644, - yarnParams.getGlobalParamsMap()); - - // yarn-env.sh - LinuxFileUtils.toFileByTemplate( - yarnEnv.get("content").toString(), - MessageFormat.format("{0}/yarn-env.sh", confDir), - yarnUser, - yarnGroup, - Constants.PERMISSION_644, - yarnParams.getGlobalParamsMap()); - - // yarn-site.xml - LinuxFileUtils.toFile( - ConfigType.XML, - MessageFormat.format("{0}/yarn-site.xml", confDir), - yarnUser, - yarnGroup, - Constants.PERMISSION_644, - yarnParams.yarnSite(), - yarnParams.getGlobalParamsMap()); - - // log4j - LinuxFileUtils.toFileByTemplate( - yarnParams.yarnLog4j().get("content").toString(), - MessageFormat.format("{0}/yarnservice-log4j.properties", confDir), - yarnUser, - yarnGroup, - Constants.PERMISSION_644, - yarnParams.getGlobalParamsMap()); - - return ShellResult.success("YARN Configure success!"); - } -} diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java index 80ccfb942..8a340f53e 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperClientScript.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.stack.bigtop.v3_3_0.zookeeper; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractClientScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; @@ -42,6 +42,11 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { - return ZookeeperSetup.config(params); + return ZookeeperSetup.configure(params); + } + + @Override + public String getComponentName() { + return "zookeeper_client"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java index 65b7429db..c7f82b3b1 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperParams.java @@ -22,20 +22,26 @@ import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.bigtop.param.BigtopParams; import org.apache.bigtop.manager.stack.core.annotations.GlobalParams; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import java.util.Map; @Getter +@AutoService(Params.class) +@NoArgsConstructor public class ZookeeperParams extends BigtopParams { private final String zookeeperLogDir = "/var/log/zookeeper"; private final String zookeeperPidDir = "/var/run/zookeeper"; - private final String zookeeperDataDir = serviceHome() + "/data"; private final String zookeeperPidFile = zookeeperPidDir + "/zookeeper_server.pid"; + private String zookeeperDataDir; + public ZookeeperParams(CommandPayload commandPayload) { super(commandPayload); globalParamsMap.put("java_home", Environments.getJavaHome()); @@ -46,13 +52,18 @@ public ZookeeperParams(CommandPayload commandPayload) { @GlobalParams public Map zooCfg() { - Map configurations = LocalSettings.configurations(serviceName(), "zoo.cfg"); - configurations.put("dataDir", zookeeperDataDir); - return configurations; + Map zooCfg = LocalSettings.configurations(getServiceName(), "zoo.cfg"); + zookeeperDataDir = (String) zooCfg.get("dataDir"); + return zooCfg; } @GlobalParams public Map zookeeperEnv() { - return LocalSettings.configurations(serviceName(), "zookeeper-env"); + return LocalSettings.configurations(getServiceName(), "zookeeper-env"); + } + + @Override + public String getServiceName() { + return "zookeeper"; } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java index 8ed887ab1..37f72a165 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperServerScript.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -46,7 +46,7 @@ public ShellResult add(Params params) { @Override public ShellResult configure(Params params) { - return ZookeeperSetup.config(params); + return ZookeeperSetup.configure(params); } @Override @@ -78,4 +78,9 @@ public ShellResult status(Params params) { ZookeeperParams zookeeperParams = (ZookeeperParams) params; return LinuxOSUtils.checkProcess(zookeeperParams.getZookeeperPidFile()); } + + @Override + public String getComponentName() { + return "zookeeper_server"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperSetup.java b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperSetup.java index 5aff530f6..1721b0f14 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/zookeeper/ZookeeperSetup.java @@ -22,7 +22,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.common.utils.NetUtils; import org.apache.bigtop.manager.stack.core.enums.ConfigType; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; @@ -39,8 +39,8 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class ZookeeperSetup { - public static ShellResult config(Params params) { - log.info("Setting zookeeper config"); + public static ShellResult configure(Params params) { + log.info("Configuring ZooKeeper"); ZookeeperParams zookeeperParams = (ZookeeperParams) params; String confDir = zookeeperParams.confDir(); @@ -66,7 +66,6 @@ public static ShellResult config(Params params) { .append("\n"); } - // myid LinuxFileUtils.toFile( ConfigType.CONTENT, MessageFormat.format("{0}/myid", zookeeperParams.getZookeeperDataDir()), @@ -75,7 +74,6 @@ public static ShellResult config(Params params) { Constants.PERMISSION_644, zkHostList.indexOf(NetUtils.getHostname()) + 1 + ""); - // zoo.cfg HashMap map = new HashMap<>(zooCfg); map.remove("content"); Map paramMap = Map.of("zk_server_str", zkServerStr.toString(), "security_enabled", false); @@ -88,7 +86,6 @@ public static ShellResult config(Params params) { Map.of("model", map), paramMap); - // zookeeper-env LinuxFileUtils.toFileByTemplate( zookeeperEnv.get("content").toString(), MessageFormat.format("{0}/zookeeper-env.sh", confDir), @@ -97,6 +94,7 @@ public static ShellResult config(Params params) { Constants.PERMISSION_644, zookeeperParams.getGlobalParamsMap()); - return ShellResult.success("ZooKeeper Server Configure success!"); + log.info("Successfully configured ZooKeeper"); + return ShellResult.success(); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java index f570bf3a0..86e588e7d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/executor/StackExecutor.java @@ -25,9 +25,9 @@ import org.apache.bigtop.manager.common.utils.CaseUtils; import org.apache.bigtop.manager.common.utils.Environments; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.PrioritySPIFactory; import org.apache.bigtop.manager.stack.core.spi.hook.Hook; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.Script; import lombok.extern.slf4j.Slf4j; @@ -40,28 +40,20 @@ public class StackExecutor { private static final Map SCRIPT_MAP = new PrioritySPIFactory<>(Script.class).getSPIMap(); + private static final Map PARAMS_MAP = new PrioritySPIFactory<>(Params.class).getSPIMap(); + private static final Map HOOK_MAP = new PrioritySPIFactory<>(Hook.class).getSPIMap(); private static Script getCommandScript(CommandPayload commandPayload) { String componentName = commandPayload.getComponentName(); - String packageName = getPackageName(commandPayload); - String scriptId = - packageName + "." + CaseUtils.toCamelCase(componentName, CaseUtils.SEPARATOR_UNDERSCORE) + "Script"; - Script script = SCRIPT_MAP.get(scriptId); + Script script = SCRIPT_MAP.get(componentName); if (script == null) { - throw new StackException("Cannot find Script Class {0}", scriptId); + throw new StackException("Cannot find script for component: {0}", componentName); } return script; } - private static String getPackageName(CommandPayload commandPayload) { - return "org.apache.bigtop.manager.stack" + "." - + CaseUtils.toLowerCase(commandPayload.getStackName()) + ".v" - + commandPayload.getStackVersion().replaceAll("\\.", "_") + "." - + CaseUtils.toLowerCase(commandPayload.getServiceName()); - } - private static void runBeforeHook(String command, Params params) { Hook hook = HOOK_MAP.get(command.toLowerCase()); if (hook != null) { @@ -86,12 +78,11 @@ public static ShellResult execute(CommandPayload commandPayload) { String methodName = CaseUtils.toCamelCase(command, CaseUtils.SEPARATOR_UNDERSCORE, false); Method method = script.getClass().getMethod(methodName, Params.class); - String paramsClassName = getPackageName(commandPayload) + "." - + CaseUtils.toCamelCase(commandPayload.getServiceName()) + "Params"; - Class paramsClass = Class.forName(paramsClassName); - Params params = (Params) - paramsClass.getDeclaredConstructor(CommandPayload.class).newInstance(commandPayload); - + Params params = PARAMS_MAP + .get(commandPayload.getServiceName()) + .getClass() + .getDeclaredConstructor(CommandPayload.class) + .newInstance(commandPayload); if (Environments.isDevMode()) { log.info("Executing {}::{} on dev mode", script.getName(), method.getName()); return ShellResult.success(); diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AbstractHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AbstractHook.java index cb4fff1d3..51a105dbf 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AbstractHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AbstractHook.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.stack.core.spi.hook; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AddHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AddHook.java index 4ec0cd6a0..8e9da6e95 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AddHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/AddHook.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.stack.core.spi.hook; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxAccountUtils; import com.google.auto.service.AutoService; diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/Hook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/Hook.java index b5ce693bc..48c6d74c9 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/Hook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/Hook.java @@ -18,8 +18,8 @@ */ package org.apache.bigtop.manager.stack.core.spi.hook; -import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.PrioritySPI; +import org.apache.bigtop.manager.stack.core.spi.param.Params; public interface Hook extends PrioritySPI { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/RestartHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/RestartHook.java index 753bf0cc5..7b111ffae 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/RestartHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/RestartHook.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.stack.core.spi.hook; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StartHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StartHook.java index 292718ad2..2f4e7dbe2 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StartHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StartHook.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.stack.core.spi.hook; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StopHook.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StopHook.java index 976daaecd..a6564222d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StopHook.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/hook/StopHook.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.stack.core.spi.hook; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java similarity index 92% rename from bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java rename to bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java index 39b1af6d9..832799018 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/BaseParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.core.param; +package org.apache.bigtop.manager.stack.core.spi.param; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; @@ -30,6 +30,7 @@ import org.apache.bigtop.manager.stack.core.utils.LocalSettings; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.lang.reflect.Method; @@ -40,6 +41,7 @@ import java.util.Map; @Slf4j +@NoArgsConstructor public abstract class BaseParams implements Params { @Getter @@ -47,7 +49,7 @@ public abstract class BaseParams implements Params { public static final String LIMITS_CONF_DIR = "/etc/security/limits.d"; - protected final CommandPayload commandPayload; + protected CommandPayload commandPayload; @SuppressWarnings("unchecked") protected BaseParams(CommandPayload commandPayload) { @@ -119,11 +121,6 @@ public List packages() { return packageInfoList; } - @Override - public String serviceName() { - return this.commandPayload.getServiceName(); - } - @Override public String stackHome() { String root = LocalSettings.cluster().getRootDir(); @@ -132,7 +129,6 @@ public String stackHome() { @Override public String serviceHome() { - String service = this.commandPayload.getServiceName(); - return MessageFormat.format("{0}/{1}", stackHome(), service.toLowerCase()); + return MessageFormat.format("{0}/{1}", stackHome(), getServiceName()); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java similarity index 80% rename from bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java rename to bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java index 102832952..cc60f35bd 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/param/Params.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/Params.java @@ -16,14 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.bigtop.manager.stack.core.param; +package org.apache.bigtop.manager.stack.core.spi.param; import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; +import org.apache.bigtop.manager.stack.core.spi.PrioritySPI; import java.util.List; -public interface Params { +public interface Params extends PrioritySPI { String confDir(); @@ -35,9 +36,14 @@ public interface Params { List packages(); - String serviceName(); - String serviceHome(); String stackHome(); + + String getServiceName(); + + @Override + default String getName() { + return getServiceName(); + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractClientScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractClientScript.java index 0d0173372..395c8dde4 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractClientScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractClientScript.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.stack.core.spi.script; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; public abstract class AbstractClientScript extends AbstractScript { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java index 14d3733eb..2bd67de1a 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/AbstractScript.java @@ -22,7 +22,7 @@ import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo; import org.apache.bigtop.manager.common.message.entity.pojo.RepoInfo; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.TarballUtils; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java index 48cfae5d1..3c95dc823 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/script/Script.java @@ -19,8 +19,8 @@ package org.apache.bigtop.manager.stack.core.spi.script; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; import org.apache.bigtop.manager.stack.core.spi.PrioritySPI; +import org.apache.bigtop.manager.stack.core.spi.param.Params; /** * Interface representing a script for component support. @@ -84,4 +84,10 @@ public interface Script extends PrioritySPI { * @return a ShellResult object indicating success */ ShellResult check(Params params); + + String getComponentName(); + + default String getName() { + return getComponentName(); + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java index e087028e8..ef3c9f7a5 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java @@ -30,6 +30,8 @@ import java.io.FileOutputStream; import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.function.Function; @Slf4j @@ -75,6 +77,10 @@ private static Boolean extract(TarArchiveInputStream tis, File destDir, Integer if (!outputFile.exists()) { outputFile.mkdirs(); } + } else if (entry.isSymbolicLink()) { + Path link = Paths.get(outputFile.getAbsolutePath()); + Path target = Paths.get(entry.getLinkName()); + Files.createSymbolicLink(link, target); } else { File parent = outputFile.getParentFile(); if (!parent.exists()) { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java index dee884a1a..623862f28 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java @@ -72,6 +72,7 @@ public static void toFile( String permissions, Object content, Object paramMap) { + log.info("Generating file: [{}]", filename); if (type == null || StringUtils.isBlank(filename) || content == null) { log.error("type, filename, content must not be null"); return; @@ -122,6 +123,7 @@ public static void toFileByTemplate( String permissions, Object modelMap, Object paramMap) { + log.info("Generating file: [{}]", filename); if (StringUtils.isBlank(filename) || modelMap == null || StringUtils.isEmpty(template)) { log.error("type, filename, content, template must not be null"); return; diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java index 89e5ad3be..0b7b68c06 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java @@ -19,11 +19,13 @@ package org.apache.bigtop.manager.stack.infra.param; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; -import org.apache.bigtop.manager.stack.core.param.BaseParams; +import org.apache.bigtop.manager.stack.core.spi.param.BaseParams; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j +@NoArgsConstructor public abstract class InfraParams extends BaseParams { protected InfraParams(CommandPayload commandPayload) { diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java index 7c96773c7..946eb7d38 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusParams.java @@ -19,15 +19,20 @@ package org.apache.bigtop.manager.stack.infra.v1_0_0.prometheus; import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.infra.param.InfraParams; +import com.google.auto.service.AutoService; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.text.MessageFormat; @Getter @Slf4j +@AutoService(Params.class) +@NoArgsConstructor public class PrometheusParams extends InfraParams { public PrometheusParams(CommandPayload commandPayload) { @@ -37,4 +42,9 @@ public PrometheusParams(CommandPayload commandPayload) { public String dataDir() { return MessageFormat.format("{0}/data", serviceHome()); } + + @Override + public String getServiceName() { + return "prometheus"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java index 69a328461..a9e3f3c88 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusServerScript.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.shell.ShellResult; import org.apache.bigtop.manager.stack.core.exception.StackException; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.spi.script.AbstractServerScript; import org.apache.bigtop.manager.stack.core.spi.script.Script; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxOSUtils; @@ -101,4 +101,9 @@ public ShellResult status(Params params) { throw new StackException(e); } } + + @Override + public String getComponentName() { + return "prometheus_server"; + } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java index 7d03b134f..50e4af6c6 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java +++ b/bigtop-manager-stack/bigtop-manager-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/prometheus/PrometheusSetup.java @@ -20,7 +20,7 @@ import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.shell.ShellResult; -import org.apache.bigtop.manager.stack.core.param.Params; +import org.apache.bigtop.manager.stack.core.spi.param.Params; import org.apache.bigtop.manager.stack.core.utils.linux.LinuxFileUtils; import lombok.AccessLevel;