在线教育离线数仓使用的脚本合集

1. 集群执行命令脚本：

1
2
3
4
5
6
7
#! /bin/bash
 
for i in hadoop102 hadoop103 hadoop104
do
    echo --------- $i ----------
    ssh $i "$*"
done

2. 集群同步脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#!/bin/bash

#1. 判断参数个数
if [ $# -lt 1 ]
then
  echo Not Enough Arguement!
  exit;
fi

#2. 遍历集群所有机器
for host in hadoop102 hadoop103 hadoop104
do
  echo ====================  $host  ====================
  #3. 遍历所有目录，挨个发送
  for file in $@
  do
    #4 判断文件是否存在
    if [ -e $file ]
    then
      #5. 获取父目录(dirname获取绝对路径父目录,相对路径获取的是.）
      pdir=$(cd -P $(dirname $file); pwd)
      #6. 获取当前文件的名称
      fname=$(basename $file)
      ssh $host "mkdir -p $pdir"
      rsync -av $pdir/$fname $host:$pdir #-av是打印进度的
    else
      echo $file does not exists!
    fi
  done
done

3. Hadoop启动脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#!/bin/bash
if [ $# -lt 1 ]
then
    echo "No Args Input..."
    exit ;
fi
case $1 in
"start")
        echo " =================== 启动 hadoop集群 ==================="

        echo " --------------- 启动 hdfs ---------------"
        ssh hadoop102 "/opt/module/hadoop/sbin/start-dfs.sh"
        echo " --------------- 启动 yarn ---------------"
        ssh hadoop103 "/opt/module/hadoop/sbin/start-yarn.sh"
        echo " --------------- 启动 historyserver ---------------"
        ssh hadoop102 "/opt/module/hadoop/bin/mapred --daemon start historyserver"
;;
"stop")
        echo " =================== 关闭 hadoop集群 ==================="

        echo " --------------- 关闭 historyserver ---------------"
        ssh hadoop102 "/opt/module/hadoop/bin/mapred --daemon stop historyserver"
        echo " --------------- 关闭 yarn ---------------"
        ssh hadoop103 "/opt/module/hadoop/sbin/stop-yarn.sh"
        echo " --------------- 关闭 hdfs ---------------"
        ssh hadoop102 "/opt/module/hadoop/sbin/stop-dfs.sh"
;;
*)
    echo "Input Args Error..."
;;
esac

4. Zookeeper启动脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#!/bin/bash

#判断是否输入参数
if [ $# -lt 1 ]; then
  echo "请输入参数"
  exit
fi

case $1 in
"start"){
        for i in hadoop102 hadoop103 hadoop104
        do
        echo ---------- zookeeper $i 启动 ------------
                ssh $i "/opt/module/zookeeper/bin/zkServer.sh start"
        done
};;
"stop"){
        for i in hadoop102 hadoop103 hadoop104
        do
        echo ---------- zookeeper $i 停止 ------------    
                ssh $i "/opt/module/zookeeper/bin/zkServer.sh stop"
        done
};;
"status"){
        for i in hadoop102 hadoop103 hadoop104
        do
        echo ---------- zookeeper $i 状态 ------------    
                ssh $i "/opt/module/zookeeper/bin/zkServer.sh status"
        done
};;
esac

5. Kafka启动脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#! /bin/bash

#判断是否输入参数
if [ $# -lt 1 ]; then
  echo "请输入参数"
  exit
fi
case $1 in
"start"){
    for i in hadoop102 hadoop103 hadoop104
    do
        echo " --------启动 $i Kafka-------"
        ssh $i "/opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties"
    done
};;
"stop"){
    for i in hadoop102 hadoop103 hadoop104
    do
        echo " --------停止 $i Kafka-------"
        ssh $i "/opt/module/kafka/bin/kafka-server-stop.sh "
    done
};;
esac

maxwell启动脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#!/bin/bash

MAXWELL_HOME=/opt/module/maxwell

#grep -v grep是为了过滤掉grep本身的进程
result=$(ps -ef | grep Maxwell | grep -v grep)

case $1 in
start )
	#判断result是否为空
    if [ -z "$result" ]; then
        echo "启动Maxwell"
        $MAXWELL_HOME/bin/maxwell --config $MAXWELL_HOME/config.properties --daemon
    else
         echo "Maxwell正在运行"
    fi
;;
stop )
    if [ -z "$result" ]; then
        echo "Maxwell未在运行"
    else
		echo "停止Maxwell"
        ps -ef | grep Maxwell | grep -v grep | awk '{print $2}' | xargs kill -9
    fi
;;
esac