1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
| #!/usr/bin/env bash
# The Shell script for setup environment before install spark cluster
#
# The install steps as below
#
# 1. vi /etc/sysconfig/network, add the below info into all cluster machines
# NETWORKING=yes
# HOSTNAME=<machineName>
#
# 2. vi /etc/ntp.conf, manual setup the information for ntpd
#
# 3. modify the machineName, master and prefix, we suppose that all the machines with the same prefix
#
# 4. run the script in master with command ./installsetup.sh
#
# 预定义说明部分，包含前缀，主机名以及机群机器列表
prefix="c650"
master="c6501"
machineName="
192.168.65.101 c6501.ambari.apache.org c6501
192.168.65.102 c6502.ambari.apache.org c6502
192.168.65.103 c6503.ambari.apache.org c6503
"
# 文件的限制数量，将被添加到 limits.conf 文件中
numfiles="
root hard nofile 65536
root soft nofile 65536
root hard nproc 65536
root soft nproc 65536
"
echo "Begin setup environment"
# 添加机器 IP，名称等到 hosts 文件
cat >> /etc/hosts << EOF
${machineName}
EOF
# add the file limits
cat >> /etc/security/limits.conf << EOF
${numfiles}
EOF
# 添加时间同步信息
cat > /tmp/sync.conf << EOF
# Add time sync:
*/20 * * * * ( /usr/sbin/ntpdate -u $master ) && ( hwclock --systohc )
EOF
# 在主机和节点之间配置 SSH 无密码登录
mkdir -p /root/.ssh; chmod 700 /root/.ssh
ssh-keygen -t dsa -N '' -f /root/.ssh/id_dsa
cp /root/.ssh/id_dsa.pub /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
nodes=" ` cat /etc/hosts | grep $prefix | awk ' { print $3 } ' ` "
echo $nodes
for target in $nodes ; do
echo "Begin setup machine: " $target "= = = = = = = = = = = = >"
ssh -o StrictHostKeyChecking=no $target " setenforce 0 "
ssh -o StrictHostKeyChecking=no $target " mkdir -p /root/.ssh ; chmod 700 /root/.ssh "
scp /root/.ssh/* $target:/root/.ssh/
# 备份原始的 hosts 信息，最好提前手动备份，以防丢失
echo "Backup original hosts = = = = = = = = = = = = >"
scp $target:/etc/hosts $target:/etc/hosts_bak_auto
# 拷贝 hosts 文件到机群内所有机器
echo "Copy master hosts to cluster = = = = = = = = = = = = >"
scp /etc/hosts $target:/etc/hosts
# 禁用 SELINUX
echo "Disable SELINUX = = = = = = = = = = = = >"
ssh $target "sed s/=enforcing/=disabled/g /etc/selinux/config -i ; grep \"SELINUX=disabled\" /etc/selinux/config"
ssh $target " setenforce 0 "
echo "SELINUX=disabled"
# 关闭 IPv6
echo "Close IPv6 = = = = = = = = = = = = >"
ssh $target " echo \" install ipv6 /bin/true \" > /etc/modprobe.d/ipv6off.conf ; cat /etc/modprobe.d/ipv6off.conf "
lsmod | grep -i ipv6
ifconfig | grep -i inet6
echo $mod "Above should be empty for IPv6 = = = = = = = = = = = = >"
echo $config "Above should be empty for IPv6 = = = = = = = = = = = = >"
# 关闭防火墙 firewall
echo "Close firewall = = = = = = = = = = = = >"
ssh $target "service iptables save"
ssh $target "service iptables stop"
ssh $target "chkconfig iptables off"
ssh $target "sed -i \" /service iptables stop/d \" /etc/rc.local ; echo \ " service iptables stop\ ">> /etc/rc.local "
# 关闭 hugepage
ssh $target "transparent_hugepage=never"
ssh $target "echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled"
ssh $target "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag"
# 调整并备份 limits.conf，并将修改信息应用到机群内所有机器
echo "Modify the limits to 65536 = = = = = = = = = = = = >"
sed -i " /pam_limits/d " /etc/pam.d/login; echo " session required pam_limits.so " >> /etc/pam.d/login
echo "Backup original limits.conf = = = = = = = = = = = = >"
scp $target:/etc/security/limits.conf $target:/etc/security/limits.conf_bak_auto
echo "Copy master limits.conf to cluster = = = = = = = = = = = = >"
scp /etc/security/limits.conf $target:/etc/security/limits.conf
ssh $target " sed -i \" /pam_limits/d \" /etc/pam.d/login "
ssh $target " echo \" session required pam_limits.so\" >> /etc/pam.d/login "
#如果存在 dnsmasq，则关闭
echo "Close dnsmasq = = = = = = = = = = = = >"
ssh $target " service dnsmasq stop "
ssh $target " chkconfig dnsmasq off "
# 检查端口是否被占用，此处只列举了部分端口，用户可以自行修改
echo "Check port = = = = = = = = = = = = >"
ssh $target " lsof -i:53,1528,2181,2182,7052,7869,7870,7871,7872,7873,7874,7875,7876,7877,7878,7879,7880,7881,7882,7883,8004,8006,8007,8080,8200,8280,8888,9000,9001,9093,9099,9999,10000,10101,10102,14000,18080,50010,50020,50030,50070,50075,50090,60000,60010,60020,60030 "
echo "The above port need to be closed if existed"
# 同步 ntpd
scp /tmp/sync.conf $target:/tmp/sync.conf
ssh $target " crontab /tmp/sync.conf ; service crond reload;"
ssh $target " /usr/sbin/ntpdate -u " $master
ssh $target " crontab -l "
ssh $target " service ntpd restart"
done
# 拷贝 known_hosts 文件到机群所有机器
for target in $nodes ; do
scp $target:/root/.ssh/known_hosts $target:/root/.ssh/known_hosts_bak
scp /root/.ssh/known_hosts $target:/root/.ssh/
ssh $target " hostname $target "
done
|
