hadoop 集群上

Haoop集群搭建

1.安装rzsz

yum install -y lrzsz

1).更新yum源 下载repo

#安装wget
yum -y istall wget
[root@localhost yum]# cd /etc/yum.repos.d/
[root@localhost yum.repos.d]# wget http://mirrors.aliyun.com/repo/Centos-7.repo
#移动并重命名
[root@localhost yum.repos.d]# mv CentOS-Base.repo /tmp && mv Centos-7.repo CentOS-CR.repo

2.常用软件

yum install -y openssh-server vim gcc gcc-c++ glibc-headers bzip2-devel lzo-devel curl wget openssh-clients zlib-devel autoconf automake cmake libtool openssl-devel fuse-devel snappy-devel telnet unzip zip net-tools.x86_64 firewalld systemd ntp

3.更新yum源

yum clean all && yum makecache && yum update -y

#更新命令
yum clean all   #清空国外源的
yum makecache   #缓存阿里云的
yum update -y   #从阿里云更新

4.关闭防火墙

# 关闭防火墙
systemctl stop firewalld.service
# 禁止防火墙开机启动
systemctl disable firewalld.service
# 查看服务是否开机启动
systemctl is-enabled firewalld.service
# 查看防火墙状态
firewall-cmd --state

5.关闭SELinux

# 查看关闭状态
/usr/sbin/sestatus -v
# 关闭方法
vim /etc/selinux/config

把文件里的SELINUX=disabled

6.安装JDK

1).解压

rpm -ivh jdk-8u144-linux-x64.rpm

2).配置JDK 环境变量

[root@localhost ]#  vi /etc/profile

export JAVA_HOME=/usr/java/jdk1.8.0_144
export JRE_HOME=$JAVA_HOME/jre
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

#使修改生效
source /etc/profile
#查看系统变量值
echo $PATH
#检查JDK 配置情况
java -version

3).crontab 定时任务

crontab [选项]
-e：编辑 crontab 定时任务；
-l：查询 crontab 定时任务；
-r：删除当前用户所有的 crontab 定时任务。如果有多个定时任务，只想删除一个，则可以使用“crontab -e”；
-u 用户名：修改或删除其他用户的 crontab 定时任务。只有 root 用户可用；

crontab -e
#编辑内容
0 * * * * /usr/sbin/ntpdate time1.aliyun.com >> /tmp/autontpdate 2>&1

7. 创建 hadoop 用户并设置 hadoop 用户密码

#创建hadoop用户
useradd hadoop
#给hadoop用户设置密码 --> 000000
passwd hadoop

8.配置用户hadoop免密切换root

1).修改配置文件

去掉两个#

[root@localhost tmp]# vim /etc/pam.d/su
#%PAM-1.0
auth            sufficient      pam_rootok.so
# Uncomment the following line to implicitly trust users in the "wheel" group.
auth           sufficient      pam_wheel.so trust use_uid   #去掉这行的井号
# Uncomment the following line to require a user to be in the "wheel" group.
auth           required        pam_wheel.so use_uid   #去掉这行的井号
auth            substack        system-auth
auth            include         postlogin
account         sufficient      pam_succeed_if.so uid = 0 use_uid quiet
account         include         system-auth
password        include         system-auth
session         include         system-auth
session         include         postlogin
session         optional        pam_xauth.so

2).使hadoop加入whell组

#把hadoop用户加到wheel组里
gpasswd -a hadoop wheel
#查看wheel组里是否有hadoop用户
cat /etc/group |grep wheel
#验证
su - root

9.配置hosts 文件

#切换root用户
su - root
#添加域名
[root@localhost hadoop]# vim /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.92.30 nn1.hadoop
192.168.92.31 nn2.hadoop
192.168.92.32 s1.hadoop
192.168.92.33 s2.hadoop
192.168.92.34 s3.hadoop

10.配置固定ip

采坑

#清空文本所有配置
ESC  g   D  
#找回系统UUID
nmcli con show

配置

[root@localhost ~]# cd /etc/sysconfig/network-scripts/
[root@localhost network-scripts]# vi ifcfg-ens33

TYPE="Ethernet"
BOOTPROTO="static"
IPADDR="192.168.92.101"
NETMASK="255.255.255.0"
GATEWAY="192.168.92.2"
DNS="192.168.92.2"
NM_CONTROLLED="no"
DEFROUTE="yes"
PEERDNS="yes"
PEERROUTES="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_PEERDNS="yes"
IPV6_PEERROUTES="yes"
IPV6_FAILURE_FATAL="no"
NAME="ens33"
UUID="2b07f404-5b28-4bbd-b3fe-9e2ad9404f22"
DEVICE="ens33"
ONBOOT="yes"
#重启网络服务
[root@localhost network-scripts]# systemctl stop NetworkManager.service
[root@localhost network-scripts]# systemctl disable NetworkManager.service
[root@localhost network-scripts]# systemctl restart network.service

查看配置

[root@localhost network-scripts]# vi /etc/resolv.conf
# Generated by NetworkManager
search localdomain
nameserver 192.168.92.2 

11.克隆

nn2.hadoop： 从节点
s1.hadoop、s2.hadoop、s3.hadoop：三个工作节点