hadoop集群搭建（一）HDFS的namenode的HA搭建-创新互联

HDFS的namenode的HA搭建，准备好机器

揭阳网站建设公司创新互联建站,揭阳网站设计制作，有大型网站制作公司丰富经验。已为揭阳上千提供企业网站建设服务。企业网站搭建\成都外贸网站建设公司要多少钱，请找那个售后服务好的揭阳做网站的公司定做！

hadoop01 IP:192.168.216.203 GATEWAY:192.168.216.2

hadoop02 IP:192.168.216.204 GATEWAY:192.168.216.2

hadoop03 IP:192.168.216.205 GATEWAY:192.168.216.2

配置网卡

[root@hadoop01 ~]# vim /etc/sysconfig/network-scripts/ifcfg-eth0

DEVICE=eth0

TYPE=Ethernet

HWADDR=00:0C:29:6B:CD:B3 网卡MAC地址

ONBOOT=yes yes表示开机启动

NM_CONTROLLED=yes

BOOTPROTO=none

IPADDR=192.168.216.203 IP地址

PREFIX=24

GATEWAY=192.168.216.2 网关

DNS1=8.8.8.8 域名解析服务器地址一

DNS2=192.168.10.254 域名解析服务器地址域名解析服务器地址二

DEFROUTE=yes

IPV4_FAILURE_FATAL=yes

IPV6INIT=no

NAME="System eth0"

安装java JDK 并配置环境变量

[root@hadoop01 jdk1.8.0_152]# vim /etc/profile

#my setting

export JAVA_HOME=/usr/local/jdk1.8.0_152/

export PATH=$PATH:$JAVA_HOME/bin:

配置hadoop01/hadoop02/hadoop03之间互相ssh免密登陆

[root@hadoop01 hadoop-2.7.1]# vim ./etc/hadoop/hadoop-env.sh

# The java implementation to use.

export JAVA_HOME=/usr/local/jdk1.8.0_152/

[root@hadoop01 ~]# vim /usr/local/hadoop-2.7.1/etc/hadoop/core-site.xml

<name>fs.defaultFS</name>

</property>

<name>ha.zookeeper.quorum</name>

<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>

</property>

</configuration>

[root@hadoop01 ~]# vim /usr/local/hadoop-2.7.1/etc/hadoop/hdfs-site.xml

<name>dfs.nameservices</name>

</property>

<name>dfs.ha.namenodes.qian</name>

</property>

<name>dfs.namenode.rpc-address.qian.nn1</name>

<value>hadoop01:9000</value>

</property>

<name>dfs.namenode.rpc-address.qian.nn2</name>

<value>hadoop02:9000</value>

</property>

<name>dfs.namenode.http-address.qian.nn1</name>

<value>hadoop01:50070</value>

</property>

<name>dfs.namenode.http-address.qian.nn2</name>

<value>hadoop02:50070</value>

</property>

<name>dfs.namenode.shared.edits.dir</name>

<value>qjournal://hadoop01:8485;hadoop02:8485;hadoop03:8485/qian</value>

</property>

<name>dfs.journalnode.edits.dir</name>

<value>/home/hadata/journalnode/data</value>

</property>

<name>dfs.ha.automatic-failover.enabled</name>

</property>

<name>dfs.client.failover.proxy.provider.qian</name>

<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

</property>

<name>dfs.ha.fencing.methods</name>

<value>sshfence</value>

</property>

<name>dfs.ha.fencing.ssh.private-key-files</name>

</property>

<name>dfs.ha.fencing.ssh.connect-timeout</name>

</property>

<name>dfs.namenode.name.dir</name>

<value>/home/hadata/dfs/name</value>

</property>

<name>dfs.datanode.data.dir</name>

<value>/home/hadata/dfs/data</value>

</property>

<name>dfs.blocksize</name>

</property>

<name>dfs.permissions.enabled</name>

<value>false</value>

</property>

<name>dfs.replication</name>

</property>

[root@hadoop01 ~]# vim /usr/local/hadoop-2.7.1/etc/hadoop/slaves

hadoop01

hadoop02

hadoop03

安装并配置zookeeper

[root@hadoop01 zookeeper-3.4.10]# tar -zxvf /home/zookeeper-3.4.10.tar.gz -C /usr/local/

[root@hadoop01 zookeeper-3.4.10]# cp ./conf/zoo_sample.cfg ./conf/zoo.cfg

# The number of milliseconds of each tick

tickTime=2000

# The number of ticks that the initial

# synchronization phase can take

initLimit=5

# The number of ticks that can pass between

# sending a request and getting an acknowledgement

syncLimit=2

# the directory where the snapshot is stored.

# do not use /tmp for storage, /tmp here is just

# example sakes.

dataDir=/home/zookeeperdata

# the port at which the clients will connect

clientPort=2181

server.1=hadoop01:2888:3888

server.2=hadoop02:2888:3888

server.3=hadoop03:2888:3888

[root@hadoop01 zookeeper-3.4.10]# scp -r /usr/local/zookeeper-3.4.10 hadoop02:/usr/local/

[root@hadoop01 zookeeper-3.4.10]# scp -r /usr/local/zookeeper-3.4.10 hadoop03:/usr/local/

配置三台机器的环境变量

[root@hadoop01 zookeeper-3.4.10]# vim /etc/profile

#my setting

export JAVA_HOME=/usr/local/jdk1.8.0_152/

export HADOOP_HOME=/usr/local/hadoop-2.7.1/

export ZK_HOME=/usr/local/zookeeper-3.4.10/

export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:

[root@hadoop01 zookeeper-3.4.10]# scp -r /etc/profile hadoop02:/etc

profile

[root@hadoop01 zookeeper-3.4.10]# scp -r /etc/profile hadoop03:/etc

profile

[root@hadoop01 ~]# source /etc/profile

[root@hadoop02 ~]# source /etc/profile

[root@hadoop03 ~]# source /etc/profile

[root@hadoop01 zookeeper-3.4.10]# mkdir /home/zookeeperdata

[root@hadoop01 zookeeper-3.4.10]# vim /home/zookeeperdata/myid myid文件里输入 1

[root@hadoop02 ~]# mkdir /home/zookeeperdata

[root@hadoop02 ~]# vim /home/zookeeperdata/myid myid文件里输入 2

[root@hadoop03 ~]# mkdir /home/zookeeperdata

[root@hadoop03 ~]# vim /home/zookeeperdata/myid myid文件里输入 3

[root@hadoop01 zookeeper-3.4.10]# zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /usr/local/zookeeper-3.4.10/bin/../conf/zoo.cfg

Mode: follower

[root@hadoop02 ~]# zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /usr/local/zookeeper-3.4.10/bin/../conf/zoo.cfg

Mode: follower

[root@hadoop03 ~]# zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /usr/local/zookeeper-3.4.10/bin/../conf/zoo.cfg

Mode: leader

[root@hadoop01 zookeeper-3.4.10]# scp -r /usr/local/hadoop-2.7.1/ hadoop02:/usr/local/

[root@hadoop01 zookeeper-3.4.10]# scp -r /usr/local/hadoop-2.7.1/ hadoop03:/usr/local/

[root@hadoop01 zookeeper-3.4.10]# hadoop-daemon.sh start journalnode

[root@hadoop02 zookeeper-3.4.10]# hadoop-daemon.sh start journalnode

[root@hadoop03 zookeeper-3.4.10]# hadoop-daemon.sh start journalnode

[root@hadoop01 zookeeper-3.4.10]# hadoop namenode -format

[root@hadoop01 zookeeper-3.4.10]# hadoop-daemon.sh start namenode

starting namenode, logging to /usr/local/hadoop-2.7.1/logs/hadoop-root-namenode-hadoop01.out

同步已启动的namenode的元数据到为启动的nomenode

[root@hadoop02 ~]# hdfs namenode -bootstrapStandby

确认zookeeper集群是否启动

[root@hadoop01 zookeeper-3.4.10]# zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /usr/local/zookeeper-3.4.10/bin/../conf/zoo.cfg

Mode: follower

[root@hadoop02 ~]# zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /usr/local/zookeeper-3.4.10/bin/../conf/zoo.cfg

Mode: follower

[root@hadoop03 ~]# zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /usr/local/zookeeper-3.4.10/bin/../conf/zoo.cfg

Mode: leader

[root@hadoop01 zookeeper-3.4.10]# hdfs zkfc -formatZK

....INFO ha.ActiveStandbyElector: Successfully created /hadoop-ha/qian in ZK.

[root@hadoop03 ~]# zkCli.sh

WatchedEvent state:SyncConnected type:None path:null

[zk: localhost:2181(CONNECTED) 0] ls /

[zookeeper, hadoop-ha]

[zk: localhost:2181(CONNECTED) 1] ls /hadoop-ha

[qian]

[zk: localhost:2181(CONNECTED) 2] ls /hadoop-ha/qian

[]

注意：退出zkCli，输入quit

[root@hadoop01 zookeeper-3.4.10]# start-dfs.sh

[root@hadoop01 zookeeper-3.4.10]# jps

3281 JournalNode

4433 Jps

3475 NameNode

4068 DataNode

3110 QuorumPeerMain

4367 DFSZKFailoverController

[root@hadoop02 ~]# jps

3489 DataNode

3715 Jps

2970 QuorumPeerMain

3162 JournalNode

3646 DFSZKFailoverController

3423 NameNode

[root@hadoop03 ~]# zkCli.sh

zkCli.sh

WATCHER::

WatchedEvent state:SyncConnected type:None path:null

[zk: localhost:2181(CONNECTED) 4] ls /hadoop-ha/qian

[ActiveBreadCrumb, ActiveStandbyElectorLock]

[zk: localhost:2181(CONNECTED) 2] get /hadoop-ha/qian/ActiveBreadCrumb

qiannn1hadoop01 �F(�>

cZxid = 0x10000000a

ctime = Sat Jan 13 01:40:21 CST 2018

mZxid = 0x10000000a

mtime = Sat Jan 13 01:40:21 CST 2018

pZxid = 0x10000000a

cversion = 0

dataVersion = 0

aclVersion = 0

ephemeralOwner = 0x0

dataLength = 31

numChildren = 0

[root@hadoop01 hadoop-2.7.1]# hdfs dfs -put ./README.txt hdfs:/

[root@hadoop01 hadoop-2.7.1]# hdfs dfs -ls /

18/01/13 01:58:24 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

Found 1 items

-rw-r--r-- 3 root supergroup 1366 2018-01-13 01:57 /README.txt

测试是否失败转移

[root@hadoop01 hadoop-2.7.1]# jps

3281 JournalNode

3475 NameNode

4644 Jps

4068 DataNode

3110 QuorumPeerMain

4367 DFSZKFailoverController

[root@hadoop01 hadoop-2.7.1]# kill -9 3475

[root@hadoop03 ~]# zkCli.sh

ActiveBreadCrumb ActiveStandbyElectorLock

[zk: localhost:2181(CONNECTED) 6] get /hadoop-ha/qian/ActiveBreadCrumb

qiannn2hadoop02 �F(�>

cZxid = 0x10000000a

ctime = Sat Jan 13 01:40:21 CST 2018

mZxid = 0x100000011

mtime = Sat Jan 13 02:01:57 CST 2018

pZxid = 0x10000000a

cversion = 0

dataVersion = 1

aclVersion = 0

ephemeralOwner = 0x0

dataLength = 31

numChildren = 0

[root@hadoop02 ~]# jps

3489 DataNode

3989 Jps

2970 QuorumPeerMain

3162 JournalNode

3646 DFSZKFailoverController

3423 NameNode

注意：一个namenode1死了会自动切换到另一个namenode2上，namenode2死后，就都死了，不会自动启动namenode1

配置集群时间同步

HA搭建完毕

另外有需要云服务器可以了解下创新互联scvps.cn，海内外云服务器15元起步，三天无理由+7*72小时售后在线，公司持有idc许可证，提供“云服务器、裸金属服务器、高防服务器、香港服务器、美国服务器、虚拟主机、免备案服务器”等云主机租用服务以及企业上云的综合解决方案，具有“安全稳定、简单易用、服务可用性高、性价比高”等特点与优势，专为企业上云打造定制，能够满足用户丰富、多元化的应用场景需求。

本文题目：hadoop集群搭建（一）HDFS的namenode的HA搭建-创新互联
网站地址：https://www.cdcxhl.com/article48/dsichp.html

成都网站建设公司_创新互联，为您提供响应式网站、建站公司、自适应网站、网站收录、定制网站、虚拟主机

声明：本网站发布的内容（图片、视频和文字）以用户投稿、用户转载内容为主，如果涉及侵权请尽快告知，我们将会在第一时间删除。文章观点不代表本网站立场，如需处理请联系客服。电话：028-86922220；邮箱：631063699@qq.com。内容未经允许不得转载，或转载时需注明来源：创新互联

猜你还喜欢下面的内容