生成yum源
cd /var/ftp/pub/cdh/5
createrepo --update .
从节点
yum clean all
配置yum库
/etc/yum.repos.d
# cat /etc/yum.repos.d/cloudera-cdh.repo
[hadoop]
name=hadoop
baseurl=ftp://192.168.34.135/pub/cdh/5/
enabled=1
gpgcheck=0
安装JDK
yum install jdk
echo "export JAVA_HOME=/usr/java/latest" >> /root/.bash_profile
echo "export PATH=$JAVA_HOME/bin:$PATH" >> /root/.bash_profile
安装namenode
yum install hadoop hadoop-hdfs hadoop-client hadoop-doc hadoop-debuginfo hadoop-hdfs-namenode
安装webhdfs
yum install hadoop-httpfs
安装secondary namenode
yum install hadoop-hdfs-secondarynamenode
安装datanode
yum install hadoop hadoop-hdfs hadoop-client hadoop-doc hadoop-debuginfo hadoop-hdfs-datanode
配置HDFS
配置文件路径
/etc/hadoop/conf
core-site.xml
<property>
<name>hadoop.tmp.dir</name>
<value>/home/yimr/var/filesystem</value>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://yi01:9000</value>
</property>
hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.permissions.superusergroup</name>
<value>yimr</value>
</property>
配置Secondary Namenode
hdfs-site.xml
<property>
<name>dfs.secondary.http.address</name>
<value>yi01:50090</value>
</property>
配置WebHdfs
core-site.xml
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
启动webhdfs
service hadoop-httpfs start
启动HDFS
同步配置到所有节点
rsync -av /etc/hadoop/conf/ 192.168.34.130:/etc/hadoop/conf/
格式化namenode
mkdir -p /home/yimr/var/filesystem
chown -R hdfs.hdfs /home/yimr/var/filesystem
sudo -u hdfs hadoop namenode -format
每个节点启动hdfs
for x in `ls /etc/init.d/|grep hadoop-hdfs` ; do service $x start ; done
service hadoop-hdfs-datanode start
service hadoop-hdfs-namenode start
安装Yarn
安装resource manager
yum install hadoop-yarn hadoop-yarn-resourcemanager
安装node manager
yum install hadoop-yarn hadoop-yarn-nodemanager hadoop-mapreduce
安装history server
yum install hadoop-mapreduce-historyserver hadoop-yarn-proxyserver
配置Yarn
mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
yarn-site.xml
<property>
<description>The hostname of the RM.</description>
<name>yarn.resourcemanager.hostname</name>
<value>yi01</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
配置history server
mapred-site.xml
<property>
<name>yarn.app.mapreduce.am.staging-dir</name>
<value>/user</value>
</property>
启动服务
启动history server
sudo -u hdfs hadoop fs -mkdir -p /user
sudo -u hdfs hadoop fs -chmod 777 /user
sudo -u hdfs hadoop fs -mkdir -p /user/history
sudo -u hdfs hadoop fs -chmod -R 1777 /user/history
sudo -u hdfs hadoop fs -chown mapred:hadoop /user/history
/etc/init.d/hadoop-mapreduce-historyserver start
启动yarn
for x in `ls /etc/init.d/|grep hadoop-yarn` ; do service $x start ; done
service hadoop-yarn-nodemanager start
HDFS界面
yi00:50070
Yarn界面
yi00:8088
history界面
yi00:19888
测试