cloudera5.16.1 离线安装

cloudera对JAVA mysql 操作系统均有要求详情可以参考下面官方文档

https://docs.cloudera.com/documentation/enterprise/release-notes/topics/rn_consolidated_pcm.html#cdh_cm_supported_os
https://docs.cloudera.com/documentation/enterprise/release-notes/topics/rn_consolidated_pcm.html#cdh_cm_supported_db
https://docs.cloudera.com/documentation/enterprise/release-notes/topics/rn_consolidated_pcm.html#pcm_jdk

官方环境要求

实验环境

收集cloudera软件资源

1.parcel下载http://archive.cloudera.com/cdh5/parcels/5.16.1/ 下载下图中3个文件

2.cloudera-manager下载http://archive.cloudera.com/cm5/cm/5/ 下载下图中1个文件

3.mysql-connector-java下载https://www.mvnjar.com/mysql/mysql-connector-java/5.1.47/detail.html下载下图中文件

4.jdk下载https://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase8-2177648.html下载下图中版本

5.mysql
链接：https://pan.baidu.com/s/1K42c6iVe3BQJNT4hLhvvPQ
提取码：qxvh

初始化安装环境

修改hostname（4台）

hostnamectl set-hostname  cdh-node1  #cdh-node1~4

修改hosts（4台）

(
cat <<EOF
10.23.215.209 cdh-node1
10.23.215.217 cdh-node2
10.23.215.218 cdh-node3
10.23.215.219 cdh-node4
EOF
) >> /etc/hosts

关闭防火墙（4台）

systemctl stop firewalld
systemctl disable firewalld

关闭selinux（4台）

setenforce 0  
sed -i 's/^SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

修改vm.swappiness（4台）

sysctl vm.swappiness=10 
echo 'sysctl vm.swappiness=10' >>/etc/sysctl.conf

禁用透明大页面（4台）

echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >>/etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag'  >>/etc/rc.local

修改时区（4台）

timedatectl set-timezone Asia/Shanghai

安装ntp psmisc libaio（4台）

yum -y install ntp psmisc libaio

设置ntp同步时间（4台）

#cdh-node1 开启ntp服务
systemctl  start ntpd
systtemctl enable ntpd
#cdh-node2~4 设置同步
echo '0 1 * * *  ntpdate cdh-node1' >>/var/spool/cron/root

安装cloudera

创建安装目录

mkdir -p /root/cloudera
wget  -P /root/cloudera    http://archive.cloudera.com/cdh5/parcels/5.16.1/CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel 
#其他文件均下载到此目录

mysql配置文件将配置文件放在/root/cloudera目录

[client]
port            = 3306
socket          = /usr/local/mysql/data/mysql.sock
default-character-set=utf8mb4
[mysqld]
port            = 3306
socket          = /usr/local/mysql/data/mysql.sock
skip-slave-start
skip-external-locking
key_buffer_size = 256M
sort_buffer_size = 2M
read_buffer_size = 2M
read_rnd_buffer_size = 4M
query_cache_size= 32M
max_allowed_packet = 16M
myisam_sort_buffer_size=128M
tmp_table_size=32M
table_open_cache = 512
thread_cache_size = 8
wait_timeout = 86400
interactive_timeout = 86400
max_connections = 600
default-storage-engine = INNODB
transaction-isolation = READ-COMMITTED
server-id  = 1739
basedir     = /usr/local/mysql
datadir     = /usr/local/mysql/data
pid-file     = /usr/local/mysql/data/hostname.pid
log-warnings
sysdate-is-now
binlog_format = ROW
log_bin_trust_function_creators=1
log-error  = /usr/local/mysql/data/hostname.err
log-bin = /usr/local/mysql/arch/mysql-bin
expire_logs_days = 7
innodb_write_io_threads=16
relay-log  = /usr/local/mysql/relay_log/relay-log
relay-log-index = /usr/local/mysql/relay_log/relay-log.index
relay_log_info_file= /usr/local/mysql/relay_log/relay-log.info
log_slave_updates=1
gtid_mode=OFF
enforce_gtid_consistency=OFF

slave-parallel-type=LOGICAL_CLOCK
slave-parallel-workers=4
master_info_repository=TABLE
relay_log_info_repository=TABLE
relay_log_recovery=ON
sync_binlog = 500

#for innodb options 
innodb_data_home_dir = /usr/local/mysql/data/
innodb_data_file_path = ibdata1:1G;ibdata2:1G:autoextend

innodb_log_group_home_dir = /usr/local/mysql/arch
innodb_log_files_in_group = 4
innodb_log_file_size = 1G
innodb_log_buffer_size = 200M


innodb_buffer_pool_size = 8G
#innodb_additional_mem_pool_size = 50M #deprecated in 5.6
tmpdir = /usr/local/mysql/tmp

innodb_lock_wait_timeout = 1000
#innodb_thread_concurrency = 0
innodb_flush_log_at_trx_commit = 2

innodb_locks_unsafe_for_binlog=1

#innodb io features: add for mysql5.5.8
performance_schema
innodb_read_io_threads=4
innodb-write-io-threads=4
innodb-io-capacity=200
#purge threads change default(0) to 1 for purge
innodb_purge_threads=1
innodb_use_native_aio=on

#case-sensitive file names and separate tablespace
innodb_file_per_table = 1
lower_case_table_names=1

[mysqldump]
quick
max_allowed_packet = 128M

[mysql]
no-auto-rehash
default-character-set=utf8mb4

[mysqlhotcopy]
interactive-timeout

[myisamchk]
key_buffer_size = 256M
sort_buffer_size = 256M
read_buffer = 2M
write_buffer = 2M

my.cnf

添加bash_profile 配置文件放在/root/cloudera目录

cat bash_profile 
if [ -f ~/.bashrc ]; then
        . ~/.bashrc
fi
# User specific environment and startup programs
export MYSQL_BASE=/usr/local/mysql
export PATH=${MYSQL_BASE}/bin:$PATH
unset USERNAME
set umask to 022
umask 022
PS1=`uname -n`":"'$USER'":"'$PWD'":>"; export PS1

View Code

安装jdk

mkdir -p /usr/java
tar zxf /root/cloudera/jdk-8u141-linux-x64.tar.gz -C /usr/java
chown -R root.root /usr/java/*
echo '#JAVA' >>/etc/profile
echo 'export JAVA_HOME=/usr/java/jdk1.8.0_141' >>/etc/profile
echo 'export PATH=$JAVA_HOME/bin:$PATH' >>/etc/profile
source /etc/profile

安装mysql-connector

mkdir -p /usr/share/java
cp /root/cloudera/mysql-connector-java-5.1.47.jar /usr/share/java/mysql-connector-java.jar

安装mysql

tar zxf /root/cloudera/mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz -C /usr/local
tar zxf /root/cloudera/config.tar.gz -C /etc/
mv /usr/local/mysql-5.7.11-linux-glibc2.5-x86_64 /usr/local/mysql
mkdir -p /usr/local/mysql/{arch,data,tmp}
groupadd -g 101 dba
useradd -u 514 -g dba -G root -d /usr/local/mysql mysqladmin
alias cp=cp
cp -rf /etc/skel/.* /usr/local/mysql/ >/dev/null
usermod -u 514 -g dba -G root -d /usr/local/mysql mysqladmin
cp -rf /root/cloudera/bash_profile /usr/local/mysql/.bash_profile
chown  mysqladmin:dba /etc/my.cnf
chmod  640 /etc/my.cnf
chown -R mysqladmin:dba /usr/local/mysql
chmod -R 755 /usr/local/mysql
cp -rf /usr/local/mysql/support-files/mysql.server /etc/rc.d/init.d/mysql
chmod +x /etc/rc.d/init.d/mysql
chkconfig --del mysql
chkconfig --add mysql
chkconfig --level 345 mysql on

初始化数据

su - mysqladmin -c "bin/mysqld --defaults-file=/etc/my.cnf --user=mysqladmin --basedir=/usr/local/mysql/ --datadir=/usr/local/mysql/data/ --initialize"

使用su 切换至mysqladmin用户执行

/usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf &
#查看mysql密码
grep "password" /usr/local/mysql/data/hostname.err |awk -F"t@" '{print $2}'|awk -F ": " '{print $2}'
#登陆mysql
mysql -uroot -p   #输入密码登陆
#在mysql执行命令
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'rzl.abc.mysql' ;
create database cmf DEFAULT CHARACTER SET utf8;
GRANT ALL PRIVILEGES ON cmf.* TO 'cmf'@'%' IDENTIFIED BY 'rzl.abc.mysql' ;
create database amon DEFAULT CHARACTER SET utf8;
GRANT ALL PRIVILEGES ON amon.* TO 'amon'@'%' IDENTIFIED BY 'rzl.abc.mysql' ;
flush privileges;
exit

exit

安装cloudera

mkdir -p /opt/cloudera-manager/
tar zxf /root/cloudera/cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz  -C /opt/cloudera-manager/
sed -i 's/server_host=localhost/server_host=cdh-node1/g' /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-agent/config.ini
sed -i 's/^#com/com/g' /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-server/db.properties
sed -i 's/=INIT/=EXTERNAL/g' /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-server/db.properties
sed -i 's/db.password=/db.password=rzl.abc.mysql/g' /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-server/db.properties
useradd --system --home=/opt/cloudera-manager/cm-5.16.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false cloudera-scm
chown -R cloudera-scm:cloudera-scm /opt/cloudera-manager/
mkdir -p /opt/cloudera/parcel-repo
mv /root/cloudera/CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel* /opt/cloudera/parcel-repo/
mv /root/cloudera/manifest.json /opt/cloudera/parcel-repo/
/opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-server start
/opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-agent start

查看cloudera日志启动大约1分钟左右看实验环境配置输出7180端口就启动完了。

tail -f /opt/cloudera-manager/cm-5.16.1/log/cloudera-scm-server/cloudera-scm-server.log

cdh-node2~4安装java （同上）

将cdh-node1安装目录中拷贝到/opt/cloudera-manager/ 修改配置启动即可

mkdir -p /opt/cloudera-manager/
tar zxf /root/cloudera/cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz  -C /opt/cloudera-manager/
sed -i 's/server_host=localhost/server_host=cdh-node1/g' /opt/cloudera-manager/cm-5.16.1/etc/cloudera-scm-agent/config.ini
useradd --system --home=/opt/cloudera-manager/cm-5.16.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false cloudera-scm
chown -R cloudera-scm:cloudera-scm /opt/cloudera-manager/
/opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-agent start

登陆 http://10.23.215.209:7180/cmf/login 默认用户名密码 admin admin