一.环境介绍
Moosefs master:192.168.55.148
Moosefs Metalogger:192.168.55.149
Moosefs Chunk-01:192.168.55.150
Moosefs Chunk-02:192.168.55.151
Moosefs Client:192.168.55.152
二.安装
#########################################################
1) master安装
$ yum install -y wget gcc zlib zlib-devel $ cd /usr/local/src $ groupadd mfs $ useradd -g mfs mfs -s /sbin/nologin -M $ wget http://www.moosefs.org/tl_files/mfscode/mfs-1.6.27-1.tar.gz $ tar zxvf mfs-1.6.27-1.tar.gz $ cd cd mfs-1.6.27 $ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfschunkserver --disable-mfsmount $ make $ make install
$ cp rpm/rh/mfsmaster.init /etc/init.d/mfsmaster $ cp rpm/rh/mfscgiserv.init /etc/init.d/mfscgiserv $ chmod 755 /etc/init.d/mfsmaster $ chmod 755 /etc/init.d/mfscgiserv $ ln -s /usr/local/mfs/sbin/mfsmaster /usr/sbin/ $ ln -s /usr/local/mfs/sbin/mfscgiserv /usr/sbin/ $ ln -s /usr/local/mfs/etc/mfs /etc/mfs $ cd /usr/local/mfs/etc/mfs $ cp mfsmaster.cfg.dist mfsmaster.cfg $ cp mfsmetalogger.cfg.dist mfsmetalogger.cfg $ cp mfsexports.cfg.dist mfsexports.cfg $ cp mfstopology.cfg.dist mfstopology.cfg
[root@localhost mfs]# more mfsexports.cfg | grep -v "^#" * / rw,alldirs,maproot=0 * . rw
$ cd /usr/local/mfs/var/mfs/ $ cp metadata.mfs.empty metadata.mfs $ vi /etc/hosts 1921.68.55.148 mfsmaster
$ vi /etc/init.d/mfsmaster #!/bin/bash # # Init file for the MooseFS master service # # chkconfig: - 92 84 # # description: MooseFS master # # processname: mfsmaster # config: @sysconfdir@/mfsmaster.cfg # Source function library. . /etc/init.d/functions # Source networking configuration. . /etc/sysconfig/network # Check that networking is up. [ "${NETWORKING}" == "no" ] && exit 0 [ -x "/usr/sbin/mfsmaster" ] || exit 1 [ -r "/etc/mfs/mfsexports.cfg" -o -r "/etc/mfs/mfsexports.cfg" ] || exit 1 RETVAL=0 prog="mfsmaster" start () { echo -n $"Starting $prog: " daemon $prog $OPTIONS RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog return $RETVAL } stop () { echo -n $"Stopping $prog: " $prog stop $OPTIONS || killproc $prog >/dev/null 2>&1 RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog return $RETVAL } restart () { stop start } case "$1" in start) start ;; stop) stop ;; restart|reload) restart ;; condrestart) [ -e /var/lock/subsys/$prog ] && restart RETVAL=$? ;; status) status $prog RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|restart|condrestart|status}" RETVAL=1 esac exit $RETVAL $ touch /usr/local/mfs/var/mfs/.mfscgiserv.lock $ chmod 777 /usr/local/mfs/var/mfs/.mfscgiserv.lock $ vi /etc/init.d/mfscgiserv #!/bin/bash # # Init file for the MooseFS CGI monitor service # # chkconfig: - 93 83 # # description: MooseFS CGI server # # processname: mfscgiserv # Source function library. . /etc/init.d/functions # Source networking configuration. . /etc/sysconfig/network MFSCGISERV_USER=nobody # Source initialization configuration. # Check that networking is up. [ "${NETWORKING}" == "no" ] && exit 0 [ -x "/usr/sbin/mfscgiserv" ] || exit 1 RETVAL=0 prog="mfscgiserv" start () { echo -n $"Starting $prog: " daemon --user $MFSCGISERV_USER $prog $OPTIONS RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog return $RETVAL } stop () { echo -n $"Stopping $prog: " $prog stop $OPTIONS || killproc $prog >/dev/null 2>&1 RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog return $RETVAL } restart () { stop start } case "$1" in start) start ;; stop) stop ;; restart|reload) restart ;; condrestart) [ -e /var/lock/subsys/$prog ] && restart RETVAL=$? ;; status) status $prog RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|restart|condrestart|status}" RETVAL=1 esac exit $RETVAL
$ /etc/init.d/mfsmaster $ /etc/init.d/mfscgiserv start $ chkconfig --add mfsmaster $ chkconfig -add mfscgiserv $ chkconfig mfsmaster on $ chkconfig mfscgiserv on
#################################################################
2)Metalogger Server install
$ yum install -y wget gcc zlib zlib-devel $ cd /usr/local/src/ $ groupadd mfs $ useradd -g mfs mfs -s /sbin/nologin -M $ tar zxvf mfs-1.6.27-1.tar.gz $ cd mfs-1.6.27 $ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfschunkserver --disable-mfsmount $ make $ make install
$ ln -s /usr/local/mfs/sbin/mfsmetalogger /usr/sbin/ $ ln -s /usr/local/mfs/etc/mfs /etc/mfs $ cp rpm/rh/mfsmetalogger.init /etc/init.d/mfsmetalogger $ chmod 755 /etc/init.d/mfsmetalogger $ cd /usr/local/mfs/etc/mfs/ $ cp mfsmetalogger.cfg.dist mfsmetalogger.cfg $ vi /etc/hosts $ 192.168.55.149 mfsmaster
$ vi /etc/init.d/mfsmetalogger #!/bin/bash # # Init file for the MooseFS metalogger service # # chkconfig: - 92 84 # # description: MooseFS metalogger # # processname: mfsmetalogger # config: @sysconfdir@/mfsmetalogger.cfg # Source function library. . /etc/init.d/functions # Source networking configuration. . /etc/sysconfig/network # Source initialization configuration. #[ -r "/usr/sbin/mfsmetalogger" ] && . /usr/sbin/mfsmetalogger # Check that networking is up. [ "${NETWORKING}" == "no" ] && exit 0 [ -x "/usr/sbin/mfsmetalogger" ] || exit 1 RETVAL=0 prog="mfsmetalogger" start () { echo -n $"Starting $prog: " daemon $prog $OPTIONS RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog return $RETVAL } stop () { echo -n $"Stopping $prog: " $prog stop $OPTIONS || killproc $prog >/dev/null 2>&1 RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog return $RETVAL } restart () { stop start } case "$1" in start) start ;; stop) stop ;; restart|reload) restart ;; condrestart) [ -e /var/lock/subsys/$prog ] && restart RETVAL=$? ;; status) status $prog RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|restart|condrestart|status}" RETVAL=1 esac exit $RETVAL $ chkconfig --add mfsmetalogger $ chkconfig mfsmetalogger on
############################################################
3) Chunk Server install
$ yum install -y wget gcc zlib zlib-devel $ cd /usr/local/src/ $ groupadd mfs $ useradd -g mfs mfs -s /sbin/nologin -M $ tar zxvf mfs-1.6.27-1.tar.gz $ cd mfs-1.6.27 $ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster $ make $ make install
$ ln -s /usr/local/mfs/etc/mfs /etc/mfs $ ln -s /usr/local/mfs/sbin/mfschunkserver /usr/sbin/ $ cp rpm/rh/mfschunkserver.init /etc/init.d/mfschunkserver $ chmod 755 /etc/init.d/mfschunkserver
$ cd /usr/local/mfs/etc/mfs
$ cp mfshdd.cfg.dist mfshdd.cfg
$ cp mfschunkserver.cfg.dist mfschunkserver.cfg
#创建2个挂载磁盘 $ mkdir -p /storage/mfschunks $ dd if=/dev/zero of=/storage/mfschunks/mfschunks1 bs=1024 count=1 seek=$((2*1024*1024-1)) $ mkfs -t ext4 /storage/mfschunks/mfschunks1 $ mkdir -p /mnt/mfschunks1 $ mount -t ext4 -o loop /storage/mfschunks/mfschunks1 /mnt/mfschunks1 $ dd if=/dev/zero of=/storage/mfschunks/mfschunks2 bs=1024 count=1 seek=$((2*1024*1024-1)) $ mkfs -t ext4 /storage/mfschunks/mfschunks2 $ mkdir -p /mnt/mfschunks2 $ mount -t ext4 -o loop /storage/mfschunks/mfschunks2 /mnt/mfschunks2 $ chown -R mfs:mfs /mnt/mfschunks1 /mnt/mfschunks2
#编辑添加挂载分区 $ vi /etc/mfs/mfshdd.cfg # mount points of HDD drives # #/mnt/hd1 #/mnt/hd2 #etc. /mnt/mfschunks1 /mnt/mfschunks2
#写入hosts $ vi /etc/hosts 192.168.55.148 mfsmaster
$ vi /etc/init.d/mfschunkserver #!/bin/bash # # Init file for the MooseFS chunkserver service # # chkconfig: - 93 83 # # description: MooseFS chunkserver # # processname: mfschunkserver # config: @sysconfdir@/mfschunkserver.cfg # Source function library. . /etc/init.d/functions # Source networking configuration. . /etc/sysconfig/network # Source initialization configuration. #[ -r "/usr/sbin/mfschunkserver" ] && . /usr/sbin/mfschunkserver # Check that networking is up. [ "${NETWORKING}" == "no" ] && exit 0 [ -x "/usr/sbin/mfschunkserver" ] || exit 1 [ -r "/etc/mfs/mfshdd.cfg" -o -r "/etc/mfs/mfshdd.cfg" ] || exit 1 RETVAL=0 prog="mfschunkserver" start () { echo -n $"Starting $prog: " daemon $prog $OPTIONS RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog return $RETVAL } stop () { echo -n $"Stopping $prog: " $prog stop $OPTIONS || killproc $prog >/dev/null 2>&1 RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog return $RETVAL } restart () { stop start } case "$1" in start) start ;; stop) stop ;; restart|reload) restart ;; condrestart) [ -e /var/lock/subsys/$prog ] && restart RETVAL=$? ;; status) status $prog RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|restart|condrestart|status}" RETVAL=1 esac exit $RETVAL
$ chkconfig --add mfschunkserver
$ chkconfig mfschunkserver on
###############################################
4) Client install
$ yum install -y wget gcc zlib zlib-devel $ cd /usr/local/src/ $ groupadd mfs $ useradd -g mfs mfs -s /sbin/nologin -M $ wget http://nchc.dl.sourceforge.net/project/buluoos/0.2/src/fuse-2.8.5.tar.gz $ tar zxvf fuse-2.8.5.tar.gz $ cd fuse-2.8.5 $ ./configure $ make && make install $ echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH' >>/etc/profile $ source /etc/profile $ lsmod |grep fuse fuse 73530 0
#################################
#lsmod |grep fuse #检查fuse是否加载到内核，若没有，执行下面命令
#modprobe fuse&&lsmod |grep fuse
###############################
$ cd /usr/local/src/ $ tar zxvf mfs-1.6.27-1.tar.gz $ cd mfs-1.6.27 $ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --$ disable-mfsmaster --disable-mfschunkserver $ make $ make install $ ln -s /usr/local/mfs/bin/* /usr/sbin/
$ vi /etc/hosts 192.168.1.1 mfsmaster $ mkdir -p /mnt/mfs $ /usr/local/mfs/bin/mfsmount /mnt/mfs -H mfsmaster $ df -h Filesystem Size Used Avail Use% Mounted on /dev/mapper/VolGroup-lv_root 11G 948M 8.8G 10% / tmpfs 116M 0 116M 0% /dev/shm /dev/sda1 485M 32M 428M 7% /boot mfsmaster:9421 6.3G 0 6.3G 0% /mnt/mfs
###TEST###
write:
time dd if=/dev/zero of=/mnt/mfs/test500m bs=1024k count=500
500+0 records in
500+0 records out
524288000 bytes (524 MB) copied, 6.14798 s, 85.3 MB/s
real 0m6.173s
user 0m0.001s
sys 0m0.973s
read:
time dd if=/mnt/mfs/test500m of=/dev/null
[root@localhost mfs]# mkdir 10 #创建目录 [root@localhost mfs]# mfssetgoal 2 /mnt/mfs/10 #设置副本份数 /mnt/mfs/10: 2 [root@localhost mfs]# mfsgetgoal -r /mnt/mfs/10 #查看副本数 /mnt/mfs/10: directories with goal 2 : 1
[root@localhost mfs]# mfssettrashtime -r 0 /mnt/mfs/10 #设置延迟删除时间 /mnt/mfs/10: inodes with trashtime changed: 1 inodes with trashtime not changed: 0 inodes with permission denied: 0 [root@localhost mfs]# mfsgettrashtime -r /mnt/mfs/10 #查看延迟删除时间 /mnt/mfs/10: directories with trashtime 0 : 1
[root@localhost mfs]# mfscheckfile /mnt/mfs/10/d.jpg #查看文件副本数 /mnt/mfs/10/d.jpg: chunks with 2 copies: 1 [root@localhost mfs]# mfsfileinfo /mnt/mfs/10/d.jpg #查看文件储存详细信息 /mnt/mfs/10/d.jpg: chunk 0: 00000000000000C5_00000001 / (id:197 ver:1) copy 1: 192.168.55.150:9422 copy 2: 192.168.55.151:9422
[root@localhost mfs]# mfsdirinfo /mnt/mfs/10 /mnt/mfs/10/: inodes: 2 directories: 1 files: 1 chunks: 1 length: 186050 size: 201728 realsize: 403456
#查看日志，默认打印在/var/log/messages下面
tail -f /var/log/messages
#故障切换(来自网络中总结)
(1)metalogger服务器中需要备份master的两个配置文件，由于配置文件不是经常变化，因此通过定时脚本进行文件同步即可。
(2)当metalogger没有下载metadata之前，不能使用期接管master。此时metarestore程序会运行失败。
(3)metarestore程序是根据metalogger中定期下载的metadata和changelog来恢复master挂掉时刻master所记录的整个mfs的信息。
metalogger的注意事项
1、部署metalogger的服务器的open files数目必须大于或者等于5000，
2、metalogger在启动时不会下载metadata，而是等到第一个下载周期的下载时间点时再去下载，metalogger下载metadata的时间点是每小时的10分30秒，时间间隔是1小时的整数倍。
3、metalogger不是在启动的时候就取下载metadata，而是必须等到下载时间点(2中所述)才会去下载。也就是说，metalogger为确保正确性，要在启动后最少一个小时以内master和metalogger都要保持良好的状态。