环境:
OS:Centos 7
说明:RAC环境的每个节点都要做同样的操作
-------------------------------------------配置multipath----------------------------------------------------
1.安装多路径
[root@localhost ios]# yum install device-mapper-multipath
2.开机自启动
[root@localhost ios]# systemctl enable multipathd.service
3.配置配置文件
拷贝一份模板配置文件到etc目录
[root@localhost /]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/
根据实际情况进行修改(后面做rac的需要做相应的配置修改)
# vi /etc/multipath.conf
blacklist {
devnode "^sda"
}
defaults {
user_friendly_names yes
path_grouping_policy multibus
failback immediate
no_path_retry fail
}
4.启动服务
systemctl start multipathd.service
5.查看相关配置文件
服务器启动之后设备的wwid会自动添加到/etc/multipath/wwids文件中,wwid自动生成到该文件中,不需要进行编辑,如下:
[root@localhost multipath]# more /etc/multipath/wwids
# Multipath wwids, Version : 1.0
# NOTE: This file is automatically maintained by multipath and multipathd.
# You should not need to edit this file in normal circumstances.
#
# Valid WWIDs:
/VBOX_HARDDISK_VBa08577cb-9ee269dc/
/VBOX_HARDDISK_VB74301632-13d08c3c/
/VBOX_HARDDISK_VBfa94873f-504b6993/
/VBOX_HARDDISK_VB4f84df6d-a94b8da6/
绑定文件/etc/multipath/bindings,该文件里面的映射关系是自动生成的,不需要进行手工编辑
[root@localhost multipath]# more /etc/multipath/bindings
# Multipath bindings, Version : 1.0
# NOTE: this file is automatically maintained by the multipath program.
# You should not need to edit this file in normal circumstances.
#
# Format:
# alias wwid
#
mpatha VBOX_HARDDISK_VBa08577cb-9ee269dc
mpathb VBOX_HARDDISK_VB74301632-13d08c3c
mpathc VBOX_HARDDISK_VBfa94873f-504b6993
mpathd VBOX_HARDDISK_VB4f84df6d-a94b8da6
6.查看服务
[root@localhost ios]# multipath -ll
mpathd (VBOX_HARDDISK_VB4f84df6d-a94b8da6) dm-4 ATA ,VBOX HARDDISK
size=40G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
mpathc (VBOX_HARDDISK_VBfa94873f-504b6993) dm-3 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
mpathb (VBOX_HARDDISK_VB74301632-13d08c3c) dm-2 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
mpatha (VBOX_HARDDISK_VBa08577cb-9ee269dc) dm-1 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
[root@localhost ios]#
执行lsblk命令就可以看到多路径磁盘mpatha了
[root@localhost ios]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 50G 0 disk
├─sda1 8:1 0 300M 0 part /boot
├─sda2 8:2 0 5.6G 0 part [SWAP]
└─sda3 8:3 0 44.1G 0 part
└─vgroot-lvroot 253:0 0 44.1G 0 lvm /
sdb 8:16 0 30G 0 disk
└─mpatha 253:1 0 30G 0 mpath
sdc 8:32 0 30G 0 disk
└─mpathb 253:2 0 30G 0 mpath
sdd 8:48 0 30G 0 disk
└─mpathc 253:3 0 30G 0 mpath
sde 8:64 0 40G 0 disk
└─mpathd 253:4 0 40G 0 mpath
sr0 11:0 1 1024M 0 rom
loop0 7:0 0 4.4G 0 loop /mnt
7.查看设备情况
[root@localhost mapper]# ls -al /dev/mapper
total 0
drwxr-xr-x. 2 root root 160 Apr 11 21:37 .
drwxr-xr-x. 19 root root 3300 Apr 11 21:37 ..
crw-------. 1 root root 10, 236 Apr 10 05:30 control
lrwxrwxrwx. 1 root root 7 Apr 11 21:43 mpatha -> ../dm-1
lrwxrwxrwx. 1 root root 7 Apr 11 21:43 mpathb -> ../dm-2
lrwxrwxrwx. 1 root root 7 Apr 11 21:43 mpathc -> ../dm-3
lrwxrwxrwx. 1 root root 7 Apr 11 21:43 mpathd -> ../dm-4
lrwxrwxrwx. 1 root root 7 Apr 10 05:30 vgroot-lvroot -> ../dm-0
8.重新编辑配置文件
编辑 /etc/multipath.conf 文件,用于安置oracle rac
multipaths {
multipath {
wwid VBOX_HARDDISK_VBa08577cb-9ee269dc
alias ocrdisk01
path_grouping_policy multibus
}
multipath {
wwid VBOX_HARDDISK_VB74301632-13d08c3c
alias ocrdisk02
path_grouping_policy multibus
}
multipath {
wwid VBOX_HARDDISK_VBfa94873f-504b6993
alias ocrdisk03
path_grouping_policy multibus
}
multipath {
wwid VBOX_HARDDISK_VB4f84df6d-a94b8da6
alias datadisk01
path_grouping_policy multibus
}
}
9.重启加载配置
[root@localhost multipath]# multipath -r
查看加载后的配置
[root@localhost multipath]# multipath -ll
ocrdisk03 (VBOX_HARDDISK_VBfa94873f-504b6993) dm-3 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (VBOX_HARDDISK_VB4f84df6d-a94b8da6) dm-4 ATA ,VBOX HARDDISK
size=40G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (VBOX_HARDDISK_VB74301632-13d08c3c) dm-2 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 4:0:0:0 sdc 8:32 active ready running
ocrdisk01 (VBOX_HARDDISK_VBa08577cb-9ee269dc) dm-1 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
`- 3:0:0:0 sdb 8:16 active ready running
可以看到mapper下面的磁盘名称也改变了
[root@localhost mapper]# ls -al /dev/mapper/
total 0
drwxr-xr-x. 2 root root 160 Apr 11 22:16 .
drwxr-xr-x. 19 root root 3300 Apr 11 21:37 ..
crw-------. 1 root root 10, 236 Apr 10 05:30 control
lrwxrwxrwx. 1 root root 7 Apr 11 22:16 datadisk01 -> ../dm-4
lrwxrwxrwx. 1 root root 7 Apr 11 22:16 ocrdisk01 -> ../dm-1
lrwxrwxrwx. 1 root root 7 Apr 11 22:16 ocrdisk02 -> ../dm-2
lrwxrwxrwx. 1 root root 7 Apr 11 22:16 ocrdisk03 -> ../dm-3
lrwxrwxrwx. 1 root root 7 Apr 10 05:30 vgroot-lvroot -> ../dm-0
[root@localhost mapper]#
10.若不想使用配置别名的话,可以修改如下文件(但是不建议修改),将映射关系写到配置文件
vi /etc/multipath/bindings
ocrdisk01 VBOX_HARDDISK_VBa08577cb-9ee269dc
ocrdisk02 VBOX_HARDDISK_VB74301632-13d08c3c
ocrdisk03 VBOX_HARDDISK_VBfa94873f-504b6993
datadisk01 VBOX_HARDDISK_VB4f84df6d-a94b8da6
然后进行重新加载配置
删除后进行加载
[root@localhost multipath]# multipath -F
[root@localhost multipath]# multipath -r
[root@localhost multipath]# multipath -ll
ocrdisk03 (VBOX_HARDDISK_VBfa94873f-504b6993) dm-3 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=enabled
`- 5:0:0:0 sdd 8:48 active ready running
datadisk01 (VBOX_HARDDISK_VB4f84df6d-a94b8da6) dm-4 ATA ,VBOX HARDDISK
size=40G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=enabled
`- 6:0:0:0 sde 8:64 active ready running
ocrdisk02 (VBOX_HARDDISK_VB74301632-13d08c3c) dm-2 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=enabled
`- 4:0:0:0 sdc 8:32 active ready running
ocrdisk01 (VBOX_HARDDISK_VBa08577cb-9ee269dc) dm-1 ATA ,VBOX HARDDISK
size=30G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=enabled
`- 3:0:0:0 sdb 8:16 active ready running
------------------------------------------添加新的lun-------------------------------------------------------
新增lun请参考
https://www.cnblogs.com/hxlasky/p/15014211.html
-------------------------------------------配置udev--------------------------------------------
1.编辑规则文件
[root@SL0101A-IVDB02 ~]# cd /etc/udev/rules.d/
[root@djp01 rules.d]# vi 99-oracle-asmdevices.rules
文件文件内容如下:
ENV{DM_NAME}=="ocrdisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="ocrdisk03", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk01", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
ENV{DM_NAME}=="datadisk02", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
这里的磁盘名称需要跟/etc/multipath.conf配置里的对应起来
2.启用
[root@localhost ~]# /sbin/udevadm trigger --type=devices --action=change
[root@localhost bin]# udevadm trigger
如执行上面两个命令不生效的情况下，可以执行如下命令
[root@localhost ~]#/sbin/udevadm control --reload
[root@localhost ~]# /sbin/udevadm trigger --type=devices --action=change
[root@localhost bin]# udevadm trigger
3.查看设备权限
[root@localhost rules.d]# ls -al /dev/dm*
brw-rw----. 1 root disk 253, 0 Apr 12 03:09 /dev/dm-0
brw-rw----. 1 grid asmadmin 253, 1 Apr 12 03:09 /dev/dm-1
brw-rw----. 1 grid asmadmin 253, 2 Apr 12 03:09 /dev/dm-2
brw-rw----. 1 grid asmadmin 253, 3 Apr 12 03:09 /dev/dm-3
brw-rw----. 1 grid asmadmin 253, 4 Apr 12 03:09 /dev/dm-4
brw-rw----. 1 grid asmadmin 253, 5 Apr 12 03:09 /dev/dm-5
4.安装rac的时候选择发现磁盘路径为:/dev/mapper/*
配置完成后建议重启动机器.