绑定前网卡驱动 [root@localhost ~]# ethtool -i enp5s0 driver: hinic version: firmware-version: expansion-rom-version: bus-info: 0000:05:00.0 supports-statistics: no supports-test: no supports-eeprom-access: no supports-register-dump: no supports-priv-flags: no [root@localhost ~]#
安装ovs遇到一个问题那就是ovs version和dpdk version不匹配，华为鲲鹏920上安装的dpdk是dpdk-19.11
ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
[root@localhost ~]# ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:05:00.0
ovs-vsctl: Error detected while setting up 'dpdk1': could not open network device dpdk1 (Address family not supported by protocol). See ovs-vswitchd log for details.
ovs-vsctl: The default log directory is "/var/log/openvswitch".
[root@localhost ~]# ovs-vsctl show
bb6aa323-f96b-4872-98f8-7aa87fc99958
Bridge "br0"
datapath_type: netdev
Port "dpdk1"
Interface "dpdk1"
type: dpdk
options: {dpdk-devargs="0000:05:00.0"}
error: "could not open network device dpdk1 (Address family not supported by protocol)"
Port "br0"
Interface "br0"
type: internal
[root@localhost ~]#
查看网卡与dpdk绑定状态
[root@localhost dpdk-19.11]# ./usertools/dpdk-devbind.py --status Network devices using DPDK-compatible driver ============================================ 0000:05:00.0 'Hi1822 Family (2*25GE) 0200' drv=vfio-pci unused=hinic 0000:06:00.0 'Hi1822 Family (2*25GE) 0200' drv=vfio-pci unused=hinic Network devices using kernel driver =================================== 0000:7d:00.0 'HNS GE/10GE/25GE RDMA Network Controller a222' if=enp125s0f0 drv=hns3 unused=hns_roce_hw_v2,vfio-pci *Active* 0000:7d:00.1 'HNS GE/10GE/25GE Network Controller a221' if=enp125s0f1 drv=hns3 unused=vfio-pci 0000:7d:00.2 'HNS GE/10GE/25GE RDMA Network Controller a222' if=enp125s0f2 drv=hns3 unused=hns_roce_hw_v2,vfio-pci 0000:7d:00.3 'HNS GE/10GE/25GE Network Controller a221' if=enp125s0f3 drv=hns3 unused=vfio-pci No 'Baseband' devices detected ============================== No 'Crypto' devices detected ============================ No 'Eventdev' devices detected ============================== No 'Mempool' devices detected ============================= No 'Compress' devices detected ============================== No 'Misc (rawdev)' devices detected =================================== [root@localhost dpdk-19.11]#
修改配置文件config/common_linuxapp，这里我们测试vhost模式，所以需要把下面两个配置项，配置为yes。 CONFIG_RTE_BUILD_COMBINE_LIBS=y CONFIG_RTE_LIBRTE_VHOST=y
[root@localhost dpdk-19.11]# grep -n CONFIG_RTE_LIBRTE_VHOST config/common_base
1030:CONFIG_RTE_LIBRTE_VHOST=y
1031:CONFIG_RTE_LIBRTE_VHOST_NUMA=n
1032:CONFIG_RTE_LIBRTE_VHOST_DEBUG=n
[root@localhost dpdk-19.11]# mkdir -p /usr/src/dpdk [root@localhost dpdk-19.11]# make install T=arm64-armv8a-linuxapp-gcc DESTDIR=/usr/src/dpdk -j 64
将dpdk安装到当前系统
make install T=arm64-armv8a-linuxapp-gcc DESTDIR=/usr -j 64
安装完成后可以使用dpdk的命令了
[root@localhost ~]# dpdk-devbind --status
Network devices using DPDK-compatible driver
============================================
0000:05:00.0 'Hi1822 Family (2*25GE) 0200' drv=vfio-pci unused=hinic
0000:06:00.0 'Hi1822 Family (2*25GE) 0200' drv=vfio-pci unused=hinic
Network devices using kernel driver
===================================
0000:7d:00.0 'HNS GE/10GE/25GE RDMA Network Controller a222' if=enp125s0f0 drv=hns3 unused=hns_roce_hw_v2,vfio-pci *Active*
0000:7d:00.1 'HNS GE/10GE/25GE Network Controller a221' if=enp125s0f1 drv=hns3 unused=vfio-pci
0000:7d:00.2 'HNS GE/10GE/25GE RDMA Network Controller a222' if=enp125s0f2 drv=hns3 unused=hns_roce_hw_v2,vfio-pci
0000:7d:00.3 'HNS GE/10GE/25GE Network Controller a221' if=enp125s0f3 drv=hns3 unused=vfio-pci
No 'Baseband' devices detected
==============================
No 'Crypto' devices detected
============================
No 'Eventdev' devices detected
==============================
No 'Mempool' devices detected
=============================
No 'Compress' devices detected
==============================
No 'Misc (rawdev)' devices detected
===================================
[root@localhost ~]# cd /usr/
[root@localhost usr]# find ./ -name dpdk*
./sbin/dpdk-devbind
./bin/dpdk-test-compress-perf
./bin/dpdk-pdump
./bin/dpdk-test-crypto-perf
./bin/dpdk-pmdinfo
./bin/dpdk-test-eventdev
./bin/dpdk-procinfo
./src/dpdk
./src/dpdk/sbin/dpdk-devbind
./src/dpdk/bin/dpdk-test-compress-perf
./src/dpdk/bin/dpdk-pdump
./src/dpdk/bin/dpdk-test-crypto-perf
./src/dpdk/bin/dpdk-pmdinfo
./src/dpdk/bin/dpdk-test-eventdev
./src/dpdk/bin/dpdk-procinfo
./src/dpdk/lib/modules/4.14.0-115.el7a.0.1.aarch64/extra/dpdk
./src/dpdk/include/dpdk
./src/dpdk/share/dpdk
./src/dpdk/share/dpdk/usertools/dpdk-pmdinfo.py
./src/dpdk/share/dpdk/usertools/dpdk-devbind.py
./src/dpdk/share/dpdk/usertools/dpdk-telemetry-client.py
./src/dpdk/share/dpdk/usertools/dpdk-setup.sh
./src/dpdk/share/dpdk/arm64-armv8a-linuxapp-gcc/app/dpdk-pmdinfogen
./lib/modules/4.14.0-115.el7a.0.1.aarch64/extra/dpdk
./include/dpdk
./share/dpdk
./share/dpdk/usertools/dpdk-pmdinfo.py
./share/dpdk/usertools/dpdk-devbind.py
./share/dpdk/usertools/dpdk-telemetry-client.py
./share/dpdk/usertools/dpdk-setup.sh
./share/dpdk/arm64-armv8a-linuxapp-gcc/app/dpdk-pmdinfogen
[root@localhost usr]# dpdk-devbind -u 0000:05:00.0 0000:06:00.0
编译ovs
[root@localhost openvswitch-2.12.0]# ls acinclude.m4 config.log Documentation Makefile ovsdb tests windows aclocal.m4 config.status include Makefile.am package.m4 third-party xenserver appveyor.yml configure ipsec Makefile.in poc tutorial AUTHORS.rst configure.ac lib manpages.mk python utilities boot.sh CONTRIBUTING.rst libtool NEWS README.rst Vagrantfile build-aux datapath LICENSE NOTICE rhel Vagrantfile-FreeBSD config.h datapath-windows m4 ofproto selinux vswitchd config.h.in debian MAINTAINERS.rst ovn stamp-h1 vtep [root@localhost openvswitch-2.12.0]# ./configure --with-dpdk=/usr/src/dpdk --prefix=/usr --sysconfdir=/etc --localstatedir=/var
[root@localhost SOURCES]# pwd /root/rpmbuild/SOURCES [root@localhost SOURCES]# rpmbuild -bb --without check openvswitch-2.12.0/rhel/openvswitch.spec
[root@localhost ~]# ls /etc/openvswitch/
system-id.conf
[root@localhost ~]# ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema [root@localhost ~]# ls /etc/openvswitch/ conf.db system-id.conf [root@localhost ~]#
ovs启动 ：
ovs.sh
#启动ovsdb server ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/var/run/openvswitch/db.sock --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir --log-file=/var/log/openvswitch/ovsdb-server.log --pidfile=/var/run/openvswitch/ovsdb-server.pid --detach --monitor #第一次启动ovs需要初始化 ovs-vsctl --no-wait init #从ovs-v2.7.0开始，开启dpdk功能已不是vswitchd进程启动时指定–dpdk等参数了，而是通过设置ovsdb来开启dpdk功能 ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true #启动vswitchd进程 ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor
What DPDK version does each Open vSwitch release work with?
A: The following table lists the DPDK version against which the given versions of Open vSwitch will successfully build.
Open vSwitch DPDK 2.2.x 1.6 2.3.x 1.6 2.4.x 2.0 2.5.x 2.2 2.6.x 16.07.2 2.7.x 16.11.9 2.8.x 17.05.2 2.9.x 17.11.4 2.10.x 17.11.4 2.11.x 18.11.6 2.12.x 18.11.6 2.13.x 19.11.0
原来是各个版本的ovs有对应的dpdk，当前ovs 版本是2.12，dpdk是19.11。所以要下载一个2.13版本的ovs
2.13需要更python版本>3.4,先安装python
wget https://www.python.org/ftp/python/3.7.0/Python-3.7.0a1.tar.xz
要安装python3.3.7,
Python-3.7.0a1.tar.xz有问题
tar -xvf Python-3.7.0a1.tar.xz ls cd Python-3.7.0a1 ls ./configure --prefix=/usr/local/python3 make && make install yum install libffi-devel make && make install ln -s /usr/local/python3/bin/python3 /usr/local/bin/python3 ln -s /usr/local/python3/bin/pip3 /usr/local/bin/pip3 python3 -V pip3 -V
[root@localhost ~]# python3 -V
Python 3.7.0a1
[root@localhost ~]# pip3 -V
pip 9.0.1 from /usr/local/python3/lib/python3.7/site-packages (python 3.7)
[root@localhost ~]#
./boot.sh
./configure
--with-dpdk=/usr/src/dpdk
--prefix=/usr
--exec-prefix=/usr
--sysconfdir=/etc
--localstatedir=/var
make -j 64
make install -j 64
mkdir -p /etc/openvswitch
mkdir -p /var/run/openvswitch
[root@localhost ovs-master]# vsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema -bash: vsdb-tool: command not found [root@localhost ovs-master]# ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema [root@localhost ovs-master]# cd - /root [root@localhost ~]# bash ovs.sh
[root@localhost ~]# ovs-appctl --version
ovs-appctl (Open vSwitch) 2.13.90 [root@localhost ~]# ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev [root@localhost ~]# ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:05:00.0 [root@localhost ~]# ovs-vsctl show 073f49bf-8e93-417b-9b46-e4431099f662 Bridge br0 datapath_type: netdev Port dpdk1 Interface dpdk1 type: dpdk options: {dpdk-devargs="0000:05:00.0"} Port br0 Interface br0 type: internal [root@localhost ~]#
[root@localhost ~]# dpdk-devbind -s Network devices using DPDK-compatible driver ============================================ 0000:05:00.0 'Hi1822 Family (2*25GE) 0200' drv=vfio-pci unused=hinic 0000:06:00.0 'Hi1822 Family (2*25GE) 0200' drv=vfio-pci unused=hinic Network devices using kernel driver =================================== 0000:7d:00.0 'HNS GE/10GE/25GE RDMA Network Controller a222' if=enp125s0f0 drv=hns3 unused=hns_roce_hw_v2,vfio-pci *Active* 0000:7d:00.1 'HNS GE/10GE/25GE Network Controller a221' if=enp125s0f1 drv=hns3 unused=vfio-pci 0000:7d:00.2 'HNS GE/10GE/25GE RDMA Network Controller a222' if=enp125s0f2 drv=hns3 unused=hns_roce_hw_v2,vfio-pci 0000:7d:00.3 'HNS GE/10GE/25GE Network Controller a221' if=enp125s0f3 drv=hns3 unused=vfio-pci No 'Baseband' devices detected ============================== No 'Crypto' devices detected ============================ No 'Eventdev' devices detected ============================== No 'Mempool' devices detected ============================= No 'Compress' devices detected ============================== No 'Misc (rawdev)' devices detected =================================== [root@localhost ~]#
[root@localhost ~]# ovs-vsctl show 073f49bf-8e93-417b-9b46-e4431099f662 Bridge br0 datapath_type: netdev Port dpdk1 Interface dpdk1 type: dpdk options: {dpdk-devargs="0000:05:00.0"} Port br0 Interface br0 type: internal [root@localhost ~]# ip a
11: ovs-netdev: <BROADCAST,MULTICAST,PROMISC> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 76:ac:03:ea:46:dc brd ff:ff:ff:ff:ff:ff
12: br0: <BROADCAST,MULTICAST,PROMISC> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 44:a1:91:a4:9b:eb brd ff:ff:ff:ff:ff:ff
[root@localhost ~]# lshw -c network -businfo
Bus info Device Class Description
========================================================
pci@0000:05:00.0 network Hi1822 Family (2*25GE)
pci@0000:06:00.0 network Hi1822 Family (2*25GE)
pci@0000:7d:00.0 enp125s0f0 network HNS GE/10GE/25GE RDMA Network Controller
pci@0000:7d:00.1 enp125s0f1 network HNS GE/10GE/25GE Network Controller
pci@0000:7d:00.2 enp125s0f2 network HNS GE/10GE/25GE RDMA Network Controller
pci@0000:7d:00.3 enp125s0f3 network HNS GE/10GE/25GE Network Controller
ovs-netdev network Ethernet interface
br0 network Ethernet interfac
