[root@localhost dpdk_lcores]# ls
main.c Makefile
[root@localhost dpdk_lcores]# cat Makefile export RTE_TARGET=arm64-armv8a-linuxapp-gcc export RTE_SDK=/data1/dpdk-19.11 include $(RTE_SDK)/mk/rte.vars.mk APP = test SRCS-y := main.c include $(RTE_SDK)/mk/rte.extapp.mk [root@localhost dpdk_lcores]# cat main.c #include <stdio.h> #include <string.h> #include <stdint.h> #include <errno.h> #include <sys/queue.h> #include <rte_memory.h> #include <rte_launch.h> #include <rte_eal.h> #include <rte_per_lcore.h> #include <rte_lcore.h> #include <rte_debug.h> static int lcore_hello(__attribute__((unused)) void *arg) { unsigned lcore_id; lcore_id = rte_lcore_id(); printf("hello from core %u ", lcore_id); while (1) { ; } return 0; } int main(int argc, char **argv) { int ret; unsigned lcore_id; ret = rte_eal_init(argc, argv); if (ret < 0) rte_panic("Cannot init EAL "); /* call lcore_hello() on every slave lcore */ RTE_LCORE_FOREACH_SLAVE(lcore_id) { rte_eal_remote_launch(lcore_hello, NULL, lcore_id); } /* call it on master lcore too */ lcore_hello(NULL); rte_eal_mp_wait_lcore(); return 0; } [root@localhost dpdk_lcores]#
[root@localhost dpdk_lcores]# ls build main.c Makefile [root@localhost dpdk_lcores]# ls build/ app _install main.o _postbuild _postinstall _preinstall test test.map [root@localhost dpdk_lcores]# ls build/app/ test test.map [root@localhost dpdk_lcores]# pwd /data1/dpdk-19.11/demo/dpdk_lcores [root@localhost dpdk_lcores]#
[root@localhost dpdk_lcores]# ./build/app/test EAL: Detected 128 lcore(s) EAL: Detected 4 NUMA nodes EAL: Multi-process socket /var/run/dpdk/rte/mp_socket EAL: Selected IOVA mode 'VA' EAL: No available hugepages reported in hugepages-2048kB EAL: Probing VFIO support... EAL: PCI device 0000:05:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: PCI device 0000:06:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: PCI device 0000:7d:00.0 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.1 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 EAL: PCI device 0000:7d:00.2 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.3 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 hello from core 1 hello from core 2 hello from core 3 hello from core 4 hello from core 7 hello from core 6 hello from core 5 hello from core 8 hello from core 9 hello from core 10 hello from core 11 hello from core 12 hello from core 13 hello from core 14 hello from core 15 hello from core 16 hello from core 17 hello from core 18 hello from core 19 hello from core 20 hello from core 21 hello from core 22 hello from core 23 hello from core 24 hello from core 25 hello from core 26 hello from core 28 hello from core 29 hello from core 27 hello from core 31 hello from core 30 hello from core 33 hello from core 34 hello from core 32 hello from core 39 hello from core 35 hello from core 45 hello from core 37 hello from core 38 hello from core 40 hello from core 41 hello from core 48 hello from core 49 hello from core 44 hello from core 50 hello from core 46 hello from core 47 hello from core 42 hello from core 43 hello from core 36 hello from core 51 hello from core 64 hello from core 65 hello from core 54 hello from core 66 hello from core 55 hello from core 56 hello from core 57 hello from core 58 hello from core 59 hello from core 67 hello from core 61 hello from core 68 hello from core 63 hello from core 69 hello from core 53 hello from core 70 hello from core 71 hello from core 60 hello from core 72 hello from core 73 hello from core 74 hello from core 52 hello from core 75 hello from core 62 hello from core 76 hello from core 77 hello from core 78 hello from core 79 hello from core 80 hello from core 81 hello from core 82 hello from core 83 hello from core 84 hello from core 85 hello from core 86 hello from core 87 hello from core 88 hello from core 89 hello from core 90 hello from core 91 hello from core 92 hello from core 93 hello from core 94 hello from core 95 hello from core 96 hello from core 97 hello from core 98 hello from core 99 hello from core 100 hello from core 101 hello from core 102 hello from core 103 hello from core 104 hello from core 105 hello from core 106 hello from core 107 hello from core 108 hello from core 109 hello from core 110 hello from core 111 hello from core 112 hello from core 113 hello from core 114 hello from core 115 hello from core 116 hello from core 117 hello from core 118 hello from core 119 hello from core 120 hello from core 121 hello from core 122 hello from core 123 hello from core 124 hello from core 125 hello from core 126 hello from core 127 hello from core 0
[root@localhost kni]# ps -elf | grep test 0 R root 18972 18466 99 80 0 - 8405522 - 04:17 pts/1 01:48:10 ./build/app/test 0 S root 19105 17774 0 80 0 - 1729 pipe_w 04:18 pts/0 00:00:00 grep --color=auto test [root@localhost kni]# ps -T -p 18466 PID SPID TTY TIME CMD 18466 18466 pts/1 00:00:00 bash [root@localhost kni]# ps -T -p 18972 PID SPID TTY TIME CMD 18972 18972 pts/1 00:00:47 test 18972 18973 pts/1 00:00:00 eal-intr-thread 18972 18974 pts/1 00:00:00 rte_mp_handle 18972 18975 pts/1 00:00:47 lcore-slave-1 18972 18976 pts/1 00:01:34 lcore-slave-2 18972 18977 pts/1 00:00:47 lcore-slave-3 18972 18978 pts/1 00:01:34 lcore-slave-4 18972 18979 pts/1 00:01:34 lcore-slave-5 18972 18980 pts/1 00:01:34 lcore-slave-6 18972 18981 pts/1 00:01:34 lcore-slave-7 18972 18982 pts/1 00:01:34 lcore-slave-8 18972 18983 pts/1 00:01:34 lcore-slave-9 18972 18984 pts/1 00:01:34 lcore-slave-10 18972 18985 pts/1 00:01:34 lcore-slave-11 18972 18986 pts/1 00:01:34 lcore-slave-12 18972 18987 pts/1 00:01:34 lcore-slave-13 18972 18988 pts/1 00:01:34 lcore-slave-14 18972 18989 pts/1 00:01:34 lcore-slave-15 18972 18990 pts/1 00:01:34 lcore-slave-16 18972 18991 pts/1 00:01:34 lcore-slave-17 18972 18992 pts/1 00:01:34 lcore-slave-18 18972 18993 pts/1 00:01:34 lcore-slave-19 18972 18994 pts/1 00:01:34 lcore-slave-20 18972 18995 pts/1 00:01:34 lcore-slave-21 18972 18996 pts/1 00:01:34 lcore-slave-22 18972 18997 pts/1 00:01:34 lcore-slave-23 18972 18998 pts/1 00:01:34 lcore-slave-24 18972 18999 pts/1 00:01:34 lcore-slave-25 18972 19000 pts/1 00:01:34 lcore-slave-26 18972 19001 pts/1 00:01:34 lcore-slave-27 18972 19002 pts/1 00:01:34 lcore-slave-28 18972 19003 pts/1 00:01:34 lcore-slave-29 18972 19004 pts/1 00:01:34 lcore-slave-30 18972 19005 pts/1 00:01:34 lcore-slave-31 18972 19006 pts/1 00:00:47 lcore-slave-32 18972 19007 pts/1 00:01:34 lcore-slave-33 18972 19008 pts/1 00:01:34 lcore-slave-34 18972 19009 pts/1 00:01:34 lcore-slave-35 18972 19010 pts/1 00:01:34 lcore-slave-36 18972 19011 pts/1 00:01:34 lcore-slave-37 18972 19012 pts/1 00:01:34 lcore-slave-38 18972 19013 pts/1 00:01:34 lcore-slave-39 18972 19014 pts/1 00:01:34 lcore-slave-40 18972 19015 pts/1 00:01:34 lcore-slave-41 18972 19016 pts/1 00:01:34 lcore-slave-42 18972 19017 pts/1 00:01:34 lcore-slave-43 18972 19018 pts/1 00:01:34 lcore-slave-44 18972 19019 pts/1 00:01:34 lcore-slave-45 18972 19020 pts/1 00:01:34 lcore-slave-46 18972 19021 pts/1 00:01:34 lcore-slave-47 18972 19022 pts/1 00:01:34 lcore-slave-48 18972 19023 pts/1 00:01:34 lcore-slave-49 18972 19024 pts/1 00:01:34 lcore-slave-50 18972 19025 pts/1 00:01:34 lcore-slave-51 18972 19026 pts/1 00:01:34 lcore-slave-52 18972 19027 pts/1 00:01:34 lcore-slave-53 18972 19028 pts/1 00:01:34 lcore-slave-54 18972 19029 pts/1 00:01:34 lcore-slave-55 18972 19030 pts/1 00:01:34 lcore-slave-56 18972 19031 pts/1 00:01:34 lcore-slave-57 18972 19032 pts/1 00:01:34 lcore-slave-58 18972 19033 pts/1 00:01:34 lcore-slave-59 18972 19034 pts/1 00:01:34 lcore-slave-60 18972 19035 pts/1 00:01:34 lcore-slave-61 18972 19036 pts/1 00:01:34 lcore-slave-62 18972 19037 pts/1 00:01:34 lcore-slave-63 18972 19038 pts/1 00:01:34 lcore-slave-64 18972 19039 pts/1 00:01:34 lcore-slave-65 18972 19040 pts/1 00:01:34 lcore-slave-66 18972 19041 pts/1 00:01:34 lcore-slave-67 18972 19042 pts/1 00:01:34 lcore-slave-68 18972 19043 pts/1 00:01:34 lcore-slave-69 18972 19044 pts/1 00:01:34 lcore-slave-70 18972 19045 pts/1 00:01:34 lcore-slave-71 18972 19046 pts/1 00:01:34 lcore-slave-72 18972 19047 pts/1 00:01:34 lcore-slave-73 18972 19048 pts/1 00:01:34 lcore-slave-74 18972 19049 pts/1 00:01:34 lcore-slave-75 18972 19050 pts/1 00:01:34 lcore-slave-76 18972 19051 pts/1 00:01:34 lcore-slave-77 18972 19052 pts/1 00:01:34 lcore-slave-78 18972 19053 pts/1 00:01:34 lcore-slave-79 18972 19054 pts/1 00:01:34 lcore-slave-80 18972 19055 pts/1 00:01:34 lcore-slave-81 18972 19056 pts/1 00:01:34 lcore-slave-82 18972 19057 pts/1 00:01:34 lcore-slave-83 18972 19058 pts/1 00:01:34 lcore-slave-84 18972 19059 pts/1 00:01:34 lcore-slave-85 18972 19060 pts/1 00:01:34 lcore-slave-86 18972 19061 pts/1 00:01:34 lcore-slave-87 18972 19062 pts/1 00:01:34 lcore-slave-88 18972 19063 pts/1 00:01:34 lcore-slave-89 18972 19064 pts/1 00:01:34 lcore-slave-90 18972 19065 pts/1 00:01:34 lcore-slave-91 18972 19066 pts/1 00:01:34 lcore-slave-92 18972 19067 pts/1 00:01:34 lcore-slave-93 18972 19068 pts/1 00:01:34 lcore-slave-94 18972 19069 pts/1 00:01:34 lcore-slave-95 18972 19070 pts/1 00:01:34 lcore-slave-96 18972 19071 pts/1 00:01:34 lcore-slave-97 18972 19072 pts/1 00:01:34 lcore-slave-98 18972 19073 pts/1 00:01:34 lcore-slave-99 18972 19074 pts/1 00:01:34 lcore-slave-100 18972 19075 pts/1 00:01:34 lcore-slave-101 18972 19076 pts/1 00:01:34 lcore-slave-102 18972 19077 pts/1 00:01:34 lcore-slave-103 18972 19078 pts/1 00:01:34 lcore-slave-104 18972 19079 pts/1 00:01:34 lcore-slave-105 18972 19080 pts/1 00:01:34 lcore-slave-106 18972 19081 pts/1 00:01:34 lcore-slave-107 18972 19082 pts/1 00:01:34 lcore-slave-108 18972 19083 pts/1 00:01:34 lcore-slave-109 18972 19084 pts/1 00:01:34 lcore-slave-110 18972 19085 pts/1 00:01:34 lcore-slave-111 18972 19086 pts/1 00:01:34 lcore-slave-112 18972 19087 pts/1 00:01:34 lcore-slave-113 18972 19088 pts/1 00:01:34 lcore-slave-114 18972 19089 pts/1 00:01:34 lcore-slave-115 18972 19090 pts/1 00:01:34 lcore-slave-116 18972 19091 pts/1 00:01:34 lcore-slave-117 18972 19092 pts/1 00:01:34 lcore-slave-118 18972 19093 pts/1 00:01:34 lcore-slave-119 18972 19094 pts/1 00:01:34 lcore-slave-120 18972 19095 pts/1 00:01:34 lcore-slave-121 18972 19096 pts/1 00:01:34 lcore-slave-122 18972 19097 pts/1 00:01:34 lcore-slave-123 18972 19098 pts/1 00:01:34 lcore-slave-124 18972 19099 pts/1 00:01:34 lcore-slave-125 18972 19100 pts/1 00:01:34 lcore-slave-126 18972 19101 pts/1 00:01:34 lcore-slave-127 [root@localhost kni]#
4个numa + 128个核
[root@localhost kni]# lscpu Architecture: aarch64 Byte Order: Little Endian CPU(s): 128 On-line CPU(s) list: 0-127 Thread(s) per core: 1 Core(s) per socket: 64 Socket(s): 2 NUMA node(s): 4 Model: 0 CPU max MHz: 2600.0000 CPU min MHz: 200.0000 BogoMIPS: 200.00 L1d cache: 64K L1i cache: 64K L2 cache: 512K L3 cache: 65536K NUMA node0 CPU(s): 0-31 NUMA node1 CPU(s): 32-63 NUMA node2 CPU(s): 64-95 NUMA node3 CPU(s): 96-127 Flags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma dcpop [root@localhost kni]#
6张网卡
[root@localhost kni]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: enp125s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether b0:08:75:5f:b7:d9 brd ff:ff:ff:ff:ff:ff inet 10.10.16.229/24 brd 10.10.16.255 scope global noprefixroute enp125s0f0 valid_lft forever preferred_lft forever inet6 fe80::f9ec:a960:7518:b745/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: enp125s0f1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast state DOWN group default qlen 1000 link/ether b0:08:75:5f:b7:da brd ff:ff:ff:ff:ff:ff 4: enp125s0f2: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast state DOWN group default qlen 1000 link/ether b0:08:75:5f:b7:db brd ff:ff:ff:ff:ff:ff 5: enp125s0f3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether b0:08:75:5f:b7:dc brd ff:ff:ff:ff:ff:ff 6: enp5s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 44:a1:91:a4:9b:eb brd ff:ff:ff:ff:ff:ff 7: enp6s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 44:a1:91:a4:9b:ec brd ff:ff:ff:ff:ff:ff
[root@localhost dpdk_lcores]# ./build/app/test --lcores "(0,1,2,3,4)@(0,1),(5,6,7,8)@3" EAL: Detected 128 lcore(s) EAL: Detected 4 NUMA nodes EAL: Multi-process socket /var/run/dpdk/rte/mp_socket EAL: Selected IOVA mode 'VA' EAL: No available hugepages reported in hugepages-2048kB EAL: Probing VFIO support... EAL: PCI device 0000:05:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: PCI device 0000:06:00.0 on NUMA socket 0 EAL: probe driver: 19e5:200 net_hinic EAL: PCI device 0000:7d:00.0 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.1 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 EAL: PCI device 0000:7d:00.2 on NUMA socket 0 EAL: probe driver: 19e5:a222 net_hns3 EAL: PCI device 0000:7d:00.3 on NUMA socket 0 EAL: probe driver: 19e5:a221 net_hns3 hello from core 1 hello from core 2 hello from core 3 hello from core 4 hello from core 5 hello from core 6 hello from core 7 hello from core 8 hello from core 0
[root@localhost kni]# ps -mo pid,tid,%cpu,psr -p 19113 PID TID %CPU PSR 19113 - 221 - - 19113 24.9 0 - 19114 0.0 39 - 19115 0.0 40 - 19116 33.3 1 - 19117 25.0 0 - 19118 33.3 1 - 19119 25.0 0 - 19120 20.0 3 - 19121 20.0 3 - 19122 20.0 3 - 19123 20.0 3
The format pattern: –lcores=’<lcore_set>[@cpu_set][,<lcore_set>[@cpu_set],...]’ ‘lcore_set’ and ‘cpu_set’ can be a single number, range or a group. A number is a “digit([0-9]+)”; a range is “<number>-<number>”; a group is “(<number|range>[,<number|range>,...])”. If a ‘@cpu_set’ value is not supplied, the value of ‘cpu_set’ will default to the value of ‘lcore_set’. For example, "--lcores='1,2@(5-7),(3-5)@(0,2),(0,6),7-8'" which means start 9 EAL thread; lcore 0 runs on cpuset 0x41 (cpu 0,6); lcore 1 runs on cpuset 0x2 (cpu 1); lcore 2 runs on cpuset 0xe0 (cpu 5,6,7); lcore 3,4,5 runs on cpuset 0x5 (cpu 0,2); lcore 6 runs on cpuset 0x41 (cpu 0,6); lcore 7 runs on cpuset 0x80 (cpu 7); lcore 8 runs on cpuset 0x100 (cpu 8). Using this option, for each given lcore ID, the associated CPUs can be assigned. It’s also compatible with the pattern of corelist(‘-l’) option.
[root@localhost kni]# ps -elf | grep test 0 R root 19113 18466 99 80 0 - 8390171 - 04:28 pts/1 00:00:53 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 0 S root 19125 17774 0 80 0 - 1729 pipe_w 04:28 pts/0 00:00:00 grep --color=auto test [root@localhost kni]# ps -T -p 19113 PID SPID TTY TIME CMD 19113 19113 pts/1 00:00:08 test 19113 19114 pts/1 00:00:00 eal-intr-thread 19113 19115 pts/1 00:00:00 rte_mp_handle 19113 19116 pts/1 00:00:11 lcore-slave-1 19113 19117 pts/1 00:00:08 lcore-slave-2 19113 19118 pts/1 00:00:11 lcore-slave-3 19113 19119 pts/1 00:00:08 lcore-slave-4 19113 19120 pts/1 00:00:06 lcore-slave-5 19113 19121 pts/1 00:00:06 lcore-slave-6 19113 19122 pts/1 00:00:06 lcore-slave-7 19113 19123 pts/1 00:00:06 lcore-slave-8 [root@localhost kni]#
[root@localhost kni]# ps -eLF | grep 19113 root 19113 18466 19113 24 11 8390171 10048 0 04:28 pts/1 00:00:31 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19114 0 11 8390171 10048 39 04:28 pts/1 00:00:00 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19115 0 11 8390171 10048 40 04:28 pts/1 00:00:00 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3
----------------------------------------------------------------------------------------------------------------------
root 19113 18466 19116 33 11 8390171 10048 1 04:28 pts/1 00:00:41 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19117 25 11 8390171 10048 0 04:28 pts/1 00:00:31 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19118 33 11 8390171 10048 1 04:28 pts/1 00:00:41 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19119 25 11 8390171 10048 0 04:28 pts/1 00:00:31 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19120 20 11 8390171 10048 3 04:28 pts/1 00:00:24 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19121 20 11 8390171 10048 3 04:28 pts/1 00:00:24 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19122 20 11 8390171 10048 3 04:28 pts/1 00:00:24 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3 root 19113 18466 19123 20 11 8390171 10048 3 04:28 pts/1 00:00:24 ./build/app/test --lcores (0,1,2,3,4)@(0,1),(5,6,7,8)@3