在工作中经常碰到需要写一些多进程/多线程的测试程序，用来测试接口的性能。本文将会从零开始一点点增加代码，最终完成一个简易的多进程测试程序编写。该程序支持实时打印测试进结果和最终测试结果的统计。
同时，本文还涵盖了以下知识点，可以作为学习参考：
- 使用
getopt_long()处理命令行选项和参数 - 使用
fork()和wait()处理多进程 - 使用
sigaction()配合alarm()处理定时信号SIGALRM - 使用
shmget()、shmat()、shmdt()、shmctl()等通过共享内存进行进程间通信 - 使用
sigaction()捕获SIGINT和SIGQUIT信号，在程序终止前做共享内存清理工作
本博客已经迁移至CatBro's Blog，那里是我自己搭建的个人博客，页面效果比这边更好，支持站内搜索，评论回复还支持邮件提醒，欢迎关注。这边只会在有时间的时候不定期搬运一下。
本文源码已开源Github
选项和参数的处理
为了使测试程序更高的可用性，我们getopt来处理选项和参数。
#include <stdio.h> // printf
#include <getopt.h> // getopt_long
#include <stdlib.h> // strtol, abort
#include <limits.h> // LONG_MIN, LONG_MAX
void ShowHelpInfo(char *name) {
printf("Usage: %s [options]
", name);
printf(" Options:
");
printf(" -p/--proc Number of processes (default: 1)
");
printf(" -d/--duration Duration of test (unit: s, default: 10)
");
printf(" -h/--help Show the help info
");
printf("
");
printf(" Example:
");
printf(" %s -p 4 -d 30
", name);
printf("
");
}
int main(int argc, char *argv[]) {
int c = 0;
int option_index = 0;
long procs = 1;
long duration = 10;
/**
* 定义命令行参数列表，option结构的含义如下（详见 man 3 getopt）：
* struct option {
* const char *name; // 参数的完整名称，对应命令中的 --xxx
* int has_arg; // 该参数是否带有一个值，如 –-config xxx.conf
* int *flag; // 一般设置为NULL
* int val; // 解析到该参数后getopt_long函数的返回值，
* // 为了方便维护，一般对应getopt_long调用时第三个参数
* };
*/
static struct option arg_options[] =
{
{"proc", 1, NULL, 'p'},
{"duration", 1, NULL, 'd'},
{"help", 0, NULL, 'h'},
{NULL, 0, NULL, 0}
};
/**
* 注意：传递给getopt_long的第三个参数对应了命令行参数的缩写形式，如-h等，
* 如果字符后面带有冒号":"，则说明该参数后跟一个值，如-c xxxxxx
* 如果开头有冒号":"，则当一个选项缺少参数时，返回":"，否则，返回"?"
*/
while ((c = getopt_long(argc, argv, ":p:d:h", arg_options, &option_index)
) != -1) {
switch (c) {
case 'h':
ShowHelpInfo(argv[0]);
//fprintf(stdout,"option is -%c, optarv is %s
", c, optarg);
return 0;
case 'p':
procs = strtol(optarg, NULL, 0);
if (procs == LONG_MIN || procs == LONG_MAX) {
fprintf(stderr, "The number of processes (%s) is overflow
",
optarg);
ShowHelpInfo(argv[0]);
return -1;
}
else if (procs <= 0) {
fprintf(stderr, "The number of processes must be > 0
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case 'd':
duration = strtol(optarg, NULL, 0);
if (duration == LONG_MIN || duration == LONG_MAX) {
fprintf(stderr, "The duration of test (%s) is overflow
",
optarg);
ShowHelpInfo(argv[0]);
return -1;
}
else if (procs <= 0) {
fprintf(stderr, "The duration of test must be > 0
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case '?':
fprintf (stderr, "Unknown option -%c
", optopt);
ShowHelpInfo(argv[0]);
return -1;
case ':':
fprintf (stderr, "Option -%c requires an argument
", optopt);
ShowHelpInfo(argv[0]);
return -1;
default:
abort();
}
}
printf("processes: %ld
", procs);
printf("duration: %lds
", duration);
printf("
-----------------------------Start Testing----------------------"
"--------
");
printf("Hello world
");
return 0;
注意：传递给getopt_long的第三个参数对应了命令行参数的缩写形式，如-h, -v, -c等。
如果字符后面带有冒号":"，则说明该参数后跟一个值，如-c xxxxxx
如果开头有冒号":"，则当一个选项缺少参数时，返回":"，否则，返回"?"
效果如下：
^_^$ make
gcc "-g" -c multi-process.c -o multi-process.o
gcc -o test multi-process.o
^_^$ ./test
processes: 1
duration: 10s
-----------------------------Start Testing------------------------------
Hello world
选项或参数错误时：
^_^$ ./test -p
Option -p requires an argument
Usage: ./test [options]
Options:
-p/--proc Number of processes (default: 1)
-d/--duration Duration of test (unit: s, default: 10)
-h/--help Show the help info
Example:
./test -p 4 -d 30
增加多进程的支持
主进程fork出n个子进程后wait子进程，子进程则通过sigaction和alarm设置一个定时器，然后进行业务测试。
为了简洁，已经把选项参数处理的部分独立出去了。
#include <stdio.h> // printf, fprintf
#include <sys/wait.h> // wait
#include <sys/types.h> // getpid, wait
#include <signal.h> // sigaction, SIGLARM
#include <limits.h> // LONG_MIN, LONG_MAX, ULLONG_MAX
#include <unistd.h> // getpid
#include <string.h> // memset
#include "multi-process.h"
int isStop = 0; // 用于标记测试终止
typedef struct param_st { // 自定义测试参数
long index;
} Param;
void handle_signal_child(int sigNum)
{
if (sigNum == SIGALRM) {
isStop = 1;
}
}
/* 实际业务测试函数 */
void doTest(void *param) {
unsigned long long i = 0;
Param *pa = (Param *)param;
for(; i < ULLONG_MAX && !isStop; ++i) {
/* DO YOUR WORK */
/* DO YOUR WORK */
}
printf("process [pid = %6u] result: %llu
", getpid(), i);
}
int main(int argc, char *argv[]) {
int rv = 0;
long i = 0;
int proc_index = 0;
Options opt;
int isParent = 1;
int wstatus = 0;
pid_t pid = 0;
struct sigaction act_child;
rv = process_options(argc, argv, &opt);
if (rv) {
return -1;
}
printf("
-----------------------------Start Testing----------------------"
"--------
");
/* COMMON INIT */
/* COMMON INIT */
while(isParent && i < opt.procs) {
pid = fork();
if(pid == -1) { /* error */
fprintf(stderr, "fork failed %d
", pid);
return -1;
}
else if(pid == 0) { /* child */
isParent = 0;
proc_index = i; // 记录进程索引
}
else { /* parent */
}
++i;
}
if(isParent) {
/* PARENT INIT */
/* PARENT INIT */
for(i =0 ; i < opt.procs; ++i) {
pid = wait(&wstatus); // 等待子进程结束
printf("process [pid = %6d] exit
", pid);
}
}
else {
/* CHILD INIT */
Param param;
memset(¶m, 0, sizeof(Param));
param.index = proc_index;
/* CHILD INIT */
act_child.sa_handler = handle_signal_child;
sigemptyset(&act_child.sa_mask);
act_child.sa_flags = SA_RESETHAND;
/* 用于测试时间到时，通知子进程结束测试 */
rv = sigaction(SIGALRM, &act_child, NULL);
if (rv) {
fprintf(stderr, "sigaction() failed
");
return -1;
}
//signal(SIGALRM, handle_signal_child);
alarm(opt.duration); // 设置测试时长
doTest(¶m);
return 0; /* child finished work */
}
printf("Hello World!
");
return 0;
}
效果如下：
^_^$ ./test -p 4 -d 2
processes: 4
duration: 2s
-----------------------------Start Testing------------------------------
process [pid = 11942] result: 446930553
process [pid = 11942] exit
process [pid = 11939] result: 434385097
process [pid = 11939] exit
process [pid = 11940] result: 442246977
process [pid = 11940] exit
process [pid = 11941] result: 442418811
process [pid = 11941] exit
这样已经可以实现简单的多进程测试，简单起见，示例代码里只是简单地进行了计数操作。读者如果想要进行自己特定的测试，只要在Param中增加需要的测试参数，接着在/* CHILD INIT */处进行参数初始化，然后在/* DO YOUR WORK */处添加实际的测试逻辑即可。
增加实时的结果统计及最终的结果汇总
为了使测试程序更加人性化，使其可以实时统计测试结果，结束时自动计算总的结果。这就需要引入父子进程间通信，我们选用共享内存的方式来实现。为了避免进程间同步对测试带来的影响，在共享内存中为每个子进程开辟了一个空间，每个子进程根据索引在自己的空间里写数据，由父进程进行结果的汇总。
#include <stdio.h> // printf, fprintf
#include <sys/wait.h> // wait
#include <sys/types.h> // getpid, wait
#include <sys/ipc.h> // shmget, shmat, shmctl, shmdt
#include <sys/shm.h> // shmget, shmat, shmctl, shmdt
#include <signal.h> // sigaction, SIGLARM
#include <limits.h> // LONG_MIN, LONG_MAX, ULLONG_MAX
#include <errno.h> // errno
#include <unistd.h> // getpid
#include <string.h> // memset
#include "multi-process.h"
typedef struct param_st { // 自定义测试参数
long index;
} Param;
typedef struct result_st { // 自定义测试结果
unsigned long long count;
} Result;
int isStop = 0; // 用于标记测试终止
Options opt; // 命令行选项
int shmid; // 共享内存id
Result *shm = NULL; // 共享内存地址，用于存放测试结果
Result res_total;
Result res_last;
void handle_signal_child(int sigNum)
{
if (sigNum == SIGALRM) {
isStop = 1;
}
}
void handle_signal_parent(int sigNum)
{
if (sigNum == SIGALRM) {
/* DO REAL-TIME STATISTICS */
memset(&res_total, 0, sizeof(Result));
for (long i = 0; i < opt.procs; ++i) {
res_total.count += shm[i].count;
}
fprintf(stderr, "total count %12llu, average %12.0lf/s
",
res_total.count, (res_total.count - res_last.count)
/ (double)opt.interval);
memcpy(&res_last, &res_total, sizeof(Result));
/* DO REAL-TIME STATISTICS */
alarm(opt.interval);
}
}
/* 实际业务测试函数 */
void doTest(void *param) {
unsigned long long i = 0;
Param *pa = (Param *)param;
for (; i < ULLONG_MAX && !isStop; ++i) {
/* DO YOUR WORK */
++shm[pa->index].count;
/* DO YOUR WORK */
}
}
int main(int argc, char *argv[]) {
int rv = 0;
long i = 0;
int proc_index = 0;
int isParent = 1;
int wstatus = 0;
pid_t pid = 0;
struct sigaction act_child;
struct sigaction act_parent;
rv = process_options(argc, argv, &opt);
if (rv) {
return -1;
}
fprintf(stderr, "
-----------------------------Start Testing-------------"
"-----------------
");
/* COMMON INIT */
shmid = shmget(IPC_PRIVATE, sizeof(sizeof(Result) * opt.procs), 0666);
if (-1 == shmid) {
fprintf(stderr, "shmget() failed
");
return -1;
}
fprintf(stderr, "shmid = %d
", shmid);
shm = (Result*)shmat(shmid, 0, 0);
if ((void *) -1 == shm) {
fprintf(stderr, "shmat() failed
");
return -1;
}
memset(shm, 0, sizeof(sizeof(Result) * opt.procs));
/* COMMON INIT */
while(isParent && i < opt.procs) {
pid = fork();
if(pid == -1) { /* error */
fprintf(stderr, "fork failed %d
", pid);
return -1;
}
else if(pid == 0) { /* child */
isParent = 0;
proc_index = i; // 记录进程索引
}
else { /* parent */
}
++i;
}
if(isParent) {
/* PARENT INIT */
memset(&act_parent, 0, sizeof(act_parent));
act_parent.sa_handler = handle_signal_parent;
/* 使wait被中断时可以自动恢复 */
act_parent.sa_flags = SA_RESTART;
rv = sigaction(SIGALRM, &act_parent, NULL); // 用于定时统计结果
//signal(SIGALRM, handle_signal_parent);
if (rv) {
fprintf(stderr, "sigaction() failed
");
return -1;
}
memset(&res_last, 0, sizeof(Result));
alarm(opt.interval);
/* PARENT INIT */
/* DO FINAL STATISTICS */
Result final;
memset(&final, 0, sizeof(Result));
for(i =0 ; i < opt.procs; ++i) {
pid = wait(&wstatus); // 等待子进程结束
alarm(0); // 终止定时器
if(pid == -1) {
fprintf(stderr, "wait() failed, errno=%d
", errno);
return -1;
}
fprintf(stderr, "process [pid = %6d] exit
", pid);
fprintf(stderr, "process [pid = %6u] count %12llu in %lus, "
"average %12.0lf/s
", pid, shm[i].count, opt.duration,
shm[i].count / (double)opt.duration);
final.count += shm[i].count;
}
fprintf(stderr, "total count %12llu in %lus, average %12.0lf/s
",
final.count, opt.duration, final.count / (double)opt.duration);
/* DO FINAL STATISTICS */
shmdt((void*)shm);
/* 子进程退出之后自动detach了, 所以这里不需要通过IPC_STAT进行判断 */
shmctl(shmid, IPC_RMID, 0);
}
else {
/* CHILD INIT */
Param param;
memset(¶m, 0, sizeof(Param));
param.index = proc_index;
/* CHILD INIT */
act_child.sa_handler = handle_signal_child;
sigemptyset(&act_child.sa_mask);
//sigaddset(&act_child.sa_mask, SIGQUIT);
//sigaddset(&act_child.sa_mask, SIGTERM);
act_child.sa_flags = SA_RESETHAND;
/* 用于测试时间到时，通知子进程结束测试 */
rv = sigaction(SIGALRM, &act_child, NULL);
if (rv) {
fprintf(stderr, "sigaction() failed
");
return -1;
}
//signal(SIGALRM, handle_signal_child);
alarm(opt.duration); // 设置测试时长
doTest(¶m);
return 0; /* child finished work */
}
return 0;
}
测试效果如下：
^_^$ ./test -p 4 -d 8 -i 1
processes: 4
duration: 8s
interval: 1s
-----------------------------Start Testing------------------------------
shmid = 2654220
total count 344235932, average 344235932/s
total count 679573681, average 335337749/s
total count 1026283924, average 346710243/s
total count 1368302354, average 342018430/s
total count 1708471662, average 340169308/s
total count 2057211138, average 348739476/s
total count 2398403059, average 341191921/s
process [pid = 25124] exit
process [pid = 25124] count 688504473 in 8s, average 86063059/s
process [pid = 25123] exit
process [pid = 25123] count 682379115 in 8s, average 85297389/s
process [pid = 25125] exit
process [pid = 25125] count 682467102 in 8s, average 85308388/s
process [pid = 25126] exit
process [pid = 25126] count 688159459 in 8s, average 86019932/s
total count 2741510149 in 8s, average 342688769/s
这里需要特别提一下wait()和sigaction()系统调用，默认情况下wait()会阻塞直到有任意一个子进程改变了其状态，或者有一个信号处理函数中断了wait()调用。所以我们程序中的wait()调用就会被自己的SIGALRM信号中断，返回-1同时errno为EINTR。这样我们就需要在wait()外面加一层循环来处理wait()被信号中断的情况。
通过在sigaction()时增加SA_RESTART标志，被中断的系统调用可以自动重开，也就省去了那个外层循环。另外，signal()封装了sigaction()，它里面默认就是设置了SA_RESTART，不过除非你有确切的理由，不然不建议使用signal()了。
增加SIGINT和SIGQUIT信号捕获
截止目前，我们已经完成了多进程的测试及结果统计。但其实还有一个潜在的问题，在实际测试中，我们经常会在测试还没完成时就手动^C终止程序执行。这样我们在程序中申请的共享内存就会得不到释放，造成内存泄漏。所以需要增加对SIGINT和SIGQUIT信号的处理函数，在里面做清理工作，释放共享内存。
如果已经不小心造成了共享内存的泄漏，可以通过如下命令手动进行删除。ipcrm shm <id>，如果是显式指定key的话也可以通过ipcrm -M <key>来进行删除。
今天，突然想到了，其实有一种更加简单的方法，即在shmat()之后立即进行shmctl(shmid, IPC_RMID, 0);。这样不仅简单，而且中间的空窗期也更短，cool！这样我们再也不用担心，^C造成内存泄漏了哈。
@@ -88,12 +88,14 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "shmget() failed
");
return -1;
}
fprintf(stderr, "shmid = %d
", shmid);
shm = (Result*)shmat(shmid, 0, 0);
if ((void *) -1 == shm) {
fprintf(stderr, "shmat() failed
");
return -1;
}
+ /* 这里直接进行IPC_RMID操作，进程退出后会自动detach了, 从而释放共享内存 */
+ shmctl(shmid, IPC_RMID, 0);
memset(shm, 0, sizeof(sizeof(Result) * opt.procs));
/* COMMON INIT */
@@ -156,10 +135,6 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "total count %12llu in %lus, average %12.0lf/s
",
final.count, opt.duration, final.count / (double)opt.duration);
/* DO FINAL STATISTICS */
-
- shmdt((void*)shm);
- /* 子进程退出之后自动detach了, 所以这里不需要通过IPC_STAT进行判断 */
- shmctl(shmid, IPC_RMID, 0);
}
else {
/* CHILD INIT */
封装错误判断函数
为了使代码看起来更加简洁，避免每个函数调用后面跟着一个if(){}判断块，我们对错误判断及日志打印函数进行了一个简单的封装。封装的函数如下：
其中mylog()单纯打印日志，fail()打印日志后退出进程，fail_if()先判断条件，如果成立打印日志退出，fail_clean_if()也是先判断条件，条件成立则打印日志，执行传入的清理函数。然后退出进程。
#include "common.h"
#include <stdio.h> // stderr
#include <stdarg.h> // va_start, vfprintf, va_end
#include <stdlib.h> // exit
void fail_if(bool condition, const char *fmt, ...) {
if (condition) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
exit(1);
}
}
void fail(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
exit(1);
}
void mylog(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
}
void fail_clean_if(bool condition, cleanup clean, void *param, const char *fmt, ...) {
if (condition) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
clean(param);
exit(1);
}
}
添加实际测试用例
接下来我们来添加实际有意义的测试用例，这里以OpenSSL引擎的性能测试为例来进行说明。为了我们的代码更清晰，已经将具体测试相关的代码独立为一个源文件。common.c封装错误函数，opt.c处理命令行选项，work.c处理具体测试，multi-process.c则负责测试的主控。完整的代码如下：
common.h
#ifndef HEADER_COMMON_H
#define HEADER_COMMON_H
#include <stdbool.h> // bool, true, false
typedef void (*cleanup) (void *);
void fail_if(bool condition, const char *fmt, ...);
void fail(const char *fmt, ...);
void mylog(const char *fmt, ...);
void fail_clean_if(bool condition, cleanup clean, void *param, const char *fmt, ...);
#endif /* HEADER_COMMON_H */
common.c
#include "common.h"
#include <stdio.h> // stderr
#include <stdarg.h> // va_start, vfprintf, va_end
#include <stdlib.h> // exit
void fail_if(bool condition, const char *fmt, ...) {
if (condition) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
exit(1);
}
}
void fail(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
exit(1);
}
void mylog(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
}
void fail_clean_if(bool condition, cleanup clean, void *param, const char *fmt, ...) {
if (condition) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
clean(param);
exit(1);
}
}
opt.h
#ifndef HEADER_OPT_H
#define HEADER_OPT_H
typedef enum test{
hash, sign, verify, enc, dec
} Test;
typedef struct options_st {
long procs; // 进程数
long duration; // 测试时间
long interval; // 统计间隔
Test test; // 测试类型
long len; // 摘要原文长度
const char *key; // 密钥文件路径
const char *cert; // 证书文件路径
const char *loglevel; // 日志等级
} Options;
/* 处理参数 */
int process_options(int argc, char *argv[], Options *opt);
#endif /* HEADER_OPT_H */
opt.c
#include "opt.h"
#include <limits.h> // LONG_MIN, LONG_MAX, ULLONG_MAX
#include <string.h> // memset
#include <stdlib.h> // strtol, abort
#include <getopt.h> // geropt_long
#include "common.h" // mylog
static void ShowHelpInfo(char *name) {
mylog("Usage: %s [options]
", name);
mylog(" Options:
");
mylog(" -p/--proc Number of processes (default: 1)
");
mylog(" -d/--duration Duration of test (unit: s, default: 10)
");
mylog(" -i/--interval Interval of statisics (unit: s, default: 1)
");
mylog(" -t/--test Test case (hash|sign|verify|enc|dec, default: hash)
");
mylog(" -l/--len Hash data len (unit: byte, default: 1024)
");
mylog(" -k/--key PEM Key file path (default: ./key.pem)
");
mylog(" -c/--cert PEM Cert file path (default: ./cert.pem)
");
mylog(" -o/--loglevel Engine log level (0-9, default: 0)
");
mylog(" -h/--help Show the help info
");
mylog("
");
mylog(" Example:
");
mylog(" %s -p 1 -d 30 -i 1 -t sign -k key.pem -c cert.pem
", name);
mylog("
");
}
/* 处理参数 */
int process_options(int argc, char *argv[], Options *opt) {
int c = 0;
int option_index = 0;
long procs = 1;
long duration = 10;
long interval = 1;
Test test = hash;
long len = 1024;
const char *key = "./key.pem";
const char *cert = "./cert.pem";
const char *loglevel = "0";
/**
* 定义命令行参数列表，option结构的含义如下（详见 man 3 getopt）：
* struct option {
* const char *name; // 参数的完整名称，对应命令中的 --xxx
* int has_arg; // 该参数是否带有一个值，如 –config xxx.conf
* int *flag; // 一般设置为NULL
* int val; // 解析到该参数后getopt_long函数的返回值，
* // 为了方便维护，一般对应getopt_long调用时第三个参数
* };
*/
static struct option arg_options[] =
{
{"proc", 1, NULL, 'p'},
{"duration", 1, NULL, 'd'},
{"interval", 1, NULL, 'i'},
{"test", 1, NULL, 't'},
{"len", 1, NULL, 'l'},
{"key", 1, NULL, 'k'},
{"cert", 1, NULL, 'c'},
{"log", 1, NULL, 'g'},
{"help", 0, NULL, 'h'},
{NULL, 0, NULL, 0}
};
/**
* 注意：传递给getopt_long的第三个参数对应了命令行参数的缩写形式，如-h等，
* 如果字符后面带有冒号":"，则说明该参数后跟一个值，如-c xxxxxx
* 如果开头有冒号":"，则当一个选项缺少参数时，返回":"，否则，返回"?"
*/
while ((c = getopt_long(argc, argv, ":p:d:i:t:l:k:c:g:h", arg_options, &option_index)
) != -1) {
switch (c) {
case 'h':
ShowHelpInfo(argv[0]);
//fprintf(stderr,"option is -%c, optarv is %s
", c, optarg);
exit(0);
case 'p':
procs = strtol(optarg, NULL, 0);
if (procs == LONG_MIN || procs == LONG_MAX) {
mylog("The number of processes (%s) is overflow
", optarg);
ShowHelpInfo(argv[0]);
return -1;
}
else if (procs <= 0) {
mylog("The number of processes must be > 0
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case 'd':
duration = strtol(optarg, NULL, 0);
if (duration == LONG_MIN || duration == LONG_MAX) {
mylog("The duration of test (%s) is overflow
", optarg);
ShowHelpInfo(argv[0]);
return -1;
}
else if (duration <= 0) {
mylog("The duration of test must be > 0
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case 'i':
interval = strtol(optarg, NULL, 0);
if (interval == LONG_MIN || interval == LONG_MAX) {
mylog("The interval of statistics (%s) is overflow
",
optarg);
ShowHelpInfo(argv[0]);
return -1;
}
else if (interval <= 0) {
mylog("The interval of statistics must be > 0
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case 't':
if(!strcasecmp("hash", optarg)) {
test = hash;
}
else if(!strcasecmp("sign", optarg)) {
test = sign;
}
else if(!strcasecmp("verify", optarg)) {
test = verify;
}
else if(!strcasecmp("enc", optarg)) {
test = enc;
}
else if(!strcasecmp("dec", optarg)) {
test = dec;
}
else {
mylog("Unknown test case type
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case 'l':
len = strtol(optarg, NULL, 0);
if (len == LONG_MIN || len == LONG_MAX) {
mylog("The len of hash data (%s) is overflow
",
optarg);
ShowHelpInfo(argv[0]);
return -1;
}
else if (len <= 0) {
mylog("The len of hash data must be > 0
");
ShowHelpInfo(argv[0]);
return -1;
}
break;
case 'k':
key = optarg;
break;
case 'c':
cert = optarg;
break;
case 'g':
loglevel = optarg;
break;
case '?':
mylog("Unknown option -%c
", optopt);
ShowHelpInfo(argv[0]);
return -1;
case ':':
mylog("Option -%c requires an argument
", optopt);
ShowHelpInfo(argv[0]);
return -1;
default:
exit(1);
}
}
mylog("processes: %ld
", procs);
mylog("duration: %lds
", duration);
mylog("interval: %lds
", interval);
mylog("test: #%d
", test);
mylog("len: %ld bytes
", len);
mylog("key: %s
", key);
mylog("cert: %s
", cert);
mylog("loglevel: %s
", loglevel);
memset(opt, 0, sizeof(Options));
opt->procs = procs;
opt->duration = duration;
opt->interval = interval;
opt->test = test;
opt->len = len;
opt->key = key;
opt->cert = cert;
opt->loglevel = loglevel;
return 0;
}
work.h
#ifndef HEADER_WORK_H
#define HEADER_WORK_H
#include <openssl/evp.h>
#include <openssl/bio.h>
#include "opt.h"
/* 自定义测试参数 */
typedef struct hash_param_st {
long index;
unsigned char *data;
unsigned int data_len;
unsigned char md[128];
unsigned int md_len;
EVP_MD_CTX *ctx;
} HashParam;
typedef struct sign_param_st {
long index;
unsigned char data[48];
size_t data_len;
unsigned char sig[256];
size_t sig_len;
EVP_PKEY_CTX *ctx;
BIO *in;
} SignParam;
typedef SignParam VerifyParam;
typedef struct enc_param_st {
long index;
unsigned char data[48];
size_t data_len;
unsigned char enc[256];
size_t enc_len;
EVP_PKEY_CTX *ctx;
BIO *in;
} EncParam;
typedef EncParam DecParam;
/* 自定义测试结果 */
typedef struct result_st {
unsigned long long count;
} Result;
typedef void (*init_fn) (Options *opt, void **param, long proc_index);
typedef void (*work_fn) (void *param);
typedef void (*clean_fn) (void *param);
extern init_fn test_init;
extern work_fn test_work;
extern clean_fn test_clean;
void global_init(Options *opt);
void global_clean();
#endif /* HEADER_WORK_H */
work.c
#include "work.h"
#include <string.h>
#include <openssl/engine.h>
#include <openssl/pem.h>
#include "common.h"
const char* so_path = "/usr/local/ssl/lib/myengine.so";
init_fn test_init = NULL;
work_fn test_work = NULL;
clean_fn test_clean = NULL;
static void hash_init(Options *opt, void **param, long proc_index);
static void hash_work(void *param);
static void hash_clean(void *param);
static void sign_init(Options *opt, void **param, long proc_index);
static void sign_work(void *param);
static void sign_clean(void *param);
static void verify_init(Options *opt, void **param, long proc_index);
static void verify_work(void *param);
static void verify_clean(void *param);
static void encrypt_init(Options *opt, void **param, long proc_index);
static void encrypt_work(void *param);
static void encrypt_clean(void *param);
static void decrypt_init(Options *opt, void **param, long proc_index);
static void decrypt_work(void *param);
static void decrypt_clean(void *param);
void global_init(Options *opt) {
ENGINE *e = NULL;
if (opt->test == hash) {
test_init = hash_init;
test_work = hash_work;
test_clean = hash_clean;
}
else if (opt->test == sign) {
test_init = sign_init;
test_work = sign_work;
test_clean = sign_clean;
}
else if (opt->test == verify) {
test_init = verify_init;
test_work = verify_work;
test_clean = verify_clean;
}
else if (opt->test == enc) {
test_init = encrypt_init;
test_work = encrypt_work;
test_clean = encrypt_clean;
}
else if (opt->test == dec) {
test_init = decrypt_init;
test_work = decrypt_work;
test_clean = decrypt_clean;
}
OpenSSL_add_all_algorithms();
/* ENGINE INIT */
ENGINE_load_dynamic();
if (!(e = ENGINE_by_id("dynamic"))) {
fail("ENGINE_by_id("dynamic") fail
");
}
if (!ENGINE_ctrl_cmd_string(e, "SO_PATH", so_path, 0)) {
fail("ENGINE_ctrl_cmd_string("SO_PATH") fail, so_path = %s
",
so_path);
}
if (!ENGINE_ctrl_cmd_string(e, "LIST_ADD", "1", 0)) {
fail("ENGINE_ctrl_cmd_string("LIST_ADD") fail
");
}
if (!ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) {
fail("ENGINE_ctrl_cmd_string("LOAD") fail
");
}
if (!ENGINE_init(e)) {
fail("ENGINE_init() fail
");
}
if (!ENGINE_ctrl_cmd_string( e, "ENGINE_SET_LOGLEVEL", opt->loglevel, 0)) {
fail("ENGINE_ctrl_cmd_string("ENGINE_SET_LOGLEVEL") fail
");
}
if (!ENGINE_set_default(e, ENGINE_METHOD_ALL)) {
fail("ENGINE_set_default() fail
");
}
ENGINE_free(e);
/* ENGINE INIT */
}
void global_clean() {
EVP_cleanup();
}
static void hash_init(Options *opt, void **param, long proc_index) {
HashParam *p;
p = OPENSSL_malloc(sizeof(HashParam));
fail_if(!p, "OPENSSL_malloc() fail
");
memset(p, 0, sizeof(HashParam));
p->ctx = EVP_MD_CTX_create();
fail_clean_if(!p->ctx, hash_clean, (void *)p, "EVP_MD_CTX_create() fail
");
EVP_MD_CTX_init(p->ctx);
p->data = OPENSSL_malloc(opt->len);
fail_clean_if(!p->data, hash_clean, (void *)p, "OPENSSL_malloc() fail
");
fail_clean_if(RAND_bytes(p->data, sizeof(opt->len)) <= 0, hash_clean, (void *)p,
"RAND_bytes() fail
");
p->data_len = opt->len;
p->index = proc_index;
*param = p;
}
static void hash_work(void *param) {
HashParam *p = (HashParam *)param;
const EVP_MD *md = EVP_get_digestbyname("SHASH");
fail_clean_if(!md, hash_clean, param,
"EVP_get_digestbyname("SHASH") fail
");
fail_clean_if(!EVP_DigestInit_ex(p->ctx, md, NULL),
hash_clean, param, "EVP_DigestInit_ex() fail
");
fail_clean_if(!EVP_DigestUpdate(p->ctx, p->data, p->data_len),
hash_clean, param, "EVP_DigestUpdate() fail
");
fail_clean_if(!EVP_DigestFinal_ex(p->ctx, p->md, &p->md_len),
hash_clean, param, "EVP_DigestFinal_ex() fail
");
}
static void hash_clean(void *param) {
HashParam *p = (HashParam *)param;
if (p->data)
OPENSSL_free(p->data);
if (p->ctx)
EVP_MD_CTX_destroy(p->ctx);
OPENSSL_free(p);
}
static void sign_init(Options *opt, void **param, long proc_index) {
SignParam *p;
EVP_PKEY *pkey;
p = OPENSSL_malloc(sizeof(SignParam));
fail_if(!p, "OPENSSL_malloc() fail
");
memset(p, 0, sizeof(SignParam));
p->in = BIO_new_file(opt->key, "r");
fail_clean_if(!p->in, sign_clean, (void *)p,
"BIO_new_file(%s) fail
", opt->key);
pkey = PEM_read_bio_PrivateKey(p->in, NULL, NULL, NULL);
fail_clean_if(!pkey, sign_clean, (void *)p,
"PEM_read_bio_PrivateKey() fail
");
p->ctx = EVP_PKEY_CTX_new(pkey, NULL);
if (!p->ctx) {
EVP_PKEY_free(pkey);
fail_clean_if(1, sign_clean, (void *)p, "EVP_PKEY_CTX_new() fail
");
}
fail_clean_if(!EVP_PKEY_sign_init(p->ctx), sign_clean, (void *)p,
"EVP_PKEY_sign_init() fail
");
fail_clean_if(RAND_bytes(p->data, sizeof(p->data)) <= 0, sign_clean, (void *)p,
"RAND_bytes() fail
");
p->data_len = sizeof(p->data);
p->sig_len = sizeof(p->sig);
p->index = proc_index;
*param = p;
}
static void sign_work(void *param) {
SignParam *p = (SignParam *)param;
fail_clean_if(!EVP_PKEY_sign(p->ctx, p->sig, &p->sig_len, p->data,
p->data_len), sign_clean, param,
"EVP_PKEY_sign() fail
");
}
static void sign_clean(void *param) {
SignParam *p = (SignParam *)param;
if (p->ctx)
EVP_PKEY_CTX_free(p->ctx);
if (p->in)
BIO_free(p->in);
OPENSSL_free(p);
}
static void verify_init(Options *opt, void **param, long proc_index) {
VerifyParam *p;
EVP_PKEY *pkey;
X509 *x;
unsigned char data[48];
size_t data_len;
unsigned char sig[256];
size_t sig_len;
/* 先签名一次，获取数据 */
SignParam *sp;
sign_init(opt, (void **)&sp, (long)0);
sign_work(sp);
memcpy(data, sp->data, sp->data_len);
data_len = sp->data_len;
memcpy(sig, sp->sig, sp->sig_len);
sig_len = sp->sig_len;
sign_clean(sp);
p = OPENSSL_malloc(sizeof(VerifyParam));
fail_if(!p, "OPENSSL_malloc() fail
");
memset(p, 0, sizeof(VerifyParam));
p->in = BIO_new_file(opt->cert, "r");
fail_clean_if(!p->in, verify_clean, (void *)p,
"BIO_new_file(%s) fail
", opt->cert);
x = PEM_read_bio_X509(p->in, NULL, 0, NULL);
fail_clean_if(!x, verify_clean, (void *)p,
"PEM_read_bio_X509() fail
");
pkey = X509_get_pubkey(x);
X509_free(x);
fail_clean_if(!pkey, verify_clean, (void *)p, "X509_get_pubkey() fail
");
p->ctx = EVP_PKEY_CTX_new(pkey, NULL);
if (!p->ctx) {
EVP_PKEY_free(pkey);
fail_clean_if(1, verify_clean, (void *)p, "EVP_PKEY_CTX_new() fail
");
}
fail_clean_if(!EVP_PKEY_verify_init(p->ctx), verify_clean, (void *)p,
"EVP_PKEY_verify_init() fail
");
memcpy(p->data, data, data_len);
p->data_len = data_len;
memcpy(p->sig, sig, sig_len);
p->sig_len = sig_len;
p->index = proc_index;
*param = p;
}
static void verify_work(void *param) {
VerifyParam *p = (VerifyParam *)param;
fail_clean_if(!EVP_PKEY_verify(p->ctx, p->sig, p->sig_len, p->data,
p->data_len), verify_clean, param,
"EVP_PKEY_verify() fail
");
}
static void verify_clean(void *param) {
VerifyParam *p = (VerifyParam *)param;
if (p->ctx)
EVP_PKEY_CTX_free(p->ctx);
if (p->in)
BIO_free(p->in);
OPENSSL_free(p);
}
static void encrypt_init(Options *opt, void **param, long proc_index) {
EncParam *p;
EVP_PKEY *pkey;
X509 *x;
p = OPENSSL_malloc(sizeof(EncParam));
fail_if(!p, "OPENSSL_malloc() fail
");
memset(p, 0, sizeof(EncParam));
p->in = BIO_new_file(opt->cert, "r");
fail_clean_if(!p->in, encrypt_clean, (void *)p,
"BIO_new_file(%s) fail
", opt->cert);
x = PEM_read_bio_X509(p->in, NULL, 0, NULL);
fail_clean_if(!x, encrypt_clean, (void *)p,
"PEM_read_bio_X509() fail
");
pkey = X509_get_pubkey(x);
X509_free(x);
fail_clean_if(!pkey, encrypt_clean, (void *)p, "X509_get_pubkey() fail
");
p->ctx = EVP_PKEY_CTX_new(pkey, NULL);
if (!p->ctx) {
EVP_PKEY_free(pkey);
fail_clean_if(1, encrypt_clean, (void *)p, "EVP_PKEY_CTX_new() fail
");
}
fail_clean_if(!EVP_PKEY_encrypt_init(p->ctx), encrypt_clean, (void *)p,
"EVP_PKEY_encrypt_init() fail
");
fail_clean_if(RAND_bytes(p->data, sizeof(p->data)) <= 0, encrypt_clean, (void *)p,
"RAND_bytes() fail
");
p->data_len = sizeof(p->data);
p->enc_len = sizeof(p->enc);
p->index = proc_index;
*param = p;
}
static void encrypt_work(void *param) {
EncParam *p = (EncParam *)param;
fail_clean_if(!EVP_PKEY_encrypt(p->ctx, p->enc, &p->enc_len, p->data,
p->data_len), encrypt_clean, param,
"EVP_PKEY_encrypt() fail
");
}
static void encrypt_clean(void *param) {
EncParam *p = (EncParam *)param;
if (p->ctx)
EVP_PKEY_CTX_free(p->ctx);
if (p->in)
BIO_free(p->in);
OPENSSL_free(p);
}
static void decrypt_init(Options *opt, void **param, long proc_index) {
DecParam *p;
EVP_PKEY *pkey;
unsigned char data[48];
size_t data_len;
unsigned char enc[256];
size_t enc_len;
/* 先加密一次，获取数据 */
EncParam *ep;
encrypt_init(opt, (void **)&ep, (long)0);
encrypt_work(ep);
memcpy(data, ep->data, ep->data_len);
data_len = ep->data_len;
memcpy(enc, ep->enc, ep->enc_len);
enc_len = ep->enc_len;
encrypt_clean(ep);
p = OPENSSL_malloc(sizeof(DecParam));
fail_if(!p, "OPENSSL_malloc() fail
");
memset(p, 0, sizeof(DecParam));
p->in = BIO_new_file(opt->key, "r");
fail_clean_if(!p->in, decrypt_clean, (void *)p,
"BIO_new_file(%s) fail
", opt->key);
pkey = PEM_read_bio_PrivateKey(p->in, NULL, NULL, PEM_AUTO_KEYPASS);
fail_clean_if(!pkey, decrypt_clean, (void *)p,
"PEM_read_bio_PrivateKey() fail
");
p->ctx = EVP_PKEY_CTX_new(pkey, NULL);
if (!p->ctx) {
EVP_PKEY_free(pkey);
fail_clean_if(1, decrypt_clean, (void *)p, "EVP_PKEY_CTX_new() fail
");
}
fail_clean_if(!EVP_PKEY_decrypt_init(p->ctx), decrypt_clean, (void *)p,
"EVP_PKEY_decrypt_init() fail
");
memcpy(p->data, data, data_len);
p->data_len = data_len;
memcpy(p->enc, enc, enc_len);
p->enc_len = enc_len;
p->index = proc_index;
*param = p;
}
static void decrypt_work(void *param) {
DecParam *p = (DecParam *)param;
fail_clean_if(!EVP_PKEY_decrypt(p->ctx, p->data, &p->data_len, p->enc,
p->enc_len), decrypt_clean, param,
"EVP_PKEY_decrypt() fail
");
}
static void decrypt_clean(void *param) {
DecParam *p = (DecParam *)param;
if (p->ctx)
EVP_PKEY_CTX_free(p->ctx);
if (p->in)
BIO_free(p->in);
OPENSSL_free(p);
}
multi-process.c
#include <sys/wait.h> // wait
#include <sys/types.h> // getpid, wait
#include <sys/ipc.h> // shmget, shmat, shmctl, shmdt
#include <sys/shm.h> // shmget, shmat, shmctl, shmdt
#include <signal.h> // sigaction, SIGLARM
#include <limits.h> // LONG_MIN, LONG_MAX, ULLONG_MAX
#include <errno.h> // errno
#include <unistd.h> // getpid
#include <string.h> // memset
#include "common.h"
#include "opt.h"
#include "work.h"
int isStop = 0; // 用于标记测试终止
Options opt; // 命令行选项
int shmid; // 共享内存id
Result *shm = NULL; // 共享内存地址，用于存放测试结果
Result res_total;
Result res_last;
void handle_signal_child(int sigNum)
{
if (sigNum == SIGALRM) {
isStop = 1;
}
}
void handle_signal_parent(int sigNum)
{
if (sigNum == SIGALRM) {
/* DO REAL-TIME STATISTICS */
memset(&res_total, 0, sizeof(Result));
long i = 0;
for (; i < opt.procs; ++i) {
res_total.count += shm[i].count;
}
mylog("total count %12llu, average %12.0lf/s
",
res_total.count, (res_total.count - res_last.count)
/ (double)opt.interval);
memcpy(&res_last, &res_total, sizeof(Result));
/* DO REAL-TIME STATISTICS */
alarm(opt.interval);
}
}
/* 执行测试主循环函数 */
void doTest(void *param) {
unsigned long long i = 0;
HashParam *pa = (HashParam *)param;
for (; i < ULLONG_MAX && !isStop; ++i) {
/* DO YOUR WORK */
test_work(param);
++shm[pa->index].count;
/* DO YOUR WORK */
}
}
int main(int argc, char *argv[]) {
int rv = 0;
long i = 0;
int proc_index = 0;
int isParent = 1;
int wstatus = 0;
pid_t pid = 0;
struct sigaction act_child;
struct sigaction act_parent;
rv = process_options(argc, argv, &opt);
if (rv) {
return -1;
}
mylog("
-----------------------------Start Testing-----------------------"
"-------
");
/* COMMON INIT */
shmid = shmget(IPC_PRIVATE, sizeof(sizeof(Result) * opt.procs), 0666);
fail_if(-1 == shmid, "shmget() failed
");
mylog("shmid = %d
", shmid);
shm = (Result*)shmat(shmid, 0, 0);
fail_if((void *) -1 == shm, "shmat() failed
");
/* 这里直接进行IPC_RMID操作，进程退出之后会自动detach了, 从而释放共享内存 */
shmctl(shmid, IPC_RMID, 0);
memset(shm, 0, sizeof(sizeof(Result) * opt.procs));
global_init(&opt);
/* COMMON INIT */
while(isParent && i < opt.procs) {
pid = fork();
fail_if(-1 == pid, "fork failed %d
", pid); /* error */
if(pid == 0) { /* child */
isParent = 0;
proc_index = i; // 记录进程索引
}
else { /* parent */
}
++i;
}
if(isParent) {
/* PARENT INIT */
memset(&act_parent, 0, sizeof(act_parent));
act_parent.sa_handler = handle_signal_parent;
/* 使wait被中断时可以自动恢复 */
act_parent.sa_flags = SA_RESTART;
rv = sigaction(SIGALRM, &act_parent, NULL); // 用于定时统计结果
fail_if(rv, "sigaction() failed
");
memset(&res_last, 0, sizeof(Result));
alarm(opt.interval);
/* PARENT INIT */
/* DO FINAL STATISTICS */
Result final;
memset(&final, 0, sizeof(Result));
for(i =0 ; i < opt.procs; ++i) {
pid = wait(&wstatus); // 等待子进程结束
alarm(0); // 终止定时器
fail_if(-1 == pid, "wait() failed, errno=%d
", errno);
mylog("process [pid = %6d] exit
", pid);
mylog("process [pid = %6u] count %12llu in %lus,"
" average %12.0lf/s
", pid, shm[i].count, opt.duration,
shm[i].count / (double)opt.duration);
final.count += shm[i].count;
}
mylog("total count %12llu in %lus, average %12.0lf/s
",
final.count, opt.duration, final.count / (double)opt.duration);
/* DO FINAL STATISTICS */
/* PARENT CLEANUP */
global_clean();
/* PARENT CLEANUP */
}
else {
/* CHILD INIT */
void *param;
test_init(&opt, ¶m, proc_index);
/* CHILD INIT */
act_child.sa_handler = handle_signal_child;
sigemptyset(&act_child.sa_mask);
act_child.sa_flags = SA_RESETHAND;
/* 用于测试时间到时，通知子进程结束测试 */
rv = sigaction(SIGALRM, &act_child, NULL);
fail_if(rv, "sigaction() failed
");
alarm(opt.duration); // 设置测试时长
doTest(param);
/* CHILD CLEANUP */
test_clean(param);
global_clean();
/* CHILD CLEANUP */
return 0; /* child finished work */
}
return 0;
}
总结
至此，一个多进程的测试程序就算完成了。读者可以根据自身测试需要，按如下步骤修改以进行自定义的测试。
- 在
opt.c和opt.h中，添加需要的命令行选项 - 在
work.c和work.h中，global_init()和global_clean()内进行全局的初始化及全局的清理工作 - 在
work.c和work.h中，添加自定义的测试函数，以及相应的测试参数和测试结果 - 主控
multi-process.c通过test_init()、test_work()、test_clean()调用测试相关的初始化、执行及清理工作 - 主控
multi-process.c中，修改测试结果的更新与统计操作。
主控multi-process.c中：
COMMON INIT、PARENT INIT和CHILD INIT代码块处分别进行公共的初始化工作和父子进程特定的初始化工作。PARENT CLEANUP和CHILD CLEANUP代码块处则分别进行对应的清理工作。DO YOUR WORK处执行具体的测试，DO REAL-TIME STATISTICS和DO FINAL STATISTICS处进行测试结果的统计。
当然本文还有一些不足之处，比如当前是用的OpenSSL1.0.2接口，没有兼容不同版本的OpenSSL。还有代码风格的问题，在函数命名和注释方式上不统一。但是考虑到这个是测试程序，就不计较这么多了（。）