https://zhuanlan.zhihu.com/p/36095768
我的推断，第二第三阶段应该不是把所有anchor进行bounding box regression，然后再选取当前条件下的所有roi，而是第一阶段选取512个roi，然后把在第一阶段匹配好的roi送到第二、三阶段
layer { name: "proposals_2nd" type: "DecodeBBox" bottom: "bbox_pred" bottom: "rois" bottom: "match_gt_boxes" top: "proposals_2nd" bbox_reg_param { bbox_mean: 0 bbox_mean: 0 bbox_mean: 0 bbox_mean: 0 bbox_std: 0.1 bbox_std: 0.1 bbox_std: 0.2 bbox_std: 0.2 } propagate_down: 0 propagate_down: 0 propagate_down: 0 }
这段代码就证明了这个想法：rois来自于第一阶段proposal_info，这些rois也是在第一阶段做roi-pooling用来训练的。
个人感觉cascade的模型就是4张图提取512个roi进行训练,然后经过第一阶段训练后,把这512个roi经过回归精修然后去除回归后x1大于x2和y1大于y2的和回归后和gt的iou大于0.95的,这样roi可能就没有512个了.把这些输入给第二阶段的proposal_info_2nd,让这个层再去决定训练样本,这样大可能训练的数据是不足512,并且3个阶段其实都是训练的同一个批roi,也就是说第一阶段进去的那些roi,后面几个阶段实际上也在训练他们,而不是新出来的框DecodeBbox层的输入是bbox_pred,rois和match_gt_boxes.首先明确一点,rpn网络会输出很多proposals出来,ProposalTarget层将这些proposals和gt算iou,确定正负样本并选取1:3的比例,然后输出rois,rois就是拿来具体训练的从rpn中获得那部分预提取框.DecodeBbox层就是将这些原本的rois回归成更精准的框,也就是在原始的rois的坐标上增加经过训练得到的回归的值,这个是通过DecodeBBoxesWithPrior函数实现.DecodeBbox层分为大致3个步骤:1.回归得到更精准的rois 2.去掉回归后x1大于x2和y1大于y2的框 3.去掉回归后和gt的iou大于0.95的框
这部分的结果
#include <cfloat> #include <vector> #include "caffe/util/bbox_util.hpp" #include "caffe/layers/decode_bbox_layer.hpp" namespace caffe { template <typename Dtype> void DecodeBBoxLayer<Dtype>::LayerSetUp( const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { // bbox mean and std BBoxRegParameter bbox_reg_param = this->layer_param_.bbox_reg_param(); bbox_mean_.Reshape(4,1,1,1); bbox_std_.Reshape(4,1,1,1); if (bbox_reg_param.bbox_mean_size() > 0 && bbox_reg_param.bbox_std_size() > 0) { int num_means = this->layer_param_.bbox_reg_param().bbox_mean_size(); int num_stds = this->layer_param_.bbox_reg_param().bbox_std_size(); CHECK_EQ(num_means,4); CHECK_EQ(num_stds,4); for (int i = 0; i < 4; i++) { bbox_mean_.mutable_cpu_data()[i] = bbox_reg_param.bbox_mean(i); bbox_std_.mutable_cpu_data()[i] = bbox_reg_param.bbox_std(i); CHECK_GT(bbox_std_.mutable_cpu_data()[i],0); } } else { caffe_set(bbox_mean_.count(), Dtype(0), bbox_mean_.mutable_cpu_data()); caffe_set(bbox_std_.count(), Dtype(1), bbox_std_.mutable_cpu_data()); } } template <typename Dtype> void DecodeBBoxLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { // bottom: bbox_blob, prior_blob, (match_gt_boxes) CHECK_EQ(bottom[0]->num(),bottom[1]->num()); if (bottom.size()>=3) { CHECK_EQ(bottom[0]->num(),bottom[2]->num()); CHECK(this->phase_ == TRAIN); } CHECK_EQ(bottom[0]->channels(),8); CHECK_EQ(bottom[1]->channels(),5); bbox_pred_.ReshapeLike(*bottom[0]); top[0]->ReshapeLike(*bottom[1]); } template <typename Dtype> void DecodeBBoxLayer<Dtype>::Forward_cpu( const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) { const int num = bottom[0]->num(); const int bbox_dim = bottom[0]->channels(); const int prior_dim = bottom[1]->channels(); //decode prior box [img_id x1 y1 x2 y2] const Dtype* prior_data = bottom[1]->cpu_data(); vector<BBox> prior_bboxes; for (int i = 0; i < num; i++) { BBox bbox; bbox.xmin = prior_data[i*prior_dim + 1]; bbox.ymin = prior_data[i*prior_dim + 2]; bbox.xmax = prior_data[i*prior_dim + 3]; bbox.ymax = prior_data[i*prior_dim + 4]; prior_bboxes.push_back(bbox); } // decode bbox predictions const Dtype* bbox_data = bottom[0]->cpu_data(); Dtype* bbox_pred_data = bbox_pred_.mutable_cpu_data(); DecodeBBoxesWithPrior(bbox_data,prior_bboxes,bbox_dim,bbox_mean_.cpu_data(), bbox_std_.cpu_data(),bbox_pred_data); vector<bool> valid_bbox_flags(num,true); // screen out mal-boxes if (this->phase_ == TRAIN) { for (int i = 0; i < num; i++) { const int base_index = i*bbox_dim+4; if (bbox_pred_data[base_index] > bbox_pred_data[base_index+2] || bbox_pred_data[base_index+1] > bbox_pred_data[base_index+3]) { valid_bbox_flags[i] = false; } } } // screen out high IoU boxes, to remove redundant gt boxes if (bottom.size()==3 && this->phase_ == TRAIN) { const Dtype* match_gt_boxes = bottom[2]->cpu_data(); const int gt_dim = bottom[2]->channels(); const float gt_iou_thr = this->layer_param_.decode_bbox_param().gt_iou_thr(); for (int i = 0; i < num; i++) { const float overlap = match_gt_boxes[i*gt_dim+gt_dim-1]; if (overlap >= gt_iou_thr) { valid_bbox_flags[i] = false; } } } vector<int> valid_bbox_ids; for (int i = 0; i < num; i++) { if (valid_bbox_flags[i]) { valid_bbox_ids.push_back(i); } } const int keep_num = valid_bbox_ids.size(); CHECK_GT(keep_num,0); top[0]->Reshape(keep_num, prior_dim, 1, 1); Dtype* decoded_bbox_data = top[0]->mutable_cpu_data(); for (int i = 0; i < keep_num; i++) { const int keep_id = valid_bbox_ids[i]; const int base_index = keep_id*bbox_dim+4; decoded_bbox_data[i*prior_dim] = prior_data[keep_id*prior_dim]; decoded_bbox_data[i*prior_dim+1] = bbox_pred_data[base_index]; decoded_bbox_data[i*prior_dim+2] = bbox_pred_data[base_index+1]; decoded_bbox_data[i*prior_dim+3] = bbox_pred_data[base_index+2]; decoded_bbox_data[i*prior_dim+4] = bbox_pred_data[base_index+3]; } } INSTANTIATE_CLASS(DecodeBBoxLayer); REGISTER_LAYER_CLASS(DecodeBBox); } // namespace caffe
DecodeBBoxesWithPrior函数在bbox_util.cpp里实现,完成的功能就是把bounding box regression的结果对输入的prior_bbox(其实就是faster中的输入的region proposal)进行回归获得更精确的框坐标,然后存储在pred_data
template <typename Dtype> void DecodeBBoxesWithPrior(const Dtype* bbox_data, const vector<BBox> prior_bboxes, const int bbox_dim, const Dtype* means, const Dtype* stds, Dtype* pred_data) { const int num = prior_bboxes.size(); const int cls_num = bbox_dim/4; for (int i = 0; i < num; i++) { Dtype pw, ph, cx, cy; pw = prior_bboxes[i].xmax-prior_bboxes[i].xmin+1; ph = prior_bboxes[i].ymax-prior_bboxes[i].ymin+1; cx = 0.5*(prior_bboxes[i].xmax+prior_bboxes[i].xmin); cy = 0.5*(prior_bboxes[i].ymax+prior_bboxes[i].ymin); for (int c = 0; c < cls_num; c++) { Dtype bx, by, bw, bh; // bbox de-normalization bx = bbox_data[i*bbox_dim+4*c]*stds[0]+means[0]; by = bbox_data[i*bbox_dim+4*c+1]*stds[1]+means[1]; bw = bbox_data[i*bbox_dim+4*c+2]*stds[2]+means[2]; bh = bbox_data[i*bbox_dim+4*c+3]*stds[3]+means[3]; Dtype tx, ty, tw, th; tx = bx*pw+cx; ty = by*ph+cy; tw = pw*exp(bw); th = ph*exp(bh); tx -= (tw-1)/2; ty -= (th-1)/2; pred_data[i*bbox_dim+4*c] = tx; pred_data[i*bbox_dim+4*c+1] = ty; pred_data[i*bbox_dim+4*c+2] = tx+tw-1; pred_data[i*bbox_dim+4*c+3] = ty+th-1; } } }