train (still not work)

demo.py

@@ -8,10 +8,14 @@
 def test():
  import os
  im_file = 'demo/004545.jpg'
+ im_file = 'data/VOCdevkit2007/VOC2007/JPEGImages/009936.jpg'
+ im_file = '/media/longc/Data/data/2DMOT2015/test/ETH-Crossing/img1/000100.jpg'
  image = cv2.imread(im_file)
 
+ # model_file = '/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5'
+ model_file = '/media/longc/Data/models/faster_rcnn_pytorch/faster_rcnn_30000.h5'
  detector = FasterRCNN()
- network.load_net('/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5', detector)
+ network.load_net(model_file, detector)
  detector.cuda()
  detector.eval()
  print('load model successfully!')
@@ -22,19 +26,19 @@ def test():
  t = Timer()
  t.tic()
  # image = np.zeros(shape=[600, 800, 3], dtype=np.uint8) + 255
- dets, scores, classes = detector.detect(image, 0.3)
+ dets, scores, classes = detector.detect(image, 0.7)
  runtime = t.toc()
  print('total spend: {}s'.format(runtime))
 
  im2show = np.copy(image)
  for i, det in enumerate(dets):
- if scores[i] < 0.3:
- continue
  det = tuple(int(x) for x in det)
  cv2.rectangle(im2show, det[0:2], det[2:4], (255, 205, 51), 2)
  cv2.putText(im2show, '%s: %.3f' % (classes[i], scores[i]), (det[0], det[1] + 15), cv2.FONT_HERSHEY_PLAIN,
  1.0, (0, 0, 255), thickness=1)
  cv2.imwrite(os.path.join('demo', 'out.jpg'), im2show)
+ # cv2.imshow('demo', im2show)
+ # cv2.waitKey(0)
 
 
 if __name__ == '__main__':

experiments/cfgs/faster_rcnn_end2end.yml

@@ -17,7 +17,7 @@ TRAIN:
  GAMMA: 0.1
  STEPSIZE: 60000
  IMS_PER_BATCH: 1
- BBOX_NORMALIZE_TARGETS_PRECOMPUTED: True
+ BBOX_NORMALIZE_TARGETS_PRECOMPUTED: False
  RPN_POSITIVE_OVERLAP: 0.7
  RPN_BATCHSIZE: 256
  PROPOSAL_METHOD: gt

faster_rcnn/datasets/factory.py

@@ -41,13 +41,13 @@ def _selective_search_IJCV_top_k(split, year, top_k):
  # Set up kittivoc
  for split in ['train', 'val', 'trainval', 'test']:
  name = 'kittivoc_{}'.format(split)
- print name
+ # print name
  __sets[name] = (lambda split=split: kittivoc(split))
 
 # # KITTI dataset
 for split in ['train', 'val', 'trainval', 'test']:
  name = 'kitti_{}'.format(split)
- print name
+ # print name
  __sets[name] = (lambda split=split: kitti(split))
 
 # Set up coco_2014_<split>
@@ -65,7 +65,7 @@ def _selective_search_IJCV_top_k(split, year, top_k):
 # NTHU dataset
 for split in ['71', '370']:
  name = 'nthu_{}'.format(split)
- print name
+ # print name
  __sets[name] = (lambda split=split: nthu(split))
 
 

faster_rcnn/datasets/kittivoc.py

@@ -8,7 +8,7 @@
 import xml.dom.minidom as minidom
 
 import os
-import PIL
+# import PIL
 import numpy as np
 import scipy.sparse
 import subprocess

faster_rcnn/faster_rcnn.py

@@ -43,7 +43,11 @@ def __init__(self):
 
  # loss
  self.cross_entropy = None
- self.loss_box = None
+ self.los_box = None
+
+ @property
+ def loss(self):
+ return self.cross_entropy + self.loss_box * 10
 
  def forward(self, im_data, im_info, gt_boxes=None, dontcare_areas=None):
  im_data = network.np_to_variable(im_data, is_cuda=True)
@@ -62,8 +66,9 @@ def forward(self, im_data, im_info, gt_boxes=None, dontcare_areas=None):
  rpn_bbox_pred = self.bbox_conv(rpn_conv1)
 
  # proposal layer
+ cfg_key = 'TRAIN' if self.training else 'TEST'
  rois = self.proposal_layer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info,
- 'TEST', self._feat_stride, self.anchor_scales)
+ cfg_key, self._feat_stride, self.anchor_scales)
 
  # generating training labels and build the rpn loss
  if self.training:
@@ -75,8 +80,7 @@ def forward(self, im_data, im_info, gt_boxes=None, dontcare_areas=None):
 
  return features, rois
 
- @staticmethod
- def build_loss(rpn_cls_score_reshape, rpn_bbox_pred, rpn_data):
+ def build_loss(self, rpn_cls_score_reshape, rpn_bbox_pred, rpn_data):
  # classification loss
  rpn_cls_score = rpn_cls_score_reshape.permute(0, 2, 3, 1).contiguous().view(-1, 2)
  rpn_label = rpn_data[0]
@@ -87,6 +91,19 @@ def build_loss(rpn_cls_score_reshape, rpn_bbox_pred, rpn_data):
  rpn_cls_score = torch.index_select(rpn_cls_score, 0, rpn_keep)
  rpn_label = torch.index_select(rpn_label, 0, rpn_keep)
 
+ fg_cnt = torch.sum(rpn_label.data.ne(0))
+ bg_cnt = rpn_label.data.numel() - fg_cnt
+ # ce_weights = torch.ones(rpn_cls_score.size()[1])
+ # ce_weights[0] = float(fg_cnt) / bg_cnt
+ # ce_weights = ce_weights.cuda()
+
+ _, predict = torch.max(rpn_cls_score.data, 1)
+ error = torch.sum(torch.abs(predict - rpn_label.data))
+ self.tp = torch.sum(predict[:fg_cnt].eq(rpn_label.data[:fg_cnt]))
+ self.tf = torch.sum(predict[fg_cnt:].eq(rpn_label.data[fg_cnt:]))
+ self.fg_cnt = fg_cnt
+ self.bg_cnt = bg_cnt
+
  rpn_cross_entropy = F.cross_entropy(rpn_cls_score, rpn_label)
  # print rpn_cross_entropy
 
@@ -95,8 +112,11 @@ def build_loss(rpn_cls_score_reshape, rpn_bbox_pred, rpn_data):
  rpn_bbox_targets = torch.mul(rpn_bbox_targets, rpn_bbox_inside_weights)
  rpn_bbox_pred = torch.mul(rpn_bbox_pred, rpn_bbox_inside_weights)
 
- rpn_loss_box = F.smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets,
- size_average=False) / torch.sum(rpn_bbox_inside_weights.data)
+ # a = rpn_bbox_pred.data.cpu().numpy()
+ # b = rpn_bbox_targets.data.cpu().numpy()
+ # s = torch.sum(rpn_bbox_inside_weights.data)
+
+ rpn_loss_box = F.smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, size_average=False) / fg_cnt
  # print rpn_loss_box
 
  return rpn_cross_entropy, rpn_loss_box
@@ -202,7 +222,11 @@ def __init__(self, classes=None):
 
  @property
  def loss(self):
- return self.cross_entropy + self.loss_box + self.rpn.cross_entropy + self.rpn.loss_box
+ # print self.cross_entropy
+ # print self.loss_box
+ # print self.rpn.cross_entropy
+ # print self.rpn.loss_box
+ return self.cross_entropy + self.loss_box * 10 + self.rpn.loss
 
  def forward(self, im_data, im_info, gt_boxes=None, dontcare_areas=None):
  features, rois = self.rpn(im_data, im_info, gt_boxes, dontcare_areas)
@@ -228,20 +252,37 @@ def forward(self, im_data, im_info, gt_boxes=None, dontcare_areas=None):
 
  return cls_prob, bbox_pred, rois
 
- @staticmethod
- def build_loss(cls_score, bbox_pred, roi_data):
+ def build_loss(self, cls_score, bbox_pred, roi_data):
  # classification loss
  label = roi_data[1].squeeze()
+ fg_cnt = torch.sum(label.data.ne(0))
+ bg_cnt = label.data.numel() - fg_cnt
+
+ # ce_weights = torch.ones(cls_score.size()[1])
+ # ce_weights[0] = float(fg_cnt) / bg_cnt / 3.0
+ # # ce_weights[0] = 1./50
+ # ce_weights = ce_weights.cuda()
+
+ maxv, predict = cls_score.data.max(1)
+ self.tp = torch.sum(predict[:fg_cnt].eq(label.data[:fg_cnt]))
+ self.tf = torch.sum(predict[fg_cnt:].eq(label.data[fg_cnt:]))
+ self.fg_cnt = fg_cnt
+ self.bg_cnt = bg_cnt
+ # print predict
  cross_entropy = F.cross_entropy(cls_score, label)
  # print cross_entropy
 
  # bounding box regression L1 loss
  bbox_targets, bbox_inside_weights, bbox_outside_weights = roi_data[2:]
 
+ # b = bbox_targets.data.cpu().numpy()
+
  bbox_targets = torch.mul(bbox_targets, bbox_inside_weights)
  bbox_pred = torch.mul(bbox_pred, bbox_inside_weights)
 
- loss_box = F.smooth_l1_loss(bbox_pred, bbox_targets, size_average=False) / torch.sum(bbox_inside_weights.data)
+ # a = bbox_pred.data.cpu().numpy()
+
+ loss_box = F.smooth_l1_loss(bbox_pred, bbox_targets, size_average=False) / fg_cnt
  # print loss_box
 
  return cross_entropy, loss_box

faster_rcnn/network.py

@@ -54,3 +54,16 @@ def np_to_variable(x, is_cuda=True, dtype=torch.FloatTensor):
  if is_cuda:
  v = v.cuda()
  return v
+
+
+def set_trainable(model, requires_grad):
+ for param in model.parameters():
+ param.requires_grad = requires_grad
+
+
+def weights_normal_init(model, dev=0.01):
+ for m in model.modules():
+ if isinstance(m, nn.Conv2d):
+ m.weight.data.normal_(0.0, dev)
+ elif isinstance(m, nn.Linear):
+ m.weight.data.normal_(0.0, dev)

faster_rcnn/pycocotools/coco.py

@@ -47,11 +47,11 @@
 import json
 import datetime
 import time
-import matplotlib.pyplot as plt
-from matplotlib.collections import PatchCollection
-from matplotlib.patches import Polygon
+# import matplotlib.pyplot as plt
+# from matplotlib.collections import PatchCollection
+# from matplotlib.patches import Polygon
 import numpy as np
-from skimage.draw import polygon
+# from skimage.draw import polygon
 import urllib
 import copy
 import itertools

faster_rcnn/rpn_msr/proposal_target_layer.py

@@ -69,15 +69,15 @@ def proposal_target_layer(rpn_rois, gt_boxes, dontcare_areas, _num_classes):
  all_rois, gt_boxes, dontcare_areas, fg_rois_per_image,
  rois_per_image, _num_classes)
 
- # if DEBUG:
- #  print 'num fg: {}'.format((labels > 0).sum())
- #  print 'num bg: {}'.format((labels == 0).sum())
- # _count += 1
- # _fg_num += (labels > 0).sum()
- # _bg_num += (labels == 0).sum()
- # print 'num fg avg: {}'.format(_fg_num / _count)
- # print 'num bg avg: {}'.format(_bg_num / _count)
- # print 'ratio: {:.3f}'.format(float(_fg_num) / float(_bg_num))
+ if DEBUG:
+ print 'num fg: {}'.format((labels > 0).sum())
+ print 'num bg: {}'.format((labels == 0).sum())
+ # _count += 1
+ # _fg_num += (labels > 0).sum()
+ # _bg_num += (labels == 0).sum()
+ # print 'num fg avg: {}'.format(_fg_num / _count)
+ # print 'num bg avg: {}'.format(_bg_num / _count)
+ # print 'ratio: {:.3f}'.format(float(_fg_num) / float(_bg_num))
 
  rois = rois.reshape(-1, 5)
  labels = labels.reshape(-1, 1)
@@ -157,21 +157,25 @@ def _sample_rois(all_rois, gt_boxes, dontcare_areas, fg_rois_per_image, rois_per
  # Select foreground RoIs as those with >= FG_THRESH overlap
  fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
  fg_inds = np.setdiff1d(fg_inds, ignore_inds)
+
+ # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
+ bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
+ (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
+ bg_inds = np.setdiff1d(bg_inds, ignore_inds)
+
  # Guard against the case when an image has fewer than fg_rois_per_image
  # foreground RoIs
- fg_rois_per_this_image = min(fg_rois_per_image, fg_inds.size)
+ fg_rois_per_this_image = int(min(fg_rois_per_image, fg_inds.size))
+ # fg_rois_per_this_image = int(min(bg_inds.size, fg_inds.size))
  # Sample foreground regions without replacement
  if fg_inds.size > 0:
  fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False)
 
- # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
- bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
- (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
- bg_inds = np.setdiff1d(bg_inds, ignore_inds)
  # Compute number of background RoIs to take from this image (guarding
  # against there being fewer than desired)
  bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
  bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size)
+ # bg_rois_per_this_image = fg_rois_per_this_image
  # Sample background regions without replacement
  if bg_inds.size > 0:
  bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False)

faster_rcnn/vgg16.py

@@ -6,6 +6,7 @@
 
 from utils.blob import im_list_to_blob
 from network import Conv2d
+import network
 
 
 class VGG16(nn.Module):
@@ -18,6 +19,9 @@ def __init__(self, bn=False):
  self.conv2 = nn.Sequential(Conv2d(64, 128, 3, same_padding=True, bn=bn),
  Conv2d(128, 128, 3, same_padding=True, bn=bn),
  nn.MaxPool2d(2))
+ network.set_trainable(self.conv1, requires_grad=False)
+ network.set_trainable(self.conv2, requires_grad=False)
+
  self.conv3 = nn.Sequential(Conv2d(128, 256, 3, same_padding=True, bn=bn),
  Conv2d(256, 256, 3, same_padding=True, bn=bn),
  Conv2d(256, 256, 3, same_padding=True, bn=bn),
@@ -61,6 +65,12 @@ def load_from_npy_file(self, fname):
  own_dict = self.state_dict()
  params = np.load(fname).item()
  for name, val in own_dict.items():
+ # # print name
+ # # print val.size()
+ # # print param.size()
+ # if name.find('bn.') >= 0:
+ # continue
+
  i, j = int(name[4]), int(name[6]) + 1
  ptype = 'weights' if name[-1] == 't' else 'biases'
  key = 'conv{}_{}'.format(i, j)