replace cuDNN alphas and betas with coefficient values

Give cuDNN {0, 1} constants for controlling accumulation through the
alpha and beta coefficients.

include/caffe/util/cudnn.hpp

@@ -50,10 +50,14 @@ template <typename Dtype> class dataType;
 template<> class dataType<float> {
  public:
  static const cudnnDataType_t type = CUDNN_DATA_FLOAT;
+ static float oneval, zeroval;
+ static const void *one, *zero;
 };
 template<> class dataType<double> {
  public:
  static const cudnnDataType_t type = CUDNN_DATA_DOUBLE;
+ static double oneval, zeroval;
+ static const void *one, *zero;
 };
 
 template <typename Dtype>
@@ -102,9 +106,9 @@ inline void setConvolutionDesc(cudnnConvolutionDescriptor_t* conv,
 }
 
 template <typename Dtype>
-inline void createPoolingDesc(cudnnPoolingDescriptor_t* conv,
+inline void createPoolingDesc(cudnnPoolingDescriptor_t* pool_desc,
  PoolingParameter_PoolMethod poolmethod, cudnnPoolingMode_t* mode,
- int h, int w, int stride_h, int stride_w) {
+ int h, int w, int pad_h, int pad_w, int stride_h, int stride_w) {
  switch (poolmethod) {
  case PoolingParameter_PoolMethod_MAX:
  *mode = CUDNN_POOLING_MAX;
@@ -115,9 +119,9 @@ inline void createPoolingDesc(cudnnPoolingDescriptor_t* conv,
  default:
  LOG(FATAL) << "Unknown pooling method.";
  }
- CUDNN_CHECK(cudnnCreatePoolingDescriptor(conv));
- CUDNN_CHECK(cudnnSetPooling2dDescriptor(*conv, *mode, h, w,
- 0, 0, stride_h, stride_w));
+ CUDNN_CHECK(cudnnCreatePoolingDescriptor(pool_desc));
+ CUDNN_CHECK(cudnnSetPooling2dDescriptor(*pool_desc, *mode, h, w,
+ pad_h, pad_w, stride_h, stride_w));
 }
 
 } // namespace cudnn

src/caffe/layers/cudnn_conv_layer.cu

@@ -21,9 +21,6 @@ void CuDNNConvolutionLayer<Dtype>::Forward_gpu(
 
  // Forward through cuDNN in parallel over groups.
  for (int g = 0; g < this->group_; g++) {
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  cudnnConvolutionFwdAlgo_t algo;
 
  // get the desired convolution algorithm
@@ -56,23 +53,21 @@ void CuDNNConvolutionLayer<Dtype>::Forward_gpu(
 
  // Filters.
  CUDNN_CHECK(cudnnConvolutionForward(handle_[g],
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  bottom_descs_[i], bottom_data + bottom_offset_ * g,
  filter_desc_, weight + weight_offset_ * g,
  conv_descs_[i],
  algo, workspace, workspaceSizeInBytes,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  top_descs_[i], top_data + top_offset_ * g));
 
  // Bias.
  if (this->bias_term_) {
  const Dtype* bias_data = this->blobs_[1]->gpu_data();
- Dtype alpha = 1.0;
- Dtype beta = 1.0;
  CUDNN_CHECK(cudnnAddTensor(handle_[g], CUDNN_ADD_SAME_C,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  bias_desc_, bias_data + bias_offset_ * g,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::one,
  top_descs_[i], top_data + top_offset_ * g));
  }
  }
@@ -105,25 +100,22 @@ void CuDNNConvolutionLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  for (int g = 0; g < this->group_; g++) {
  // Gradient w.r.t. bias.
  if (this->bias_term_ && this->param_propagate_down_[1]) {
- Dtype alpha = 1.0;
- Dtype beta = 1.0;
  CUDNN_CHECK(cudnnConvolutionBackwardBias(handle_[0*this->group_ + g],
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  top_descs_[i], top_diff + top_offset_ * g,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::one,
  bias_desc_, bias_diff + bias_offset_ * g));
  }
 
  // Gradient w.r.t. weights.
  if (this->param_propagate_down_[0]) {
  const Dtype* bottom_data = bottom[i]->gpu_data();
- Dtype alpha = 1.0;
- Dtype beta = 1.0;
  CUDNN_CHECK(cudnnConvolutionBackwardFilter(handle_[1*this->group_ + g],
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  bottom_descs_[i], bottom_data + bottom_offset_ * g,
  top_descs_[i], top_diff + top_offset_ * g,
- conv_descs_[i], reinterpret_cast<void *>(&beta),
+ conv_descs_[i],
+ cudnn::dataType<Dtype>::one,
  filter_desc_, weight_diff + weight_offset_ * g));
  }
 
@@ -133,13 +125,12 @@ void CuDNNConvolutionLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  weight = this->blobs_[0]->gpu_data();
  }
  Dtype* bottom_diff = bottom[i]->mutable_gpu_diff();
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
  CUDNN_CHECK(cudnnConvolutionBackwardData(handle_[2*this->group_ + g],
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  filter_desc_, weight + weight_offset_ * g,
- top_descs_[i], top_diff + top_offset_ * g,
- conv_descs_[i], reinterpret_cast<void *>(&beta),
+ top_descs_[i], top_diff + top_offset_ * g,
+ conv_descs_[i],
+ cudnn::dataType<Dtype>::zero,
  bottom_descs_[i], bottom_diff + bottom_offset_ * g));
  }
  }

src/caffe/layers/cudnn_pooling_layer.cu

@@ -14,14 +14,10 @@ void CuDNNPoolingLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
  const vector<Blob<Dtype>*>& top) {
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* top_data = top[0]->mutable_gpu_data();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnPoolingForward(handle_, pooling_desc_,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  top_desc_, top_data));
 }
 
@@ -35,15 +31,11 @@ void CuDNNPoolingLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  const Dtype* top_data = top[0]->gpu_data();
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnPoolingBackward(handle_, pooling_desc_,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  top_desc_, top_data, top_desc_, top_diff,
  bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  bottom_desc_, bottom_diff));
 }
 

src/caffe/layers/cudnn_relu_layer.cu

@@ -17,15 +17,11 @@ void CuDNNReLULayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
 
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* top_data = top[0]->mutable_gpu_data();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnActivationForward(this->handle_,
  CUDNN_ACTIVATION_RELU,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  this->bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  this->top_desc_, top_data));
 }
 
@@ -46,16 +42,12 @@ void CuDNNReLULayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  const Dtype* top_diff = top[0]->gpu_diff();
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnActivationBackward(this->handle_,
  CUDNN_ACTIVATION_RELU,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  this->top_desc_, top_data, this->top_desc_, top_diff,
  this->bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  this->bottom_desc_, bottom_diff));
 }
 

src/caffe/layers/cudnn_sigmoid_layer.cu

@@ -12,15 +12,11 @@ void CuDNNSigmoidLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
  const vector<Blob<Dtype>*>& top) {
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* top_data = top[0]->mutable_gpu_data();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnActivationForward(this->handle_,
  CUDNN_ACTIVATION_SIGMOID,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  this->bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  this->top_desc_, top_data));
 }
 
@@ -36,16 +32,12 @@ void CuDNNSigmoidLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  const Dtype* top_diff = top[0]->gpu_diff();
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnActivationBackward(this->handle_,
  CUDNN_ACTIVATION_SIGMOID,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  this->top_desc_, top_data, this->top_desc_, top_diff,
  this->bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  this->bottom_desc_, bottom_diff));
 }
 

src/caffe/layers/cudnn_softmax_layer.cu

@@ -16,15 +16,11 @@ void CuDNNSoftmaxLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
  const vector<Blob<Dtype>*>& top) {
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* top_data = top[0]->mutable_gpu_data();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnSoftmaxForward(handle_, CUDNN_SOFTMAX_ACCURATE,
  CUDNN_SOFTMAX_MODE_CHANNEL,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  top_desc_, top_data));
 }
 
@@ -37,14 +33,11 @@ void CuDNNSoftmaxLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
 
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnSoftmaxBackward(handle_, CUDNN_SOFTMAX_ACCURATE,
  CUDNN_SOFTMAX_MODE_CHANNEL,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  top_desc_, top_data, top_desc_, top_diff,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  bottom_desc_, bottom_diff));
  }
 }

src/caffe/layers/cudnn_tanh_layer.cu

@@ -12,15 +12,11 @@ void CuDNNTanHLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
  const vector<Blob<Dtype>*>& top) {
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* top_data = top[0]->mutable_gpu_data();
-
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnActivationForward(this->handle_,
  CUDNN_ACTIVATION_TANH,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  this->bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  this->top_desc_, top_data));
 }
 
@@ -37,15 +33,12 @@ void CuDNNTanHLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
  const Dtype* bottom_data = bottom[0]->gpu_data();
  Dtype* bottom_diff = bottom[0]->mutable_gpu_diff();
 
- Dtype alpha = 1.0;
- Dtype beta = 0.0;
-
  CUDNN_CHECK(cudnnActivationBackward(this->handle_,
  CUDNN_ACTIVATION_TANH,
- reinterpret_cast<void *>(&alpha),
+ cudnn::dataType<Dtype>::one,
  this->top_desc_, top_data, this->top_desc_, top_diff,
  this->bottom_desc_, bottom_data,
- reinterpret_cast<void *>(&beta),
+ cudnn::dataType<Dtype>::zero,
  this->bottom_desc_, bottom_diff));
 }
 

src/caffe/util/cudnn.cpp

@@ -0,0 +1,23 @@
+#ifdef USE_CUDNN
+#include "caffe/util/cudnn.hpp"
+
+namespace caffe {
+namespace cudnn {
+
+float dataType<float>::oneval = 1.0;
+float dataType<float>::zeroval = 0.0;
+const void* dataType<float>::one =
+ static_cast<void *>(&dataType<float>::oneval);
+const void* dataType<float>::zero =
+ static_cast<void *>(&dataType<float>::zeroval);
+
+double dataType<double>::oneval = 1.0;
+double dataType<double>::zeroval = 0.0;
+const void* dataType<double>::one =
+ static_cast<void *>(&dataType<double>::oneval);
+const void* dataType<double>::zero =
+ static_cast<void *>(&dataType<double>::zeroval);
+
+} // namespace cudnn
+} // namespace caffe
+#endif