RNNOp only call cuda/cudnn if GPU ctx is requested (#16632)

Author: leezu

src/operator/rnn-inl.h

@@ -422,6 +422,8 @@ class RNNOp {
     init_mem_ = false;
     reserve_mem_size_ = 0;
 #endif
+
+    if (ctx_.dev_type == kGPU) {
 #if MXNET_USE_CUDNN == 1
     init_cudnn_ = false;
     dtype_ = mshadow::DataType<DType>::kCudnnFlag;
@@ -505,6 +507,7 @@ class RNNOp {
       LOG(FATAL) << "RNN on GPU is only available for cuDNN at the moment.";
     }
 #endif  // MXNET_USE_CUDNN == 1
+    }
 
     if (ctx_.dev_type == kCPU) {
       this->init_space_ = false;
@@ -523,6 +526,7 @@ class RNNOp {
   }
 
   ~RNNOp() {
+    if (ctx_.dev_type == kGPU) {
 #if MXNET_USE_CUDNN == 1
     CUDNN_CALL(cudnnDestroyTensorDescriptor(hx_desc_));
     CUDNN_CALL(cudnnDestroyTensorDescriptor(cx_desc_));
@@ -557,6 +561,7 @@ class RNNOp {
     CUDNN_CALL(cudnnDestroyRNNDataDescriptor(dy_data_desc_));
 #endif  // MXNET_USE_CUDNN_GE_7200
 #endif  // MXNET_USE_CUDNN
+    }
   }
 
   void Forward(const OpContext &ctx, const std::vector<TBlob> &in_data,