warning fix (pytorch#142)

Represent empty tensors as size {0} tensors and fix scalar checks.

This gets rid of kUndefinedDimensions and has nice properties like:
- the dimensionality always matches the length of the sizes and strides.
- the number of elements is always the product of the sizes (starting at the identity)
- the shape you pass to factory functions (e.g. randn) matches the shape that is returned
etc.

In addition to the empty tensor change, this makes some related changes:
1) expand is now a native function, because it needs to operate on the ATen view of the size/strides.
2) adds tests for a number of functions operating on empty, scalar, non-scalar tensors.
This uncovered a number of scalar_check bugs; some of these are fixed in the generated code,
some that need to be manually specified can be specified by a 'scalar_check' argument in the cwrap.
3) fixes the formatting of empty tensors
4) changes the THLongStorageView API; the public API was getting overly complicated, so now you call
'makeFromSize', 'makeFromStride', 'makeFromLength' and it just handles the correct mapping for that type.

Address review comments.

Add comment explaining return of dim() when tensor is a scalar.

tighten hasCUDA check

Update ExpandUtils.h

Include what you use, otherwise compilation may break.
@prigoyal reported compilation errors with gcc-4.9 I believe.

Correct dimensions for reduction functions, squeeze, unsqueeze.

Reduction functions that take a dimension now properly reduce
down to scalars if passed a 1-dimensional tensor.

Squeeze now properly reduces down to scalars as well (and is implemented
as a native function).

Unsqueeze now handles scalar inputs correctly (so unsqueezing a scalar
returns a dim 1 tensor, rather than a dim 2 tensor).

update dlpack header and convertors

Add an at() method for indexing. (pytorch#152)

Signed-off-by: Edward Z. Yang <[email protected]>

Fix handling of inf and nan (pytorch#153)

Implement stack as a native function.

update nn

v2

Fix for THD squeeze_out and NativeFunction stack.

Author: zdevito

ExpandUtils.cpp

@@ -0,0 +1,76 @@
+#include "ATen/ExpandUtils.h"
+
+namespace at {
+
+std::vector<int64_t> infer_size(IntList a, IntList b) {
+  auto dimsA = a.size();
+  auto dimsB = b.size();
+  ptrdiff_t ndim = dimsA > dimsB ? dimsA : dimsB;
+  std::vector<int64_t> expandedSizes(ndim);
+
+  for (long i = ndim - 1; i >= 0; --i) {
+    long offset = ndim - 1 - i;
+    long dimA = dimsA - 1 - offset;
+    long dimB = dimsB - 1 - offset;
+    long sizeA = (dimA >= 0) ? a[dimA] : 1;
+    long sizeB = (dimB >= 0) ? b[dimB] : 1;
+    if (sizeA == sizeB || sizeA == 1 || sizeB == 1) {
+      expandedSizes[i] = std::max(sizeA, sizeB);
+    } else {
+      std::ostringstream oss;
+      oss << "The size of tensor a (" << sizeA << ") must match the size of tensor b ("
+          << sizeB << ") at non-singleton dimension " << i;
+      throw std::runtime_error(oss.str());
+    }
+  }
+
+  return expandedSizes;
+}
+
+std::tuple<std::vector<int64_t>, std::vector<int64_t> >
+inferExpandGeometry(const Tensor &tensor, IntList sizes) {
+  int64_t ndim = sizes.size();
+
+  if (tensor.dim() == 0) {
+    std::vector<int64_t> expandedStrides(ndim, 0);
+    return std::tuple<std::vector<int64_t>, std::vector<int64_t>>(sizes.vec(), expandedStrides);
+  }
+  std::vector<int64_t> expandedSizes(ndim);
+  std::vector<int64_t> expandedStrides(ndim);
+
+  // create a new geometry for the tensors
+  for (int64_t i = ndim - 1; i >= 0; --i) {
+    int64_t offset = ndim - 1 - i;
+    int64_t dim = tensor.dim() - 1 - offset;
+    int64_t size = (dim >= 0) ? tensor.sizes()[dim] : 1;
+    int64_t stride = (dim >= 0) ?
+        tensor.strides()[dim] : expandedSizes[i + 1] * expandedStrides[i + 1];
+    int64_t targetSize = sizes[i];
+    if (targetSize == -1) {
+      if (dim < 0) {
+        std::ostringstream oss;
+        oss << "The expanded size of the tensor (" << targetSize << ") isn't allowed in a leading, "
+            << "non-existing dimension " << i;
+        throw std::runtime_error(oss.str());
+      } else {
+        targetSize = size;
+      }
+    }
+    if (size != targetSize) {
+      if (size == 1) {
+        size = targetSize;
+        stride = 0;
+      } else {
+        std::ostringstream oss;
+        oss << "The expanded size of the tensor (" << targetSize << ") must match the existing size (" << size 
+            << ") at non-singleton dimension " << i;
+        throw std::runtime_error(oss.str());
+      }
+    }
+    expandedSizes[i] = size;
+    expandedStrides[i] = stride;
+  }
+  return std::tuple<std::vector<int64_t>, std::vector<int64_t>>(expandedSizes, expandedStrides);
+}
+
+}

test/scalar_tensor_test.cpp

@@ -0,0 +1,294 @@
+#include "ATen/ATen.h"
+#include <iostream>
+#include <numeric>
+
+using namespace at;
+
+void assert_equal_size_dim(const Tensor &lhs, const Tensor &rhs) {
+  assert(lhs.dim() == rhs.dim());
+  assert(lhs.sizes().equals(rhs.sizes()));
+}
+
+bool should_expand(const IntList &from_size, const IntList &to_size) {
+  if(from_size.size() > to_size.size()) {
+    return false;
+  }
+  for (auto from_dim_it = from_size.rbegin(); from_dim_it != from_size.rend(); ++from_dim_it) {
+    for (auto to_dim_it = to_size.rbegin(); to_dim_it != to_size.rend(); ++to_dim_it) {
+      if (*from_dim_it != 1 && *from_dim_it != *to_dim_it) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+int main() {
+  Type & T = CPU(kFloat);
+
+  std::vector<std::vector<int64_t> > sizes = { {}, {0}, {1}, {1, 1}, {2}};
+
+  // single-tensor/size tests
+  for (auto s = sizes.begin(); s != sizes.end(); ++s) {
+    // verify that the dim, sizes, strides, etc match what was requested.
+    auto t = T.ones(*s);
+    assert(t.dim() == s->size());
+    assert(t.ndimension() == s->size());
+    assert(t.sizes().equals(*s));
+    assert(t.strides().size() == s->size());
+    auto numel = std::accumulate(s->begin(), s->end(), 1, std::multiplies<int64_t>());
+    assert(t.numel() == numel);
+    // verify we can output
+    std::cout << t << std::endl;
+
+    // set_
+    auto t2 = T.ones(*s);
+    t2.set_();
+    assert_equal_size_dim(t2, T.ones({0}));
+
+    // unsqueeze
+    if (t.numel() != 0) {
+      assert(t.unsqueeze(0).dim() == t.dim() + 1);
+    } else {
+      try {
+        // can't unsqueeze empty tensor
+        t.unsqueeze(0);
+        assert (false);
+      } catch (std::runtime_error &e) {}
+    }
+
+    // unsqueeze_
+    {
+      auto t2 = T.ones(*s);
+      if (t2.numel() != 0) {
+        auto r = t2.unsqueeze_(0);
+        assert(r.dim() == t.dim() + 1);
+      } else {
+        try {
+          // can't unsqueeze empty tensor
+          t2.unsqueeze_(0);
+          assert (false);
+        } catch (std::runtime_error &e) {}
+      }
+    }
+
+    // squeeze (with dimension argument)
+    if (t.dim() > 0 && t.sizes()[0] == 1) {
+      assert(t.squeeze(0).dim() == t.dim() - 1);
+    } else if (t.dim() == 0) {
+      try {
+        t.squeeze(0);
+        assert(false);
+      } catch (std::runtime_error &e) {}
+    } else {
+      // In PyTorch, it is a no-op to try to squeeze a dimension that has size != 1;
+      // in NumPy this is an error.
+      assert(t.squeeze(0).dim() == t.dim());
+    }
+
+    // squeeze (with no dimension argument)
+    {
+      std::vector<int64_t> size_without_ones;
+      for (auto size : *s) {
+        if (size != 1) {
+          size_without_ones.push_back(size);
+        }
+      }
+      auto result = t.squeeze();
+      assert_equal_size_dim(result, T.ones(size_without_ones));
+    }
+
+    {
+      // squeeze_ (with dimension argument)
+      auto t2 = T.ones(*s);
+      if (t2.dim() > 0 && t2.sizes()[0] == 1) {
+        assert(t2.squeeze_(0).dim() == t.dim() - 1);
+      } else if (t2.dim() == 0) {
+        try {
+          t2.squeeze_(0);
+          assert(false);
+        } catch (std::runtime_error &e) {}
+      } else {
+        // In PyTorch, it is a no-op to try to squeeze a dimension that has size != 1;
+        // in NumPy this is an error.
+        assert(t2.squeeze_(0).dim() == t.dim());
+      }
+    }
+
+    // squeeze_ (with no dimension argument)
+    {
+      auto t2 = T.ones(*s);
+      std::vector<int64_t> size_without_ones;
+      for (auto size : *s) {
+        if (size != 1) {
+          size_without_ones.push_back(size);
+        }
+      }
+      auto r = t2.squeeze_();
+      assert_equal_size_dim(t2, T.ones(size_without_ones));
+    }
+
+    // reduce (with dimension argument and with 1 return argument)
+    if (t.dim() > 0 && t.numel() != 0) {
+      assert(t.sum(0).dim() == t.dim() - 1);
+    } else if (t.dim() == 0) {
+      try {
+        t.sum(0);
+        assert(false);
+      } catch (std::runtime_error &e) {}
+    } else {
+      // FIXME: you should be able to reduce over size {0}
+      try {
+        t.sum(0);
+        assert(false);
+      } catch (std::runtime_error &e) {}
+    }
+
+    // reduce (with dimension argument and with 2 return arguments)
+    if (t.dim() > 0 && t.numel() != 0) {
+      auto ret = t.min(0);
+      assert(std::get<0>(ret).dim() == t.dim() - 1);
+      assert(std::get<1>(ret).dim() == t.dim() - 1);
+    } else if (t.dim() == 0) {
+      try {
+        t.sum(0);
+        assert(false);
+      } catch (std::runtime_error &e) {}
+    } else {
+      // FIXME: you should be able to reduce over size {0}
+      try {
+        t.sum(0);
+        assert(false);
+      } catch (std::runtime_error &e) {}
+    }
+
+    // simple indexing
+    if (t.dim() > 0 && t.numel() != 0) {
+      assert(t[0].dim() == std::max<int64_t>(t.dim() - 1, 0));
+    } else if (t.dim() == 0) {
+      try {
+        t[0];
+        assert(false);
+      } catch (std::runtime_error &e) {}
+    }
+  }
+
+  for (auto lhs_it = sizes.begin(); lhs_it != sizes.end(); ++lhs_it) {
+    for (auto rhs_it = sizes.begin(); rhs_it != sizes.end(); ++rhs_it) {
+      // is_same_size should only match if they are the same shape
+      {
+          auto lhs = T.ones(*lhs_it);
+          auto rhs = T.ones(*rhs_it);
+          if(*lhs_it != *rhs_it) {
+            assert(!lhs.is_same_size(rhs));
+            assert(!rhs.is_same_size(lhs));
+          }
+      }
+      // forced size functions (resize_, resize_as, set_)
+      {
+        // resize_
+        {
+          auto lhs = T.ones(*lhs_it);
+          auto rhs = T.ones(*rhs_it);
+          lhs.resize_(*rhs_it);
+          assert_equal_size_dim(lhs, rhs);
+        }
+        // resize_as_
+        {
+          auto lhs = T.ones(*lhs_it);
+          auto rhs = T.ones(*rhs_it);
+          lhs.resize_as_(rhs);
+          assert_equal_size_dim(lhs, rhs);
+        }
+        // set_
+        {
+          {
+            // with tensor
+            auto lhs = T.ones(*lhs_it);
+            auto rhs = T.ones(*rhs_it);
+            lhs.set_(rhs);
+            assert_equal_size_dim(lhs, rhs);
+          }
+          {
+            // with storage
+            auto lhs = T.ones(*lhs_it);
+            auto rhs = T.ones(*rhs_it);
+            auto storage = T.storage(rhs.numel());
+            lhs.set_(*storage);
+            // should not be dim 0 because an empty storage is dim 1; all other storages aren't scalars
+            assert(lhs.dim() != 0);
+          }
+          {
+            // with storage, offset, sizes, strides
+            auto lhs = T.ones(*lhs_it);
+            auto rhs = T.ones(*rhs_it);
+            auto storage = T.storage(rhs.numel());
+            lhs.set_(*storage, rhs.storage_offset(), rhs.sizes(), rhs.strides());
+            assert_equal_size_dim(lhs, rhs);
+          }
+        }
+
+        // assign_
+        {
+          auto lhs = T.ones(*lhs_it);
+          auto lhs_save = T.ones(*lhs_it);
+          auto rhs = T.ones(*rhs_it);
+          try {
+            lhs.assign_(rhs);
+            assert(lhs_save.numel() == rhs.numel());
+            // ensure didn't change shape
+            assert_equal_size_dim(lhs, lhs_save);
+          } catch (std::runtime_error &e) {
+            assert(lhs_save.numel() != rhs.numel());
+          }
+        }
+      }
+
+      // view
+      {
+        auto lhs = T.ones(*lhs_it);
+        auto rhs = T.ones(*rhs_it);
+        auto rhs_size = *rhs_it;
+        try {
+          auto result = lhs.view(rhs_size);
+          assert(lhs.numel() == rhs.numel());
+          assert_equal_size_dim(result, rhs);
+        } catch (std::runtime_error &e) {
+          assert(lhs.numel() != rhs.numel());
+        }
+      }
+
+      // expand
+      {
+        auto lhs = T.ones(*lhs_it);
+        auto lhs_size = *lhs_it;
+        auto rhs = T.ones(*rhs_it);
+        auto rhs_size = *rhs_it;
+        bool should_pass = should_expand(lhs_size, rhs_size);
+        try {
+          auto result = lhs.expand(rhs_size);
+          assert(should_pass);
+          assert_equal_size_dim(result, rhs);
+        } catch (std::runtime_error &e) {
+          assert(!should_pass);
+        }
+
+        // in-place functions (would be good if we can also do a non-broadcasting one, b/c
+        // broadcasting functions will always end up operating on tensors of same size;
+        // is there an example of this outside of assign_ ?)
+        {
+          bool should_pass_inplace = should_expand(rhs_size, lhs_size);
+          try {
+            lhs.add_(rhs);
+            assert(should_pass_inplace);
+            assert_equal_size_dim(lhs, T.ones(*lhs_it));
+          } catch (std::runtime_error &e) {
+            assert(!should_pass_inplace);
+          }
+        }
+      }
+    }
+  }
+
+  return 0;
+}

torch/lib/ATen/ArrayRef.h

@@ -139,6 +139,12 @@ namespace at {
       return Data[Index];
     }
 
+    /// Vector compatibility
+    const T &at(size_t Index) const {
+      assert(Index < Length && "Invalid index!");
+      return Data[Index];
+    }
+
     /// Disallow accidental assignment from a temporary.
     ///
     /// The declaration here is extra complicated so that "arrayRef = {}"

torch/lib/ATen/Context.cpp

@@ -55,6 +55,11 @@ Context & globalContext() {
 
 bool Context::hasCUDA() const {
 #ifdef AT_CUDA_ENABLED
+  int count;
+  cudaError_t err = cudaGetDeviceCount(&count);
+  if (err == cudaErrorInsufficientDriver) {
+    return false;
+  }
   return true;
 #else
   return false;