#pragma once #include #include #include #include "torch/csrc/jit/tensorexpr/eval.h" namespace torch { namespace jit { namespace tensorexpr { template struct DefaultPaddedValue; template <> struct DefaultPaddedValue { static const int kValue = static_cast(0xDEADBEEF); }; template <> struct DefaultPaddedValue { static const int8_t kValue = static_cast(0xBE); }; template <> struct DefaultPaddedValue { static const uint8_t kValue = static_cast(0xBE); }; template <> struct DefaultPaddedValue { static const int16_t kValue = static_cast(0xBEEF); }; template <> struct DefaultPaddedValue { static const int64_t kValue = static_cast(0xDEADBEEF); }; template <> struct DefaultPaddedValue { static constexpr float kValue = 0.1357; }; template <> struct DefaultPaddedValue { // at::Half ctor isn't constexpr, so just fill it with bits. static constexpr uint16_t kValue = 1357; }; template <> struct DefaultPaddedValue { static constexpr double kValue = 0.1357; }; // A concrete base to be used in PaddedBase. class PaddedBufferBase { public: const std::string& name() const { return name_; } int size() const { return total_size_; } int raw_size() const { return total_size_ + 2 * kPaddingSize; } virtual ~PaddedBufferBase() {} protected: explicit PaddedBufferBase( const std::vector& dims, const std::string& name); int Index(const std::vector& indices) const; std::vector dims_; std::string name_; std::vector strides_; int total_size_; // total number of useful element, does not include the // paddings static constexpr int kPaddingSize = 64; }; // A padded buffer with wartermarks for testing. // The buffer carries padded watermarks on both sides to catch potential // out-of-bounds writes. For read-only data that are not supposed to change, it // can also make a backup and be compared later. template class PaddedBuffer : public PaddedBufferBase { public: PaddedBuffer(int d0, const std::string& name = "") : PaddedBuffer(std::vector({d0}), name) {} PaddedBuffer(int d0, int d1, const std::string& name = "") : PaddedBuffer(std::vector({d0, d1}), name) {} PaddedBuffer(int d0, int d1, int d2, const std::string& name = "") : PaddedBuffer(std::vector({d0, d1, d2}), name) {} PaddedBuffer(int d0, int d1, int d2, int d3, const std::string& name = "") : PaddedBuffer(std::vector({d0, d1, d2, d3}), name) {} PaddedBuffer(const std::vector& dims, const std::string& name = "") : PaddedBufferBase(dims, name) { data_.resize(total_size_ + 2 * kPaddingSize, kPaddingValue); } PaddedBuffer(const PaddedBuffer& other, const std::string& name) : PaddedBuffer(other) { this->name_ = name; } T* data() { return data_.data() + kPaddingSize; } const T* data() const { return const_cast(this)->data(); } T* raw_data() { return data_.data(); } const T* raw_data() const { return const_cast(this)->raw_data(); } T& operator()(int i0) { // There is a bit performance impact with forming a vector here. But this // data structure is for testing only, and not performance critical. return this->operator()(std::vector({i0})); } const T& operator()(int i0) const { return const_cast(this)->operator()(i0); } T& operator()(int i0, int i1) { return this->operator()(std::vector({i0, i1})); } const T& operator()(int i0, int i1) const { return const_cast(this)->operator()(i0, i1); } T& operator()(int i0, int i1, int i2) { return this->operator()(std::vector({i0, i1, i2})); } const T& operator()(int i0, int i1, int i2) const { return const_cast(this)->operator()(i0, i1, i2); } T& operator()(int i0, int i1, int i2, int i3) { return this->operator()(std::vector({i0, i1, i2, i3})); } const T& operator()(int i0, int i1, int i2, int i3) const { return const_cast(this)->operator()(i0, i1, i2, i3); } T& operator()(const std::vector& indices) { return data_[kPaddingSize + Index(indices)]; } const T& operator()(const std::vector& indices) const { return const_cast(this)->operator()(indices); } template friend void ExpectAllNear( const PaddedBuffer& v1, const PaddedBuffer& v2, float abs_error); template friend void ExpectAllEqual( const PaddedBuffer& v1, const PaddedBuffer& v2); void Backup() { backup_data_ = data_; } // Verify the watermarks in the paddings are intact. void ValidateWatermark() const { for (const auto i : c10::irange(kPaddingSize)) { ASSERT_EQ(data_[i], kPaddingValue); ASSERT_EQ(data_[i + total_size_ + kPaddingSize], kPaddingValue); } } void CheckBackup() const { ValidateWatermark(); DCHECK(backup_data_.size() == data_.size()) << "Please make sure you have call Backup() before calling CheckBackup()"; for (const auto i : c10::irange(total_size_)) { ASSERT_EQ(data_[i + kPaddingSize], backup_data_[i + kPaddingSize]); } } private: std::vector data_; std::vector backup_data_; T kPaddingValue = DefaultPaddedValue::kValue; }; template inline CodeGen::CallArg::CallArg(const PaddedBuffer& buffer) : data_(const_cast(buffer.data())) {} template std::string CompareErrorMsg( const PaddedBuffer& v1, const PaddedBuffer& v2, int index) { std::ostringstream oss; oss << "index: " << index << ", v1: (" << v1.name() << ", " << v1(index) << ")" << ", v2: (" << v2.name() << ", " << v2(index) << ")"; return oss.str(); } template void ExpectAllEqual(const PaddedBuffer& f1, const PaddedBuffer& f2) { const std::vector& v1 = f1.data_; const std::vector& v2 = f2.data_; const int kPaddingSize = f1.kPaddingSize; const int total_size = f1.total_size_; ASSERT_EQ(v1.size(), v2.size()); f1.ValidateWatermark(); f2.ValidateWatermark(); for (const auto i : c10::irange(total_size)) { ASSERT_EQ(v1[kPaddingSize + i], v2[kPaddingSize + i]); } } template void ExpectAllNear( const PaddedBuffer& f1, const PaddedBuffer& f2, float abs_error) { const std::vector& v1 = f1.data_; const std::vector& v2 = f2.data_; const int kPaddingSize = f1.kPaddingSize; const int total_size = f1.total_size_; ASSERT_EQ(v1.size(), v2.size()); f1.ValidateWatermark(); f2.ValidateWatermark(); for (const auto i : c10::irange(total_size)) { ASSERT_NEAR(v1[kPaddingSize + i], v2[kPaddingSize + i], abs_error); } } } // namespace tensorexpr } // namespace jit } // namespace torch