1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DEVICE_MEMORY_ALLOCATOR_H_ 17 #define TENSORFLOW_COMPILER_XLA_SERVICE_DEVICE_MEMORY_ALLOCATOR_H_ 18 19 #include <vector> 20 21 #include "absl/types/span.h" 22 #include "tensorflow/compiler/xla/service/owning_device_memory.h" 23 #include "tensorflow/compiler/xla/statusor.h" 24 #include "tensorflow/compiler/xla/types.h" 25 #include "tensorflow/core/platform/stream_executor_no_cuda.h" 26 #include "tensorflow/core/platform/types.h" 27 28 namespace xla { 29 30 // Interface for device memory allocators used within the XLA service. An 31 // allocator is responsible for allocating memory on all devices of a particular 32 // platform. 33 class DeviceMemoryAllocator { 34 public: 35 // Parameter platform indicates which platform the allocator allocates memory 36 // on. Must be non-null. DeviceMemoryAllocator(const se::Platform * platform)37 explicit DeviceMemoryAllocator(const se::Platform* platform) 38 : platform_(platform) {} ~DeviceMemoryAllocator()39 virtual ~DeviceMemoryAllocator() {} 40 41 // Allocates memory on the device. 42 // 43 // If size > 0 and the returned StatusOr is OK, the wrapped OwningDeviceMemory 44 // must not be null. If size == 0, must return a null OwningDeviceMemory. 45 // 46 // 'retry_on_failure': If false, and the first attempt to allocate the memory 47 // fails, the allocation should return immediately without retrying. An 48 // example use case is optional scratch spaces where a failure has only 49 // performance impact. 50 virtual StatusOr<OwningDeviceMemory> Allocate(int device_ordinal, uint64 size, 51 bool retry_on_failure) = 0; 52 53 // Two-arg version of Allocate(), which sets retry-on-failure to true. 54 // 55 // (We don't simply use a default argument on the virtual Allocate function 56 // because default args on virtual functions are disallowed by the Google 57 // style guide.) Allocate(int device_ordinal,uint64 size)58 StatusOr<OwningDeviceMemory> Allocate(int device_ordinal, uint64 size) { 59 return Allocate(device_ordinal, size, /*retry_on_failure=*/true); 60 } 61 62 // Must be a nop for null pointers. 63 virtual Status Deallocate(int device_ordinal, se::DeviceMemoryBase mem) = 0; 64 65 // Return the platform that the allocator allocates memory on. platform()66 const se::Platform* platform() const { return platform_; } 67 68 // Can we call Deallocate() as soon as a computation has been scheduled on 69 // a stream, or do we have to wait for the computation to complete first? 70 virtual bool AllowsAsynchronousDeallocation() const = 0; 71 72 protected: 73 friend class OwningDeviceMemory; 74 const se::Platform* platform_; 75 }; 76 77 // Default memory allocator for a platform which uses 78 // StreamExecutor::Allocate/Deallocate. 79 class StreamExecutorMemoryAllocator : public DeviceMemoryAllocator { 80 public: 81 StreamExecutorMemoryAllocator( 82 const se::Platform* platform, 83 absl::Span<se::StreamExecutor* const> stream_executors); 84 85 StatusOr<OwningDeviceMemory> Allocate(int device_ordinal, uint64 size, 86 bool retry_on_failure) override; 87 88 // Pull in two-arg overload that sets retry_on_failure to true. 89 using DeviceMemoryAllocator::Allocate; 90 91 Status Deallocate(int device_ordinal, se::DeviceMemoryBase mem) override; 92 93 bool AllowsAsynchronousDeallocation() const override; 94 95 private: 96 StatusOr<se::StreamExecutor*> GetStreamExecutor(int device_ordinal); 97 98 // A vector indexed by device ordinal of StreamExecutors for each device of 99 // the allocator's platform type. If an element is nullptr, then the device 100 // with the respective device ordinal is not supported by XLA. 101 std::vector<se::StreamExecutor*> stream_executors_; 102 }; 103 104 } // namespace xla 105 106 #endif // TENSORFLOW_COMPILER_XLA_SERVICE_DEVICE_MEMORY_ALLOCATOR_H_ 107