#pragma once #include namespace c10d { class FakeWork : public Work { public: bool wait(std::chrono::milliseconds timeout) override { return true; } c10::intrusive_ptr getFuture() override { auto fut = c10::make_intrusive(c10::NoneType::get()); fut->markCompleted(); return fut; } }; class FakeProcessGroup : public Backend { public: FakeProcessGroup(int rank, int size) : Backend(rank, size) {} c10::intrusive_ptr broadcast( std::vector& /* tensors */, const BroadcastOptions& /* opts */ = BroadcastOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr allreduce( std::vector& /* tensors */, const AllreduceOptions& /* opts */ = AllreduceOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr allreduce_sparse( std::vector& /* tensors */, const AllreduceOptions& /* opts */ = AllreduceOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr allreduce_coalesced( std::vector& /* tensors */, const AllreduceCoalescedOptions& /* opts */ = AllreduceCoalescedOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr reduce( std::vector& /* tensors */, const ReduceOptions& /* opts */ = ReduceOptions()) override { return c10::make_intrusive(); } // NOTE [allgather on FakeProcessGroup] // Assume each rank have the same input tensor so we just copy to the results // since it's not a real allgather, we simply make this copying logic to let // some simple validation works (i.e. calling allgather to see if each rank // have the same tensor or not). // // NOTE: in general it's not good form to try to make FakeProcessGroup work // with real data, but the reasoning here is that we want FakeProcessGroup to // work with DeviceMesh's init code that have the data validation, which // makes it worth the tradeoff. c10::intrusive_ptr allgather( std::vector>& outputTensors, std::vector& inputTensors, const AllgatherOptions& /* opts */ = AllgatherOptions()) override { for (auto& tensor : outputTensors[0]) { tensor.copy_(inputTensors[0]); } return c10::make_intrusive(); } c10::intrusive_ptr _allgather_base( at::Tensor& outputBuffer, at::Tensor& inputBuffer, const AllgatherOptions& /* opts */ = AllgatherOptions()) override { auto chunks = outputBuffer.chunk(size_); for (auto& tensor : chunks) { tensor.copy_(inputBuffer); } return c10::make_intrusive(); } c10::intrusive_ptr allgather_coalesced( std::vector>& /* outputTensorLists */, std::vector& /* inputTensors */, const AllgatherOptions& /* opts */ = AllgatherOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr allgather_into_tensor_coalesced( std::vector& outputs, std::vector& inputs, const AllgatherOptions& /* opts */ = AllgatherOptions()) override { for (size_t i = 0; i < outputs.size(); ++i) { auto chunks = outputs[i].chunk(size_); for (auto& chunk : chunks) { chunk.copy_(inputs[i]); } } return c10::make_intrusive(); } c10::intrusive_ptr gather( std::vector>& /* outputTensors */, std::vector& /* inputTensors */, const GatherOptions& /* opts */ = GatherOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr scatter( std::vector& /* outputTensors */, std::vector>& /* inputTensors */, const ScatterOptions& /* opts */ = ScatterOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr reduce_scatter( std::vector& /* outputTensors */, std::vector>& /* inputTensors */, const ReduceScatterOptions& /* opts */ = ReduceScatterOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr _reduce_scatter_base( at::Tensor& /* outputBuffer */, at::Tensor& /* inputBuffer */, const ReduceScatterOptions& /* opts */ = ReduceScatterOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr reduce_scatter_tensor_coalesced( std::vector& /* outputs */, std::vector& /* inputs */, const ReduceScatterOptions& /* opts */ = ReduceScatterOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr alltoall_base( at::Tensor& /* outputBuffer */, at::Tensor& /* inputBuffer */, std::vector& /* outputSplitSizes */, std::vector& /* inputSplitSizes */, const AllToAllOptions& /* opts */ = AllToAllOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr alltoall( std::vector& /* outputTensors */, std::vector& /* inputTensors */, const AllToAllOptions& opts = AllToAllOptions()) override { return c10::make_intrusive(); } c10::intrusive_ptr send( std::vector& /* tensors */, int /* dstRank */, int /* tag */) override { return c10::make_intrusive(); } c10::intrusive_ptr recv( std::vector& /* tensors */, int /* srcRank */, int /* tag */) override { return c10::make_intrusive(); } c10::intrusive_ptr recvAnysource( std::vector& /* tensors */, int /* tag */) override { return c10::make_intrusive(); } c10::intrusive_ptr barrier( const BarrierOptions& /* opts */ = BarrierOptions()) override { return c10::make_intrusive(); } }; } // namespace c10d