1 /* -*- mesa-c++ -*- 2 * 3 * Copyright (c) 2021 Collabora LTD 4 * 5 * Author: Gert Wollny <gert.wollny@collabora.com> 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * on the rights to use, copy, modify, merge, publish, distribute, sub 11 * license, and/or sell copies of the Software, and to permit persons to whom 12 * the Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 */ 26 27 #pragma once 28 29 #include "sfn_alu_defines.h" 30 #include "sfn_defines.h" 31 #include "sfn_virtualvalues.h" 32 33 #include <iostream> 34 #include <list> 35 #include <set> 36 37 namespace r600 { 38 39 class ConstInstrVisitor; 40 41 class InstrVisitor; 42 class AluInstr; 43 class AluGroup; 44 class TexInstr; 45 class ExportInstr; 46 class FetchInstr; 47 class ControlFlowInstr; 48 class IfInstr; 49 class ScratchIOInstr; 50 class StreamOutInstr; 51 class MemRingOutInstr; 52 class EmitVertexInstr; 53 class GDSInstr; 54 class WriteTFInstr; 55 class LDSAtomicInstr; 56 class LDSReadInstr; 57 class RatInstr; 58 59 bool 60 int_from_string_with_prefix_optional(const std::string& str, 61 const std::string& prefix, 62 int& value); 63 int 64 int_from_string_with_prefix(const std::string& str, const std::string& prefix); 65 int 66 sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle& swz, bool& is_ssa); 67 68 class Instr : public Allocate { 69 public: 70 enum Flags { 71 always_keep, 72 dead, 73 scheduled, 74 vpm, 75 force_cf, 76 ack_rat_return_write, 77 helper, 78 no_lds_or_addr_group, 79 nflags 80 }; 81 82 Instr(); 83 84 Instr(const Instr& orig) = default; 85 86 virtual ~Instr(); 87 88 using Pointer = R600_POINTER_TYPE(Instr); 89 90 void print(std::ostream& os) const; 91 bool equal_to(const Instr& lhs) const; 92 93 virtual void accept(ConstInstrVisitor& visitor) const = 0; 94 virtual void accept(InstrVisitor& visitor) = 0; end_group()95 virtual bool end_group() const { return true; } 96 97 virtual bool is_last() const; 98 set_always_keep()99 void set_always_keep() { m_instr_flags.set(always_keep); } 100 bool set_dead(); set_scheduled()101 virtual void set_scheduled() 102 { 103 m_instr_flags.set(scheduled); 104 forward_set_scheduled(); 105 } is_dead()106 bool is_dead() const { return m_instr_flags.test(dead); } is_scheduled()107 bool is_scheduled() const { return m_instr_flags.test(scheduled); } keep()108 bool keep() const { return m_instr_flags.test(always_keep); } can_start_alu_block()109 bool can_start_alu_block() { return m_instr_flags.test(no_lds_or_addr_group);} group_force_alu_cf()110 bool group_force_alu_cf() { return m_instr_flags.test(force_cf);} 111 has_instr_flag(Flags f)112 bool has_instr_flag(Flags f) const { return m_instr_flags.test(f); } set_instr_flag(Flags f)113 void set_instr_flag(Flags f) { m_instr_flags.set(f); } 114 115 virtual bool replace_source(PRegister old_src, PVirtualValue new_src); 116 virtual bool replace_dest(PRegister new_dest, AluInstr *move_instr); 117 nesting_corr()118 virtual int nesting_corr() const { return 0; } 119 end_block()120 virtual bool end_block() const { return false; } nesting_offset()121 virtual int nesting_offset() const { return 0; } 122 123 void set_blockid(int id, int index); block_id()124 int block_id() const { return m_block_id; } index()125 int index() const { return m_index; } 126 127 void add_required_instr(Instr *instr); 128 void replace_required_instr(Instr *old_instr, Instr *new_instr); 129 130 bool ready() const; 131 slots()132 virtual uint32_t slots() const { return 0; }; 133 134 using InstrList = std::list<Instr *, Allocator<Instr *>>; 135 dependend_instr()136 const InstrList& dependend_instr() { return m_dependend_instr; } 137 as_alu()138 virtual AluInstr *as_alu() { return nullptr; } allowed_src_chan_mask()139 virtual uint8_t allowed_src_chan_mask() const { return 0; } 140 update_indirect_addr(PRegister old_reg,PRegister addr)141 virtual void update_indirect_addr(PRegister old_reg, PRegister addr) { 142 (void)old_reg; 143 (void)addr; 144 unreachable("Instruction type has no indirect addess"); 145 }; required_instr()146 const InstrList& required_instr() const { return m_required_instr; } 147 as_alu_group()148 virtual AluGroup *as_alu_group() { return nullptr;} 149 150 protected: 151 152 153 private: 154 virtual void forward_set_blockid(int id, int index); 155 156 virtual bool do_ready() const = 0; 157 158 virtual void do_print(std::ostream& os) const = 0; 159 virtual bool propagate_death(); forward_set_scheduled()160 virtual void forward_set_scheduled() {} 161 162 InstrList m_required_instr; 163 InstrList m_dependend_instr; 164 165 int m_use_count; 166 int m_block_id; 167 int m_index; 168 std::bitset<nflags> m_instr_flags{0}; 169 }; 170 using PInst = Instr::Pointer; 171 172 class Block : public Instr { 173 public: 174 enum Type { 175 cf, 176 alu, 177 tex, 178 vtx, 179 gds, 180 unknown 181 }; 182 183 using Instructions = std::list<Instr *, Allocator<Instr *>>; 184 using Pointer = R600_POINTER_TYPE(Block); 185 using iterator = Instructions::iterator; 186 using reverse_iterator = Instructions::reverse_iterator; 187 using const_iterator = Instructions::const_iterator; 188 189 Block(int nesting_depth, int id); 190 Block(const Block& orig) = delete; 191 192 void push_back(PInst instr); begin()193 iterator begin() { return m_instructions.begin(); } end()194 iterator end() { return m_instructions.end(); } rbegin()195 reverse_iterator rbegin() { return m_instructions.rbegin(); } rend()196 reverse_iterator rend() { return m_instructions.rend(); } 197 begin()198 const_iterator begin() const { return m_instructions.begin(); } end()199 const_iterator end() const { return m_instructions.end(); } 200 empty()201 bool empty() const { return m_instructions.empty(); } 202 203 void erase(iterator node); 204 205 iterator insert(const iterator pos, Instr *instr); 206 207 bool is_equal_to(const Block& lhs) const; 208 209 void accept(ConstInstrVisitor& visitor) const override; 210 void accept(InstrVisitor& visitor) override; 211 nesting_depth()212 int nesting_depth() const { return m_nesting_depth; } 213 id()214 int id() const { return m_id; } 215 type()216 auto type() const {return m_block_type; } 217 void set_type(Type t, r600_chip_class chip_class); remaining_slots()218 int32_t remaining_slots() const { return m_remaining_slots;} 219 220 bool try_reserve_kcache(const AluGroup& instr); 221 bool try_reserve_kcache(const AluInstr& group); 222 last_lds_instr()223 auto last_lds_instr() { return m_last_lds_instr; } set_last_lds_instr(Instr * instr)224 void set_last_lds_instr(Instr *instr) { m_last_lds_instr = instr; } 225 226 void lds_group_start(AluInstr *alu); 227 void lds_group_end(); lds_group_active()228 bool lds_group_active() { return m_lds_group_start != nullptr; } 229 size()230 size_t size() const { return m_instructions.size(); } 231 kcache_reservation_failed()232 bool kcache_reservation_failed() const { return m_kcache_alloc_failed; } 233 inc_rat_emitted()234 int inc_rat_emitted() { return ++m_emitted_rat_instr; } 235 set_expected_ar_uses(uint32_t n)236 void set_expected_ar_uses(uint32_t n) {m_expected_ar_uses = n;} expected_ar_uses()237 auto expected_ar_uses() const {return m_expected_ar_uses;} dec_expected_ar_uses()238 void dec_expected_ar_uses() { 239 assert(m_expected_ar_uses > 0); 240 --m_expected_ar_uses; 241 } 242 243 static void set_chipclass(r600_chip_class chip_class); 244 245 private: 246 bool try_reserve_kcache(const UniformValue& u, 247 std::array<KCacheLine, 4>& kcache) const; 248 do_ready()249 bool do_ready() const override { return true; }; 250 void do_print(std::ostream& os) const override; 251 Instructions m_instructions; 252 int m_nesting_depth; 253 int m_id; 254 int m_next_index; 255 256 Type m_block_type{unknown}; 257 uint32_t m_remaining_slots{0xffff}; 258 259 std::array<KCacheLine, 4> m_kcache; 260 bool m_kcache_alloc_failed{false}; 261 262 Instr *m_last_lds_instr{nullptr}; 263 264 int m_lds_group_requirement{0}; 265 AluInstr *m_lds_group_start{nullptr}; 266 static unsigned s_max_kcache_banks; 267 int m_emitted_rat_instr{0}; 268 uint32_t m_expected_ar_uses{0}; 269 }; 270 271 class Resource { 272 public: Resource(Instr * user,int base,PRegister offset)273 Resource(Instr *user, int base, PRegister offset): 274 m_base(base), 275 m_offset(offset), 276 m_user(user) 277 { 278 if (m_offset) { 279 m_offset->add_use(m_user); 280 } 281 } replace_resource_offset(PRegister old_offset,PRegister new_offset)282 bool replace_resource_offset(PRegister old_offset, PRegister new_offset) 283 { 284 if (m_offset && old_offset->equal_to(*m_offset)) { 285 m_offset->del_use(m_user); 286 m_offset = new_offset; 287 m_offset->add_use(m_user); 288 return true; 289 } 290 return false; 291 } set_resource_offset(PRegister offset)292 void set_resource_offset(PRegister offset) 293 { 294 if (m_offset) 295 m_offset->del_use(m_user); 296 m_offset = offset; 297 if (m_offset) { 298 m_offset->add_use(m_user); 299 } 300 } 301 resource_is_equal(const Resource & other)302 bool resource_is_equal(const Resource& other) const 303 { 304 if (m_base != other.m_base) 305 return false; 306 if (m_offset && other.m_offset) 307 return m_offset->equal_to(*other.m_offset); 308 return !m_offset && !other.m_offset; 309 } 310 resource_id()311 auto resource_id() const { return m_base; } 312 resource_offset()313 auto resource_offset() const { return m_offset; } 314 315 auto resource_index_mode() const -> EBufferIndexMode 316 { 317 if (!m_offset || !m_offset->has_flag(Register::addr_or_idx)) 318 return bim_none; 319 320 switch (m_offset->sel()) { 321 case 1: 322 return bim_zero; 323 case 2: 324 return bim_one; 325 default: 326 unreachable("Invalid resource offset, scheduler must substitute registers"); 327 } 328 } 329 resource_ready(int block_id,int index)330 bool resource_ready(int block_id, int index) const 331 { 332 return !m_offset || m_offset->ready(block_id, index); 333 } 334 335 protected: print_resource_offset(std::ostream & os)336 void print_resource_offset(std::ostream& os) const 337 { 338 if (m_offset) 339 os << " + " << *m_offset; 340 } 341 342 private: 343 int m_base{0}; 344 PRegister m_offset{nullptr}; 345 Instr *m_user; 346 }; 347 348 class InstrWithVectorResult : public Instr, public Resource { 349 public: 350 InstrWithVectorResult(const RegisterVec4& dest, 351 const RegisterVec4::Swizzle& dest_swizzle, 352 int resource_base, 353 PRegister resource_offset); 354 set_dest_swizzle(const RegisterVec4::Swizzle & swz)355 void set_dest_swizzle(const RegisterVec4::Swizzle& swz) { m_dest_swizzle = swz; } dest_swizzle(int i)356 int dest_swizzle(int i) const { return m_dest_swizzle[i]; } all_dest_swizzle()357 const RegisterVec4::Swizzle& all_dest_swizzle() const { return m_dest_swizzle; } dst()358 const RegisterVec4& dst() const { return m_dest; } 359 360 void update_indirect_addr(PRegister old_reg, PRegister addr) override; 361 362 protected: 363 InstrWithVectorResult(const InstrWithVectorResult& orig); 364 365 void print_dest(std::ostream& os) const; 366 bool comp_dest(const RegisterVec4& dest, 367 const RegisterVec4::Swizzle& dest_swizzle) const; 368 369 private: 370 RegisterVec4 m_dest; 371 RegisterVec4::Swizzle m_dest_swizzle; 372 }; 373 374 inline bool 375 operator==(const Instr& lhs, const Instr& rhs) 376 { 377 return lhs.equal_to(rhs); 378 } 379 380 inline bool 381 operator!=(const Instr& lhs, const Instr& rhs) 382 { 383 return !(lhs == rhs); 384 } 385 386 inline std::ostream& 387 operator<<(std::ostream& os, const Instr& instr) 388 { 389 instr.print(os); 390 return os; 391 } 392 393 template <typename T, typename = std::enable_if_t<std::is_base_of_v<Instr, T>>> 394 std::ostream& 395 operator<<(std::ostream& os, const T& instr) 396 { 397 instr.print(os); 398 return os; 399 } 400 401 class ConstInstrVisitor { 402 public: 403 virtual void visit(const AluInstr& instr) = 0; 404 virtual void visit(const AluGroup& instr) = 0; 405 virtual void visit(const TexInstr& instr) = 0; 406 virtual void visit(const ExportInstr& instr) = 0; 407 virtual void visit(const FetchInstr& instr) = 0; 408 virtual void visit(const Block& instr) = 0; 409 virtual void visit(const ControlFlowInstr& instr) = 0; 410 virtual void visit(const IfInstr& instr) = 0; 411 virtual void visit(const ScratchIOInstr& instr) = 0; 412 virtual void visit(const StreamOutInstr& instr) = 0; 413 virtual void visit(const MemRingOutInstr& instr) = 0; 414 virtual void visit(const EmitVertexInstr& instr) = 0; 415 virtual void visit(const GDSInstr& instr) = 0; 416 virtual void visit(const WriteTFInstr& instr) = 0; 417 virtual void visit(const LDSAtomicInstr& instr) = 0; 418 virtual void visit(const LDSReadInstr& instr) = 0; 419 virtual void visit(const RatInstr& instr) = 0; 420 }; 421 422 class InstrVisitor { 423 public: 424 virtual void visit(AluInstr *instr) = 0; 425 virtual void visit(AluGroup *instr) = 0; 426 virtual void visit(TexInstr *instr) = 0; 427 virtual void visit(ExportInstr *instr) = 0; 428 virtual void visit(FetchInstr *instr) = 0; 429 virtual void visit(Block *instr) = 0; 430 virtual void visit(ControlFlowInstr *instr) = 0; 431 virtual void visit(IfInstr *instr) = 0; 432 virtual void visit(ScratchIOInstr *instr) = 0; 433 virtual void visit(StreamOutInstr *instr) = 0; 434 virtual void visit(MemRingOutInstr *instr) = 0; 435 virtual void visit(EmitVertexInstr *instr) = 0; 436 virtual void visit(GDSInstr *instr) = 0; 437 virtual void visit(WriteTFInstr *instr) = 0; 438 virtual void visit(LDSAtomicInstr *instr) = 0; 439 virtual void visit(LDSReadInstr *instr) = 0; 440 virtual void visit(RatInstr *instr) = 0; 441 }; 442 443 } // namespace r600 444