392 lower_load(nir_intrinsic_instr *intrin, struct lower_io_state *state,  in lower_load()  argument
396    assert(intrin->dest.is_ssa);  in lower_load()
397    if (intrin->dest.ssa.bit_size == 64 &&  in lower_load()
406       while (dest_comp < intrin->dest.ssa.num_components) {  in lower_load()
408             MIN2(intrin->dest.ssa.num_components - dest_comp,  in lower_load()
425       return nir_vec(b, comp64, intrin->dest.ssa.num_components);  in lower_load()
426    } else if (intrin->dest.ssa.bit_size == 1) {  in lower_load()
431                                 intrin->dest.ssa.num_components, 32,  in lower_load()
435                        intrin->dest.ssa.num_components,  in lower_load()
436                        intrin->dest.ssa.bit_size,  in lower_load()
504 lower_store(nir_intrinsic_instr *intrin, struct lower_io_state *state,  in lower_store()  argument
508    assert(intrin->src[1].is_ssa);  in lower_store()
509    if (intrin->src[1].ssa->bit_size == 64 &&  in lower_store()
517       nir_component_mask_t write_mask = nir_intrinsic_write_mask(intrin);  in lower_store()
518       while (src_comp < intrin->num_components) {  in lower_store()
520             MIN2(intrin->num_components - src_comp,  in lower_store()
525                nir_channels(b, intrin->src[1].ssa,  in lower_store()
546    } else if (intrin->dest.ssa.bit_size == 1) {  in lower_store()
549       nir_ssa_def *b32_val = nir_b2b32(&state->builder, intrin->src[1].ssa);  in lower_store()
551                  component, intrin->num_components,  in lower_store()
552                  nir_intrinsic_write_mask(intrin),  in lower_store()
555       emit_store(state, intrin->src[1].ssa, array_index, var, offset,  in lower_store()
556                  component, intrin->num_components,  in lower_store()
557                  nir_intrinsic_write_mask(intrin),  in lower_store()
563 lower_interpolate_at(nir_intrinsic_instr *intrin, struct lower_io_state *state,  in lower_interpolate_at()  argument
578          assert(intrin->intrinsic == nir_intrinsic_interp_deref_at_vertex);  in lower_interpolate_at()
579          vertex_index = intrin->src[1].ssa;  in lower_interpolate_at()
582       return lower_load(intrin, state, vertex_index, var, offset, component, type);  in lower_interpolate_at()
586    assert(intrin->dest.is_ssa && intrin->dest.ssa.bit_size <= 32);  in lower_interpolate_at()
589    switch (intrin->intrinsic) {  in lower_interpolate_at()
611    if (intrin->intrinsic == nir_intrinsic_interp_deref_at_sample ||  in lower_interpolate_at()
612        intrin->intrinsic == nir_intrinsic_interp_deref_at_offset ||  in lower_interpolate_at()
613        intrin->intrinsic == nir_intrinsic_interp_deref_at_vertex)  in lower_interpolate_at()
614       nir_src_copy(&bary_setup->src[0], &intrin->src[1]);  in lower_interpolate_at()
625    assert(intrin->dest.is_ssa);  in lower_interpolate_at()
628                                   intrin->dest.ssa.num_components,  in lower_interpolate_at()
629                                   intrin->dest.ssa.bit_size,  in lower_interpolate_at()
651       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);  in nir_lower_io_block()  local
653       switch (intrin->intrinsic) {  in nir_lower_io_block()
672       nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in nir_lower_io_block()
707          if (intrin->intrinsic != nir_intrinsic_store_deref) {  in nir_lower_io_block()
709                nir_imm_zero(b, intrin->dest.ssa.num_components,  in nir_lower_io_block()
710                              intrin->dest.ssa.bit_size);  in nir_lower_io_block()
711             nir_ssa_def_rewrite_uses(&intrin->dest.ssa,  in nir_lower_io_block()
715          nir_instr_remove(&intrin->instr);  in nir_lower_io_block()
726       switch (intrin->intrinsic) {  in nir_lower_io_block()
728          replacement = lower_load(intrin, state, array_index, var, offset,  in nir_lower_io_block()
733          lower_store(intrin, state, array_index, var, offset,  in nir_lower_io_block()
742          replacement = lower_interpolate_at(intrin, state, var, offset,  in nir_lower_io_block()
751          nir_ssa_def_rewrite_uses(&intrin->dest.ssa,  in nir_lower_io_block()
754       nir_instr_remove(&intrin->instr);  in nir_lower_io_block()
1330 build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin,  in build_explicit_io_load()  argument
1336    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in build_explicit_io_load()
1341          return build_explicit_io_load(b, intrin, addr, addr_format,  in build_explicit_io_load()
1349             build_explicit_io_load(b, intrin, addr, addr_format,  in build_explicit_io_load()
1355             build_explicit_io_load(b, intrin, addr, addr_format,  in build_explicit_io_load()
1366             build_explicit_io_load(b, intrin, addr, addr_format,  in build_explicit_io_load()
1373             build_explicit_io_load(b, intrin, addr, addr_format,  in build_explicit_io_load()
1386    switch (intrin->intrinsic) {  in build_explicit_io_load()
1495       nir_intrinsic_set_access(load, nir_intrinsic_access(intrin));  in build_explicit_io_load()
1509    unsigned bit_size = intrin->dest.ssa.bit_size;  in build_explicit_io_load()
1525    assert(intrin->dest.is_ssa);  in build_explicit_io_load()
1558    if (intrin->dest.ssa.bit_size == 1) {  in build_explicit_io_load()
1575 build_explicit_io_store(nir_builder *b, nir_intrinsic_instr *intrin,  in build_explicit_io_store()  argument
1585          build_explicit_io_store(b, intrin, addr, addr_format,  in build_explicit_io_store()
1592          build_explicit_io_store(b, intrin, addr, addr_format,  in build_explicit_io_store()
1597          build_explicit_io_store(b, intrin, addr, addr_format,  in build_explicit_io_store()
1606          build_explicit_io_store(b, intrin, addr, addr_format,  in build_explicit_io_store()
1612          build_explicit_io_store(b, intrin, addr, addr_format,  in build_explicit_io_store()
1625    switch (intrin->intrinsic) {  in build_explicit_io_store()
1718       nir_intrinsic_set_access(store, nir_intrinsic_access(intrin));  in build_explicit_io_store()
1723           value->num_components == intrin->num_components);  in build_explicit_io_store()
1742 build_explicit_io_atomic(nir_builder *b, nir_intrinsic_instr *intrin,  in build_explicit_io_atomic()  argument
1750          return build_explicit_io_atomic(b, intrin, addr, addr_format,  in build_explicit_io_atomic()
1756             build_explicit_io_atomic(b, intrin, addr, addr_format,  in build_explicit_io_atomic()
1760             build_explicit_io_atomic(b, intrin, addr, addr_format,  in build_explicit_io_atomic()
1769             build_explicit_io_atomic(b, intrin, addr, addr_format,  in build_explicit_io_atomic()
1774             build_explicit_io_atomic(b, intrin, addr, addr_format,  in build_explicit_io_atomic()
1785       nir_intrinsic_infos[intrin->intrinsic].num_srcs - 1;  in build_explicit_io_atomic()
1791          op = global_atomic_for_deref(addr_format, intrin->intrinsic);  in build_explicit_io_atomic()
1793          op = ssbo_atomic_for_deref(intrin->intrinsic);  in build_explicit_io_atomic()
1797       op = global_atomic_for_deref(addr_format, intrin->intrinsic);  in build_explicit_io_atomic()
1801       op = shared_atomic_for_deref(intrin->intrinsic);  in build_explicit_io_atomic()
1805       op = task_payload_atomic_for_deref(intrin->intrinsic);  in build_explicit_io_atomic()
1824       atomic->src[src++] = nir_src_for_ssa(intrin->src[1 + i].ssa);  in build_explicit_io_atomic()
1831       nir_intrinsic_set_access(atomic, nir_intrinsic_access(intrin));  in build_explicit_io_atomic()
1833    assert(intrin->dest.ssa.num_components == 1);  in build_explicit_io_atomic()
1835                      1, intrin->dest.ssa.bit_size, NULL);  in build_explicit_io_atomic()
1913                             nir_intrinsic_instr *intrin,  in nir_lower_explicit_io_instr()  argument
1917    b->cursor = nir_after_instr(&intrin->instr);  in nir_lower_explicit_io_instr()
1919    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in nir_lower_explicit_io_instr()
1932    switch (intrin->intrinsic) {  in nir_lower_explicit_io_instr()
1937          for (unsigned i = 0; i < intrin->num_components; i++) {  in nir_lower_explicit_io_instr()
1942             comps[i] = build_explicit_io_load(b, intrin, comp_addr,  in nir_lower_explicit_io_instr()
1949          value = nir_vec(b, comps, intrin->num_components);  in nir_lower_explicit_io_instr()
1951          value = build_explicit_io_load(b, intrin, addr, addr_format,  in nir_lower_explicit_io_instr()
1953                                         intrin->num_components);  in nir_lower_explicit_io_instr()
1955       nir_ssa_def_rewrite_uses(&intrin->dest.ssa, value);  in nir_lower_explicit_io_instr()
1960       assert(intrin->src[1].is_ssa);  in nir_lower_explicit_io_instr()
1961       nir_ssa_def *value = intrin->src[1].ssa;  in nir_lower_explicit_io_instr()
1962       nir_component_mask_t write_mask = nir_intrinsic_write_mask(intrin);  in nir_lower_explicit_io_instr()
1964          for (unsigned i = 0; i < intrin->num_components; i++) {  in nir_lower_explicit_io_instr()
1972             build_explicit_io_store(b, intrin, comp_addr, addr_format,  in nir_lower_explicit_io_instr()
1978          build_explicit_io_store(b, intrin, addr, addr_format,  in nir_lower_explicit_io_instr()
1986       nir_ssa_def *value = build_explicit_io_load(b, intrin, addr, addr_format,  in nir_lower_explicit_io_instr()
1989                                                   intrin->num_components);  in nir_lower_explicit_io_instr()
1990       nir_ssa_def_rewrite_uses(&intrin->dest.ssa, value);  in nir_lower_explicit_io_instr()
1995       assert(intrin->src[1].is_ssa);  in nir_lower_explicit_io_instr()
1996       nir_ssa_def *value = intrin->src[1].ssa;  in nir_lower_explicit_io_instr()
1998       build_explicit_io_store(b, intrin, addr, addr_format,  in nir_lower_explicit_io_instr()
2006          build_explicit_io_atomic(b, intrin, addr, addr_format, deref->modes);  in nir_lower_explicit_io_instr()
2007       nir_ssa_def_rewrite_uses(&intrin->dest.ssa, value);  in nir_lower_explicit_io_instr()
2012    nir_instr_remove(&intrin->instr);  in nir_lower_explicit_io_instr()
2145 lower_explicit_io_access(nir_builder *b, nir_intrinsic_instr *intrin,  in lower_explicit_io_access()  argument
2148    assert(intrin->src[0].is_ssa);  in lower_explicit_io_access()
2149    nir_lower_explicit_io_instr(b, intrin, intrin->src[0].ssa, addr_format);  in lower_explicit_io_access()
2153 lower_explicit_io_array_length(nir_builder *b, nir_intrinsic_instr *intrin,  in lower_explicit_io_array_length()  argument
2156    b->cursor = nir_after_instr(&intrin->instr);  in lower_explicit_io_array_length()
2158    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in lower_explicit_io_array_length()
2169    unsigned access = nir_intrinsic_access(intrin);  in lower_explicit_io_array_length()
2175    nir_ssa_def_rewrite_uses(&intrin->dest.ssa, arr_size);  in lower_explicit_io_array_length()
2176    nir_instr_remove(&intrin->instr);  in lower_explicit_io_array_length()
2180 lower_explicit_io_mode_check(nir_builder *b, nir_intrinsic_instr *intrin,  in lower_explicit_io_mode_check()  argument
2189       intrin->intrinsic = nir_intrinsic_addr_mode_is;  in lower_explicit_io_mode_check()
2193    assert(intrin->src[0].is_ssa);  in lower_explicit_io_mode_check()
2194    nir_ssa_def *addr = intrin->src[0].ssa;  in lower_explicit_io_mode_check()
2196    b->cursor = nir_instr_remove(&intrin->instr);  in lower_explicit_io_mode_check()
2200                                     nir_intrinsic_memory_modes(intrin));  in lower_explicit_io_mode_check()
2202    nir_ssa_def_rewrite_uses(&intrin->dest.ssa, is_mode);  in lower_explicit_io_mode_check()
2231             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);  in nir_lower_explicit_io_impl()  local
2232             switch (intrin->intrinsic) {  in nir_lower_explicit_io_impl()
2251                nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in nir_lower_explicit_io_impl()
2253                   lower_explicit_io_access(&b, intrin, addr_format);  in nir_lower_explicit_io_impl()
2260                nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in nir_lower_explicit_io_impl()
2262                   lower_explicit_io_array_length(&b, intrin, addr_format);  in nir_lower_explicit_io_impl()
2269                nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in nir_lower_explicit_io_impl()
2271                   lower_explicit_io_mode_check(&b, intrin, addr_format);  in nir_lower_explicit_io_impl()
2835 is_input(nir_intrinsic_instr *intrin)  in is_input()  argument
2837    return intrin->intrinsic == nir_intrinsic_load_input ||  in is_input()
2838           intrin->intrinsic == nir_intrinsic_load_per_vertex_input ||  in is_input()
2839           intrin->intrinsic == nir_intrinsic_load_interpolated_input ||  in is_input()
2840           intrin->intrinsic == nir_intrinsic_load_fs_input_interp_deltas;  in is_input()
2844 is_output(nir_intrinsic_instr *intrin)  in is_output()  argument
2846    return intrin->intrinsic == nir_intrinsic_load_output ||  in is_output()
2847           intrin->intrinsic == nir_intrinsic_load_per_vertex_output ||  in is_output()
2848           intrin->intrinsic == nir_intrinsic_load_per_primitive_output ||  in is_output()
2849           intrin->intrinsic == nir_intrinsic_store_output ||  in is_output()
2850           intrin->intrinsic == nir_intrinsic_store_per_vertex_output ||  in is_output()
2851           intrin->intrinsic == nir_intrinsic_store_per_primitive_output;  in is_output()
2854 static bool is_dual_slot(nir_intrinsic_instr *intrin)  in is_dual_slot()  argument
2856    if (intrin->intrinsic == nir_intrinsic_store_output ||  in is_dual_slot()
2857        intrin->intrinsic == nir_intrinsic_store_per_vertex_output ||  in is_dual_slot()
2858        intrin->intrinsic == nir_intrinsic_store_per_primitive_output) {  in is_dual_slot()
2859       return nir_src_bit_size(intrin->src[0]) == 64 &&  in is_dual_slot()
2860              nir_src_num_components(intrin->src[0]) >= 3;  in is_dual_slot()
2863    return nir_dest_bit_size(intrin->dest) == 64 &&  in is_dual_slot()
2864           nir_dest_num_components(intrin->dest) >= 3;  in is_dual_slot()
2885       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);  in add_const_offset_to_base_block()  local
2887       if (((modes & nir_var_shader_in) && is_input(intrin)) ||  in add_const_offset_to_base_block()
2888           ((modes & nir_var_shader_out) && is_output(intrin))) {  in add_const_offset_to_base_block()
2889          nir_io_semantics sem = nir_intrinsic_io_semantics(intrin);  in add_const_offset_to_base_block()
2898          nir_src *offset = nir_get_io_offset_src(intrin);  in add_const_offset_to_base_block()
2902              !nir_intrinsic_io_semantics(intrin).per_view) {  in add_const_offset_to_base_block()
2905             nir_intrinsic_set_base(intrin, nir_intrinsic_base(intrin) + off);  in add_const_offset_to_base_block()
2909             sem.num_slots = is_dual_slot(intrin) ? 2 : 1;  in add_const_offset_to_base_block()
2910             nir_intrinsic_set_io_semantics(intrin, sem);  in add_const_offset_to_base_block()
2912             b->cursor = nir_before_instr(&intrin->instr);  in add_const_offset_to_base_block()
2913             nir_instr_rewrite_src(&intrin->instr, offset,  in add_const_offset_to_base_block()
2961          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);  in nir_lower_color_inputs()  local
2963          if (intrin->intrinsic != nir_intrinsic_load_deref)  in nir_lower_color_inputs()
2966          nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);  in nir_lower_color_inputs()
2988          nir_ssa_def_rewrite_uses(&intrin->dest.ssa, def);  in nir_lower_color_inputs()