Lines Matching +full:static +full:- +full:libclc
40 static int to_llvm_address_space(SpvStorageClass mode) in to_llvm_address_space()
50 default: return -1; in to_llvm_address_space()
55 static void
65 const struct glsl_type *type = src_types[i]->type; in vtn_opencl_mangle()
66 enum vtn_base_type base_type = src_types[i]->base_type; in vtn_opencl_mangle()
67 if (src_types[i]->base_type == vtn_base_type_pointer) { in vtn_opencl_mangle()
69 int address_space = to_llvm_address_space(src_types[i]->storage_class); in vtn_opencl_mangle()
73 type = src_types[i]->deref->type; in vtn_opencl_mangle()
74 base_type = src_types[i]->deref->base_type; in vtn_opencl_mangle()
82 /* Vectors are not treated as built-ins for mangling, so check for substitution. in vtn_opencl_mangle()
84 * the functions we need from libclc only support 1 in vtn_opencl_mangle()
88 const struct glsl_type *other_type = src_types[j]->base_type == vtn_base_type_pointer ? in vtn_opencl_mangle()
89 src_types[j]->deref->type : src_types[j]->type; in vtn_opencl_mangle()
135 static nir_function *mangle_and_find(struct vtn_builder *b, in mangle_and_find()
146 nir_foreach_function(funcs, b->shader) { in mangle_and_find()
147 if (!strcmp(funcs->name, mname)) { in mangle_and_find()
153 if (!found && b->options->clc_shader && b->options->clc_shader != b->shader) { in mangle_and_find()
154 nir_foreach_function(funcs, b->options->clc_shader) { in mangle_and_find()
155 if (!strcmp(funcs->name, mname)) { in mangle_and_find()
161 nir_function *decl = nir_function_create(b->shader, mname); in mangle_and_find()
162 decl->num_params = found->num_params; in mangle_and_find()
163 decl->params = ralloc_array(b->shader, nir_parameter, decl->num_params); in mangle_and_find()
164 for (unsigned i = 0; i < decl->num_params; i++) { in mangle_and_find()
165 decl->params[i] = found->params[i]; in mangle_and_find()
176 static bool call_mangled_function(struct vtn_builder *b, in call_mangled_function()
189 nir_call_instr *call = nir_call_instr_create(b->shader, found); in call_mangled_function()
194 nir_variable *ret_tmp = nir_local_variable_create(b->nb.impl, in call_mangled_function()
195 glsl_get_bare_type(dest_type->type), in call_mangled_function()
197 ret_deref = nir_build_deref_var(&b->nb, ret_tmp); in call_mangled_function()
198 call->params[param_idx++] = nir_src_for_ssa(&ret_deref->dest.ssa); in call_mangled_function()
202 call->params[param_idx++] = nir_src_for_ssa(srcs[i]); in call_mangled_function()
203 nir_builder_instr_insert(&b->nb, &call->instr); in call_mangled_function()
209 static void
221 srcs[i] = ssa->def; in handle_instr()
222 src_types[i] = val->type; in handle_instr()
233 static nir_op
283 static nir_ssa_def *
288 …nir_ssa_def *ret = nir_build_alu(&b->nb, nir_alu_op_for_opencl_opcode(b, (enum OpenCLstd_Entrypoin… in handle_alu()
291 ret = nir_u2u(&b->nb, ret, glsl_get_bit_size(dest_type->type)); in handle_alu()
296 static const struct {
388 static const char *remap_clc_opcode(enum OpenCLstd_Entrypoints opcode) in remap_clc_opcode()
395 static struct vtn_type *
400 ret->type = type; in get_vtn_type_for_glsl_type()
401 ret->length = glsl_get_vector_elements(type); in get_vtn_type_for_glsl_type()
402 ret->base_type = glsl_type_is_vector(type) ? vtn_base_type_vector : vtn_base_type_scalar; in get_vtn_type_for_glsl_type()
406 static struct vtn_type *
410 ret->type = nir_address_format_to_glsl_type( in get_pointer_type()
413 ret->base_type = vtn_base_type_pointer; in get_pointer_type()
414 ret->storage_class = storage_class; in get_pointer_type()
415 ret->deref = t; in get_pointer_type()
419 static struct vtn_type *
422 if (t->base_type == vtn_base_type_pointer) { in get_signed_type()
423 return get_pointer_type(b, get_signed_type(b, t->deref), t->storage_class); in get_signed_type()
426 b, glsl_vector_type(glsl_signed_base_type_of(glsl_get_base_type(t->type)), in get_signed_type()
427 glsl_get_vector_elements(t->type))); in get_signed_type()
430 static nir_ssa_def *
441 /* Some functions which take params end up with uint (or pointer-to-uint) being passed, in handle_clc_fn()
442 * which doesn't mangle correctly when the function expects int or pointer-to-int. in handle_clc_fn()
443 …* See https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_a_id_unsignedsigned_a_uns… in handle_clc_fn()
445 int signed_param = -1; in handle_clc_fn()
475 return ret_deref ? nir_load_deref(&b->nb, ret_deref) : NULL; in handle_clc_fn()
478 static nir_ssa_def *
483 nir_builder *nb = &b->nb; in handle_special()
515 if (dest_type->length == 4) in handle_special()
521 if (nb->shader->options->lower_fmod) in handle_special()
544 /* SPIR-V and CL have different defs for upsample, just implement in nir */ in handle_special()
557 if (nb->shader->options->lower_ldexp) in handle_special()
562 if (nb->shader->options->lower_ffma32 && srcs[0]->bit_size == 32) in handle_special()
576 static nir_ssa_def *
585 /* Libclc doesn't include 3-component overloads of the async copy functions. in handle_core()
587 * async_work_group_copy and async_work_group_strided_copy for 3-component vector types in handle_core()
588 … * behave as async_work_group_copy and async_work_group_strided_copy respectively for 4-component in handle_core()
592 if (src_types[i]->base_type == vtn_base_type_pointer && in handle_core()
593 src_types[i]->deref->base_type == vtn_base_type_vector && in handle_core()
594 src_types[i]->deref->length == 3) { in handle_core()
597 … get_vtn_type_for_glsl_type(b, glsl_replace_vector_type(src_types[i]->deref->type, 4)), in handle_core()
598 src_types[i]->storage_class); in handle_core()
615 return ret_deref ? nir_load_deref(&b->nb, ret_deref) : NULL; in handle_core()
619 static void
631 enum glsl_base_type base_type = glsl_get_base_type(type->type); in _handle_v_load_store()
632 unsigned components = glsl_get_vector_elements(type->type); in _handle_v_load_store()
640 nir_ssa_def *moffset = nir_imul_imm(&b->nb, offset, in _handle_v_load_store()
642 nir_deref_instr *deref = vtn_pointer_to_deref(b, p->pointer); in _handle_v_load_store()
644 unsigned alignment = vec_aligned ? glsl_get_cl_alignment(type->type) : in _handle_v_load_store()
645 glsl_get_bit_size(type->type) / 8; in _handle_v_load_store()
647 glsl_get_base_type(p->pointer->type->type); in _handle_v_load_store()
654 "floating-point types."); in _handle_v_load_store()
656 /* Above-computed alignment was for floats/doubles, not halves */ in _handle_v_load_store()
657 alignment /= glsl_get_bit_size(type->type) / glsl_base_type_get_bit_size(ptr_base_type); in _handle_v_load_store()
660 deref = nir_alignment_deref_cast(&b->nb, deref, alignment, 0); in _handle_v_load_store()
663 nir_ssa_def *coffset = nir_iadd_imm(&b->nb, moffset, i); in _handle_v_load_store()
664 nir_deref_instr *arr_deref = nir_build_deref_ptr_as_array(&b->nb, deref, coffset); in _handle_v_load_store()
667 comps[i] = vtn_local_load(b, arr_deref, p->type->access); in _handle_v_load_store()
668 ncomps[i] = comps[i]->def; in _handle_v_load_store()
673 ncomps[i] = nir_f2fN(&b->nb, ncomps[i], in _handle_v_load_store()
679 ssa->def = nir_channel(&b->nb, val->def, i); in _handle_v_load_store()
685 ssa->def = nir_f2f16(&b->nb, ssa->def); in _handle_v_load_store()
687 ssa->def = nir_convert_alu_types(&b->nb, 16, ssa->def, in _handle_v_load_store()
688 nir_type_float | ssa->def->bit_size, in _handle_v_load_store()
693 vtn_local_store(b, ssa, arr_deref, p->type->access); in _handle_v_load_store()
697 vtn_push_nir_ssa(b, w[2], nir_vec(&b->nb, ncomps, components)); in _handle_v_load_store()
701 static void
710 static void
719 static void
728 static unsigned
733 while (deref && deref->deref_type != nir_deref_type_var) in vtn_add_printf_string()
738 vtn_fail_if(deref->var->constant_initializer == NULL, in vtn_add_printf_string()
740 vtn_fail_if(!glsl_type_is_array(deref->var->type), in vtn_add_printf_string()
742 const struct glsl_type *char_type = glsl_get_array_element(deref->var->type); in vtn_add_printf_string()
747 nir_constant *c = deref->var->constant_initializer; in vtn_add_printf_string()
748 assert(c->num_elements == glsl_get_length(deref->var->type)); in vtn_add_printf_string()
750 unsigned idx = info->string_size; in vtn_add_printf_string()
751 info->strings = reralloc_size(b->shader, info->strings, in vtn_add_printf_string()
752 idx + c->num_elements); in vtn_add_printf_string()
753 info->string_size += c->num_elements; in vtn_add_printf_string()
755 char *str = &info->strings[idx]; in vtn_add_printf_string()
757 for (unsigned i = 0; i < c->num_elements; i++) { in vtn_add_printf_string()
758 memcpy((char *)str + i, c->elements[i]->values, 1); in vtn_add_printf_string()
767 static void
771 if (!b->options->caps.printf) { in handle_printf()
772 vtn_push_nir_ssa(b, w_dest[1], nir_imm_int(&b->nb, -1)); in handle_printf()
779 * info_idx is 1-based to match clover/llvm in handle_printf()
780 * the backend indexes the info table at info_idx - 1. in handle_printf()
782 b->shader->printf_info_count++; in handle_printf()
783 unsigned info_idx = b->shader->printf_info_count; in handle_printf()
785 b->shader->printf_info = reralloc(b->shader, b->shader->printf_info, in handle_printf()
787 nir_printf_info *info = &b->shader->printf_info[info_idx - 1]; in handle_printf()
789 info->strings = NULL; in handle_printf()
790 info->string_size = 0; in handle_printf()
794 info->num_args = num_srcs - 1; in handle_printf()
795 info->arg_sizes = ralloc_array(b->shader, unsigned, info->num_args); in handle_printf()
797 /* Step 2, build an ad-hoc struct type out of the args */ in handle_printf()
800 rzalloc_array(b, struct glsl_struct_field, num_srcs - 1); in handle_printf()
803 struct vtn_type *src_type = val->type; in handle_printf()
804 fields[i - 1].type = src_type->type; in handle_printf()
805 fields[i - 1].name = ralloc_asprintf(b->shader, "arg_%u", i); in handle_printf()
807 fields[i - 1].offset = field_offset; in handle_printf()
808 info->arg_sizes[i - 1] = glsl_get_cl_size(src_type->type); in handle_printf()
809 field_offset += glsl_get_cl_size(src_type->type); in handle_printf()
812 glsl_struct_type(fields, num_srcs - 1, "printf", true); in handle_printf()
815 nir_variable *var = nir_local_variable_create(b->nb.impl, struct_type, NULL); in handle_printf()
816 nir_deref_instr *deref_var = nir_build_deref_var(&b->nb, var); in handle_printf()
820 nir_build_deref_struct(&b->nb, deref_var, i - 1); in handle_printf()
821 nir_ssa_def *field_src = vtn_ssa_value(b, w_src[i])->def; in handle_printf()
823 fmt_pos = util_printf_next_spec_pos(info->strings, fmt_pos); in handle_printf()
824 if (fmt_pos != -1 && info->strings[fmt_pos] == 's') { in handle_printf()
826 nir_store_deref(&b->nb, field_deref, in handle_printf()
827 nir_imm_intN_t(&b->nb, idx, field_src->bit_size), in handle_printf()
830 nir_store_deref(&b->nb, field_deref, field_src, ~0); in handle_printf()
834 nir_ssa_def *fmt_idx = nir_imm_int(&b->nb, info_idx); in handle_printf()
835 nir_ssa_def *ret = nir_printf(&b->nb, fmt_idx, &deref_var->dest.ssa); in handle_printf()
839 static nir_ssa_def *
845 nir_builder *nb = &b->nb; in handle_round()
846 nir_ssa_def *half = nir_imm_floatN_t(nb, 0.5, src->bit_size); in handle_round()
854 static nir_ssa_def *
862 unsigned out_elems = dest_type->length; in handle_shuffle()
864 unsigned in_elems = input->num_components; in handle_shuffle()
865 if (mask->bit_size != 32) in handle_shuffle()
866 mask = nir_u2u32(&b->nb, mask); in handle_shuffle()
867 mask = nir_iand(&b->nb, mask, nir_imm_intN_t(&b->nb, in_elems - 1, mask->bit_size)); in handle_shuffle()
869 outres[i] = nir_vector_extract(&b->nb, input, nir_channel(&b->nb, mask, i)); in handle_shuffle()
871 return nir_vec(&b->nb, outres, out_elems); in handle_shuffle()
874 static nir_ssa_def *
883 unsigned out_elems = dest_type->length; in handle_shuffle2()
885 unsigned in_elems = input0->num_components; in handle_shuffle2()
886 unsigned total_mask = 2 * in_elems - 1; in handle_shuffle2()
887 unsigned half_mask = in_elems - 1; in handle_shuffle2()
888 if (mask->bit_size != 32) in handle_shuffle2()
889 mask = nir_u2u32(&b->nb, mask); in handle_shuffle2()
890 mask = nir_iand(&b->nb, mask, nir_imm_intN_t(&b->nb, total_mask, mask->bit_size)); in handle_shuffle2()
892 nir_ssa_def *this_mask = nir_channel(&b->nb, mask, i); in handle_shuffle2()
893 …nir_ssa_def *vmask = nir_iand(&b->nb, this_mask, nir_imm_intN_t(&b->nb, half_mask, mask->bit_size)… in handle_shuffle2()
894 nir_ssa_def *val0 = nir_vector_extract(&b->nb, input0, vmask); in handle_shuffle2()
895 nir_ssa_def *val1 = nir_vector_extract(&b->nb, input1, vmask); in handle_shuffle2()
896 … nir_ssa_def *sel = nir_ilt(&b->nb, this_mask, nir_imm_intN_t(&b->nb, in_elems, mask->bit_size)); in handle_shuffle2()
897 outres[i] = nir_bcsel(&b->nb, sel, val0, val1); in handle_shuffle2()
899 return nir_vec(&b->nb, outres, out_elems); in handle_shuffle2()
948 handle_instr(b, ext_opcode, w + 5, count - 5, w + 1, handle_alu); in vtn_handle_opencl_instruction()
1055 handle_instr(b, ext_opcode, w + 5, count - 5, w + 1, handle_special); in vtn_handle_opencl_instruction()
1075 handle_instr(b, ext_opcode, w + 5, count - 5, w + 1, handle_shuffle); in vtn_handle_opencl_instruction()
1078 handle_instr(b, ext_opcode, w + 5, count - 5, w + 1, handle_shuffle2); in vtn_handle_opencl_instruction()
1081 handle_instr(b, ext_opcode, w + 5, count - 5, w + 1, handle_round); in vtn_handle_opencl_instruction()
1084 handle_printf(b, ext_opcode, w + 5, count - 5, w + 1); in vtn_handle_opencl_instruction()
1101 handle_instr(b, opcode, w + 4, count - 4, w + 1, handle_core); in vtn_handle_opencl_core_instruction()
1104 handle_instr(b, opcode, w + 2, count - 2, NULL, handle_core); in vtn_handle_opencl_core_instruction()