1 /*
2 * Copyright © 2016 Red Hat
3 *
4 * based on anv driver:
5 * Copyright © 2016 Intel Corporation
6 *
7 * SPDX-License-Identifier: MIT
8 */
9
10 #include "nir/nir_builder.h"
11 #include "radv_entrypoints.h"
12 #include "radv_meta.h"
13 #include "vk_common_entrypoints.h"
14 #include "vk_format.h"
15 #include "vk_shader_module.h"
16
17 enum blit2d_src_type {
18 BLIT2D_SRC_TYPE_IMAGE,
19 BLIT2D_SRC_TYPE_IMAGE_3D,
20 BLIT2D_SRC_TYPE_BUFFER,
21 BLIT2D_NUM_SRC_TYPES,
22 };
23
24 static VkResult get_color_pipeline(struct radv_device *device, enum blit2d_src_type src_type, VkFormat format,
25 uint32_t log2_samples, VkPipeline *pipeline_out, VkPipelineLayout *layout_out);
26
27 static VkResult get_depth_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
28 uint32_t log2_samples, VkPipeline *pipeline_out, VkPipelineLayout *layout_out);
29
30 static VkResult get_stencil_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type,
31 uint32_t log2_samples, VkPipeline *pipeline_out,
32 VkPipelineLayout *layout_out);
33
34 static void
create_iview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * surf,struct radv_image_view * iview,VkFormat depth_format,VkImageAspectFlagBits aspects)35 create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf, struct radv_image_view *iview,
36 VkFormat depth_format, VkImageAspectFlagBits aspects)
37 {
38 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
39 VkFormat format;
40
41 if (depth_format)
42 format = depth_format;
43 else
44 format = surf->format;
45
46 radv_image_view_init(iview, device,
47 &(VkImageViewCreateInfo){
48 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
49 .image = radv_image_to_handle(surf->image),
50 .viewType = radv_meta_get_view_type(surf->image),
51 .format = format,
52 .subresourceRange = {.aspectMask = aspects,
53 .baseMipLevel = surf->level,
54 .levelCount = 1,
55 .baseArrayLayer = surf->layer,
56 .layerCount = 1},
57 },
58 &(struct radv_image_view_extra_create_info){.disable_dcc_mrt = surf->disable_compression});
59 }
60
61 static void
create_bview(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_buffer * src,struct radv_buffer_view * bview,VkFormat depth_format)62 create_bview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src, struct radv_buffer_view *bview,
63 VkFormat depth_format)
64 {
65 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
66 VkFormat format;
67
68 if (depth_format)
69 format = depth_format;
70 else
71 format = src->format;
72 radv_buffer_view_init(bview, device,
73 &(VkBufferViewCreateInfo){
74 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
75 .flags = 0,
76 .buffer = radv_buffer_to_handle(src->buffer),
77 .format = format,
78 .offset = src->offset,
79 .range = VK_WHOLE_SIZE,
80 });
81 }
82
83 struct blit2d_src_temps {
84 struct radv_image_view iview;
85 struct radv_buffer_view bview;
86 };
87
88 static void
blit2d_bind_src(struct radv_cmd_buffer * cmd_buffer,VkPipelineLayout layout,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct blit2d_src_temps * tmp,enum blit2d_src_type src_type,VkFormat depth_format,VkImageAspectFlagBits aspects,uint32_t log2_samples)89 blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer, VkPipelineLayout layout, struct radv_meta_blit2d_surf *src_img,
90 struct radv_meta_blit2d_buffer *src_buf, struct blit2d_src_temps *tmp, enum blit2d_src_type src_type,
91 VkFormat depth_format, VkImageAspectFlagBits aspects, uint32_t log2_samples)
92 {
93 if (src_type == BLIT2D_SRC_TYPE_BUFFER) {
94 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format);
95
96 radv_meta_push_descriptor_set(
97 cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, 1,
98 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
99 .dstBinding = 0,
100 .dstArrayElement = 0,
101 .descriptorCount = 1,
102 .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER,
103 .pTexelBufferView = (VkBufferView[]){radv_buffer_view_to_handle(&tmp->bview)}}});
104
105 vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), layout, VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
106 &src_buf->pitch);
107 } else {
108 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects);
109
110 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D)
111 vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), layout, VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4,
112 &src_img->layer);
113
114 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, 1,
115 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
116 .dstBinding = 0,
117 .dstArrayElement = 0,
118 .descriptorCount = 1,
119 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
120 .pImageInfo = (VkDescriptorImageInfo[]){
121 {
122 .sampler = VK_NULL_HANDLE,
123 .imageView = radv_image_view_to_handle(&tmp->iview),
124 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
125 },
126 }}});
127 }
128 }
129
130 static void
radv_meta_blit2d_normal_dst(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,struct radv_meta_blit2d_rect * rect,enum blit2d_src_type src_type,uint32_t log2_samples)131 radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
132 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
133 struct radv_meta_blit2d_rect *rect, enum blit2d_src_type src_type, uint32_t log2_samples)
134 {
135 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
136 VkPipelineLayout layout;
137 VkPipeline pipeline;
138 VkResult result;
139
140 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
141 &(VkViewport){.x = rect->dst_x,
142 .y = rect->dst_y,
143 .width = rect->width,
144 .height = rect->height,
145 .minDepth = 0.0f,
146 .maxDepth = 1.0f});
147
148 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
149 &(VkRect2D){
150 .offset = (VkOffset2D){rect->dst_x, rect->dst_y},
151 .extent = (VkExtent2D){rect->width, rect->height},
152 });
153
154 u_foreach_bit (i, dst->aspect_mask) {
155 unsigned aspect_mask = 1u << i;
156 unsigned src_aspect_mask = aspect_mask;
157 VkFormat depth_format = 0;
158 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT)
159 depth_format = vk_format_stencil_only(dst->image->vk.format);
160 else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT)
161 depth_format = vk_format_depth_only(dst->image->vk.format);
162 else if (src_img)
163 src_aspect_mask = src_img->aspect_mask;
164
165 /* Adjust the aspect for color to depth/stencil image copies. */
166 if (src_img) {
167 if (vk_format_is_color(src_img->image->vk.format) && vk_format_is_depth_or_stencil(dst->image->vk.format)) {
168 assert(src_img->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT);
169 src_aspect_mask = src_img->aspect_mask;
170 }
171 }
172
173 struct radv_image_view dst_iview;
174 create_iview(cmd_buffer, dst, &dst_iview, depth_format, aspect_mask);
175
176 const VkRenderingAttachmentInfo att_info = {
177 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
178 .imageView = radv_image_view_to_handle(&dst_iview),
179 .imageLayout = dst->current_layout,
180 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
181 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
182 };
183
184 VkRenderingInfo rendering_info = {
185 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
186 .flags = VK_RENDERING_INPUT_ATTACHMENT_NO_CONCURRENT_WRITES_BIT_MESA,
187 .renderArea =
188 {
189 .offset = {rect->dst_x, rect->dst_y},
190 .extent = {rect->width, rect->height},
191 },
192 .layerCount = 1,
193 };
194
195 if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT || aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT ||
196 aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT || aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) {
197 result = get_color_pipeline(device, src_type, dst_iview.vk.format, log2_samples, &pipeline, &layout);
198 if (result != VK_SUCCESS) {
199 vk_command_buffer_set_error(&cmd_buffer->vk, result);
200 goto fail_pipeline;
201 }
202
203 rendering_info.colorAttachmentCount = 1;
204 rendering_info.pColorAttachments = &att_info;
205
206 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
207 } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
208 result = get_depth_only_pipeline(device, src_type, log2_samples, &pipeline, &layout);
209 if (result != VK_SUCCESS) {
210 vk_command_buffer_set_error(&cmd_buffer->vk, result);
211 goto fail_pipeline;
212 }
213
214 rendering_info.pDepthAttachment = &att_info,
215 rendering_info.pStencilAttachment = (dst->image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? &att_info : NULL,
216
217 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
218
219 } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
220 result = get_stencil_only_pipeline(device, src_type, log2_samples, &pipeline, &layout);
221 if (result != VK_SUCCESS) {
222 vk_command_buffer_set_error(&cmd_buffer->vk, result);
223 goto fail_pipeline;
224 }
225
226 rendering_info.pDepthAttachment = (dst->image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) ? &att_info : NULL,
227 rendering_info.pStencilAttachment = &att_info,
228
229 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
230 } else
231 unreachable("Processing blit2d with multiple aspects.");
232
233 float vertex_push_constants[4] = {
234 rect->src_x,
235 rect->src_y,
236 rect->src_x + rect->width,
237 rect->src_y + rect->height,
238 };
239
240 vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), layout, VK_SHADER_STAGE_VERTEX_BIT, 0, 16,
241 vertex_push_constants);
242
243 struct blit2d_src_temps src_temps;
244 blit2d_bind_src(cmd_buffer, layout, src_img, src_buf, &src_temps, src_type,
245 (src_aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) ? depth_format : 0,
246 src_aspect_mask, log2_samples);
247
248 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
249
250 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
251
252 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer));
253
254 fail_pipeline:
255
256 if (src_type == BLIT2D_SRC_TYPE_BUFFER)
257 radv_buffer_view_finish(&src_temps.bview);
258 else
259 radv_image_view_finish(&src_temps.iview);
260
261 radv_image_view_finish(&dst_iview);
262 }
263 }
264
265 void
radv_meta_blit2d(struct radv_cmd_buffer * cmd_buffer,struct radv_meta_blit2d_surf * src_img,struct radv_meta_blit2d_buffer * src_buf,struct radv_meta_blit2d_surf * dst,struct radv_meta_blit2d_rect * rect)266 radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
267 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
268 struct radv_meta_blit2d_rect *rect)
269 {
270 bool use_3d = (src_img && src_img->image->vk.image_type == VK_IMAGE_TYPE_3D);
271 enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER
272 : use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D
273 : BLIT2D_SRC_TYPE_IMAGE;
274 radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst, rect, src_type,
275 src_img ? util_logbase2(src_img->image->vk.samples) : 0);
276 }
277
278 static nir_shader *
build_nir_vertex_shader(struct radv_device * device)279 build_nir_vertex_shader(struct radv_device *device)
280 {
281 const struct glsl_type *vec4 = glsl_vec4_type();
282 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
283 nir_builder b = radv_meta_init_shader(device, MESA_SHADER_VERTEX, "meta_blit2d_vs");
284
285 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
286 pos_out->data.location = VARYING_SLOT_POS;
287
288 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec2, "v_tex_pos");
289 tex_pos_out->data.location = VARYING_SLOT_VAR0;
290 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
291
292 nir_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL);
293 nir_store_var(&b, pos_out, outvec, 0xf);
294
295 nir_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16);
296 nir_def *vertex_id = nir_load_vertex_id_zero_base(&b);
297
298 /* vertex 0 - src_x, src_y */
299 /* vertex 1 - src_x, src_y+h */
300 /* vertex 2 - src_x+w, src_y */
301 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
302 channel 1 is vertex id != 1 ? src_y : src_y + w */
303
304 nir_def *c0cmp = nir_ine_imm(&b, vertex_id, 2);
305 nir_def *c1cmp = nir_ine_imm(&b, vertex_id, 1);
306
307 nir_def *comp[2];
308 comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2));
309
310 comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3));
311 nir_def *out_tex_vec = nir_vec(&b, comp, 2);
312 nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3);
313 return b.shader;
314 }
315
316 typedef nir_def *(*texel_fetch_build_func)(struct nir_builder *, struct radv_device *, nir_def *, bool, bool);
317
318 static nir_def *
build_nir_texel_fetch(struct nir_builder * b,struct radv_device * device,nir_def * tex_pos,bool is_3d,bool is_multisampled)319 build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device, nir_def *tex_pos, bool is_3d,
320 bool is_multisampled)
321 {
322 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D
323 : is_multisampled ? GLSL_SAMPLER_DIM_MS
324 : GLSL_SAMPLER_DIM_2D;
325 const struct glsl_type *sampler_type = glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT);
326 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex");
327 sampler->data.descriptor_set = 0;
328 sampler->data.binding = 0;
329
330 nir_def *tex_pos_3d = NULL;
331 nir_def *sample_idx = NULL;
332 if (is_3d) {
333 nir_def *layer = nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4);
334
335 nir_def *chans[3];
336 chans[0] = nir_channel(b, tex_pos, 0);
337 chans[1] = nir_channel(b, tex_pos, 1);
338 chans[2] = layer;
339 tex_pos_3d = nir_vec(b, chans, 3);
340 }
341 if (is_multisampled) {
342 sample_idx = nir_load_sample_id(b);
343 }
344
345 nir_deref_instr *tex_deref = nir_build_deref_var(b, sampler);
346
347 if (is_multisampled) {
348 return nir_txf_ms_deref(b, tex_deref, tex_pos, sample_idx);
349 } else {
350 return nir_txf_deref(b, tex_deref, is_3d ? tex_pos_3d : tex_pos, NULL);
351 }
352 }
353
354 static nir_def *
build_nir_buffer_fetch(struct nir_builder * b,struct radv_device * device,nir_def * tex_pos,bool is_3d,bool is_multisampled)355 build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device, nir_def *tex_pos, bool is_3d,
356 bool is_multisampled)
357 {
358 const struct glsl_type *sampler_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT);
359 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex");
360 sampler->data.descriptor_set = 0;
361 sampler->data.binding = 0;
362
363 nir_def *width = nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4);
364
365 nir_def *pos_x = nir_channel(b, tex_pos, 0);
366 nir_def *pos_y = nir_channel(b, tex_pos, 1);
367 pos_y = nir_imul(b, pos_y, width);
368 pos_x = nir_iadd(b, pos_x, pos_y);
369
370 nir_deref_instr *tex_deref = nir_build_deref_var(b, sampler);
371 return nir_txf_deref(b, tex_deref, pos_x, NULL);
372 }
373
374 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)375 build_nir_copy_fragment_shader(struct radv_device *device, texel_fetch_build_func txf_func, const char *name,
376 bool is_3d, bool is_multisampled)
377 {
378 const struct glsl_type *vec4 = glsl_vec4_type();
379 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
380 nir_builder b = radv_meta_init_shader(device, MESA_SHADER_FRAGMENT, "%s", name);
381
382 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
383 tex_pos_in->data.location = VARYING_SLOT_VAR0;
384
385 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
386 color_out->data.location = FRAG_RESULT_DATA0;
387
388 nir_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
389 nir_def *tex_pos = nir_trim_vector(&b, pos_int, 2);
390
391 nir_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
392 nir_store_var(&b, color_out, color, 0xf);
393
394 b.shader->info.fs.uses_sample_shading = is_multisampled;
395
396 return b.shader;
397 }
398
399 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)400 build_nir_copy_fragment_shader_depth(struct radv_device *device, texel_fetch_build_func txf_func, const char *name,
401 bool is_3d, bool is_multisampled)
402 {
403 const struct glsl_type *vec4 = glsl_vec4_type();
404 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
405 nir_builder b = radv_meta_init_shader(device, MESA_SHADER_FRAGMENT, "%s", name);
406
407 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
408 tex_pos_in->data.location = VARYING_SLOT_VAR0;
409
410 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
411 color_out->data.location = FRAG_RESULT_DEPTH;
412
413 nir_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
414 nir_def *tex_pos = nir_trim_vector(&b, pos_int, 2);
415
416 nir_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
417 nir_store_var(&b, color_out, color, 0x1);
418
419 b.shader->info.fs.uses_sample_shading = is_multisampled;
420
421 return b.shader;
422 }
423
424 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * device,texel_fetch_build_func txf_func,const char * name,bool is_3d,bool is_multisampled)425 build_nir_copy_fragment_shader_stencil(struct radv_device *device, texel_fetch_build_func txf_func, const char *name,
426 bool is_3d, bool is_multisampled)
427 {
428 const struct glsl_type *vec4 = glsl_vec4_type();
429 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2);
430 nir_builder b = radv_meta_init_shader(device, MESA_SHADER_FRAGMENT, "%s", name);
431
432 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos");
433 tex_pos_in->data.location = VARYING_SLOT_VAR0;
434
435 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
436 color_out->data.location = FRAG_RESULT_STENCIL;
437
438 nir_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in));
439 nir_def *tex_pos = nir_trim_vector(&b, pos_int, 2);
440
441 nir_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled);
442 nir_store_var(&b, color_out, color, 0x1);
443
444 b.shader->info.fs.uses_sample_shading = is_multisampled;
445
446 return b.shader;
447 }
448
449 struct radv_blit2d_key {
450 enum radv_meta_object_key_type type;
451 uint32_t index;
452 };
453
454 static VkResult
create_layout(struct radv_device * device,int idx,VkPipelineLayout * layout_out)455 create_layout(struct radv_device *device, int idx, VkPipelineLayout *layout_out)
456 {
457 const VkDescriptorType desc_type =
458 (idx == BLIT2D_SRC_TYPE_BUFFER) ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
459
460 struct radv_blit2d_key key;
461
462 memset(&key, 0, sizeof(key));
463 key.type = RADV_META_OBJECT_KEY_BLIT2D;
464 key.index = idx;
465
466 const VkDescriptorSetLayoutBinding binding = {
467 .binding = 0,
468 .descriptorType = desc_type,
469 .descriptorCount = 1,
470 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
471 };
472
473 const VkDescriptorSetLayoutCreateInfo desc_info = {
474 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
475 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT,
476 .bindingCount = 1,
477 .pBindings = &binding,
478 };
479
480 const VkPushConstantRange pc_range = {
481 .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT,
482 .size = 20,
483 };
484
485 return vk_meta_get_pipeline_layout(&device->vk, &device->meta_state.device, &desc_info, &pc_range, &key, sizeof(key),
486 layout_out);
487 }
488
489 struct radv_blit2d_color_key {
490 enum radv_meta_object_key_type type;
491 enum blit2d_src_type src_type;
492 uint32_t log2_samples;
493 uint32_t fs_key;
494 };
495
496 static VkResult
get_color_pipeline(struct radv_device * device,enum blit2d_src_type src_type,VkFormat format,uint32_t log2_samples,VkPipeline * pipeline_out,VkPipelineLayout * layout_out)497 get_color_pipeline(struct radv_device *device, enum blit2d_src_type src_type, VkFormat format, uint32_t log2_samples,
498 VkPipeline *pipeline_out, VkPipelineLayout *layout_out)
499 {
500 struct radv_blit2d_color_key key;
501 const char *name;
502 VkResult result;
503
504 result = create_layout(device, src_type, layout_out);
505 if (result != VK_SUCCESS)
506 return result;
507
508 memset(&key, 0, sizeof(key));
509 key.type = RADV_META_OBJECT_KEY_BLIT2D_COLOR;
510 key.src_type = src_type;
511 key.log2_samples = log2_samples;
512 key.fs_key = radv_format_meta_fs_key(device, format);
513
514 VkPipeline pipeline_from_cache = vk_meta_lookup_pipeline(&device->meta_state.device, &key, sizeof(key));
515 if (pipeline_from_cache != VK_NULL_HANDLE) {
516 *pipeline_out = pipeline_from_cache;
517 return VK_SUCCESS;
518 }
519
520 texel_fetch_build_func src_func;
521 switch (src_type) {
522 case BLIT2D_SRC_TYPE_IMAGE:
523 src_func = build_nir_texel_fetch;
524 name = "meta_blit2d_image_fs";
525 break;
526 case BLIT2D_SRC_TYPE_IMAGE_3D:
527 src_func = build_nir_texel_fetch;
528 name = "meta_blit3d_image_fs";
529 break;
530 case BLIT2D_SRC_TYPE_BUFFER:
531 src_func = build_nir_buffer_fetch;
532 name = "meta_blit2d_buffer_fs";
533 break;
534 default:
535 unreachable("unknown blit src type\n");
536 break;
537 }
538
539 nir_shader *vs_module = build_nir_vertex_shader(device);
540 nir_shader *fs_module =
541 build_nir_copy_fragment_shader(device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
542
543 const VkGraphicsPipelineCreateInfo pipeline_create_info = {
544 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
545 .stageCount = 2,
546 .pStages =
547 (VkPipelineShaderStageCreateInfo[]){
548 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
549 .stage = VK_SHADER_STAGE_VERTEX_BIT,
550 .module = vk_shader_module_handle_from_nir(vs_module),
551 .pName = "main",
552 .pSpecializationInfo = NULL},
553 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
554 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
555 .module = vk_shader_module_handle_from_nir(fs_module),
556 .pName = "main",
557 .pSpecializationInfo = NULL},
558 },
559 .pVertexInputState =
560 &(VkPipelineVertexInputStateCreateInfo){
561 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
562 .vertexBindingDescriptionCount = 0,
563 .vertexAttributeDescriptionCount = 0,
564 },
565 .pInputAssemblyState =
566 &(VkPipelineInputAssemblyStateCreateInfo){
567 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
568 .topology = VK_PRIMITIVE_TOPOLOGY_META_RECT_LIST_MESA,
569 .primitiveRestartEnable = false,
570 },
571 .pViewportState =
572 &(VkPipelineViewportStateCreateInfo){
573 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
574 .viewportCount = 1,
575 .scissorCount = 1,
576 },
577 .pRasterizationState =
578 &(VkPipelineRasterizationStateCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
579 .rasterizerDiscardEnable = false,
580 .polygonMode = VK_POLYGON_MODE_FILL,
581 .cullMode = VK_CULL_MODE_NONE,
582 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
583 .depthBiasConstantFactor = 0.0f,
584 .depthBiasClamp = 0.0f,
585 .depthBiasSlopeFactor = 0.0f,
586 .lineWidth = 1.0f},
587 .pMultisampleState =
588 &(VkPipelineMultisampleStateCreateInfo){
589 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
590 .rasterizationSamples = 1 << log2_samples,
591 .sampleShadingEnable = log2_samples > 1,
592 .minSampleShading = 1.0,
593 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
594 },
595 .pColorBlendState =
596 &(VkPipelineColorBlendStateCreateInfo){
597 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
598 .attachmentCount = 1,
599 .pAttachments =
600 (VkPipelineColorBlendAttachmentState[]){
601 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
602 VK_COLOR_COMPONENT_B_BIT},
603 },
604 .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}},
605 .pDynamicState =
606 &(VkPipelineDynamicStateCreateInfo){
607 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
608 .dynamicStateCount = 2,
609 .pDynamicStates =
610 (VkDynamicState[]){
611 VK_DYNAMIC_STATE_VIEWPORT,
612 VK_DYNAMIC_STATE_SCISSOR,
613 },
614 },
615 .layout = *layout_out,
616 };
617
618 struct vk_meta_rendering_info render = {
619 .color_attachment_count = 1,
620 .color_attachment_formats = {format},
621 };
622
623 result = vk_meta_create_graphics_pipeline(&device->vk, &device->meta_state.device, &pipeline_create_info, &render,
624 &key, sizeof(key), pipeline_out);
625
626 ralloc_free(vs_module);
627 ralloc_free(fs_module);
628 return result;
629 }
630
631 struct radv_blit2d_ds_key {
632 enum radv_meta_object_key_type type;
633 enum blit2d_src_type src_type;
634 uint32_t log2_samples;
635 };
636
637 static VkResult
get_depth_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples,VkPipeline * pipeline_out,VkPipelineLayout * layout_out)638 get_depth_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type, uint32_t log2_samples,
639 VkPipeline *pipeline_out, VkPipelineLayout *layout_out)
640 {
641 struct radv_blit2d_ds_key key;
642 const char *name;
643 VkResult result;
644
645 result = create_layout(device, src_type, layout_out);
646 if (result != VK_SUCCESS)
647 return result;
648
649 memset(&key, 0, sizeof(key));
650 key.type = RADV_META_OBJECT_KEY_BLIT2D_DEPTH;
651 key.src_type = src_type;
652 key.log2_samples = log2_samples;
653
654 VkPipeline pipeline_from_cache = vk_meta_lookup_pipeline(&device->meta_state.device, &key, sizeof(key));
655 if (pipeline_from_cache != VK_NULL_HANDLE) {
656 *pipeline_out = pipeline_from_cache;
657 return VK_SUCCESS;
658 }
659
660 texel_fetch_build_func src_func;
661 switch (src_type) {
662 case BLIT2D_SRC_TYPE_IMAGE:
663 src_func = build_nir_texel_fetch;
664 name = "meta_blit2d_depth_image_fs";
665 break;
666 case BLIT2D_SRC_TYPE_IMAGE_3D:
667 src_func = build_nir_texel_fetch;
668 name = "meta_blit3d_depth_image_fs";
669 break;
670 case BLIT2D_SRC_TYPE_BUFFER:
671 src_func = build_nir_buffer_fetch;
672 name = "meta_blit2d_depth_buffer_fs";
673 break;
674 default:
675 unreachable("unknown blit src type\n");
676 break;
677 }
678
679 nir_shader *vs_module = build_nir_vertex_shader(device);
680 nir_shader *fs_module = build_nir_copy_fragment_shader_depth(device, src_func, name,
681 src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
682
683 const VkGraphicsPipelineCreateInfo pipeline_create_info = {
684 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
685 .stageCount = 2,
686 .pStages =
687 (VkPipelineShaderStageCreateInfo[]){
688 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
689 .stage = VK_SHADER_STAGE_VERTEX_BIT,
690 .module = vk_shader_module_handle_from_nir(vs_module),
691 .pName = "main",
692 .pSpecializationInfo = NULL},
693 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
694 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
695 .module = vk_shader_module_handle_from_nir(fs_module),
696 .pName = "main",
697 .pSpecializationInfo = NULL},
698 },
699 .pVertexInputState =
700 &(VkPipelineVertexInputStateCreateInfo){
701 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
702 .vertexBindingDescriptionCount = 0,
703 .vertexAttributeDescriptionCount = 0,
704 },
705 .pInputAssemblyState =
706 &(VkPipelineInputAssemblyStateCreateInfo){
707 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
708 .topology = VK_PRIMITIVE_TOPOLOGY_META_RECT_LIST_MESA,
709 .primitiveRestartEnable = false,
710 },
711 .pViewportState =
712 &(VkPipelineViewportStateCreateInfo){
713 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
714 .viewportCount = 1,
715 .scissorCount = 1,
716 },
717 .pRasterizationState =
718 &(VkPipelineRasterizationStateCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
719 .rasterizerDiscardEnable = false,
720 .polygonMode = VK_POLYGON_MODE_FILL,
721 .cullMode = VK_CULL_MODE_NONE,
722 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
723 .depthBiasConstantFactor = 0.0f,
724 .depthBiasClamp = 0.0f,
725 .depthBiasSlopeFactor = 0.0f,
726 .lineWidth = 1.0f},
727 .pMultisampleState =
728 &(VkPipelineMultisampleStateCreateInfo){
729 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
730 .rasterizationSamples = 1 << log2_samples,
731 .sampleShadingEnable = false,
732 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
733 },
734 .pColorBlendState =
735 &(VkPipelineColorBlendStateCreateInfo){
736 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
737 .attachmentCount = 0,
738 .pAttachments = NULL,
739 .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
740 },
741 .pDepthStencilState =
742 &(VkPipelineDepthStencilStateCreateInfo){
743 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
744 .depthTestEnable = true,
745 .depthWriteEnable = true,
746 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
747 .front =
748 {
749 .failOp = VK_STENCIL_OP_KEEP,
750 .passOp = VK_STENCIL_OP_KEEP,
751 .depthFailOp = VK_STENCIL_OP_KEEP,
752 .compareOp = VK_COMPARE_OP_NEVER,
753 .compareMask = UINT32_MAX,
754 .writeMask = UINT32_MAX,
755 .reference = 0u,
756 },
757 .back =
758 {
759 .failOp = VK_STENCIL_OP_KEEP,
760 .passOp = VK_STENCIL_OP_KEEP,
761 .depthFailOp = VK_STENCIL_OP_KEEP,
762 .compareOp = VK_COMPARE_OP_NEVER,
763 .compareMask = UINT32_MAX,
764 .writeMask = UINT32_MAX,
765 .reference = 0u,
766 },
767 .minDepthBounds = 0.0f,
768 .maxDepthBounds = 1.0f,
769 },
770 .pDynamicState =
771 &(VkPipelineDynamicStateCreateInfo){
772 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
773 .dynamicStateCount = 2,
774 .pDynamicStates =
775 (VkDynamicState[]){
776 VK_DYNAMIC_STATE_VIEWPORT,
777 VK_DYNAMIC_STATE_SCISSOR,
778 },
779 },
780 .layout = *layout_out,
781 };
782
783 struct vk_meta_rendering_info render = {
784 .depth_attachment_format = VK_FORMAT_D32_SFLOAT,
785 };
786
787 result = vk_meta_create_graphics_pipeline(&device->vk, &device->meta_state.device, &pipeline_create_info, &render,
788 &key, sizeof(key), pipeline_out);
789
790 ralloc_free(vs_module);
791 ralloc_free(fs_module);
792 return result;
793 }
794
795 static VkResult
get_stencil_only_pipeline(struct radv_device * device,enum blit2d_src_type src_type,uint32_t log2_samples,VkPipeline * pipeline_out,VkPipelineLayout * layout_out)796 get_stencil_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type, uint32_t log2_samples,
797 VkPipeline *pipeline_out, VkPipelineLayout *layout_out)
798 {
799 struct radv_blit2d_ds_key key;
800 const char *name;
801 VkResult result;
802
803 result = create_layout(device, src_type, layout_out);
804 if (result != VK_SUCCESS)
805 return result;
806
807 memset(&key, 0, sizeof(key));
808 key.type = RADV_META_OBJECT_KEY_BLIT2D_STENCIL;
809 key.src_type = src_type;
810 key.log2_samples = log2_samples;
811
812 VkPipeline pipeline_from_cache = vk_meta_lookup_pipeline(&device->meta_state.device, &key, sizeof(key));
813 if (pipeline_from_cache != VK_NULL_HANDLE) {
814 *pipeline_out = pipeline_from_cache;
815 return VK_SUCCESS;
816 }
817
818 texel_fetch_build_func src_func;
819 switch (src_type) {
820 case BLIT2D_SRC_TYPE_IMAGE:
821 src_func = build_nir_texel_fetch;
822 name = "meta_blit2d_stencil_image_fs";
823 break;
824 case BLIT2D_SRC_TYPE_IMAGE_3D:
825 src_func = build_nir_texel_fetch;
826 name = "meta_blit3d_stencil_image_fs";
827 break;
828 case BLIT2D_SRC_TYPE_BUFFER:
829 src_func = build_nir_buffer_fetch;
830 name = "meta_blit2d_stencil_buffer_fs";
831 break;
832 default:
833 unreachable("unknown blit src type\n");
834 break;
835 }
836
837 nir_shader *vs_module = build_nir_vertex_shader(device);
838 nir_shader *fs_module = build_nir_copy_fragment_shader_stencil(
839 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0);
840
841 const VkGraphicsPipelineCreateInfo pipeline_create_info = {
842 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
843 .stageCount = 2,
844 .pStages =
845 (VkPipelineShaderStageCreateInfo[]){
846 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
847 .stage = VK_SHADER_STAGE_VERTEX_BIT,
848 .module = vk_shader_module_handle_from_nir(vs_module),
849 .pName = "main",
850 .pSpecializationInfo = NULL},
851 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
852 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
853 .module = vk_shader_module_handle_from_nir(fs_module),
854 .pName = "main",
855 .pSpecializationInfo = NULL},
856 },
857 .pVertexInputState =
858 &(VkPipelineVertexInputStateCreateInfo){
859 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
860 .vertexBindingDescriptionCount = 0,
861 .vertexAttributeDescriptionCount = 0,
862 },
863 .pInputAssemblyState =
864 &(VkPipelineInputAssemblyStateCreateInfo){
865 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
866 .topology = VK_PRIMITIVE_TOPOLOGY_META_RECT_LIST_MESA,
867 .primitiveRestartEnable = false,
868 },
869 .pViewportState =
870 &(VkPipelineViewportStateCreateInfo){
871 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
872 .viewportCount = 1,
873 .scissorCount = 1,
874 },
875 .pRasterizationState =
876 &(VkPipelineRasterizationStateCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
877 .rasterizerDiscardEnable = false,
878 .polygonMode = VK_POLYGON_MODE_FILL,
879 .cullMode = VK_CULL_MODE_NONE,
880 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
881 .depthBiasConstantFactor = 0.0f,
882 .depthBiasClamp = 0.0f,
883 .depthBiasSlopeFactor = 0.0f,
884 .lineWidth = 1.0f},
885 .pMultisampleState =
886 &(VkPipelineMultisampleStateCreateInfo){
887 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
888 .rasterizationSamples = 1 << log2_samples,
889 .sampleShadingEnable = false,
890 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
891 },
892 .pColorBlendState =
893 &(VkPipelineColorBlendStateCreateInfo){
894 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
895 .attachmentCount = 0,
896 .pAttachments = NULL,
897 .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
898 },
899 .pDepthStencilState =
900 &(VkPipelineDepthStencilStateCreateInfo){
901 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
902 .depthTestEnable = false,
903 .depthWriteEnable = false,
904 .stencilTestEnable = true,
905 .front = {.failOp = VK_STENCIL_OP_REPLACE,
906 .passOp = VK_STENCIL_OP_REPLACE,
907 .depthFailOp = VK_STENCIL_OP_REPLACE,
908 .compareOp = VK_COMPARE_OP_ALWAYS,
909 .compareMask = 0xff,
910 .writeMask = 0xff,
911 .reference = 0},
912 .back = {.failOp = VK_STENCIL_OP_REPLACE,
913 .passOp = VK_STENCIL_OP_REPLACE,
914 .depthFailOp = VK_STENCIL_OP_REPLACE,
915 .compareOp = VK_COMPARE_OP_ALWAYS,
916 .compareMask = 0xff,
917 .writeMask = 0xff,
918 .reference = 0},
919 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
920 .minDepthBounds = 0.0f,
921 .maxDepthBounds = 1.0f,
922 },
923 .pDynamicState =
924 &(VkPipelineDynamicStateCreateInfo){
925 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
926 .dynamicStateCount = 2,
927 .pDynamicStates =
928 (VkDynamicState[]){
929 VK_DYNAMIC_STATE_VIEWPORT,
930 VK_DYNAMIC_STATE_SCISSOR,
931 },
932 },
933 .layout = *layout_out,
934 };
935
936 struct vk_meta_rendering_info render = {
937 .stencil_attachment_format = VK_FORMAT_S8_UINT,
938 };
939
940 result = vk_meta_create_graphics_pipeline(&device->vk, &device->meta_state.device, &pipeline_create_info, &render,
941 &key, sizeof(key), pipeline_out);
942
943 ralloc_free(vs_module);
944 ralloc_free(fs_module);
945 return result;
946 }
947