1 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "SpirvShader.hpp"
16
17 #include "System/Types.hpp"
18
19 #include "Vulkan/VkDescriptorSetLayout.hpp"
20 #include "Vulkan/VkPipelineLayout.hpp"
21
22 #include <spirv/unified1/spirv.hpp>
23
24 namespace sw {
25
SpirvFormatToVulkanFormat(spv::ImageFormat format)26 static vk::Format SpirvFormatToVulkanFormat(spv::ImageFormat format)
27 {
28 switch(format)
29 {
30 case spv::ImageFormatUnknown: return VK_FORMAT_UNDEFINED;
31 case spv::ImageFormatRgba32f: return VK_FORMAT_R32G32B32A32_SFLOAT;
32 case spv::ImageFormatRgba16f: return VK_FORMAT_R16G16B16A16_SFLOAT;
33 case spv::ImageFormatR32f: return VK_FORMAT_R32_SFLOAT;
34 case spv::ImageFormatRgba8: return VK_FORMAT_R8G8B8A8_UNORM;
35 case spv::ImageFormatRgba8Snorm: return VK_FORMAT_R8G8B8A8_SNORM;
36 case spv::ImageFormatRg32f: return VK_FORMAT_R32G32_SFLOAT;
37 case spv::ImageFormatRg16f: return VK_FORMAT_R16G16_SFLOAT;
38 case spv::ImageFormatR11fG11fB10f: return VK_FORMAT_B10G11R11_UFLOAT_PACK32;
39 case spv::ImageFormatR16f: return VK_FORMAT_R16_SFLOAT;
40 case spv::ImageFormatRgba16: return VK_FORMAT_R16G16B16A16_UNORM;
41 case spv::ImageFormatRgb10A2: return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
42 case spv::ImageFormatRg16: return VK_FORMAT_R16G16_UNORM;
43 case spv::ImageFormatRg8: return VK_FORMAT_R8G8_UNORM;
44 case spv::ImageFormatR16: return VK_FORMAT_R16_UNORM;
45 case spv::ImageFormatR8: return VK_FORMAT_R8_UNORM;
46 case spv::ImageFormatRgba16Snorm: return VK_FORMAT_R16G16B16A16_SNORM;
47 case spv::ImageFormatRg16Snorm: return VK_FORMAT_R16G16_SNORM;
48 case spv::ImageFormatRg8Snorm: return VK_FORMAT_R8G8_SNORM;
49 case spv::ImageFormatR16Snorm: return VK_FORMAT_R16_SNORM;
50 case spv::ImageFormatR8Snorm: return VK_FORMAT_R8_SNORM;
51 case spv::ImageFormatRgba32i: return VK_FORMAT_R32G32B32A32_SINT;
52 case spv::ImageFormatRgba16i: return VK_FORMAT_R16G16B16A16_SINT;
53 case spv::ImageFormatRgba8i: return VK_FORMAT_R8G8B8A8_SINT;
54 case spv::ImageFormatR32i: return VK_FORMAT_R32_SINT;
55 case spv::ImageFormatRg32i: return VK_FORMAT_R32G32_SINT;
56 case spv::ImageFormatRg16i: return VK_FORMAT_R16G16_SINT;
57 case spv::ImageFormatRg8i: return VK_FORMAT_R8G8_SINT;
58 case spv::ImageFormatR16i: return VK_FORMAT_R16_SINT;
59 case spv::ImageFormatR8i: return VK_FORMAT_R8_SINT;
60 case spv::ImageFormatRgba32ui: return VK_FORMAT_R32G32B32A32_UINT;
61 case spv::ImageFormatRgba16ui: return VK_FORMAT_R16G16B16A16_UINT;
62 case spv::ImageFormatRgba8ui: return VK_FORMAT_R8G8B8A8_UINT;
63 case spv::ImageFormatR32ui: return VK_FORMAT_R32_UINT;
64 case spv::ImageFormatRgb10a2ui: return VK_FORMAT_A2B10G10R10_UINT_PACK32;
65 case spv::ImageFormatRg32ui: return VK_FORMAT_R32G32_UINT;
66 case spv::ImageFormatRg16ui: return VK_FORMAT_R16G16_UINT;
67 case spv::ImageFormatRg8ui: return VK_FORMAT_R8G8_UINT;
68 case spv::ImageFormatR16ui: return VK_FORMAT_R16_UINT;
69 case spv::ImageFormatR8ui: return VK_FORMAT_R8_UINT;
70
71 default:
72 UNSUPPORTED("SPIR-V ImageFormat %u", format);
73 return VK_FORMAT_UNDEFINED;
74 }
75 }
76
ImageInstruction(InsnIterator insn,const Spirv & shader,const SpirvEmitter & state)77 SpirvEmitter::ImageInstruction::ImageInstruction(InsnIterator insn, const Spirv &shader, const SpirvEmitter &state)
78 : ImageInstructionSignature(parseVariantAndMethod(insn))
79 , position(insn.distanceFrom(shader.begin()))
80 {
81 if(samplerMethod == Write)
82 {
83 imageId = insn.word(1);
84 coordinateId = insn.word(2);
85 texelId = insn.word(3);
86 }
87 else
88 {
89 resultTypeId = insn.resultTypeId(); // word(1)
90 resultId = insn.resultId(); // word(2)
91
92 if(samplerMethod == Fetch || samplerMethod == Read || samplerMethod == TexelPointer) // Samplerless
93 {
94 imageId = insn.word(3);
95 }
96 else
97 {
98 // sampledImageId is either the result of an OpSampledImage instruction or
99 // an externally combined sampler and image.
100 Object::ID sampledImageId = insn.word(3);
101
102 if(state.isSampledImage(sampledImageId)) // Result of an OpSampledImage instruction
103 {
104 const SampledImagePointer &sampledImage = state.getSampledImage(sampledImageId);
105 imageId = shader.getObject(sampledImageId).definition.word(3);
106 samplerId = sampledImage.samplerId;
107 }
108 else // Combined image/sampler
109 {
110 imageId = sampledImageId;
111 samplerId = sampledImageId;
112 }
113 }
114
115 coordinateId = insn.word(4);
116 }
117
118 // `imageId` can represent either a Sampled Image, a samplerless Image, or a pointer to an Image.
119 // To get to the OpTypeImage operands, traverse the OpTypeSampledImage or OpTypePointer.
120 const Type &imageObjectType = shader.getObjectType(imageId);
121 const Type &imageReferenceType = (imageObjectType.opcode() == spv::OpTypeSampledImage)
122 ? shader.getType(imageObjectType.definition.word(2))
123 : imageObjectType;
124 const Type &imageType = ((imageReferenceType.opcode() == spv::OpTypePointer)
125 ? shader.getType(imageReferenceType.element)
126 : imageReferenceType);
127
128 ASSERT(imageType.opcode() == spv::OpTypeImage);
129 dim = imageType.definition.word(3);
130 arrayed = imageType.definition.word(5);
131 imageFormat = imageType.definition.word(8);
132
133 const Object &coordinateObject = shader.getObject(coordinateId);
134 const Type &coordinateType = shader.getType(coordinateObject);
135 coordinates = coordinateType.componentCount - (isProj() ? 1 : 0);
136
137 if(samplerMethod == TexelPointer)
138 {
139 sampleId = insn.word(5);
140 sample = !shader.getObject(sampleId).isConstantZero();
141 }
142
143 if(isDref())
144 {
145 drefId = insn.word(5);
146 }
147
148 if(samplerMethod == Gather)
149 {
150 gatherComponent = !isDref() ? shader.getObject(insn.word(5)).constantValue[0] : 0;
151 }
152
153 uint32_t operandsIndex = getImageOperandsIndex(insn);
154 uint32_t imageOperands = (operandsIndex != 0) ? insn.word(operandsIndex) : 0; // The mask which indicates which operands are provided.
155
156 operandsIndex += 1; // Advance to the first actual operand <id> location.
157
158 if(imageOperands & spv::ImageOperandsBiasMask)
159 {
160 ASSERT(samplerMethod == Bias);
161 lodOrBiasId = insn.word(operandsIndex);
162 operandsIndex += 1;
163 imageOperands &= ~spv::ImageOperandsBiasMask;
164 }
165
166 if(imageOperands & spv::ImageOperandsLodMask)
167 {
168 ASSERT(samplerMethod == Lod || samplerMethod == Fetch);
169 lodOrBiasId = insn.word(operandsIndex);
170 operandsIndex += 1;
171 imageOperands &= ~spv::ImageOperandsLodMask;
172 }
173
174 if(imageOperands & spv::ImageOperandsGradMask)
175 {
176 ASSERT(samplerMethod == Grad);
177 gradDxId = insn.word(operandsIndex + 0);
178 gradDyId = insn.word(operandsIndex + 1);
179 operandsIndex += 2;
180 imageOperands &= ~spv::ImageOperandsGradMask;
181
182 grad = shader.getObjectType(gradDxId).componentCount;
183 }
184
185 if(imageOperands & spv::ImageOperandsConstOffsetMask)
186 {
187 offsetId = insn.word(operandsIndex);
188 operandsIndex += 1;
189 imageOperands &= ~spv::ImageOperandsConstOffsetMask;
190
191 offset = shader.getObjectType(offsetId).componentCount;
192 }
193
194 if(imageOperands & spv::ImageOperandsSampleMask)
195 {
196 ASSERT(samplerMethod == Fetch || samplerMethod == Read || samplerMethod == Write);
197 sampleId = insn.word(operandsIndex);
198 operandsIndex += 1;
199 imageOperands &= ~spv::ImageOperandsSampleMask;
200
201 sample = !shader.getObject(sampleId).isConstantZero();
202 }
203
204 // TODO(b/174475384)
205 if(imageOperands & spv::ImageOperandsZeroExtendMask)
206 {
207 ASSERT(samplerMethod == Read || samplerMethod == Write);
208 imageOperands &= ~spv::ImageOperandsZeroExtendMask;
209 }
210 else if(imageOperands & spv::ImageOperandsSignExtendMask)
211 {
212 ASSERT(samplerMethod == Read || samplerMethod == Write);
213 imageOperands &= ~spv::ImageOperandsSignExtendMask;
214 }
215
216 [[maybe_unused]] spv::Scope scope = spv::ScopeCrossDevice; // "Whilst the CrossDevice scope is defined in SPIR-V, it is disallowed in Vulkan."
217
218 if(imageOperands & spv::ImageOperandsMakeTexelAvailableMask)
219 {
220 scope = static_cast<spv::Scope>(insn.word(operandsIndex));
221 operandsIndex += 1;
222 imageOperands &= ~spv::ImageOperandsMakeTexelAvailableMask;
223 }
224
225 if(imageOperands & spv::ImageOperandsMakeTexelVisibleMask)
226 {
227 scope = static_cast<spv::Scope>(insn.word(operandsIndex));
228 operandsIndex += 1;
229 imageOperands &= ~spv::ImageOperandsMakeTexelVisibleMask;
230 }
231
232 if(imageOperands & spv::ImageOperandsNonPrivateTexelMask)
233 {
234 imageOperands &= ~spv::ImageOperandsNonPrivateTexelMask;
235 }
236
237 if(imageOperands & spv::ImageOperandsVolatileTexelMask)
238 {
239 UNIMPLEMENTED("b/176819536");
240 imageOperands &= ~spv::ImageOperandsVolatileTexelMask;
241 }
242
243 if(imageOperands & spv::ImageOperandsNontemporalMask)
244 {
245 // Hints that the accessed texels are not likely
246 // to be accessed again in the near future.
247 imageOperands &= ~spv::ImageOperandsNontemporalMask;
248 }
249
250 // There should be no remaining image operands.
251 if(imageOperands != 0)
252 {
253 UNSUPPORTED("Image operands 0x%08X", imageOperands);
254 }
255 }
256
parseVariantAndMethod(InsnIterator insn)257 SpirvEmitter::ImageInstructionSignature SpirvEmitter::ImageInstruction::parseVariantAndMethod(InsnIterator insn)
258 {
259 uint32_t imageOperands = getImageOperandsMask(insn);
260 bool bias = imageOperands & spv::ImageOperandsBiasMask;
261 bool grad = imageOperands & spv::ImageOperandsGradMask;
262
263 switch(insn.opcode())
264 {
265 case spv::OpImageSampleImplicitLod: return { None, bias ? Bias : Implicit };
266 case spv::OpImageSampleExplicitLod: return { None, grad ? Grad : Lod };
267 case spv::OpImageSampleDrefImplicitLod: return { Dref, bias ? Bias : Implicit };
268 case spv::OpImageSampleDrefExplicitLod: return { Dref, grad ? Grad : Lod };
269 case spv::OpImageSampleProjImplicitLod: return { Proj, bias ? Bias : Implicit };
270 case spv::OpImageSampleProjExplicitLod: return { Proj, grad ? Grad : Lod };
271 case spv::OpImageSampleProjDrefImplicitLod: return { ProjDref, bias ? Bias : Implicit };
272 case spv::OpImageSampleProjDrefExplicitLod: return { ProjDref, grad ? Grad : Lod };
273 case spv::OpImageGather: return { None, Gather };
274 case spv::OpImageDrefGather: return { Dref, Gather };
275 case spv::OpImageFetch: return { None, Fetch };
276 case spv::OpImageQueryLod: return { None, Query };
277 case spv::OpImageRead: return { None, Read };
278 case spv::OpImageWrite: return { None, Write };
279 case spv::OpImageTexelPointer: return { None, TexelPointer };
280
281 default:
282 ASSERT(false);
283 return { None, Implicit };
284 }
285 }
286
287 // Returns the instruction word index at which the Image Operands mask is located, or 0 if not present.
getImageOperandsIndex(InsnIterator insn)288 uint32_t SpirvEmitter::ImageInstruction::getImageOperandsIndex(InsnIterator insn)
289 {
290 switch(insn.opcode())
291 {
292 case spv::OpImageSampleImplicitLod:
293 case spv::OpImageSampleProjImplicitLod:
294 return insn.wordCount() > 5 ? 5 : 0; // Optional
295 case spv::OpImageSampleExplicitLod:
296 case spv::OpImageSampleProjExplicitLod:
297 return 5; // "Either Lod or Grad image operands must be present."
298 case spv::OpImageSampleDrefImplicitLod:
299 case spv::OpImageSampleProjDrefImplicitLod:
300 return insn.wordCount() > 6 ? 6 : 0; // Optional
301 case spv::OpImageSampleDrefExplicitLod:
302 case spv::OpImageSampleProjDrefExplicitLod:
303 return 6; // "Either Lod or Grad image operands must be present."
304 case spv::OpImageGather:
305 case spv::OpImageDrefGather:
306 return insn.wordCount() > 6 ? 6 : 0; // Optional
307 case spv::OpImageFetch:
308 return insn.wordCount() > 5 ? 5 : 0; // Optional
309 case spv::OpImageQueryLod:
310 ASSERT(insn.wordCount() == 5);
311 return 0; // No image operands.
312 case spv::OpImageRead:
313 return insn.wordCount() > 5 ? 5 : 0; // Optional
314 case spv::OpImageWrite:
315 return insn.wordCount() > 4 ? 4 : 0; // Optional
316 case spv::OpImageTexelPointer:
317 ASSERT(insn.wordCount() == 6);
318 return 0; // No image operands.
319
320 default:
321 ASSERT(false);
322 return 0;
323 }
324 }
325
getImageOperandsMask(InsnIterator insn)326 uint32_t SpirvEmitter::ImageInstruction::getImageOperandsMask(InsnIterator insn)
327 {
328 uint32_t operandsIndex = getImageOperandsIndex(insn);
329 return (operandsIndex != 0) ? insn.word(operandsIndex) : 0;
330 }
331
EmitImageSample(const ImageInstruction & instruction)332 void SpirvEmitter::EmitImageSample(const ImageInstruction &instruction)
333 {
334 auto &resultType = shader.getType(instruction.resultTypeId);
335 auto &result = createIntermediate(instruction.resultId, resultType.componentCount);
336 Array<SIMD::Float> out(4);
337
338 // TODO(b/153380916): When we're in a code path that is always executed,
339 // i.e. post-dominators of the entry block, we don't have to dynamically
340 // check whether any lanes are active, and can elide the jump.
341 If(AnyTrue(activeLaneMask()))
342 {
343 EmitImageSampleUnconditional(out, instruction);
344 }
345
346 for(auto i = 0u; i < resultType.componentCount; i++) { result.move(i, out[i]); }
347 }
348
EmitImageSampleUnconditional(Array<SIMD::Float> & out,const ImageInstruction & instruction) const349 void SpirvEmitter::EmitImageSampleUnconditional(Array<SIMD::Float> &out, const ImageInstruction &instruction) const
350 {
351 auto decorations = shader.GetDecorationsForId(instruction.imageId);
352
353 if(decorations.NonUniform)
354 {
355 SIMD::Int activeLaneMask = this->activeLaneMask();
356 SIMD::Pointer imagePointer = getImage(instruction.imageId);
357 // PerLane output
358 for(int laneIdx = 0; laneIdx < SIMD::Width; laneIdx++)
359 {
360 Array<SIMD::Float> laneOut(out.getArraySize());
361 If(Extract(activeLaneMask, laneIdx) != 0)
362 {
363 Pointer<Byte> imageDescriptor = imagePointer.getPointerForLane(laneIdx); // vk::SampledImageDescriptor*
364 Pointer<Byte> samplerDescriptor = getSamplerDescriptor(imageDescriptor, instruction, laneIdx);
365
366 Pointer<Byte> samplerFunction = lookupSamplerFunction(imageDescriptor, samplerDescriptor, instruction);
367
368 callSamplerFunction(samplerFunction, laneOut, imageDescriptor, instruction);
369 }
370
371 for(int outIdx = 0; outIdx < out.getArraySize(); outIdx++)
372 {
373 out[outIdx] = Insert(out[outIdx], Extract(laneOut[outIdx], laneIdx), laneIdx);
374 }
375 }
376 }
377 else
378 {
379 Pointer<Byte> imageDescriptor = getImage(instruction.imageId).getUniformPointer(); // vk::SampledImageDescriptor*
380 Pointer<Byte> samplerDescriptor = getSamplerDescriptor(imageDescriptor, instruction);
381
382 Pointer<Byte> samplerFunction = lookupSamplerFunction(imageDescriptor, samplerDescriptor, instruction);
383
384 callSamplerFunction(samplerFunction, out, imageDescriptor, instruction);
385 }
386 }
387
getSamplerDescriptor(Pointer<Byte> imageDescriptor,const ImageInstruction & instruction) const388 Pointer<Byte> SpirvEmitter::getSamplerDescriptor(Pointer<Byte> imageDescriptor, const ImageInstruction &instruction) const
389 {
390 return ((instruction.samplerId == instruction.imageId) || (instruction.samplerId == 0)) ? imageDescriptor : getImage(instruction.samplerId).getUniformPointer();
391 }
392
getSamplerDescriptor(Pointer<Byte> imageDescriptor,const ImageInstruction & instruction,int laneIdx) const393 Pointer<Byte> SpirvEmitter::getSamplerDescriptor(Pointer<Byte> imageDescriptor, const ImageInstruction &instruction, int laneIdx) const
394 {
395 return ((instruction.samplerId == instruction.imageId) || (instruction.samplerId == 0)) ? imageDescriptor : getImage(instruction.samplerId).getPointerForLane(laneIdx);
396 }
397
lookupSamplerFunction(Pointer<Byte> imageDescriptor,Pointer<Byte> samplerDescriptor,const ImageInstruction & instruction) const398 Pointer<Byte> SpirvEmitter::lookupSamplerFunction(Pointer<Byte> imageDescriptor, Pointer<Byte> samplerDescriptor, const ImageInstruction &instruction) const
399 {
400 Int samplerId = (instruction.samplerId != 0) ? *Pointer<rr::Int>(samplerDescriptor + OFFSET(vk::SampledImageDescriptor, samplerId)) : Int(0);
401
402 auto &cache = routine->samplerCache.at(instruction.position);
403 Bool cacheHit = (cache.imageDescriptor == imageDescriptor) && (cache.samplerId == samplerId); // TODO(b/205566405): Skip sampler ID check for samplerless instructions.
404
405 If(!cacheHit)
406 {
407 rr::Int imageViewId = *Pointer<rr::Int>(imageDescriptor + OFFSET(vk::ImageDescriptor, imageViewId));
408 cache.function = Call(getImageSampler, routine->device, instruction.signature, samplerId, imageViewId);
409 cache.imageDescriptor = imageDescriptor;
410 cache.samplerId = samplerId;
411 }
412
413 return cache.function;
414 }
415
callSamplerFunction(Pointer<Byte> samplerFunction,Array<SIMD::Float> & out,Pointer<Byte> imageDescriptor,const ImageInstruction & instruction) const416 void SpirvEmitter::callSamplerFunction(Pointer<Byte> samplerFunction, Array<SIMD::Float> &out, Pointer<Byte> imageDescriptor, const ImageInstruction &instruction) const
417 {
418 Array<SIMD::Float> in(16); // Maximum 16 input parameter components.
419
420 auto coordinate = Operand(shader, *this, instruction.coordinateId);
421
422 uint32_t i = 0;
423 for(; i < instruction.coordinates; i++)
424 {
425 if(instruction.isProj())
426 {
427 in[i] = coordinate.Float(i) / coordinate.Float(instruction.coordinates); // TODO(b/129523279): Optimize using reciprocal.
428 }
429 else
430 {
431 in[i] = coordinate.Float(i);
432 }
433 }
434
435 if(instruction.isDref())
436 {
437 auto drefValue = Operand(shader, *this, instruction.drefId);
438
439 if(instruction.isProj())
440 {
441 in[i] = drefValue.Float(0) / coordinate.Float(instruction.coordinates); // TODO(b/129523279): Optimize using reciprocal.
442 }
443 else
444 {
445 in[i] = drefValue.Float(0);
446 }
447
448 i++;
449 }
450
451 if(instruction.lodOrBiasId != 0)
452 {
453 auto lodValue = Operand(shader, *this, instruction.lodOrBiasId);
454 in[i] = lodValue.Float(0);
455 i++;
456 }
457 else if(instruction.gradDxId != 0)
458 {
459 auto dxValue = Operand(shader, *this, instruction.gradDxId);
460 auto dyValue = Operand(shader, *this, instruction.gradDyId);
461 ASSERT(dxValue.componentCount == dxValue.componentCount);
462
463 for(uint32_t j = 0; j < dxValue.componentCount; j++, i++)
464 {
465 in[i] = dxValue.Float(j);
466 }
467
468 for(uint32_t j = 0; j < dxValue.componentCount; j++, i++)
469 {
470 in[i] = dyValue.Float(j);
471 }
472 }
473 else if(instruction.samplerMethod == Fetch)
474 {
475 // The instruction didn't provide a lod operand, but the sampler's Fetch
476 // function requires one to be present. If no lod is supplied, the default
477 // is zero.
478 in[i] = As<SIMD::Float>(SIMD::Int(0));
479 i++;
480 }
481
482 if(instruction.offsetId != 0)
483 {
484 auto offsetValue = Operand(shader, *this, instruction.offsetId);
485
486 for(uint32_t j = 0; j < offsetValue.componentCount; j++, i++)
487 {
488 in[i] = As<SIMD::Float>(offsetValue.Int(j)); // Integer values, but transfered as float.
489 }
490 }
491
492 if(instruction.sample)
493 {
494 auto sampleValue = Operand(shader, *this, instruction.sampleId);
495 in[i] = As<SIMD::Float>(sampleValue.Int(0));
496 }
497
498 Pointer<Byte> texture = imageDescriptor + OFFSET(vk::SampledImageDescriptor, texture); // sw::Texture*
499
500 Call<ImageSampler>(samplerFunction, texture, &in, &out, routine->constants);
501 }
502
EmitImageQuerySizeLod(InsnIterator insn)503 void SpirvEmitter::EmitImageQuerySizeLod(InsnIterator insn)
504 {
505 auto &resultTy = shader.getType(insn.resultTypeId());
506 auto imageId = Object::ID(insn.word(3));
507 auto lodId = Object::ID(insn.word(4));
508
509 auto &dst = createIntermediate(insn.resultId(), resultTy.componentCount);
510 GetImageDimensions(resultTy, imageId, lodId, dst);
511 }
512
EmitImageQuerySize(InsnIterator insn)513 void SpirvEmitter::EmitImageQuerySize(InsnIterator insn)
514 {
515 auto &resultTy = shader.getType(insn.resultTypeId());
516 auto imageId = Object::ID(insn.word(3));
517 auto lodId = Object::ID(0);
518
519 auto &dst = createIntermediate(insn.resultId(), resultTy.componentCount);
520 GetImageDimensions(resultTy, imageId, lodId, dst);
521 }
522
GetImageDimensions(const Type & resultTy,Object::ID imageId,Object::ID lodId,Intermediate & dst) const523 void SpirvEmitter::GetImageDimensions(const Type &resultTy, Object::ID imageId, Object::ID lodId, Intermediate &dst) const
524 {
525 auto &image = shader.getObject(imageId);
526 auto &imageType = shader.getType(image);
527
528 ASSERT(imageType.definition.opcode() == spv::OpTypeImage);
529 bool isArrayed = imageType.definition.word(5) != 0;
530 uint32_t dimensions = resultTy.componentCount - (isArrayed ? 1 : 0);
531
532 const Spirv::DescriptorDecorations &d = shader.descriptorDecorations.at(imageId);
533 auto descriptorType = routine->pipelineLayout->getDescriptorType(d.DescriptorSet, d.Binding);
534
535 Pointer<Byte> descriptor = getPointer(imageId).getUniformPointer();
536
537 Int width;
538 Int height;
539 Int depth;
540
541 switch(descriptorType)
542 {
543 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
544 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
545 width = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, width));
546 height = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, height));
547 depth = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, depth));
548 break;
549 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
550 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
551 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
552 width = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, width));
553 height = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, height));
554 depth = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, depth));
555 break;
556 default:
557 UNREACHABLE("Image descriptorType: %d", int(descriptorType));
558 }
559
560 if(lodId != 0)
561 {
562 auto lodVal = Operand(shader, *this, lodId);
563 ASSERT(lodVal.componentCount == 1);
564 auto lod = lodVal.Int(0);
565 auto one = SIMD::Int(1);
566
567 if(dimensions >= 1) dst.move(0, Max(SIMD::Int(width) >> lod, one));
568 if(dimensions >= 2) dst.move(1, Max(SIMD::Int(height) >> lod, one));
569 if(dimensions >= 3) dst.move(2, Max(SIMD::Int(depth) >> lod, one));
570 }
571 else
572 {
573
574 if(dimensions >= 1) dst.move(0, SIMD::Int(width));
575 if(dimensions >= 2) dst.move(1, SIMD::Int(height));
576 if(dimensions >= 3) dst.move(2, SIMD::Int(depth));
577 }
578
579 if(isArrayed)
580 {
581 dst.move(dimensions, SIMD::Int(depth));
582 }
583 }
584
EmitImageQueryLevels(InsnIterator insn)585 void SpirvEmitter::EmitImageQueryLevels(InsnIterator insn)
586 {
587 auto &resultTy = shader.getType(insn.resultTypeId());
588 ASSERT(resultTy.componentCount == 1);
589 auto imageId = Object::ID(insn.word(3));
590
591 const Spirv::DescriptorDecorations &d = shader.descriptorDecorations.at(imageId);
592 auto descriptorType = routine->pipelineLayout->getDescriptorType(d.DescriptorSet, d.Binding);
593
594 Pointer<Byte> descriptor = getPointer(imageId).getUniformPointer();
595 Int mipLevels = 0;
596 switch(descriptorType)
597 {
598 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
599 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
600 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
601 mipLevels = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, mipLevels)); // uint32_t
602 break;
603 default:
604 UNREACHABLE("Image descriptorType: %d", int(descriptorType));
605 }
606
607 auto &dst = createIntermediate(insn.resultId(), 1);
608 dst.move(0, SIMD::Int(mipLevels));
609 }
610
EmitImageQuerySamples(InsnIterator insn)611 void SpirvEmitter::EmitImageQuerySamples(InsnIterator insn)
612 {
613 auto &resultTy = shader.getType(insn.resultTypeId());
614 ASSERT(resultTy.componentCount == 1);
615 auto imageId = Object::ID(insn.word(3));
616 auto imageTy = shader.getObjectType(imageId);
617 ASSERT(imageTy.definition.opcode() == spv::OpTypeImage);
618 ASSERT(imageTy.definition.word(3) == spv::Dim2D);
619 ASSERT(imageTy.definition.word(6 /* MS */) == 1);
620
621 const Spirv::DescriptorDecorations &d = shader.descriptorDecorations.at(imageId);
622 auto descriptorType = routine->pipelineLayout->getDescriptorType(d.DescriptorSet, d.Binding);
623
624 Pointer<Byte> descriptor = getPointer(imageId).getUniformPointer();
625 Int sampleCount = 0;
626 switch(descriptorType)
627 {
628 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
629 sampleCount = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, sampleCount)); // uint32_t
630 break;
631 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
632 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
633 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
634 sampleCount = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, sampleCount)); // uint32_t
635 break;
636 default:
637 UNREACHABLE("Image descriptorType: %d", int(descriptorType));
638 }
639
640 auto &dst = createIntermediate(insn.resultId(), 1);
641 dst.move(0, SIMD::Int(sampleCount));
642 }
643
setupTexelAddressData(SIMD::Int rowPitch,SIMD::Int slicePitch,SIMD::Int samplePitch,ImageInstructionSignature instruction,SIMD::Int coordinate[],SIMD::Int sample,vk::Format imageFormat,const SpirvRoutine * routine)644 SpirvEmitter::TexelAddressData SpirvEmitter::setupTexelAddressData(SIMD::Int rowPitch, SIMD::Int slicePitch, SIMD::Int samplePitch, ImageInstructionSignature instruction, SIMD::Int coordinate[], SIMD::Int sample, vk::Format imageFormat, const SpirvRoutine *routine)
645 {
646 TexelAddressData data;
647
648 data.isArrayed = instruction.arrayed;
649 data.dim = static_cast<spv::Dim>(instruction.dim);
650 data.texelSize = imageFormat.bytes();
651 data.dims = instruction.coordinates - (data.isArrayed ? 1 : 0);
652
653 data.u = coordinate[0];
654 data.v = SIMD::Int(0);
655
656 if(data.dims > 1)
657 {
658 data.v = coordinate[1];
659 }
660
661 if(data.dim == spv::DimSubpassData)
662 {
663 data.u += routine->windowSpacePosition[0];
664 data.v += routine->windowSpacePosition[1];
665 }
666
667 data.ptrOffset = data.u * SIMD::Int(data.texelSize);
668
669 if(data.dims > 1)
670 {
671 data.ptrOffset += data.v * rowPitch;
672 }
673
674 data.w = 0;
675 if((data.dims > 2) || data.isArrayed)
676 {
677 if(data.dims > 2)
678 {
679 data.w += coordinate[2];
680 }
681
682 if(data.isArrayed)
683 {
684 data.w += coordinate[data.dims];
685 }
686
687 data.ptrOffset += data.w * slicePitch;
688 }
689
690 if(data.dim == spv::DimSubpassData)
691 {
692 // Multiview input attachment access is to the layer corresponding to the current view
693 data.ptrOffset += SIMD::Int(routine->layer) * slicePitch;
694 }
695
696 if(instruction.sample)
697 {
698 data.ptrOffset += sample * samplePitch;
699 }
700
701 return data;
702 }
703
GetNonUniformTexelAddress(ImageInstructionSignature instruction,SIMD::Pointer descriptor,SIMD::Int coordinate[],SIMD::Int sample,vk::Format imageFormat,OutOfBoundsBehavior outOfBoundsBehavior,SIMD::Int activeLaneMask,const SpirvRoutine * routine)704 SIMD::Pointer SpirvEmitter::GetNonUniformTexelAddress(ImageInstructionSignature instruction, SIMD::Pointer descriptor, SIMD::Int coordinate[], SIMD::Int sample, vk::Format imageFormat, OutOfBoundsBehavior outOfBoundsBehavior, SIMD::Int activeLaneMask, const SpirvRoutine *routine)
705 {
706 const bool useStencilAspect = (imageFormat == VK_FORMAT_S8_UINT);
707 auto rowPitch = (descriptor + (useStencilAspect
708 ? OFFSET(vk::StorageImageDescriptor, stencilRowPitchBytes)
709 : OFFSET(vk::StorageImageDescriptor, rowPitchBytes)))
710 .Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask);
711 auto slicePitch = (descriptor + (useStencilAspect
712 ? OFFSET(vk::StorageImageDescriptor, stencilSlicePitchBytes)
713 : OFFSET(vk::StorageImageDescriptor, slicePitchBytes)))
714 .Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask);
715 auto samplePitch = (descriptor + (useStencilAspect
716 ? OFFSET(vk::StorageImageDescriptor, stencilSamplePitchBytes)
717 : OFFSET(vk::StorageImageDescriptor, samplePitchBytes)))
718 .Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask);
719
720 auto texelData = setupTexelAddressData(rowPitch, slicePitch, samplePitch, instruction, coordinate, sample, imageFormat, routine);
721
722 // If the out-of-bounds behavior is set to nullify, then each coordinate must be tested individually.
723 // Other out-of-bounds behaviors work properly by just comparing the offset against the total size.
724 if(outOfBoundsBehavior == OutOfBoundsBehavior::Nullify)
725 {
726 SIMD::UInt width = (descriptor + OFFSET(vk::StorageImageDescriptor, width)).Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask);
727 SIMD::Int oobMask = As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(texelData.u), width));
728
729 if(texelData.dims > 1)
730 {
731 SIMD::UInt height = As<SIMD::UInt>((descriptor + OFFSET(vk::StorageImageDescriptor, height)).Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask));
732 oobMask |= As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(texelData.v), height));
733 }
734
735 if((texelData.dims > 2) || texelData.isArrayed)
736 {
737 SIMD::UInt depth = As<SIMD::UInt>((descriptor + OFFSET(vk::StorageImageDescriptor, depth)).Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask));
738 if(texelData.dim == spv::DimCube) { depth *= 6; }
739 oobMask |= As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(texelData.w), depth));
740 }
741
742 if(instruction.sample)
743 {
744 SIMD::UInt sampleCount = As<SIMD::UInt>((descriptor + OFFSET(vk::StorageImageDescriptor, sampleCount)).Load<SIMD::Int>(outOfBoundsBehavior, activeLaneMask));
745 oobMask |= As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(sample), sampleCount));
746 }
747
748 constexpr int32_t OOB_OFFSET = 0x7FFFFFFF - 16; // SIMD pointer offsets are signed 32-bit, so this is the largest offset (for 16-byte texels).
749 static_assert(OOB_OFFSET >= vk::MAX_MEMORY_ALLOCATION_SIZE, "the largest offset must be guaranteed to be out-of-bounds");
750
751 texelData.ptrOffset = (texelData.ptrOffset & ~oobMask) | (oobMask & SIMD::Int(OOB_OFFSET)); // oob ? OOB_OFFSET : ptrOffset // TODO: IfThenElse()
752 }
753
754 std::vector<Pointer<Byte>> imageBase(SIMD::Width);
755 for(int i = 0; i < SIMD::Width; i++)
756 {
757 imageBase[i] = *Pointer<Pointer<Byte>>(descriptor.getPointerForLane(i) + (useStencilAspect
758 ? OFFSET(vk::StorageImageDescriptor, stencilPtr)
759 : OFFSET(vk::StorageImageDescriptor, ptr)));
760 }
761
762 return SIMD::Pointer(imageBase) + texelData.ptrOffset;
763 }
764
GetTexelAddress(ImageInstructionSignature instruction,Pointer<Byte> descriptor,SIMD::Int coordinate[],SIMD::Int sample,vk::Format imageFormat,OutOfBoundsBehavior outOfBoundsBehavior,const SpirvRoutine * routine)765 SIMD::Pointer SpirvEmitter::GetTexelAddress(ImageInstructionSignature instruction, Pointer<Byte> descriptor, SIMD::Int coordinate[], SIMD::Int sample, vk::Format imageFormat, OutOfBoundsBehavior outOfBoundsBehavior, const SpirvRoutine *routine)
766 {
767 const bool useStencilAspect = (imageFormat == VK_FORMAT_S8_UINT);
768 auto rowPitch = SIMD::Int(*Pointer<Int>(descriptor + (useStencilAspect
769 ? OFFSET(vk::StorageImageDescriptor, stencilRowPitchBytes)
770 : OFFSET(vk::StorageImageDescriptor, rowPitchBytes))));
771 auto slicePitch = SIMD::Int(
772 *Pointer<Int>(descriptor + (useStencilAspect
773 ? OFFSET(vk::StorageImageDescriptor, stencilSlicePitchBytes)
774 : OFFSET(vk::StorageImageDescriptor, slicePitchBytes))));
775 auto samplePitch = SIMD::Int(
776 *Pointer<Int>(descriptor + (useStencilAspect
777 ? OFFSET(vk::StorageImageDescriptor, stencilSamplePitchBytes)
778 : OFFSET(vk::StorageImageDescriptor, samplePitchBytes))));
779
780 auto texelData = setupTexelAddressData(rowPitch, slicePitch, samplePitch, instruction, coordinate, sample, imageFormat, routine);
781
782 // If the out-of-bounds behavior is set to nullify, then each coordinate must be tested individually.
783 // Other out-of-bounds behaviors work properly by just comparing the offset against the total size.
784 if(outOfBoundsBehavior == OutOfBoundsBehavior::Nullify)
785 {
786 SIMD::UInt width = *Pointer<UInt>(descriptor + OFFSET(vk::StorageImageDescriptor, width));
787 SIMD::Int oobMask = As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(texelData.u), width));
788
789 if(texelData.dims > 1)
790 {
791 SIMD::UInt height = *Pointer<UInt>(descriptor + OFFSET(vk::StorageImageDescriptor, height));
792 oobMask |= As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(texelData.v), height));
793 }
794
795 if((texelData.dims > 2) || texelData.isArrayed)
796 {
797 UInt depth = *Pointer<UInt>(descriptor + OFFSET(vk::StorageImageDescriptor, depth));
798 if(texelData.dim == spv::DimCube) { depth *= 6; }
799 oobMask |= As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(texelData.w), SIMD::UInt(depth)));
800 }
801
802 if(instruction.sample)
803 {
804 SIMD::UInt sampleCount = *Pointer<UInt>(descriptor + OFFSET(vk::StorageImageDescriptor, sampleCount));
805 oobMask |= As<SIMD::Int>(CmpNLT(As<SIMD::UInt>(sample), sampleCount));
806 }
807
808 constexpr int32_t OOB_OFFSET = 0x7FFFFFFF - 16; // SIMD pointer offsets are signed 32-bit, so this is the largest offset (for 16-byte texels).
809 static_assert(OOB_OFFSET >= vk::MAX_MEMORY_ALLOCATION_SIZE, "the largest offset must be guaranteed to be out-of-bounds");
810
811 texelData.ptrOffset = (texelData.ptrOffset & ~oobMask) | (oobMask & SIMD::Int(OOB_OFFSET)); // oob ? OOB_OFFSET : ptrOffset // TODO: IfThenElse()
812 }
813
814 Pointer<Byte> imageBase = *Pointer<Pointer<Byte>>(descriptor + (useStencilAspect
815 ? OFFSET(vk::StorageImageDescriptor, stencilPtr)
816 : OFFSET(vk::StorageImageDescriptor, ptr)));
817
818 Int imageSizeInBytes = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, sizeInBytes));
819
820 return SIMD::Pointer(imageBase, imageSizeInBytes, texelData.ptrOffset);
821 }
822
EmitImageRead(const ImageInstruction & instruction)823 void SpirvEmitter::EmitImageRead(const ImageInstruction &instruction)
824 {
825 auto &resultType = shader.getObjectType(instruction.resultId);
826 auto &image = shader.getObject(instruction.imageId);
827 auto &imageType = shader.getType(image);
828
829 ASSERT(imageType.definition.opcode() == spv::OpTypeImage);
830 auto dim = static_cast<spv::Dim>(instruction.dim);
831
832 auto coordinate = Operand(shader, *this, instruction.coordinateId);
833 const Spirv::DescriptorDecorations &d = shader.descriptorDecorations.at(instruction.imageId);
834
835 // For subpass data, format in the instruction is spv::ImageFormatUnknown. Get it from
836 // the renderpass data instead. In all other cases, we can use the format in the instruction.
837 vk::Format imageFormat = (dim == spv::DimSubpassData)
838 ? shader.getInputAttachmentFormat(d.InputAttachmentIndex)
839 : SpirvFormatToVulkanFormat(static_cast<spv::ImageFormat>(instruction.imageFormat));
840
841 // Depth+Stencil image attachments select aspect based on the Sampled Type of the
842 // OpTypeImage. If float, then we want the depth aspect. If int, we want the stencil aspect.
843 bool useStencilAspect = (imageFormat == VK_FORMAT_D32_SFLOAT_S8_UINT &&
844 shader.getType(imageType.definition.word(2)).opcode() == spv::OpTypeInt);
845
846 if(useStencilAspect)
847 {
848 imageFormat = VK_FORMAT_S8_UINT;
849 }
850
851 auto &dst = createIntermediate(instruction.resultId, resultType.componentCount);
852 SIMD::Pointer ptr = getPointer(instruction.imageId);
853
854 SIMD::Int uvwa[4];
855 SIMD::Int sample;
856 const int texelSize = imageFormat.bytes();
857 // VK_EXT_image_robustness requires replacing out-of-bounds access with zero.
858 // TODO(b/162327166): Only perform bounds checks when VK_EXT_image_robustness is enabled.
859 auto robustness = OutOfBoundsBehavior::Nullify;
860
861 for(uint32_t i = 0; i < instruction.coordinates; i++)
862 {
863 uvwa[i] = coordinate.Int(i);
864 }
865 if(instruction.sample)
866 {
867 sample = Operand(shader, *this, instruction.sampleId).Int(0);
868 }
869
870 // Gather packed texel data. Texels larger than 4 bytes occupy multiple SIMD::Int elements.
871 // TODO(b/160531165): Provide gather abstractions for various element sizes.
872 SIMD::Int packed[4];
873
874 SIMD::Pointer texelPtr = ptr.isBasePlusOffset
875 ? GetTexelAddress(instruction, ptr.getUniformPointer(), uvwa, sample, imageFormat, robustness, routine)
876 : GetNonUniformTexelAddress(instruction, ptr, uvwa, sample, imageFormat, robustness, activeLaneMask(), routine);
877 if(texelSize == 4 || texelSize == 8 || texelSize == 16)
878 {
879 for(auto i = 0; i < texelSize / 4; i++)
880 {
881 packed[i] = texelPtr.Load<SIMD::Int>(robustness, activeLaneMask());
882 texelPtr += sizeof(float);
883 }
884 }
885 else if(texelSize == 2)
886 {
887 SIMD::Int mask = activeLaneMask() & texelPtr.isInBounds(2, robustness);
888
889 for(int i = 0; i < SIMD::Width; i++)
890 {
891 If(Extract(mask, i) != 0)
892 {
893 packed[0] = Insert(packed[0], Int(*Pointer<Short>(texelPtr.getPointerForLane(i))), i);
894 }
895 }
896 }
897 else if(texelSize == 1)
898 {
899 SIMD::Int mask = activeLaneMask() & texelPtr.isInBounds(1, robustness);
900 for(int i = 0; i < SIMD::Width; i++)
901 {
902 If(Extract(mask, i) != 0)
903 {
904 packed[0] = Insert(packed[0], Int(*Pointer<Byte>(texelPtr.getPointerForLane(i))), i);
905 }
906 }
907 }
908 else
909 UNREACHABLE("texelSize: %d", int(texelSize));
910
911 // Format support requirements here come from two sources:
912 // - Minimum required set of formats for loads from storage images
913 // - Any format supported as a color or depth/stencil attachment, for input attachments
914 switch(imageFormat)
915 {
916 case VK_FORMAT_R32G32B32A32_SFLOAT:
917 case VK_FORMAT_R32G32B32A32_SINT:
918 case VK_FORMAT_R32G32B32A32_UINT:
919 dst.move(0, packed[0]);
920 dst.move(1, packed[1]);
921 dst.move(2, packed[2]);
922 dst.move(3, packed[3]);
923 break;
924 case VK_FORMAT_R32_SINT:
925 case VK_FORMAT_R32_UINT:
926 dst.move(0, packed[0]);
927 // Fill remaining channels with 0,0,1 (of the correct type)
928 dst.move(1, SIMD::Int(0));
929 dst.move(2, SIMD::Int(0));
930 dst.move(3, SIMD::Int(1));
931 break;
932 case VK_FORMAT_R32_SFLOAT:
933 case VK_FORMAT_D32_SFLOAT:
934 case VK_FORMAT_D32_SFLOAT_S8_UINT:
935 dst.move(0, packed[0]);
936 // Fill remaining channels with 0,0,1 (of the correct type)
937 dst.move(1, SIMD::Float(0.0f));
938 dst.move(2, SIMD::Float(0.0f));
939 dst.move(3, SIMD::Float(1.0f));
940 break;
941 case VK_FORMAT_D16_UNORM:
942 dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
943 dst.move(1, SIMD::Float(0.0f));
944 dst.move(2, SIMD::Float(0.0f));
945 dst.move(3, SIMD::Float(1.0f));
946 break;
947 case VK_FORMAT_R16G16B16A16_UNORM:
948 dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
949 dst.move(1, SIMD::Float((packed[0] >> 16) & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
950 dst.move(2, SIMD::Float(packed[1] & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
951 dst.move(3, SIMD::Float((packed[1] >> 16) & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
952 break;
953 case VK_FORMAT_R16G16B16A16_SNORM:
954 dst.move(0, Max(SIMD::Float((packed[0] << 16) & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
955 dst.move(1, Max(SIMD::Float(packed[0] & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
956 dst.move(2, Max(SIMD::Float((packed[1] << 16) & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
957 dst.move(3, Max(SIMD::Float(packed[1] & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
958 break;
959 case VK_FORMAT_R16G16B16A16_SINT:
960 dst.move(0, (packed[0] << 16) >> 16);
961 dst.move(1, packed[0] >> 16);
962 dst.move(2, (packed[1] << 16) >> 16);
963 dst.move(3, packed[1] >> 16);
964 break;
965 case VK_FORMAT_R16G16B16A16_UINT:
966 dst.move(0, packed[0] & SIMD::Int(0xFFFF));
967 dst.move(1, (packed[0] >> 16) & SIMD::Int(0xFFFF));
968 dst.move(2, packed[1] & SIMD::Int(0xFFFF));
969 dst.move(3, (packed[1] >> 16) & SIMD::Int(0xFFFF));
970 break;
971 case VK_FORMAT_R16G16B16A16_SFLOAT:
972 dst.move(0, halfToFloatBits(As<SIMD::UInt>(packed[0]) & SIMD::UInt(0x0000FFFF)));
973 dst.move(1, halfToFloatBits((As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFFFF0000)) >> 16));
974 dst.move(2, halfToFloatBits(As<SIMD::UInt>(packed[1]) & SIMD::UInt(0x0000FFFF)));
975 dst.move(3, halfToFloatBits((As<SIMD::UInt>(packed[1]) & SIMD::UInt(0xFFFF0000)) >> 16));
976 break;
977 case VK_FORMAT_R8G8B8A8_SNORM:
978 case VK_FORMAT_A8B8G8R8_SNORM_PACK32:
979 dst.move(0, Max(SIMD::Float((packed[0] << 24) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
980 dst.move(1, Max(SIMD::Float((packed[0] << 16) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
981 dst.move(2, Max(SIMD::Float((packed[0] << 8) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
982 dst.move(3, Max(SIMD::Float((packed[0]) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
983 break;
984 case VK_FORMAT_R8G8B8A8_UNORM:
985 case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
986 dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
987 dst.move(1, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
988 dst.move(2, SIMD::Float((packed[0] >> 16) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
989 dst.move(3, SIMD::Float((packed[0] >> 24) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
990 break;
991 case VK_FORMAT_R8G8B8A8_SRGB:
992 case VK_FORMAT_A8B8G8R8_SRGB_PACK32:
993 dst.move(0, sRGBtoLinear(SIMD::Float(packed[0] & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF)));
994 dst.move(1, sRGBtoLinear(SIMD::Float((packed[0] >> 8) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF)));
995 dst.move(2, sRGBtoLinear(SIMD::Float((packed[0] >> 16) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF)));
996 dst.move(3, SIMD::Float((packed[0] >> 24) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
997 break;
998 case VK_FORMAT_B8G8R8A8_UNORM:
999 dst.move(0, SIMD::Float((packed[0] >> 16) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1000 dst.move(1, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1001 dst.move(2, SIMD::Float(packed[0] & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1002 dst.move(3, SIMD::Float((packed[0] >> 24) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1003 break;
1004 case VK_FORMAT_B8G8R8A8_SRGB:
1005 dst.move(0, sRGBtoLinear(SIMD::Float((packed[0] >> 16) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF)));
1006 dst.move(1, sRGBtoLinear(SIMD::Float((packed[0] >> 8) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF)));
1007 dst.move(2, sRGBtoLinear(SIMD::Float(packed[0] & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF)));
1008 dst.move(3, SIMD::Float((packed[0] >> 24) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1009 break;
1010 case VK_FORMAT_R8G8B8A8_UINT:
1011 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
1012 dst.move(0, As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFF));
1013 dst.move(1, (As<SIMD::UInt>(packed[0]) >> 8) & SIMD::UInt(0xFF));
1014 dst.move(2, (As<SIMD::UInt>(packed[0]) >> 16) & SIMD::UInt(0xFF));
1015 dst.move(3, (As<SIMD::UInt>(packed[0]) >> 24) & SIMD::UInt(0xFF));
1016 break;
1017 case VK_FORMAT_R8G8B8A8_SINT:
1018 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
1019 dst.move(0, (packed[0] << 24) >> 24);
1020 dst.move(1, (packed[0] << 16) >> 24);
1021 dst.move(2, (packed[0] << 8) >> 24);
1022 dst.move(3, packed[0] >> 24);
1023 break;
1024 case VK_FORMAT_R8_UNORM:
1025 dst.move(0, SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 0xFF));
1026 dst.move(1, SIMD::Float(0.0f));
1027 dst.move(2, SIMD::Float(0.0f));
1028 dst.move(3, SIMD::Float(1.0f));
1029 break;
1030 case VK_FORMAT_R8_SNORM:
1031 dst.move(0, Max(SIMD::Float((packed[0] << 24) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
1032 dst.move(1, SIMD::Float(0.0f));
1033 dst.move(2, SIMD::Float(0.0f));
1034 dst.move(3, SIMD::Float(1.0f));
1035 break;
1036 case VK_FORMAT_R8_UINT:
1037 case VK_FORMAT_S8_UINT:
1038 dst.move(0, As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFF));
1039 dst.move(1, SIMD::UInt(0));
1040 dst.move(2, SIMD::UInt(0));
1041 dst.move(3, SIMD::UInt(1));
1042 break;
1043 case VK_FORMAT_R8_SINT:
1044 dst.move(0, (packed[0] << 24) >> 24);
1045 dst.move(1, SIMD::Int(0));
1046 dst.move(2, SIMD::Int(0));
1047 dst.move(3, SIMD::Int(1));
1048 break;
1049 case VK_FORMAT_R8G8_UNORM:
1050 dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1051 dst.move(1, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xFF)) * SIMD::Float(1.0f / 0xFF));
1052 dst.move(2, SIMD::Float(0.0f));
1053 dst.move(3, SIMD::Float(1.0f));
1054 break;
1055 case VK_FORMAT_R8G8_SNORM:
1056 dst.move(0, Max(SIMD::Float((packed[0] << 24) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
1057 dst.move(1, Max(SIMD::Float((packed[0] << 16) & SIMD::Int(0xFF000000)) * SIMD::Float(1.0f / 0x7F000000), SIMD::Float(-1.0f)));
1058 dst.move(2, SIMD::Float(0.0f));
1059 dst.move(3, SIMD::Float(1.0f));
1060 break;
1061 case VK_FORMAT_R8G8_UINT:
1062 dst.move(0, As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFF));
1063 dst.move(1, (As<SIMD::UInt>(packed[0]) >> 8) & SIMD::UInt(0xFF));
1064 dst.move(2, SIMD::UInt(0));
1065 dst.move(3, SIMD::UInt(1));
1066 break;
1067 case VK_FORMAT_R8G8_SINT:
1068 dst.move(0, (packed[0] << 24) >> 24);
1069 dst.move(1, (packed[0] << 16) >> 24);
1070 dst.move(2, SIMD::Int(0));
1071 dst.move(3, SIMD::Int(1));
1072 break;
1073 case VK_FORMAT_R16_SFLOAT:
1074 dst.move(0, halfToFloatBits(As<SIMD::UInt>(packed[0]) & SIMD::UInt(0x0000FFFF)));
1075 dst.move(1, SIMD::Float(0.0f));
1076 dst.move(2, SIMD::Float(0.0f));
1077 dst.move(3, SIMD::Float(1.0f));
1078 break;
1079 case VK_FORMAT_R16_UNORM:
1080 dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
1081 dst.move(1, SIMD::Float(0.0f));
1082 dst.move(2, SIMD::Float(0.0f));
1083 dst.move(3, SIMD::Float(1.0f));
1084 break;
1085 case VK_FORMAT_R16_SNORM:
1086 dst.move(0, Max(SIMD::Float((packed[0] << 16) & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
1087 dst.move(1, SIMD::Float(0.0f));
1088 dst.move(2, SIMD::Float(0.0f));
1089 dst.move(3, SIMD::Float(1.0f));
1090 break;
1091 case VK_FORMAT_R16_UINT:
1092 dst.move(0, packed[0] & SIMD::Int(0xFFFF));
1093 dst.move(1, SIMD::UInt(0));
1094 dst.move(2, SIMD::UInt(0));
1095 dst.move(3, SIMD::UInt(1));
1096 break;
1097 case VK_FORMAT_R16_SINT:
1098 dst.move(0, (packed[0] << 16) >> 16);
1099 dst.move(1, SIMD::Int(0));
1100 dst.move(2, SIMD::Int(0));
1101 dst.move(3, SIMD::Int(1));
1102 break;
1103 case VK_FORMAT_R16G16_SFLOAT:
1104 dst.move(0, halfToFloatBits(As<SIMD::UInt>(packed[0]) & SIMD::UInt(0x0000FFFF)));
1105 dst.move(1, halfToFloatBits((As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFFFF0000)) >> 16));
1106 dst.move(2, SIMD::Float(0.0f));
1107 dst.move(3, SIMD::Float(1.0f));
1108 break;
1109 case VK_FORMAT_R16G16_UNORM:
1110 dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xFFFF)) * SIMD::Float(1.0f / 0xFFFF));
1111 dst.move(1, SIMD::Float(As<SIMD::UInt>(packed[0]) >> 16) * SIMD::Float(1.0f / 0xFFFF));
1112 dst.move(2, SIMD::Float(0.0f));
1113 dst.move(3, SIMD::Float(1.0f));
1114 break;
1115 case VK_FORMAT_R16G16_SNORM:
1116 dst.move(0, Max(SIMD::Float((packed[0] << 16) & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
1117 dst.move(1, Max(SIMD::Float(packed[0] & SIMD::Int(0xFFFF0000)) * SIMD::Float(1.0f / 0x7FFF0000), SIMD::Float(-1.0f)));
1118 dst.move(2, SIMD::Float(0.0f));
1119 dst.move(3, SIMD::Float(1.0f));
1120 break;
1121 case VK_FORMAT_R16G16_UINT:
1122 dst.move(0, packed[0] & SIMD::Int(0xFFFF));
1123 dst.move(1, (packed[0] >> 16) & SIMD::Int(0xFFFF));
1124 dst.move(2, SIMD::UInt(0));
1125 dst.move(3, SIMD::UInt(1));
1126 break;
1127 case VK_FORMAT_R16G16_SINT:
1128 dst.move(0, (packed[0] << 16) >> 16);
1129 dst.move(1, packed[0] >> 16);
1130 dst.move(2, SIMD::Int(0));
1131 dst.move(3, SIMD::Int(1));
1132 break;
1133 case VK_FORMAT_R32G32_SINT:
1134 case VK_FORMAT_R32G32_UINT:
1135 dst.move(0, packed[0]);
1136 dst.move(1, packed[1]);
1137 dst.move(2, SIMD::Int(0));
1138 dst.move(3, SIMD::Int(1));
1139 break;
1140 case VK_FORMAT_R32G32_SFLOAT:
1141 dst.move(0, packed[0]);
1142 dst.move(1, packed[1]);
1143 dst.move(2, SIMD::Float(0.0f));
1144 dst.move(3, SIMD::Float(1.0f));
1145 break;
1146 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
1147 dst.move(0, packed[0] & SIMD::Int(0x3FF));
1148 dst.move(1, (packed[0] >> 10) & SIMD::Int(0x3FF));
1149 dst.move(2, (packed[0] >> 20) & SIMD::Int(0x3FF));
1150 dst.move(3, (packed[0] >> 30) & SIMD::Int(0x3));
1151 break;
1152 case VK_FORMAT_A2R10G10B10_UINT_PACK32:
1153 dst.move(2, packed[0] & SIMD::Int(0x3FF));
1154 dst.move(1, (packed[0] >> 10) & SIMD::Int(0x3FF));
1155 dst.move(0, (packed[0] >> 20) & SIMD::Int(0x3FF));
1156 dst.move(3, (packed[0] >> 30) & SIMD::Int(0x3));
1157 break;
1158 case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
1159 dst.move(0, SIMD::Float((packed[0]) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF));
1160 dst.move(1, SIMD::Float((packed[0] >> 10) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF));
1161 dst.move(2, SIMD::Float((packed[0] >> 20) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF));
1162 dst.move(3, SIMD::Float((packed[0] >> 30) & SIMD::Int(0x3)) * SIMD::Float(1.0f / 0x3));
1163 break;
1164 case VK_FORMAT_A2R10G10B10_UNORM_PACK32:
1165 dst.move(2, SIMD::Float((packed[0]) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF));
1166 dst.move(1, SIMD::Float((packed[0] >> 10) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF));
1167 dst.move(0, SIMD::Float((packed[0] >> 20) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF));
1168 dst.move(3, SIMD::Float((packed[0] >> 30) & SIMD::Int(0x3)) * SIMD::Float(1.0f / 0x3));
1169 break;
1170 case VK_FORMAT_R4G4B4A4_UNORM_PACK16:
1171 dst.move(0, SIMD::Float((packed[0] >> 12) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1172 dst.move(1, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1173 dst.move(2, SIMD::Float((packed[0] >> 4) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1174 dst.move(3, SIMD::Float((packed[0]) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1175 break;
1176 case VK_FORMAT_B4G4R4A4_UNORM_PACK16:
1177 dst.move(0, SIMD::Float((packed[0] >> 4) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1178 dst.move(1, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1179 dst.move(2, SIMD::Float((packed[0] >> 12) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1180 dst.move(3, SIMD::Float((packed[0]) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1181 break;
1182 case VK_FORMAT_A4R4G4B4_UNORM_PACK16:
1183 dst.move(0, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1184 dst.move(1, SIMD::Float((packed[0] >> 4) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1185 dst.move(2, SIMD::Float((packed[0]) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1186 dst.move(3, SIMD::Float((packed[0] >> 12) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1187 break;
1188 case VK_FORMAT_A4B4G4R4_UNORM_PACK16:
1189 dst.move(0, SIMD::Float((packed[0]) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1190 dst.move(1, SIMD::Float((packed[0] >> 4) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1191 dst.move(2, SIMD::Float((packed[0] >> 8) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1192 dst.move(3, SIMD::Float((packed[0] >> 12) & SIMD::Int(0xF)) * SIMD::Float(1.0f / 0xF));
1193 break;
1194 case VK_FORMAT_R5G6B5_UNORM_PACK16:
1195 dst.move(0, SIMD::Float((packed[0] >> 11) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1196 dst.move(1, SIMD::Float((packed[0] >> 5) & SIMD::Int(0x3F)) * SIMD::Float(1.0f / 0x3F));
1197 dst.move(2, SIMD::Float((packed[0]) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1198 dst.move(3, SIMD::Float(1.0f));
1199 break;
1200 case VK_FORMAT_B5G6R5_UNORM_PACK16:
1201 dst.move(0, SIMD::Float((packed[0]) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1202 dst.move(1, SIMD::Float((packed[0] >> 5) & SIMD::Int(0x3F)) * SIMD::Float(1.0f / 0x3F));
1203 dst.move(2, SIMD::Float((packed[0] >> 11) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1204 dst.move(3, SIMD::Float(1.0f));
1205 break;
1206 case VK_FORMAT_R5G5B5A1_UNORM_PACK16:
1207 dst.move(0, SIMD::Float((packed[0] >> 11) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1208 dst.move(1, SIMD::Float((packed[0] >> 6) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1209 dst.move(2, SIMD::Float((packed[0] >> 1) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1210 dst.move(3, SIMD::Float((packed[0]) & SIMD::Int(0x1)));
1211 break;
1212 case VK_FORMAT_B5G5R5A1_UNORM_PACK16:
1213 dst.move(0, SIMD::Float((packed[0] >> 1) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1214 dst.move(1, SIMD::Float((packed[0] >> 6) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1215 dst.move(2, SIMD::Float((packed[0] >> 11) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1216 dst.move(3, SIMD::Float((packed[0]) & SIMD::Int(0x1)));
1217 break;
1218 case VK_FORMAT_A1R5G5B5_UNORM_PACK16:
1219 dst.move(0, SIMD::Float((packed[0] >> 10) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1220 dst.move(1, SIMD::Float((packed[0] >> 5) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1221 dst.move(2, SIMD::Float((packed[0]) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F));
1222 dst.move(3, SIMD::Float((packed[0] >> 15) & SIMD::Int(0x1)));
1223 break;
1224 case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
1225 dst.move(0, halfToFloatBits((packed[0] << 4) & SIMD::Int(0x7FF0)));
1226 dst.move(1, halfToFloatBits((packed[0] >> 7) & SIMD::Int(0x7FF0)));
1227 dst.move(2, halfToFloatBits((packed[0] >> 17) & SIMD::Int(0x7FE0)));
1228 dst.move(3, SIMD::Float(1.0f));
1229 break;
1230 default:
1231 UNSUPPORTED("VkFormat %d", int(imageFormat));
1232 break;
1233 }
1234 }
1235
EmitImageWrite(const ImageInstruction & instruction)1236 void SpirvEmitter::EmitImageWrite(const ImageInstruction &instruction)
1237 {
1238 auto &image = shader.getObject(instruction.imageId);
1239 auto &imageType = shader.getType(image);
1240
1241 ASSERT(imageType.definition.opcode() == spv::OpTypeImage);
1242 ASSERT(static_cast<spv::Dim>(instruction.dim) != spv::DimSubpassData); // "Its Dim operand must not be SubpassData."
1243
1244 auto coordinate = Operand(shader, *this, instruction.coordinateId);
1245 auto texel = Operand(shader, *this, instruction.texelId);
1246
1247 Array<SIMD::Int> coord(5); // uvwa & sample
1248
1249 uint32_t i = 0;
1250 for(; i < instruction.coordinates; i++)
1251 {
1252 coord[i] = coordinate.Int(i);
1253 }
1254
1255 if(instruction.sample)
1256 {
1257 coord[i] = Operand(shader, *this, instruction.sampleId).Int(0);
1258 }
1259
1260 Array<SIMD::Int> texelAndMask(5);
1261 texelAndMask[0] = texel.Int(0);
1262 texelAndMask[1] = texel.Int(1);
1263 texelAndMask[2] = texel.Int(2);
1264 texelAndMask[3] = texel.Int(3);
1265 texelAndMask[4] = activeStoresAndAtomicsMask();
1266
1267 vk::Format imageFormat = SpirvFormatToVulkanFormat(static_cast<spv::ImageFormat>(instruction.imageFormat));
1268
1269 SIMD::Pointer ptr = getPointer(instruction.imageId);
1270 if(ptr.isBasePlusOffset)
1271 {
1272 Pointer<Byte> imageDescriptor = ptr.getUniformPointer(); // vk::StorageImageDescriptor* or vk::SampledImageDescriptor*
1273 Pointer<Byte> samplerDescriptor = getSamplerDescriptor(imageDescriptor, instruction);
1274
1275 if(imageFormat == VK_FORMAT_UNDEFINED) // spv::ImageFormatUnknown
1276 {
1277 Pointer<Byte> samplerFunction = lookupSamplerFunction(imageDescriptor, samplerDescriptor, instruction);
1278
1279 Call<ImageSampler>(samplerFunction, imageDescriptor, &coord, &texelAndMask, routine->constants);
1280 }
1281 else
1282 {
1283 WriteImage(instruction, imageDescriptor, &coord, &texelAndMask, imageFormat);
1284 }
1285 }
1286 else
1287 {
1288 for(int j = 0; j < SIMD::Width; j++)
1289 {
1290 SIMD::Int singleLaneMask = 0;
1291 singleLaneMask = Insert(singleLaneMask, 0xffffffff, j);
1292 texelAndMask[4] = activeStoresAndAtomicsMask() & singleLaneMask;
1293 Pointer<Byte> imageDescriptor = ptr.getPointerForLane(j);
1294 Pointer<Byte> samplerDescriptor = getSamplerDescriptor(imageDescriptor, instruction, j);
1295
1296 if(imageFormat == VK_FORMAT_UNDEFINED) // spv::ImageFormatUnknown
1297 {
1298 Pointer<Byte> samplerFunction = lookupSamplerFunction(imageDescriptor, samplerDescriptor, instruction);
1299
1300 Call<ImageSampler>(samplerFunction, imageDescriptor, &coord, &texelAndMask, routine->constants);
1301 }
1302 else
1303 {
1304 WriteImage(instruction, imageDescriptor, &coord, &texelAndMask, imageFormat);
1305 }
1306 }
1307 }
1308 }
1309
WriteImage(ImageInstructionSignature instruction,Pointer<Byte> descriptor,const Pointer<SIMD::Int> & coord,const Pointer<SIMD::Int> & texelAndMask,vk::Format imageFormat)1310 void SpirvEmitter::WriteImage(ImageInstructionSignature instruction, Pointer<Byte> descriptor, const Pointer<SIMD::Int> &coord, const Pointer<SIMD::Int> &texelAndMask, vk::Format imageFormat)
1311 {
1312 SIMD::Int texel[4];
1313 texel[0] = texelAndMask[0];
1314 texel[1] = texelAndMask[1];
1315 texel[2] = texelAndMask[2];
1316 texel[3] = texelAndMask[3];
1317 SIMD::Int mask = texelAndMask[4];
1318
1319 SIMD::Int packed[4];
1320 switch(imageFormat)
1321 {
1322 case VK_FORMAT_R32G32B32A32_SFLOAT:
1323 case VK_FORMAT_R32G32B32A32_SINT:
1324 case VK_FORMAT_R32G32B32A32_UINT:
1325 packed[0] = texel[0];
1326 packed[1] = texel[1];
1327 packed[2] = texel[2];
1328 packed[3] = texel[3];
1329 break;
1330 case VK_FORMAT_R32_SFLOAT:
1331 case VK_FORMAT_R32_SINT:
1332 case VK_FORMAT_R32_UINT:
1333 packed[0] = texel[0];
1334 break;
1335 case VK_FORMAT_R8G8B8A8_UNORM:
1336 case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
1337 packed[0] = (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) |
1338 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 8) |
1339 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[2]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 16) |
1340 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 24);
1341 break;
1342 case VK_FORMAT_B8G8R8A8_UNORM:
1343 packed[0] = (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[2]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) |
1344 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 8) |
1345 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 16) |
1346 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 24);
1347 break;
1348 case VK_FORMAT_B8G8R8A8_SRGB:
1349 packed[0] = (SIMD::UInt(Round(Min(Max(linearToSRGB(As<SIMD::Float>(texel[2])), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) |
1350 ((SIMD::UInt(Round(Min(Max(linearToSRGB(As<SIMD::Float>(texel[1])), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 8) |
1351 ((SIMD::UInt(Round(Min(Max(linearToSRGB(As<SIMD::Float>(texel[0])), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 16) |
1352 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 24);
1353 break;
1354 case VK_FORMAT_R8G8B8A8_SNORM:
1355 case VK_FORMAT_A8B8G8R8_SNORM_PACK32:
1356 packed[0] = (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) &
1357 SIMD::Int(0xFF)) |
1358 ((SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) &
1359 SIMD::Int(0xFF))
1360 << 8) |
1361 ((SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[2]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) &
1362 SIMD::Int(0xFF))
1363 << 16) |
1364 ((SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) &
1365 SIMD::Int(0xFF))
1366 << 24);
1367 break;
1368 case VK_FORMAT_R8G8B8A8_SINT:
1369 case VK_FORMAT_R8G8B8A8_UINT:
1370 case VK_FORMAT_A8B8G8R8_SINT_PACK32:
1371 case VK_FORMAT_A8B8G8R8_UINT_PACK32:
1372 packed[0] = (SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0xff))) |
1373 (SIMD::UInt(As<SIMD::UInt>(texel[1]) & SIMD::UInt(0xff)) << 8) |
1374 (SIMD::UInt(As<SIMD::UInt>(texel[2]) & SIMD::UInt(0xff)) << 16) |
1375 (SIMD::UInt(As<SIMD::UInt>(texel[3]) & SIMD::UInt(0xff)) << 24);
1376 break;
1377 case VK_FORMAT_R16G16B16A16_SFLOAT:
1378 packed[0] = floatToHalfBits(As<SIMD::UInt>(texel[0]), false) | floatToHalfBits(As<SIMD::UInt>(texel[1]), true);
1379 packed[1] = floatToHalfBits(As<SIMD::UInt>(texel[2]), false) | floatToHalfBits(As<SIMD::UInt>(texel[3]), true);
1380 break;
1381 case VK_FORMAT_R16G16B16A16_SINT:
1382 case VK_FORMAT_R16G16B16A16_UINT:
1383 packed[0] = SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0xFFFF)) | (SIMD::UInt(As<SIMD::UInt>(texel[1]) & SIMD::UInt(0xFFFF)) << 16);
1384 packed[1] = SIMD::UInt(As<SIMD::UInt>(texel[2]) & SIMD::UInt(0xFFFF)) | (SIMD::UInt(As<SIMD::UInt>(texel[3]) & SIMD::UInt(0xFFFF)) << 16);
1385 break;
1386 case VK_FORMAT_R32G32_SFLOAT:
1387 case VK_FORMAT_R32G32_SINT:
1388 case VK_FORMAT_R32G32_UINT:
1389 packed[0] = texel[0];
1390 packed[1] = texel[1];
1391 break;
1392 case VK_FORMAT_R16G16_SFLOAT:
1393 packed[0] = floatToHalfBits(As<SIMD::UInt>(texel[0]), false) | floatToHalfBits(As<SIMD::UInt>(texel[1]), true);
1394 break;
1395 case VK_FORMAT_R16G16_SINT:
1396 case VK_FORMAT_R16G16_UINT:
1397 packed[0] = SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0xFFFF)) | (SIMD::UInt(As<SIMD::UInt>(texel[1]) & SIMD::UInt(0xFFFF)) << 16);
1398 break;
1399 case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
1400 // Truncates instead of rounding. See b/147900455
1401 packed[0] = ((floatToHalfBits(As<SIMD::UInt>(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f))), false) & SIMD::UInt(0x7FF0)) >> 4) |
1402 ((floatToHalfBits(As<SIMD::UInt>(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f))), false) & SIMD::UInt(0x7FF0)) << 7) |
1403 ((floatToHalfBits(As<SIMD::UInt>(Max(As<SIMD::Float>(texel[2]), SIMD::Float(0.0f))), false) & SIMD::UInt(0x7FE0)) << 17);
1404 break;
1405 case VK_FORMAT_R16_SFLOAT:
1406 packed[0] = floatToHalfBits(As<SIMD::UInt>(texel[0]), false);
1407 break;
1408 case VK_FORMAT_R16G16B16A16_UNORM:
1409 packed[0] = SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF))) |
1410 (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF))) << 16);
1411 packed[1] = SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[2]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF))) |
1412 (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF))) << 16);
1413 break;
1414 case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
1415 packed[0] = (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x3FF)))) |
1416 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x3FF)))) << 10) |
1417 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[2]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x3FF)))) << 20) |
1418 ((SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x3)))) << 30);
1419 break;
1420 case VK_FORMAT_R16G16_UNORM:
1421 packed[0] = SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF))) |
1422 (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF))) << 16);
1423 break;
1424 case VK_FORMAT_R8G8_UNORM:
1425 packed[0] = SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFF))) |
1426 (SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFF))) << 8);
1427 break;
1428 case VK_FORMAT_R16_UNORM:
1429 packed[0] = SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFFFF)));
1430 break;
1431 case VK_FORMAT_R8_UNORM:
1432 packed[0] = SIMD::UInt(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(0xFF)));
1433 break;
1434 case VK_FORMAT_R16G16B16A16_SNORM:
1435 packed[0] = (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF))) & SIMD::Int(0xFFFF)) |
1436 (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF))) << 16);
1437 packed[1] = (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[2]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF))) & SIMD::Int(0xFFFF)) |
1438 (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[3]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF))) << 16);
1439 break;
1440 case VK_FORMAT_R16G16_SNORM:
1441 packed[0] = (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF))) & SIMD::Int(0xFFFF)) |
1442 (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF))) << 16);
1443 break;
1444 case VK_FORMAT_R8G8_SNORM:
1445 packed[0] = (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7F))) & SIMD::Int(0xFF)) |
1446 (SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[1]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7F))) << 8);
1447 break;
1448 case VK_FORMAT_R16_SNORM:
1449 packed[0] = SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7FFF)));
1450 break;
1451 case VK_FORMAT_R8_SNORM:
1452 packed[0] = SIMD::Int(Round(Min(Max(As<SIMD::Float>(texel[0]), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(0x7F)));
1453 break;
1454 case VK_FORMAT_R8G8_SINT:
1455 case VK_FORMAT_R8G8_UINT:
1456 packed[0] = SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0xFF)) | (SIMD::UInt(As<SIMD::UInt>(texel[1]) & SIMD::UInt(0xFF)) << 8);
1457 break;
1458 case VK_FORMAT_R16_SINT:
1459 case VK_FORMAT_R16_UINT:
1460 packed[0] = SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0xFFFF));
1461 break;
1462 case VK_FORMAT_R8_SINT:
1463 case VK_FORMAT_R8_UINT:
1464 packed[0] = SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0xFF));
1465 break;
1466 case VK_FORMAT_A2B10G10R10_UINT_PACK32:
1467 packed[0] = (SIMD::UInt(As<SIMD::UInt>(texel[0]) & SIMD::UInt(0x3FF))) |
1468 (SIMD::UInt(As<SIMD::UInt>(texel[1]) & SIMD::UInt(0x3FF)) << 10) |
1469 (SIMD::UInt(As<SIMD::UInt>(texel[2]) & SIMD::UInt(0x3FF)) << 20) |
1470 (SIMD::UInt(As<SIMD::UInt>(texel[3]) & SIMD::UInt(0x3)) << 30);
1471 break;
1472 default:
1473 UNSUPPORTED("VkFormat %d", int(imageFormat));
1474 break;
1475 }
1476
1477 // "The integer texel coordinates are validated according to the same rules as for texel input coordinate
1478 // validation. If the texel fails integer texel coordinate validation, then the write has no effect."
1479 // - https://www.khronos.org/registry/vulkan/specs/1.2/html/chap16.html#textures-output-coordinate-validation
1480 auto robustness = OutOfBoundsBehavior::Nullify;
1481 // GetTexelAddress() only needs the SpirvRoutine* for SubpassData accesses (i.e. input attachments).
1482 const SpirvRoutine *routine = nullptr;
1483
1484 SIMD::Int uvwa[4];
1485 SIMD::Int sample;
1486
1487 uint32_t i = 0;
1488 for(; i < instruction.coordinates; i++)
1489 {
1490 uvwa[i] = As<SIMD::Int>(coord[i]);
1491 }
1492
1493 if(instruction.sample)
1494 {
1495 sample = As<SIMD::Int>(coord[i]);
1496 }
1497
1498 auto texelPtr = GetTexelAddress(instruction, descriptor, uvwa, sample, imageFormat, robustness, routine);
1499
1500 const int texelSize = imageFormat.bytes();
1501
1502 // Scatter packed texel data.
1503 // TODO(b/160531165): Provide scatter abstractions for various element sizes.
1504 if(texelSize == 4 || texelSize == 8 || texelSize == 16)
1505 {
1506 for(auto i = 0; i < texelSize / 4; i++)
1507 {
1508 texelPtr.Store(packed[i], robustness, mask);
1509 texelPtr += sizeof(float);
1510 }
1511 }
1512 else if(texelSize == 2)
1513 {
1514 mask = mask & texelPtr.isInBounds(2, robustness);
1515
1516 for(int i = 0; i < SIMD::Width; i++)
1517 {
1518 If(Extract(mask, i) != 0)
1519 {
1520 *Pointer<Short>(texelPtr.getPointerForLane(i)) = Short(Extract(packed[0], i));
1521 }
1522 }
1523 }
1524 else if(texelSize == 1)
1525 {
1526 mask = mask & texelPtr.isInBounds(1, robustness);
1527
1528 for(int i = 0; i < SIMD::Width; i++)
1529 {
1530 If(Extract(mask, i) != 0)
1531 {
1532 *Pointer<Byte>(texelPtr.getPointerForLane(i)) = Byte(Extract(packed[0], i));
1533 }
1534 }
1535 }
1536 else
1537 UNREACHABLE("texelSize: %d", int(texelSize));
1538 }
1539
EmitImageTexelPointer(const ImageInstruction & instruction)1540 void SpirvEmitter::EmitImageTexelPointer(const ImageInstruction &instruction)
1541 {
1542 auto coordinate = Operand(shader, *this, instruction.coordinateId);
1543
1544 SIMD::Pointer ptr = getPointer(instruction.imageId);
1545
1546 // VK_EXT_image_robustness requires checking for out-of-bounds accesses.
1547 // TODO(b/162327166): Only perform bounds checks when VK_EXT_image_robustness is enabled.
1548 auto robustness = OutOfBoundsBehavior::Nullify;
1549 vk::Format imageFormat = SpirvFormatToVulkanFormat(static_cast<spv::ImageFormat>(instruction.imageFormat));
1550
1551 SIMD::Int uvwa[4];
1552
1553 for(uint32_t i = 0; i < instruction.coordinates; i++)
1554 {
1555 uvwa[i] = coordinate.Int(i);
1556 }
1557
1558 SIMD::Int sample = Operand(shader, *this, instruction.sampleId).Int(0);
1559
1560 auto texelPtr = ptr.isBasePlusOffset
1561 ? GetTexelAddress(instruction, ptr.getUniformPointer(), uvwa, sample, imageFormat, robustness, routine)
1562 : GetNonUniformTexelAddress(instruction, ptr, uvwa, sample, imageFormat, robustness, activeLaneMask(), routine);
1563
1564 createPointer(instruction.resultId, texelPtr);
1565 }
1566
EmitSampledImage(InsnIterator insn)1567 void SpirvEmitter::EmitSampledImage(InsnIterator insn)
1568 {
1569 Object::ID resultId = insn.word(2);
1570 Object::ID imageId = insn.word(3);
1571 Object::ID samplerId = insn.word(4);
1572
1573 // Create a sampled image, containing both a sampler and an image
1574 createSampledImage(resultId, { getPointer(imageId), samplerId });
1575 }
1576
EmitImage(InsnIterator insn)1577 void SpirvEmitter::EmitImage(InsnIterator insn)
1578 {
1579 Object::ID resultId = insn.word(2);
1580 Object::ID imageId = insn.word(3);
1581
1582 // Extract the image from a sampled image.
1583 createPointer(resultId, getImage(imageId));
1584 }
1585
1586 } // namespace sw
1587