• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //    http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "SpirvShader.hpp"
16 #include "SpirvShaderDebug.hpp"
17 
18 #include "ShaderCore.hpp"
19 #include "Reactor/Assert.hpp"
20 #include "Vulkan/VkPipelineLayout.hpp"
21 
22 #include <spirv/unified1/spirv.hpp>
23 
24 namespace sw {
25 
EmitLoad(InsnIterator insn)26 void SpirvEmitter::EmitLoad(InsnIterator insn)
27 {
28 	bool atomic = (insn.opcode() == spv::OpAtomicLoad);
29 	Object::ID resultId = insn.word(2);
30 	Object::ID pointerId = insn.word(3);
31 	auto &result = shader.getObject(resultId);
32 	auto &resultTy = shader.getType(result);
33 	auto &pointer = shader.getObject(pointerId);
34 	auto &pointerTy = shader.getType(pointer);
35 	std::memory_order memoryOrder = std::memory_order_relaxed;
36 
37 	ASSERT(shader.getType(pointer).element == result.typeId());
38 	ASSERT(Type::ID(insn.word(1)) == result.typeId());
39 	ASSERT(!atomic || shader.getType(shader.getType(pointer).element).opcode() == spv::OpTypeInt);  // Vulkan 1.1: "Atomic instructions must declare a scalar 32-bit integer type, for the value pointed to by Pointer."
40 
41 	if(pointerTy.storageClass == spv::StorageClassUniformConstant)
42 	{
43 		// Just propagate the pointer.
44 		auto &ptr = getPointer(pointerId);
45 		createPointer(resultId, ptr);
46 	}
47 
48 	if(atomic)
49 	{
50 		Object::ID semanticsId = insn.word(5);
51 		auto memorySemantics = static_cast<spv::MemorySemanticsMask>(shader.getObject(semanticsId).constantValue[0]);
52 		memoryOrder = shader.MemoryOrder(memorySemantics);
53 	}
54 
55 	auto ptr = GetPointerToData(pointerId, 0, false);
56 	auto robustness = shader.getOutOfBoundsBehavior(pointerId, routine->pipelineLayout);
57 
58 	if(result.kind == Object::Kind::Pointer)
59 	{
60 		shader.VisitMemoryObject(pointerId, true, [&](const Spirv::MemoryElement &el) {
61 			ASSERT(el.index == 0);
62 			auto p = GetElementPointer(ptr, el.offset, pointerTy.storageClass);
63 			createPointer(resultId, p.Load<SIMD::Pointer>(robustness, activeLaneMask(), atomic, memoryOrder, sizeof(void *)));
64 		});
65 
66 		SPIRV_SHADER_DBG("Load(atomic: {0}, order: {1}, ptr: {2}, mask: {3})", atomic, int(memoryOrder), ptr, activeLaneMask());
67 	}
68 	else
69 	{
70 		auto &dst = createIntermediate(resultId, resultTy.componentCount);
71 		shader.VisitMemoryObject(pointerId, false, [&](const Spirv::MemoryElement &el) {
72 			auto p = GetElementPointer(ptr, el.offset, pointerTy.storageClass);
73 			dst.move(el.index, p.Load<SIMD::Float>(robustness, activeLaneMask(), atomic, memoryOrder));
74 		});
75 
76 		SPIRV_SHADER_DBG("Load(atomic: {0}, order: {1}, ptr: {2}, val: {3}, mask: {4})", atomic, int(memoryOrder), ptr, dst, activeLaneMask());
77 	}
78 }
79 
EmitStore(InsnIterator insn)80 void SpirvEmitter::EmitStore(InsnIterator insn)
81 {
82 	bool atomic = (insn.opcode() == spv::OpAtomicStore);
83 	Object::ID pointerId = insn.word(1);
84 	Object::ID objectId = insn.word(atomic ? 4 : 2);
85 	std::memory_order memoryOrder = std::memory_order_relaxed;
86 
87 	if(atomic)
88 	{
89 		Object::ID semanticsId = insn.word(3);
90 		auto memorySemantics = static_cast<spv::MemorySemanticsMask>(shader.getObject(semanticsId).constantValue[0]);
91 		memoryOrder = shader.MemoryOrder(memorySemantics);
92 	}
93 
94 	const auto &value = Operand(shader, *this, objectId);
95 
96 	Store(pointerId, value, atomic, memoryOrder);
97 }
98 
Store(Object::ID pointerId,const Operand & value,bool atomic,std::memory_order memoryOrder) const99 void SpirvEmitter::Store(Object::ID pointerId, const Operand &value, bool atomic, std::memory_order memoryOrder) const
100 {
101 	auto &pointer = shader.getObject(pointerId);
102 	auto &pointerTy = shader.getType(pointer);
103 	auto &elementTy = shader.getType(pointerTy.element);
104 
105 	ASSERT(!atomic || elementTy.opcode() == spv::OpTypeInt);  // Vulkan 1.1: "Atomic instructions must declare a scalar 32-bit integer type, for the value pointed to by Pointer."
106 
107 	auto ptr = GetPointerToData(pointerId, 0, false);
108 	auto robustness = shader.getOutOfBoundsBehavior(pointerId, routine->pipelineLayout);
109 
110 	SIMD::Int mask = activeLaneMask();
111 	if(shader.StoresInHelperInvocationsHaveNoEffect(pointerTy.storageClass))
112 	{
113 		mask = mask & storesAndAtomicsMask();
114 	}
115 
116 	SPIRV_SHADER_DBG("Store(atomic: {0}, order: {1}, ptr: {2}, val: {3}, mask: {4}", atomic, int(memoryOrder), ptr, value, mask);
117 
118 	if(value.isPointer())
119 	{
120 		shader.VisitMemoryObject(pointerId, true, [&](const Spirv::MemoryElement &el) {
121 			ASSERT(el.index == 0);
122 			auto p = GetElementPointer(ptr, el.offset, pointerTy.storageClass);
123 			p.Store(value.Pointer(), robustness, mask, atomic, memoryOrder);
124 		});
125 	}
126 	else
127 	{
128 		shader.VisitMemoryObject(pointerId, false, [&](const Spirv::MemoryElement &el) {
129 			auto p = GetElementPointer(ptr, el.offset, pointerTy.storageClass);
130 			p.Store(value.Float(el.index), robustness, mask, atomic, memoryOrder);
131 		});
132 	}
133 }
134 
EmitVariable(InsnIterator insn)135 void SpirvEmitter::EmitVariable(InsnIterator insn)
136 {
137 	Object::ID resultId = insn.word(2);
138 	auto &object = shader.getObject(resultId);
139 	auto &objectTy = shader.getType(object);
140 
141 	switch(objectTy.storageClass)
142 	{
143 	case spv::StorageClassOutput:
144 	case spv::StorageClassPrivate:
145 	case spv::StorageClassFunction:
146 		{
147 			ASSERT(objectTy.opcode() == spv::OpTypePointer);
148 			auto base = &routine->getVariable(resultId)[0];
149 			auto elementTy = shader.getType(objectTy.element);
150 			auto size = elementTy.componentCount * static_cast<uint32_t>(sizeof(float)) * SIMD::Width;
151 			createPointer(resultId, SIMD::Pointer(base, size));
152 		}
153 		break;
154 	case spv::StorageClassWorkgroup:
155 		{
156 			ASSERT(objectTy.opcode() == spv::OpTypePointer);
157 			auto base = &routine->workgroupMemory[0];
158 			auto size = shader.workgroupMemory.size();
159 			createPointer(resultId, SIMD::Pointer(base, size, shader.workgroupMemory.offsetOf(resultId)));
160 		}
161 		break;
162 	case spv::StorageClassInput:
163 		{
164 			if(object.kind == Object::Kind::InterfaceVariable)
165 			{
166 				auto &dst = routine->getVariable(resultId);
167 				int offset = 0;
168 				shader.VisitInterface(resultId,
169 				                      [&](const Decorations &d, Spirv::AttribType type) {
170 					                      auto scalarSlot = d.Location << 2 | d.Component;
171 					                      dst[offset++] = routine->inputs[scalarSlot];
172 				                      });
173 			}
174 			ASSERT(objectTy.opcode() == spv::OpTypePointer);
175 			auto base = &routine->getVariable(resultId)[0];
176 			auto elementTy = shader.getType(objectTy.element);
177 			auto size = elementTy.componentCount * static_cast<uint32_t>(sizeof(float)) * SIMD::Width;
178 			createPointer(resultId, SIMD::Pointer(base, size));
179 		}
180 		break;
181 	case spv::StorageClassUniformConstant:
182 		{
183 			const auto &d = shader.descriptorDecorations.at(resultId);
184 			ASSERT(d.DescriptorSet >= 0);
185 			ASSERT(d.Binding >= 0);
186 
187 			uint32_t bindingOffset = routine->pipelineLayout->getBindingOffset(d.DescriptorSet, d.Binding);
188 			Pointer<Byte> set = routine->descriptorSets[d.DescriptorSet];  // DescriptorSet*
189 			Pointer<Byte> binding = Pointer<Byte>(set + bindingOffset);    // vk::SampledImageDescriptor*
190 			auto size = 0;                                                 // Not required as this pointer is not directly used by SIMD::Read or SIMD::Write.
191 			createPointer(resultId, SIMD::Pointer(binding, size));
192 		}
193 		break;
194 	case spv::StorageClassUniform:
195 	case spv::StorageClassStorageBuffer:
196 	case spv::StorageClassPhysicalStorageBuffer:
197 		{
198 			const auto &d = shader.descriptorDecorations.at(resultId);
199 			ASSERT(d.DescriptorSet >= 0);
200 			auto size = 0;  // Not required as this pointer is not directly used by SIMD::Read or SIMD::Write.
201 			// Note: the module may contain descriptor set references that are not suitable for this implementation -- using a set index higher than the number
202 			// of descriptor set binding points we support. As long as the selected entrypoint doesn't actually touch the out of range binding points, this
203 			// is valid. In this case make the value nullptr to make it easier to diagnose an attempt to dereference it.
204 			if(static_cast<uint32_t>(d.DescriptorSet) < vk::MAX_BOUND_DESCRIPTOR_SETS)
205 			{
206 				createPointer(resultId, SIMD::Pointer(routine->descriptorSets[d.DescriptorSet], size));
207 			}
208 			else
209 			{
210 				createPointer(resultId, SIMD::Pointer(nullptr, 0));
211 			}
212 		}
213 		break;
214 	case spv::StorageClassPushConstant:
215 		{
216 			createPointer(resultId, SIMD::Pointer(routine->pushConstants, vk::MAX_PUSH_CONSTANT_SIZE));
217 		}
218 		break;
219 	default:
220 		UNREACHABLE("Storage class %d", objectTy.storageClass);
221 		break;
222 	}
223 
224 	if(insn.wordCount() > 4)
225 	{
226 		Object::ID initializerId = insn.word(4);
227 		if(shader.getObject(initializerId).kind != Object::Kind::Constant)
228 		{
229 			UNIMPLEMENTED("b/148241854: Non-constant initializers not yet implemented");  // FIXME(b/148241854)
230 		}
231 
232 		switch(objectTy.storageClass)
233 		{
234 		case spv::StorageClassOutput:
235 		case spv::StorageClassPrivate:
236 		case spv::StorageClassFunction:
237 		case spv::StorageClassWorkgroup:
238 			{
239 				auto ptr = GetPointerToData(resultId, 0, false);
240 				Operand initialValue(shader, *this, initializerId);
241 
242 				shader.VisitMemoryObject(resultId, false, [&](const Spirv::MemoryElement &el) {
243 					auto p = GetElementPointer(ptr, el.offset, objectTy.storageClass);
244 					auto robustness = OutOfBoundsBehavior::UndefinedBehavior;  // Local variables are always within bounds.
245 					p.Store(initialValue.Float(el.index), robustness, activeLaneMask());
246 				});
247 
248 				if(objectTy.storageClass == spv::StorageClassWorkgroup)
249 				{
250 					// Initialization of workgroup memory is done by each subgroup and requires waiting on a barrier.
251 					// TODO(b/221242292): Initialize just once per workgroup and eliminate the barrier.
252 					Yield(YieldResult::ControlBarrier);
253 				}
254 			}
255 			break;
256 		default:
257 			ASSERT_MSG(initializerId == 0, "Vulkan does not permit variables of storage class %d to have initializers", int(objectTy.storageClass));
258 		}
259 	}
260 }
261 
EmitCopyMemory(InsnIterator insn)262 void SpirvEmitter::EmitCopyMemory(InsnIterator insn)
263 {
264 	Object::ID dstPtrId = insn.word(1);
265 	Object::ID srcPtrId = insn.word(2);
266 	auto &dstPtrTy = shader.getObjectType(dstPtrId);
267 	auto &srcPtrTy = shader.getObjectType(srcPtrId);
268 	ASSERT(dstPtrTy.element == srcPtrTy.element);
269 
270 	auto dstPtr = GetPointerToData(dstPtrId, 0, false);
271 	auto srcPtr = GetPointerToData(srcPtrId, 0, false);
272 
273 	std::unordered_map<uint32_t, uint32_t> srcOffsets;
274 
275 	shader.VisitMemoryObject(srcPtrId, false, [&](const Spirv::MemoryElement &el) { srcOffsets[el.index] = el.offset; });
276 
277 	shader.VisitMemoryObject(dstPtrId, false, [&](const Spirv::MemoryElement &el) {
278 		auto it = srcOffsets.find(el.index);
279 		ASSERT(it != srcOffsets.end());
280 		auto srcOffset = it->second;
281 		auto dstOffset = el.offset;
282 
283 		auto dst = GetElementPointer(dstPtr, dstOffset, dstPtrTy.storageClass);
284 		auto src = GetElementPointer(srcPtr, srcOffset, srcPtrTy.storageClass);
285 
286 		// TODO(b/131224163): Optimize based on src/dst storage classes.
287 		auto robustness = OutOfBoundsBehavior::RobustBufferAccess;
288 
289 		auto value = src.Load<SIMD::Float>(robustness, activeLaneMask());
290 		dst.Store(value, robustness, activeLaneMask());
291 	});
292 }
293 
EmitMemoryBarrier(InsnIterator insn)294 void SpirvEmitter::EmitMemoryBarrier(InsnIterator insn)
295 {
296 	auto semantics = spv::MemorySemanticsMask(shader.GetConstScalarInt(insn.word(2)));
297 	// TODO(b/176819536): We probably want to consider the memory scope here.
298 	// For now, just always emit the full fence.
299 	Fence(semantics);
300 }
301 
VisitMemoryObjectInner(Type::ID id,Decorations d,uint32_t & index,uint32_t offset,bool resultIsPointer,const MemoryVisitor & f) const302 void Spirv::VisitMemoryObjectInner(Type::ID id, Decorations d, uint32_t &index, uint32_t offset, bool resultIsPointer, const MemoryVisitor &f) const
303 {
304 	ApplyDecorationsForId(&d, id);
305 	const auto &type = getType(id);
306 
307 	if(d.HasOffset)
308 	{
309 		offset += d.Offset;
310 		d.HasOffset = false;
311 	}
312 
313 	switch(type.opcode())
314 	{
315 	case spv::OpTypePointer:
316 		if(resultIsPointer)
317 		{
318 			// Load/Store the pointer itself, rather than the structure pointed to by the pointer
319 			f(MemoryElement{ index++, offset, type });
320 		}
321 		else
322 		{
323 			VisitMemoryObjectInner(type.definition.word(3), d, index, offset, resultIsPointer, f);
324 		}
325 		break;
326 	case spv::OpTypeInt:
327 	case spv::OpTypeFloat:
328 	case spv::OpTypeRuntimeArray:
329 		f(MemoryElement{ index++, offset, type });
330 		break;
331 	case spv::OpTypeVector:
332 		{
333 			auto elemStride = (d.InsideMatrix && d.HasRowMajor && d.RowMajor) ? d.MatrixStride : static_cast<int32_t>(sizeof(float));
334 			for(auto i = 0u; i < type.definition.word(3); i++)
335 			{
336 				VisitMemoryObjectInner(type.definition.word(2), d, index, offset + elemStride * i, resultIsPointer, f);
337 			}
338 		}
339 		break;
340 	case spv::OpTypeMatrix:
341 		{
342 			auto columnStride = (d.HasRowMajor && d.RowMajor) ? static_cast<int32_t>(sizeof(float)) : d.MatrixStride;
343 			d.InsideMatrix = true;
344 			for(auto i = 0u; i < type.definition.word(3); i++)
345 			{
346 				ASSERT(d.HasMatrixStride);
347 				VisitMemoryObjectInner(type.definition.word(2), d, index, offset + columnStride * i, resultIsPointer, f);
348 			}
349 		}
350 		break;
351 	case spv::OpTypeStruct:
352 		for(auto i = 0u; i < type.definition.wordCount() - 2; i++)
353 		{
354 			ApplyDecorationsForIdMember(&d, id, i);
355 			VisitMemoryObjectInner(type.definition.word(i + 2), d, index, offset, resultIsPointer, f);
356 		}
357 		break;
358 	case spv::OpTypeArray:
359 		{
360 			auto arraySize = GetConstScalarInt(type.definition.word(3));
361 			for(auto i = 0u; i < arraySize; i++)
362 			{
363 				ASSERT(d.HasArrayStride);
364 				VisitMemoryObjectInner(type.definition.word(2), d, index, offset + i * d.ArrayStride, resultIsPointer, f);
365 			}
366 		}
367 		break;
368 	default:
369 		UNREACHABLE("%s", OpcodeName(type.opcode()));
370 	}
371 }
372 
VisitMemoryObject(Object::ID id,bool resultIsPointer,const MemoryVisitor & f) const373 void Spirv::VisitMemoryObject(Object::ID id, bool resultIsPointer, const MemoryVisitor &f) const
374 {
375 	auto typeId = getObject(id).typeId();
376 	const auto &type = getType(typeId);
377 
378 	if(IsExplicitLayout(type.storageClass))
379 	{
380 		Decorations d = GetDecorationsForId(id);
381 		uint32_t index = 0;
382 		VisitMemoryObjectInner(typeId, d, index, 0, resultIsPointer, f);
383 	}
384 	else
385 	{
386 		// Objects without explicit layout are tightly packed.
387 		auto &elType = getType(type.element);
388 		for(auto index = 0u; index < elType.componentCount; index++)
389 		{
390 			auto offset = static_cast<uint32_t>(index * sizeof(float));
391 			f({ index, offset, elType });
392 		}
393 	}
394 }
395 
GetPointerToData(Object::ID id,SIMD::Int arrayIndices,bool nonUniform) const396 SIMD::Pointer SpirvEmitter::GetPointerToData(Object::ID id, SIMD::Int arrayIndices, bool nonUniform) const
397 {
398 	auto &object = shader.getObject(id);
399 	switch(object.kind)
400 	{
401 	case Object::Kind::Pointer:
402 	case Object::Kind::InterfaceVariable:
403 		return getPointer(id);
404 
405 	case Object::Kind::DescriptorSet:
406 		{
407 			const auto &d = shader.descriptorDecorations.at(id);
408 			ASSERT(d.DescriptorSet >= 0 && static_cast<uint32_t>(d.DescriptorSet) < vk::MAX_BOUND_DESCRIPTOR_SETS);
409 			ASSERT(d.Binding >= 0);
410 			ASSERT(routine->pipelineLayout->getDescriptorCount(d.DescriptorSet, d.Binding) != 0);  // "If descriptorCount is zero this binding entry is reserved and the resource must not be accessed from any stage via this binding within any pipeline using the set layout."
411 
412 			uint32_t bindingOffset = routine->pipelineLayout->getBindingOffset(d.DescriptorSet, d.Binding);
413 			uint32_t descriptorSize = routine->pipelineLayout->getDescriptorSize(d.DescriptorSet, d.Binding);
414 
415 			auto set = getPointer(id);
416 			if(nonUniform)
417 			{
418 				SIMD::Int descriptorOffset = bindingOffset + descriptorSize * arrayIndices;
419 				auto robustness = shader.getOutOfBoundsBehavior(id, routine->pipelineLayout);
420 				ASSERT(routine->pipelineLayout->getDescriptorType(d.DescriptorSet, d.Binding) != VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT);
421 
422 				std::vector<Pointer<Byte>> pointers(SIMD::Width);
423 				for(int i = 0; i < SIMD::Width; i++)
424 				{
425 					pointers[i] = *Pointer<Pointer<Byte>>(set.getPointerForLane(i) + Extract(descriptorOffset, i) + OFFSET(vk::BufferDescriptor, ptr));
426 				}
427 
428 				SIMD::Pointer ptr(pointers);
429 
430 				if(routine->pipelineLayout->isDescriptorDynamic(d.DescriptorSet, d.Binding))
431 				{
432 					SIMD::Int dynamicOffsetIndex = SIMD::Int(routine->pipelineLayout->getDynamicOffsetIndex(d.DescriptorSet, d.Binding) + arrayIndices);
433 					SIMD::Pointer routineDynamicOffsets = SIMD::Pointer(routine->descriptorDynamicOffsets, 0, sizeof(int) * dynamicOffsetIndex);
434 					SIMD::Int dynamicOffsets = routineDynamicOffsets.Load<SIMD::Int>(robustness, activeLaneMask());
435 					ptr += dynamicOffsets;
436 				}
437 				return ptr;
438 			}
439 			else
440 			{
441 				rr::Int arrayIdx = Extract(arrayIndices, 0);
442 				rr::Int descriptorOffset = bindingOffset + descriptorSize * arrayIdx;
443 				Pointer<Byte> descriptor = set.getUniformPointer() + descriptorOffset;  // BufferDescriptor* or inline uniform block
444 
445 				auto descriptorType = routine->pipelineLayout->getDescriptorType(d.DescriptorSet, d.Binding);
446 				if(descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT)
447 				{
448 					// Note: there is no bounds checking for inline uniform blocks.
449 					// MAX_INLINE_UNIFORM_BLOCK_SIZE represents the maximum size of
450 					// an inline uniform block, but this value should remain unused.
451 					return SIMD::Pointer(descriptor, vk::MAX_INLINE_UNIFORM_BLOCK_SIZE);
452 				}
453 				else
454 				{
455 					Pointer<Byte> data = *Pointer<Pointer<Byte>>(descriptor + OFFSET(vk::BufferDescriptor, ptr));  // void*
456 					rr::Int size = *Pointer<Int>(descriptor + OFFSET(vk::BufferDescriptor, sizeInBytes));
457 
458 					if(routine->pipelineLayout->isDescriptorDynamic(d.DescriptorSet, d.Binding))
459 					{
460 						rr::Int dynamicOffsetIndex =
461 						    routine->pipelineLayout->getDynamicOffsetIndex(d.DescriptorSet, d.Binding) +
462 						    arrayIdx;
463 						rr::Int offset = routine->descriptorDynamicOffsets[dynamicOffsetIndex];
464 						rr::Int robustnessSize = *Pointer<rr::Int>(descriptor + OFFSET(vk::BufferDescriptor, robustnessSize));
465 
466 						return SIMD::Pointer(data + offset, Min(size, robustnessSize - offset));
467 					}
468 					else
469 					{
470 						return SIMD::Pointer(data, size);
471 					}
472 				}
473 			}
474 		}
475 
476 	default:
477 		UNREACHABLE("Invalid pointer kind %d", int(object.kind));
478 		return SIMD::Pointer(Pointer<Byte>(), 0);
479 	}
480 }
481 
OffsetToElement(SIMD::Pointer & ptr,Object::ID elementId,int32_t arrayStride) const482 void SpirvEmitter::OffsetToElement(SIMD::Pointer &ptr, Object::ID elementId, int32_t arrayStride) const
483 {
484 	if(elementId != 0 && arrayStride != 0)
485 	{
486 		auto &elementObject = shader.getObject(elementId);
487 		ASSERT(elementObject.kind == Object::Kind::Constant || elementObject.kind == Object::Kind::Intermediate);
488 
489 		if(elementObject.kind == Object::Kind::Constant)
490 		{
491 			ptr += shader.GetConstScalarInt(elementId) * arrayStride;
492 		}
493 		else
494 		{
495 			ptr += getIntermediate(elementId).Int(0) * arrayStride;
496 		}
497 	}
498 }
499 
Fence(spv::MemorySemanticsMask semantics) const500 void SpirvEmitter::Fence(spv::MemorySemanticsMask semantics) const
501 {
502 	if(semantics != spv::MemorySemanticsMaskNone)
503 	{
504 		rr::Fence(shader.MemoryOrder(semantics));
505 	}
506 }
507 
MemoryOrder(spv::MemorySemanticsMask memorySemantics)508 std::memory_order Spirv::MemoryOrder(spv::MemorySemanticsMask memorySemantics)
509 {
510 	uint32_t control = static_cast<uint32_t>(memorySemantics) & static_cast<uint32_t>(
511 	                                                                spv::MemorySemanticsAcquireMask |
512 	                                                                spv::MemorySemanticsReleaseMask |
513 	                                                                spv::MemorySemanticsAcquireReleaseMask |
514 	                                                                spv::MemorySemanticsSequentiallyConsistentMask);
515 	switch(control)
516 	{
517 	case spv::MemorySemanticsMaskNone: return std::memory_order_relaxed;
518 	case spv::MemorySemanticsAcquireMask: return std::memory_order_acquire;
519 	case spv::MemorySemanticsReleaseMask: return std::memory_order_release;
520 	case spv::MemorySemanticsAcquireReleaseMask: return std::memory_order_acq_rel;
521 	case spv::MemorySemanticsSequentiallyConsistentMask: return std::memory_order_acq_rel;  // Vulkan 1.1: "SequentiallyConsistent is treated as AcquireRelease"
522 	default:
523 		// "it is invalid for more than one of these four bits to be set:
524 		//  Acquire, Release, AcquireRelease, or SequentiallyConsistent."
525 		UNREACHABLE("MemorySemanticsMask: %x", int(control));
526 		return std::memory_order_acq_rel;
527 	}
528 }
529 
StoresInHelperInvocationsHaveNoEffect(spv::StorageClass storageClass)530 bool Spirv::StoresInHelperInvocationsHaveNoEffect(spv::StorageClass storageClass)
531 {
532 	switch(storageClass)
533 	{
534 	// "Stores and atomics performed by helper invocations must not have any effect on memory..."
535 	default:
536 		return true;
537 	// "...except for the Function, Private and Output storage classes".
538 	case spv::StorageClassFunction:
539 	case spv::StorageClassPrivate:
540 	case spv::StorageClassOutput:
541 		return false;
542 	}
543 }
544 
IsExplicitLayout(spv::StorageClass storageClass)545 bool Spirv::IsExplicitLayout(spv::StorageClass storageClass)
546 {
547 	// From the Vulkan spec:
548 	// "Composite objects in the StorageBuffer, PhysicalStorageBuffer, Uniform,
549 	//  and PushConstant Storage Classes must be explicitly laid out."
550 	switch(storageClass)
551 	{
552 	case spv::StorageClassUniform:
553 	case spv::StorageClassStorageBuffer:
554 	case spv::StorageClassPhysicalStorageBuffer:
555 	case spv::StorageClassPushConstant:
556 		return true;
557 	default:
558 		return false;
559 	}
560 }
561 
GetElementPointer(sw::SIMD::Pointer structure,uint32_t offset,spv::StorageClass storageClass)562 sw::SIMD::Pointer SpirvEmitter::GetElementPointer(sw::SIMD::Pointer structure, uint32_t offset, spv::StorageClass storageClass)
563 {
564 	if(IsStorageInterleavedByLane(storageClass))
565 	{
566 		for(int i = 0; i < SIMD::Width; i++)
567 		{
568 			structure.staticOffsets[i] += i * sizeof(float);
569 		}
570 
571 		return structure + offset * sw::SIMD::Width;
572 	}
573 	else
574 	{
575 		return structure + offset;
576 	}
577 }
578 
IsStorageInterleavedByLane(spv::StorageClass storageClass)579 bool SpirvEmitter::IsStorageInterleavedByLane(spv::StorageClass storageClass)
580 {
581 	switch(storageClass)
582 	{
583 	case spv::StorageClassUniform:
584 	case spv::StorageClassStorageBuffer:
585 	case spv::StorageClassPhysicalStorageBuffer:
586 	case spv::StorageClassPushConstant:
587 	case spv::StorageClassWorkgroup:
588 	case spv::StorageClassImage:
589 		return false;
590 	default:
591 		return true;
592 	}
593 }
594 
595 }  // namespace sw