• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2016 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //    http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "Renderer.hpp"
16 
17 #include "Clipper.hpp"
18 #include "Polygon.hpp"
19 #include "Primitive.hpp"
20 #include "Vertex.hpp"
21 #include "Pipeline/Constants.hpp"
22 #include "Pipeline/SpirvShader.hpp"
23 #include "Reactor/Reactor.hpp"
24 #include "System/Debug.hpp"
25 #include "System/Half.hpp"
26 #include "System/Math.hpp"
27 #include "System/Memory.hpp"
28 #include "System/Timer.hpp"
29 #include "Vulkan/VkConfig.hpp"
30 #include "Vulkan/VkDescriptorSet.hpp"
31 #include "Vulkan/VkDevice.hpp"
32 #include "Vulkan/VkFence.hpp"
33 #include "Vulkan/VkImageView.hpp"
34 #include "Vulkan/VkPipelineLayout.hpp"
35 #include "Vulkan/VkQueryPool.hpp"
36 
37 #include "marl/containers.h"
38 #include "marl/defer.h"
39 #include "marl/trace.h"
40 
41 #undef max
42 
43 #ifndef NDEBUG
44 unsigned int minPrimitives = 1;
45 unsigned int maxPrimitives = 1 << 21;
46 #endif
47 
48 namespace sw {
49 
50 template<typename T>
setBatchIndices(unsigned int batch[128][3],VkPrimitiveTopology topology,VkProvokingVertexModeEXT provokingVertexMode,T indices,unsigned int start,unsigned int triangleCount)51 inline bool setBatchIndices(unsigned int batch[128][3], VkPrimitiveTopology topology, VkProvokingVertexModeEXT provokingVertexMode, T indices, unsigned int start, unsigned int triangleCount)
52 {
53 	bool provokeFirst = (provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT);
54 
55 	switch(topology)
56 	{
57 		case VK_PRIMITIVE_TOPOLOGY_POINT_LIST:
58 		{
59 			auto index = start;
60 			auto pointBatch = &(batch[0][0]);
61 			for(unsigned int i = 0; i < triangleCount; i++)
62 			{
63 				*pointBatch++ = indices[index++];
64 			}
65 
66 			// Repeat the last index to allow for SIMD width overrun.
67 			index--;
68 			for(unsigned int i = 0; i < 3; i++)
69 			{
70 				*pointBatch++ = indices[index];
71 			}
72 			break;
73 		}
74 		case VK_PRIMITIVE_TOPOLOGY_LINE_LIST:
75 		{
76 			auto index = 2 * start;
77 			for(unsigned int i = 0; i < triangleCount; i++)
78 			{
79 				batch[i][0] = indices[index + (provokeFirst ? 0 : 1)];
80 				batch[i][1] = indices[index + (provokeFirst ? 1 : 0)];
81 				batch[i][2] = indices[index + 1];
82 
83 				index += 2;
84 			}
85 			break;
86 		}
87 		case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP:
88 		{
89 			auto index = start;
90 			for(unsigned int i = 0; i < triangleCount; i++)
91 			{
92 				batch[i][0] = indices[index + (provokeFirst ? 0 : 1)];
93 				batch[i][1] = indices[index + (provokeFirst ? 1 : 0)];
94 				batch[i][2] = indices[index + 1];
95 
96 				index += 1;
97 			}
98 			break;
99 		}
100 		case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST:
101 		{
102 			auto index = 3 * start;
103 			for(unsigned int i = 0; i < triangleCount; i++)
104 			{
105 				batch[i][0] = indices[index + (provokeFirst ? 0 : 2)];
106 				batch[i][1] = indices[index + (provokeFirst ? 1 : 0)];
107 				batch[i][2] = indices[index + (provokeFirst ? 2 : 1)];
108 
109 				index += 3;
110 			}
111 			break;
112 		}
113 		case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP:
114 		{
115 			auto index = start;
116 			for(unsigned int i = 0; i < triangleCount; i++)
117 			{
118 				batch[i][0] = indices[index + (provokeFirst ? 0 : 2)];
119 				batch[i][1] = indices[index + ((start + i) & 1) + (provokeFirst ? 1 : 0)];
120 				batch[i][2] = indices[index + (~(start + i) & 1) + (provokeFirst ? 1 : 0)];
121 
122 				index += 1;
123 			}
124 			break;
125 		}
126 		case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN:
127 		{
128 			auto index = start + 1;
129 			for(unsigned int i = 0; i < triangleCount; i++)
130 			{
131 				batch[i][provokeFirst ? 0 : 2] = indices[index + 0];
132 				batch[i][provokeFirst ? 1 : 0] = indices[index + 1];
133 				batch[i][provokeFirst ? 2 : 1] = indices[0];
134 
135 				index += 1;
136 			}
137 			break;
138 		}
139 		default:
140 			ASSERT(false);
141 			return false;
142 	}
143 
144 	return true;
145 }
146 
DrawCall()147 DrawCall::DrawCall()
148 {
149 	data = (DrawData *)allocate(sizeof(DrawData));
150 	data->constants = &Constants::Get();
151 }
152 
~DrawCall()153 DrawCall::~DrawCall()
154 {
155 	deallocate(data);
156 }
157 
Renderer(vk::Device * device)158 Renderer::Renderer(vk::Device *device)
159     : device(device)
160 {
161 	vertexProcessor.setRoutineCacheSize(1024);
162 	pixelProcessor.setRoutineCacheSize(1024);
163 	setupProcessor.setRoutineCacheSize(1024);
164 }
165 
~Renderer()166 Renderer::~Renderer()
167 {
168 	drawTickets.take().wait();
169 }
170 
171 // Renderer objects have to be mem aligned to the alignment provided in the class declaration
operator new(size_t size)172 void *Renderer::operator new(size_t size)
173 {
174 	ASSERT(size == sizeof(Renderer));  // This operator can't be called from a derived class
175 	return vk::allocate(sizeof(Renderer), alignof(Renderer), vk::DEVICE_MEMORY, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
176 }
177 
operator delete(void * mem)178 void Renderer::operator delete(void *mem)
179 {
180 	vk::deallocate(mem, vk::DEVICE_MEMORY);
181 }
182 
draw(const vk::GraphicsPipeline * pipeline,const vk::DynamicState & dynamicState,unsigned int count,int baseVertex,CountedEvent * events,int instanceID,int viewID,void * indexBuffer,const VkExtent3D & framebufferExtent,vk::Pipeline::PushConstantStorage const & pushConstants,bool update)183 void Renderer::draw(const vk::GraphicsPipeline *pipeline, const vk::DynamicState &dynamicState, unsigned int count, int baseVertex,
184                     CountedEvent *events, int instanceID, int viewID, void *indexBuffer, const VkExtent3D &framebufferExtent,
185                     vk::Pipeline::PushConstantStorage const &pushConstants, bool update)
186 {
187 	if(count == 0) { return; }
188 
189 	auto id = nextDrawID++;
190 	MARL_SCOPED_EVENT("draw %d", id);
191 
192 	marl::Pool<sw::DrawCall>::Loan draw;
193 	{
194 		MARL_SCOPED_EVENT("drawCallPool.borrow()");
195 		draw = drawCallPool.borrow();
196 	}
197 	draw->id = id;
198 
199 	const vk::GraphicsState &pipelineState = pipeline->getState(dynamicState);
200 	pixelProcessor.setBlendConstant(pipelineState.getBlendConstants());
201 
202 	const vk::Inputs &inputs = pipeline->getInputs();
203 
204 	if(update)
205 	{
206 		MARL_SCOPED_EVENT("update");
207 
208 		const sw::SpirvShader *fragmentShader = pipeline->getShader(VK_SHADER_STAGE_FRAGMENT_BIT).get();
209 		const sw::SpirvShader *vertexShader = pipeline->getShader(VK_SHADER_STAGE_VERTEX_BIT).get();
210 
211 		const vk::Attachments attachments = pipeline->getAttachments();
212 
213 		vertexState = vertexProcessor.update(pipelineState, vertexShader, inputs);
214 		setupState = setupProcessor.update(pipelineState, fragmentShader, vertexShader, attachments);
215 		pixelState = pixelProcessor.update(pipelineState, fragmentShader, vertexShader, attachments, hasOcclusionQuery());
216 
217 		vertexRoutine = vertexProcessor.routine(vertexState, pipelineState.getPipelineLayout(), vertexShader, inputs.getDescriptorSets());
218 		setupRoutine = setupProcessor.routine(setupState);
219 		pixelRoutine = pixelProcessor.routine(pixelState, pipelineState.getPipelineLayout(), fragmentShader, inputs.getDescriptorSets());
220 	}
221 
222 	draw->containsImageWrite = pipeline->containsImageWrite();
223 
224 	DrawCall::SetupFunction setupPrimitives = nullptr;
225 	int ms = pipelineState.getSampleCount();
226 	unsigned int numPrimitivesPerBatch = MaxBatchSize / ms;
227 
228 	if(pipelineState.isDrawTriangle(false))
229 	{
230 		switch(pipelineState.getPolygonMode())
231 		{
232 			case VK_POLYGON_MODE_FILL:
233 				setupPrimitives = &DrawCall::setupSolidTriangles;
234 				break;
235 			case VK_POLYGON_MODE_LINE:
236 				setupPrimitives = &DrawCall::setupWireframeTriangles;
237 				numPrimitivesPerBatch /= 3;
238 				break;
239 			case VK_POLYGON_MODE_POINT:
240 				setupPrimitives = &DrawCall::setupPointTriangles;
241 				numPrimitivesPerBatch /= 3;
242 				break;
243 			default:
244 				UNSUPPORTED("polygon mode: %d", int(pipelineState.getPolygonMode()));
245 				return;
246 		}
247 	}
248 	else if(pipelineState.isDrawLine(false))
249 	{
250 		setupPrimitives = &DrawCall::setupLines;
251 	}
252 	else  // Point primitive topology
253 	{
254 		setupPrimitives = &DrawCall::setupPoints;
255 	}
256 
257 	DrawData *data = draw->data;
258 	draw->device = device;
259 	draw->occlusionQuery = occlusionQuery;
260 	draw->batchDataPool = &batchDataPool;
261 	draw->numPrimitives = count;
262 	draw->numPrimitivesPerBatch = numPrimitivesPerBatch;
263 	draw->numBatches = (count + draw->numPrimitivesPerBatch - 1) / draw->numPrimitivesPerBatch;
264 	draw->topology = pipelineState.getTopology();
265 	draw->provokingVertexMode = pipelineState.getProvokingVertexMode();
266 	draw->indexType = pipeline->getIndexBuffer().getIndexType();
267 	draw->lineRasterizationMode = pipelineState.getLineRasterizationMode();
268 	draw->descriptorSetObjects = inputs.getDescriptorSetObjects();
269 	draw->pipelineLayout = pipelineState.getPipelineLayout();
270 
271 	draw->vertexRoutine = vertexRoutine;
272 	draw->setupRoutine = setupRoutine;
273 	draw->pixelRoutine = pixelRoutine;
274 	draw->setupPrimitives = setupPrimitives;
275 	draw->setupState = setupState;
276 
277 	data->descriptorSets = inputs.getDescriptorSets();
278 	data->descriptorDynamicOffsets = inputs.getDescriptorDynamicOffsets();
279 
280 	for(int i = 0; i < MAX_INTERFACE_COMPONENTS / 4; i++)
281 	{
282 		const sw::Stream &stream = inputs.getStream(i);
283 		data->input[i] = stream.buffer;
284 		data->robustnessSize[i] = stream.robustnessSize;
285 		data->stride[i] = stream.vertexStride;
286 	}
287 
288 	data->indices = indexBuffer;
289 	data->viewID = viewID;
290 	data->instanceID = instanceID;
291 	data->baseVertex = baseVertex;
292 
293 	if(pixelState.stencilActive)
294 	{
295 		data->stencil[0].set(pipelineState.getFrontStencil().reference, pipelineState.getFrontStencil().compareMask, pipelineState.getFrontStencil().writeMask);
296 		data->stencil[1].set(pipelineState.getBackStencil().reference, pipelineState.getBackStencil().compareMask, pipelineState.getBackStencil().writeMask);
297 	}
298 
299 	data->lineWidth = pipelineState.getLineWidth();
300 
301 	data->factor = pixelProcessor.factor;
302 
303 	if(pixelState.alphaToCoverage)
304 	{
305 		if(ms == 4)
306 		{
307 			data->a2c0 = float4(0.2f);
308 			data->a2c1 = float4(0.4f);
309 			data->a2c2 = float4(0.6f);
310 			data->a2c3 = float4(0.8f);
311 		}
312 		else if(ms == 2)
313 		{
314 			data->a2c0 = float4(0.25f);
315 			data->a2c1 = float4(0.75f);
316 		}
317 		else if(ms == 1)
318 		{
319 			data->a2c0 = float4(0.5f);
320 		}
321 		else
322 			ASSERT(false);
323 	}
324 
325 	if(pixelState.occlusionEnabled)
326 	{
327 		for(int cluster = 0; cluster < MaxClusterCount; cluster++)
328 		{
329 			data->occlusion[cluster] = 0;
330 		}
331 	}
332 
333 	// Viewport
334 	{
335 		const VkViewport &viewport = pipelineState.getViewport();
336 
337 		float W = 0.5f * viewport.width;
338 		float H = 0.5f * viewport.height;
339 		float X0 = viewport.x + W;
340 		float Y0 = viewport.y + H;
341 		float N = viewport.minDepth;
342 		float F = viewport.maxDepth;
343 		float Z = F - N;
344 		constexpr float subPixF = vk::SUBPIXEL_PRECISION_FACTOR;
345 
346 		data->WxF = float4(W * subPixF);
347 		data->HxF = float4(H * subPixF);
348 		data->X0xF = float4(X0 * subPixF - subPixF / 2);
349 		data->Y0xF = float4(Y0 * subPixF - subPixF / 2);
350 		data->halfPixelX = float4(0.5f / W);
351 		data->halfPixelY = float4(0.5f / H);
352 		data->viewportHeight = abs(viewport.height);
353 		data->depthRange = Z;
354 		data->depthNear = N;
355 		data->constantDepthBias = pipelineState.getConstantDepthBias();
356 		data->slopeDepthBias = pipelineState.getSlopeDepthBias();
357 		data->depthBiasClamp = pipelineState.getDepthBiasClamp();
358 
359 		const vk::Attachments attachments = pipeline->getAttachments();
360 		if(attachments.depthBuffer)
361 		{
362 			switch(attachments.depthBuffer->getFormat(VK_IMAGE_ASPECT_DEPTH_BIT))
363 			{
364 				case VK_FORMAT_D16_UNORM:
365 					data->minimumResolvableDepthDifference = 1.0f / 0xFFFF;
366 					break;
367 				case VK_FORMAT_D32_SFLOAT:
368 					// The minimum resolvable depth difference is determined per-polygon for floating-point depth
369 					// buffers. DrawData::minimumResolvableDepthDifference is unused.
370 					break;
371 				default:
372 					UNSUPPORTED("Depth format: %d", int(attachments.depthBuffer->getFormat(VK_IMAGE_ASPECT_DEPTH_BIT)));
373 			}
374 		}
375 	}
376 
377 	// Target
378 	{
379 		const vk::Attachments attachments = pipeline->getAttachments();
380 
381 		for(int index = 0; index < RENDERTARGETS; index++)
382 		{
383 			draw->renderTarget[index] = attachments.renderTarget[index];
384 
385 			if(draw->renderTarget[index])
386 			{
387 				data->colorBuffer[index] = (unsigned int *)attachments.renderTarget[index]->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_COLOR_BIT, 0, data->viewID);
388 				data->colorPitchB[index] = attachments.renderTarget[index]->rowPitchBytes(VK_IMAGE_ASPECT_COLOR_BIT, 0);
389 				data->colorSliceB[index] = attachments.renderTarget[index]->slicePitchBytes(VK_IMAGE_ASPECT_COLOR_BIT, 0);
390 			}
391 		}
392 
393 		draw->depthBuffer = attachments.depthBuffer;
394 		draw->stencilBuffer = attachments.stencilBuffer;
395 
396 		if(draw->depthBuffer)
397 		{
398 			data->depthBuffer = (float *)attachments.depthBuffer->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_DEPTH_BIT, 0, data->viewID);
399 			data->depthPitchB = attachments.depthBuffer->rowPitchBytes(VK_IMAGE_ASPECT_DEPTH_BIT, 0);
400 			data->depthSliceB = attachments.depthBuffer->slicePitchBytes(VK_IMAGE_ASPECT_DEPTH_BIT, 0);
401 		}
402 
403 		if(draw->stencilBuffer)
404 		{
405 			data->stencilBuffer = (unsigned char *)attachments.stencilBuffer->getOffsetPointer({ 0, 0, 0 }, VK_IMAGE_ASPECT_STENCIL_BIT, 0, data->viewID);
406 			data->stencilPitchB = attachments.stencilBuffer->rowPitchBytes(VK_IMAGE_ASPECT_STENCIL_BIT, 0);
407 			data->stencilSliceB = attachments.stencilBuffer->slicePitchBytes(VK_IMAGE_ASPECT_STENCIL_BIT, 0);
408 		}
409 	}
410 
411 	// Scissor
412 	{
413 		const VkRect2D &scissor = pipelineState.getScissor();
414 
415 		data->scissorX0 = clamp<int>(scissor.offset.x, 0, framebufferExtent.width);
416 		data->scissorX1 = clamp<int>(scissor.offset.x + scissor.extent.width, 0, framebufferExtent.width);
417 		data->scissorY0 = clamp<int>(scissor.offset.y, 0, framebufferExtent.height);
418 		data->scissorY1 = clamp<int>(scissor.offset.y + scissor.extent.height, 0, framebufferExtent.height);
419 	}
420 
421 	// Push constants
422 	{
423 		data->pushConstants = pushConstants;
424 	}
425 
426 	draw->events = events;
427 
428 	vk::DescriptorSet::PrepareForSampling(draw->descriptorSetObjects, draw->pipelineLayout, device);
429 
430 	DrawCall::run(draw, &drawTickets, clusterQueues);
431 }
432 
setup()433 void DrawCall::setup()
434 {
435 	if(occlusionQuery != nullptr)
436 	{
437 		occlusionQuery->start();
438 	}
439 
440 	if(events)
441 	{
442 		events->add();
443 	}
444 }
445 
teardown()446 void DrawCall::teardown()
447 {
448 	if(events)
449 	{
450 		events->done();
451 		events = nullptr;
452 	}
453 
454 	if(occlusionQuery != nullptr)
455 	{
456 		for(int cluster = 0; cluster < MaxClusterCount; cluster++)
457 		{
458 			occlusionQuery->add(data->occlusion[cluster]);
459 		}
460 		occlusionQuery->finish();
461 	}
462 
463 	vertexRoutine = {};
464 	setupRoutine = {};
465 	pixelRoutine = {};
466 
467 	for(auto *rt : renderTarget)
468 	{
469 		if(rt)
470 		{
471 			rt->contentsChanged();
472 		}
473 	}
474 
475 	if(containsImageWrite)
476 	{
477 		vk::DescriptorSet::ContentsChanged(descriptorSetObjects, pipelineLayout, device);
478 	}
479 }
480 
run(const marl::Loan<DrawCall> & draw,marl::Ticket::Queue * tickets,marl::Ticket::Queue clusterQueues[MaxClusterCount])481 void DrawCall::run(const marl::Loan<DrawCall> &draw, marl::Ticket::Queue *tickets, marl::Ticket::Queue clusterQueues[MaxClusterCount])
482 {
483 	draw->setup();
484 
485 	auto const numPrimitives = draw->numPrimitives;
486 	auto const numPrimitivesPerBatch = draw->numPrimitivesPerBatch;
487 	auto const numBatches = draw->numBatches;
488 
489 	auto ticket = tickets->take();
490 	auto finally = marl::make_shared_finally([draw, ticket] {
491 		MARL_SCOPED_EVENT("FINISH draw %d", draw->id);
492 		draw->teardown();
493 		ticket.done();
494 	});
495 
496 	for(unsigned int batchId = 0; batchId < numBatches; batchId++)
497 	{
498 		auto batch = draw->batchDataPool->borrow();
499 		batch->id = batchId;
500 		batch->firstPrimitive = batch->id * numPrimitivesPerBatch;
501 		batch->numPrimitives = std::min(batch->firstPrimitive + numPrimitivesPerBatch, numPrimitives) - batch->firstPrimitive;
502 
503 		for(int cluster = 0; cluster < MaxClusterCount; cluster++)
504 		{
505 			batch->clusterTickets[cluster] = std::move(clusterQueues[cluster].take());
506 		}
507 
508 		marl::schedule([draw, batch, finally] {
509 			processVertices(draw.get(), batch.get());
510 
511 			if(!draw->setupState.rasterizerDiscard)
512 			{
513 				processPrimitives(draw.get(), batch.get());
514 
515 				if(batch->numVisible > 0)
516 				{
517 					processPixels(draw, batch, finally);
518 					return;
519 				}
520 			}
521 
522 			for(int cluster = 0; cluster < MaxClusterCount; cluster++)
523 			{
524 				batch->clusterTickets[cluster].done();
525 			}
526 		});
527 	}
528 }
529 
processVertices(DrawCall * draw,BatchData * batch)530 void DrawCall::processVertices(DrawCall *draw, BatchData *batch)
531 {
532 	MARL_SCOPED_EVENT("VERTEX draw %d, batch %d", draw->id, batch->id);
533 
534 	unsigned int triangleIndices[MaxBatchSize + 1][3];  // One extra for SIMD width overrun. TODO: Adjust to dynamic batch size.
535 	{
536 		MARL_SCOPED_EVENT("processPrimitiveVertices");
537 		processPrimitiveVertices(
538 		    triangleIndices,
539 		    draw->data->indices,
540 		    draw->indexType,
541 		    batch->firstPrimitive,
542 		    batch->numPrimitives,
543 		    draw->topology,
544 		    draw->provokingVertexMode);
545 	}
546 
547 	auto &vertexTask = batch->vertexTask;
548 	vertexTask.primitiveStart = batch->firstPrimitive;
549 	// We're only using batch compaction for points, not lines
550 	vertexTask.vertexCount = batch->numPrimitives * ((draw->topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST) ? 1 : 3);
551 	if(vertexTask.vertexCache.drawCall != draw->id)
552 	{
553 		vertexTask.vertexCache.clear();
554 		vertexTask.vertexCache.drawCall = draw->id;
555 	}
556 
557 	draw->vertexRoutine(&batch->triangles.front().v0, &triangleIndices[0][0], &vertexTask, draw->data);
558 }
559 
processPrimitives(DrawCall * draw,BatchData * batch)560 void DrawCall::processPrimitives(DrawCall *draw, BatchData *batch)
561 {
562 	MARL_SCOPED_EVENT("PRIMITIVES draw %d batch %d", draw->id, batch->id);
563 	auto triangles = &batch->triangles[0];
564 	auto primitives = &batch->primitives[0];
565 	batch->numVisible = draw->setupPrimitives(triangles, primitives, draw, batch->numPrimitives);
566 }
567 
processPixels(const marl::Loan<DrawCall> & draw,const marl::Loan<BatchData> & batch,const std::shared_ptr<marl::Finally> & finally)568 void DrawCall::processPixels(const marl::Loan<DrawCall> &draw, const marl::Loan<BatchData> &batch, const std::shared_ptr<marl::Finally> &finally)
569 {
570 	struct Data
571 	{
572 		Data(const marl::Loan<DrawCall> &draw, const marl::Loan<BatchData> &batch, const std::shared_ptr<marl::Finally> &finally)
573 		    : draw(draw)
574 		    , batch(batch)
575 		    , finally(finally)
576 		{}
577 		marl::Loan<DrawCall> draw;
578 		marl::Loan<BatchData> batch;
579 		std::shared_ptr<marl::Finally> finally;
580 	};
581 	auto data = std::make_shared<Data>(draw, batch, finally);
582 	for(int cluster = 0; cluster < MaxClusterCount; cluster++)
583 	{
584 		batch->clusterTickets[cluster].onCall([data, cluster] {
585 			auto &draw = data->draw;
586 			auto &batch = data->batch;
587 			MARL_SCOPED_EVENT("PIXEL draw %d, batch %d, cluster %d", draw->id, batch->id, cluster);
588 			draw->pixelRoutine(&batch->primitives.front(), batch->numVisible, cluster, MaxClusterCount, draw->data);
589 			batch->clusterTickets[cluster].done();
590 		});
591 	}
592 }
593 
synchronize()594 void Renderer::synchronize()
595 {
596 	MARL_SCOPED_EVENT("synchronize");
597 	auto ticket = drawTickets.take();
598 	ticket.wait();
599 	device->updateSamplingRoutineSnapshotCache();
600 	ticket.done();
601 }
602 
processPrimitiveVertices(unsigned int triangleIndicesOut[MaxBatchSize+1][3],const void * primitiveIndices,VkIndexType indexType,unsigned int start,unsigned int triangleCount,VkPrimitiveTopology topology,VkProvokingVertexModeEXT provokingVertexMode)603 void DrawCall::processPrimitiveVertices(
604     unsigned int triangleIndicesOut[MaxBatchSize + 1][3],
605     const void *primitiveIndices,
606     VkIndexType indexType,
607     unsigned int start,
608     unsigned int triangleCount,
609     VkPrimitiveTopology topology,
610     VkProvokingVertexModeEXT provokingVertexMode)
611 {
612 	if(!primitiveIndices)
613 	{
614 		struct LinearIndex
615 		{
616 			unsigned int operator[](unsigned int i) { return i; }
617 		};
618 
619 		if(!setBatchIndices(triangleIndicesOut, topology, provokingVertexMode, LinearIndex(), start, triangleCount))
620 		{
621 			return;
622 		}
623 	}
624 	else
625 	{
626 		switch(indexType)
627 		{
628 			case VK_INDEX_TYPE_UINT16:
629 				if(!setBatchIndices(triangleIndicesOut, topology, provokingVertexMode, static_cast<const uint16_t *>(primitiveIndices), start, triangleCount))
630 				{
631 					return;
632 				}
633 				break;
634 			case VK_INDEX_TYPE_UINT32:
635 				if(!setBatchIndices(triangleIndicesOut, topology, provokingVertexMode, static_cast<const uint32_t *>(primitiveIndices), start, triangleCount))
636 				{
637 					return;
638 				}
639 				break;
640 				break;
641 			default:
642 				ASSERT(false);
643 				return;
644 		}
645 	}
646 
647 	// setBatchIndices() takes care of the point case, since it's different due to the compaction
648 	if(topology != VK_PRIMITIVE_TOPOLOGY_POINT_LIST)
649 	{
650 		// Repeat the last index to allow for SIMD width overrun.
651 		triangleIndicesOut[triangleCount][0] = triangleIndicesOut[triangleCount - 1][2];
652 		triangleIndicesOut[triangleCount][1] = triangleIndicesOut[triangleCount - 1][2];
653 		triangleIndicesOut[triangleCount][2] = triangleIndicesOut[triangleCount - 1][2];
654 	}
655 }
656 
setupSolidTriangles(Triangle * triangles,Primitive * primitives,const DrawCall * drawCall,int count)657 int DrawCall::setupSolidTriangles(Triangle *triangles, Primitive *primitives, const DrawCall *drawCall, int count)
658 {
659 	auto &state = drawCall->setupState;
660 
661 	int ms = state.multiSampleCount;
662 	const DrawData *data = drawCall->data;
663 	int visible = 0;
664 
665 	for(int i = 0; i < count; i++, triangles++)
666 	{
667 		Vertex &v0 = triangles->v0;
668 		Vertex &v1 = triangles->v1;
669 		Vertex &v2 = triangles->v2;
670 
671 		Polygon polygon(&v0.position, &v1.position, &v2.position);
672 
673 		if((v0.cullMask | v1.cullMask | v2.cullMask) == 0)
674 		{
675 			continue;
676 		}
677 
678 		if((v0.clipFlags & v1.clipFlags & v2.clipFlags) != Clipper::CLIP_FINITE)
679 		{
680 			continue;
681 		}
682 
683 		int clipFlagsOr = v0.clipFlags | v1.clipFlags | v2.clipFlags;
684 		if(clipFlagsOr != Clipper::CLIP_FINITE)
685 		{
686 			if(!Clipper::Clip(polygon, clipFlagsOr, *drawCall))
687 			{
688 				continue;
689 			}
690 		}
691 
692 		if(drawCall->setupRoutine(primitives, triangles, &polygon, data))
693 		{
694 			primitives += ms;
695 			visible++;
696 		}
697 	}
698 
699 	return visible;
700 }
701 
setupWireframeTriangles(Triangle * triangles,Primitive * primitives,const DrawCall * drawCall,int count)702 int DrawCall::setupWireframeTriangles(Triangle *triangles, Primitive *primitives, const DrawCall *drawCall, int count)
703 {
704 	auto &state = drawCall->setupState;
705 
706 	int ms = state.multiSampleCount;
707 	int visible = 0;
708 
709 	for(int i = 0; i < count; i++)
710 	{
711 		const Vertex &v0 = triangles[i].v0;
712 		const Vertex &v1 = triangles[i].v1;
713 		const Vertex &v2 = triangles[i].v2;
714 
715 		float A = ((float)v0.projected.y - (float)v2.projected.y) * (float)v1.projected.x +
716 		          ((float)v2.projected.y - (float)v1.projected.y) * (float)v0.projected.x +
717 		          ((float)v1.projected.y - (float)v0.projected.y) * (float)v2.projected.x;  // Area
718 
719 		int w0w1w2 = bit_cast<int>(v0.w) ^
720 		             bit_cast<int>(v1.w) ^
721 		             bit_cast<int>(v2.w);
722 
723 		A = w0w1w2 < 0 ? -A : A;
724 
725 		bool frontFacing = (state.frontFace == VK_FRONT_FACE_COUNTER_CLOCKWISE) ? (A >= 0.0f) : (A <= 0.0f);
726 
727 		if(state.cullMode & VK_CULL_MODE_FRONT_BIT)
728 		{
729 			if(frontFacing) continue;
730 		}
731 		if(state.cullMode & VK_CULL_MODE_BACK_BIT)
732 		{
733 			if(!frontFacing) continue;
734 		}
735 
736 		Triangle lines[3];
737 		lines[0].v0 = v0;
738 		lines[0].v1 = v1;
739 		lines[1].v0 = v1;
740 		lines[1].v1 = v2;
741 		lines[2].v0 = v2;
742 		lines[2].v1 = v0;
743 
744 		for(int i = 0; i < 3; i++)
745 		{
746 			if(setupLine(*primitives, lines[i], *drawCall))
747 			{
748 				primitives += ms;
749 				visible++;
750 			}
751 		}
752 	}
753 
754 	return visible;
755 }
756 
setupPointTriangles(Triangle * triangles,Primitive * primitives,const DrawCall * drawCall,int count)757 int DrawCall::setupPointTriangles(Triangle *triangles, Primitive *primitives, const DrawCall *drawCall, int count)
758 {
759 	auto &state = drawCall->setupState;
760 
761 	int ms = state.multiSampleCount;
762 	int visible = 0;
763 
764 	for(int i = 0; i < count; i++)
765 	{
766 		const Vertex &v0 = triangles[i].v0;
767 		const Vertex &v1 = triangles[i].v1;
768 		const Vertex &v2 = triangles[i].v2;
769 
770 		float d = (v0.y * v1.x - v0.x * v1.y) * v2.w +
771 		          (v0.x * v2.y - v0.y * v2.x) * v1.w +
772 		          (v2.x * v1.y - v1.x * v2.y) * v0.w;
773 
774 		bool frontFacing = (state.frontFace == VK_FRONT_FACE_COUNTER_CLOCKWISE) ? (d > 0) : (d < 0);
775 		if(state.cullMode & VK_CULL_MODE_FRONT_BIT)
776 		{
777 			if(frontFacing) continue;
778 		}
779 		if(state.cullMode & VK_CULL_MODE_BACK_BIT)
780 		{
781 			if(!frontFacing) continue;
782 		}
783 
784 		Triangle points[3];
785 		points[0].v0 = v0;
786 		points[1].v0 = v1;
787 		points[2].v0 = v2;
788 
789 		for(int i = 0; i < 3; i++)
790 		{
791 			if(setupPoint(*primitives, points[i], *drawCall))
792 			{
793 				primitives += ms;
794 				visible++;
795 			}
796 		}
797 	}
798 
799 	return visible;
800 }
801 
setupLines(Triangle * triangles,Primitive * primitives,const DrawCall * drawCall,int count)802 int DrawCall::setupLines(Triangle *triangles, Primitive *primitives, const DrawCall *drawCall, int count)
803 {
804 	auto &state = drawCall->setupState;
805 
806 	int visible = 0;
807 	int ms = state.multiSampleCount;
808 
809 	for(int i = 0; i < count; i++)
810 	{
811 		if(setupLine(*primitives, *triangles, *drawCall))
812 		{
813 			primitives += ms;
814 			visible++;
815 		}
816 
817 		triangles++;
818 	}
819 
820 	return visible;
821 }
822 
setupPoints(Triangle * triangles,Primitive * primitives,const DrawCall * drawCall,int count)823 int DrawCall::setupPoints(Triangle *triangles, Primitive *primitives, const DrawCall *drawCall, int count)
824 {
825 	auto &state = drawCall->setupState;
826 
827 	int visible = 0;
828 	int ms = state.multiSampleCount;
829 
830 	for(int i = 0; i < count; i++)
831 	{
832 		if(setupPoint(*primitives, *triangles, *drawCall))
833 		{
834 			primitives += ms;
835 			visible++;
836 		}
837 
838 		triangles++;
839 	}
840 
841 	return visible;
842 }
843 
setupLine(Primitive & primitive,Triangle & triangle,const DrawCall & draw)844 bool DrawCall::setupLine(Primitive &primitive, Triangle &triangle, const DrawCall &draw)
845 {
846 	const DrawData &data = *draw.data;
847 
848 	float lineWidth = data.lineWidth;
849 
850 	Vertex &v0 = triangle.v0;
851 	Vertex &v1 = triangle.v1;
852 
853 	if((v0.cullMask | v1.cullMask) == 0)
854 	{
855 		return false;
856 	}
857 
858 	const float4 &P0 = v0.position;
859 	const float4 &P1 = v1.position;
860 
861 	if(P0.w <= 0 && P1.w <= 0)
862 	{
863 		return false;
864 	}
865 
866 	constexpr float subPixF = vk::SUBPIXEL_PRECISION_FACTOR;
867 
868 	const float W = data.WxF[0] * (1.0f / subPixF);
869 	const float H = data.HxF[0] * (1.0f / subPixF);
870 
871 	float dx = W * (P1.x / P1.w - P0.x / P0.w);
872 	float dy = H * (P1.y / P1.w - P0.y / P0.w);
873 
874 	if(dx == 0 && dy == 0)
875 	{
876 		return false;
877 	}
878 
879 	if(draw.lineRasterizationMode != VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT)
880 	{
881 		// Rectangle centered on the line segment
882 
883 		float4 P[4];
884 		int C[4];
885 
886 		P[0] = P0;
887 		P[1] = P1;
888 		P[2] = P1;
889 		P[3] = P0;
890 
891 		float scale = lineWidth * 0.5f / sqrt(dx * dx + dy * dy);
892 
893 		dx *= scale;
894 		dy *= scale;
895 
896 		float dx0h = dx * P0.w / H;
897 		float dy0w = dy * P0.w / W;
898 
899 		float dx1h = dx * P1.w / H;
900 		float dy1w = dy * P1.w / W;
901 
902 		P[0].x += -dy0w;
903 		P[0].y += +dx0h;
904 		C[0] = Clipper::ComputeClipFlags(P[0]);
905 
906 		P[1].x += -dy1w;
907 		P[1].y += +dx1h;
908 		C[1] = Clipper::ComputeClipFlags(P[1]);
909 
910 		P[2].x += +dy1w;
911 		P[2].y += -dx1h;
912 		C[2] = Clipper::ComputeClipFlags(P[2]);
913 
914 		P[3].x += +dy0w;
915 		P[3].y += -dx0h;
916 		C[3] = Clipper::ComputeClipFlags(P[3]);
917 
918 		if((C[0] & C[1] & C[2] & C[3]) == Clipper::CLIP_FINITE)
919 		{
920 			Polygon polygon(P, 4);
921 
922 			int clipFlagsOr = C[0] | C[1] | C[2] | C[3];
923 
924 			if(clipFlagsOr != Clipper::CLIP_FINITE)
925 			{
926 				if(!Clipper::Clip(polygon, clipFlagsOr, draw))
927 				{
928 					return false;
929 				}
930 			}
931 
932 			return draw.setupRoutine(&primitive, &triangle, &polygon, &data);
933 		}
934 	}
935 	else if(false)  // TODO(b/80135519): Deprecate
936 	{
937 		// Connecting diamonds polygon
938 		// This shape satisfies the diamond test convention, except for the exit rule part.
939 		// Line segments with overlapping endpoints have duplicate fragments.
940 		// The ideal algorithm requires half-open line rasterization (b/80135519).
941 
942 		float4 P[8];
943 		int C[8];
944 
945 		P[0] = P0;
946 		P[1] = P0;
947 		P[2] = P0;
948 		P[3] = P0;
949 		P[4] = P1;
950 		P[5] = P1;
951 		P[6] = P1;
952 		P[7] = P1;
953 
954 		float dx0 = lineWidth * 0.5f * P0.w / W;
955 		float dy0 = lineWidth * 0.5f * P0.w / H;
956 
957 		float dx1 = lineWidth * 0.5f * P1.w / W;
958 		float dy1 = lineWidth * 0.5f * P1.w / H;
959 
960 		P[0].x += -dx0;
961 		C[0] = Clipper::ComputeClipFlags(P[0]);
962 
963 		P[1].y += +dy0;
964 		C[1] = Clipper::ComputeClipFlags(P[1]);
965 
966 		P[2].x += +dx0;
967 		C[2] = Clipper::ComputeClipFlags(P[2]);
968 
969 		P[3].y += -dy0;
970 		C[3] = Clipper::ComputeClipFlags(P[3]);
971 
972 		P[4].x += -dx1;
973 		C[4] = Clipper::ComputeClipFlags(P[4]);
974 
975 		P[5].y += +dy1;
976 		C[5] = Clipper::ComputeClipFlags(P[5]);
977 
978 		P[6].x += +dx1;
979 		C[6] = Clipper::ComputeClipFlags(P[6]);
980 
981 		P[7].y += -dy1;
982 		C[7] = Clipper::ComputeClipFlags(P[7]);
983 
984 		if((C[0] & C[1] & C[2] & C[3] & C[4] & C[5] & C[6] & C[7]) == Clipper::CLIP_FINITE)
985 		{
986 			float4 L[6];
987 
988 			if(dx > -dy)
989 			{
990 				if(dx > dy)  // Right
991 				{
992 					L[0] = P[0];
993 					L[1] = P[1];
994 					L[2] = P[5];
995 					L[3] = P[6];
996 					L[4] = P[7];
997 					L[5] = P[3];
998 				}
999 				else  // Down
1000 				{
1001 					L[0] = P[0];
1002 					L[1] = P[4];
1003 					L[2] = P[5];
1004 					L[3] = P[6];
1005 					L[4] = P[2];
1006 					L[5] = P[3];
1007 				}
1008 			}
1009 			else
1010 			{
1011 				if(dx > dy)  // Up
1012 				{
1013 					L[0] = P[0];
1014 					L[1] = P[1];
1015 					L[2] = P[2];
1016 					L[3] = P[6];
1017 					L[4] = P[7];
1018 					L[5] = P[4];
1019 				}
1020 				else  // Left
1021 				{
1022 					L[0] = P[1];
1023 					L[1] = P[2];
1024 					L[2] = P[3];
1025 					L[3] = P[7];
1026 					L[4] = P[4];
1027 					L[5] = P[5];
1028 				}
1029 			}
1030 
1031 			Polygon polygon(L, 6);
1032 
1033 			int clipFlagsOr = C[0] | C[1] | C[2] | C[3] | C[4] | C[5] | C[6] | C[7];
1034 
1035 			if(clipFlagsOr != Clipper::CLIP_FINITE)
1036 			{
1037 				if(!Clipper::Clip(polygon, clipFlagsOr, draw))
1038 				{
1039 					return false;
1040 				}
1041 			}
1042 
1043 			return draw.setupRoutine(&primitive, &triangle, &polygon, &data);
1044 		}
1045 	}
1046 	else
1047 	{
1048 		// Parallelogram approximating Bresenham line
1049 		// This algorithm does not satisfy the ideal diamond-exit rule, but does avoid the
1050 		// duplicate fragment rasterization problem and satisfies all of Vulkan's minimum
1051 		// requirements for Bresenham line segment rasterization.
1052 
1053 		float4 P[8];
1054 		P[0] = P0;
1055 		P[1] = P0;
1056 		P[2] = P0;
1057 		P[3] = P0;
1058 		P[4] = P1;
1059 		P[5] = P1;
1060 		P[6] = P1;
1061 		P[7] = P1;
1062 
1063 		float dx0 = lineWidth * 0.5f * P0.w / W;
1064 		float dy0 = lineWidth * 0.5f * P0.w / H;
1065 
1066 		float dx1 = lineWidth * 0.5f * P1.w / W;
1067 		float dy1 = lineWidth * 0.5f * P1.w / H;
1068 
1069 		P[0].x += -dx0;
1070 		P[1].y += +dy0;
1071 		P[2].x += +dx0;
1072 		P[3].y += -dy0;
1073 		P[4].x += -dx1;
1074 		P[5].y += +dy1;
1075 		P[6].x += +dx1;
1076 		P[7].y += -dy1;
1077 
1078 		float4 L[4];
1079 
1080 		if(dx > -dy)
1081 		{
1082 			if(dx > dy)  // Right
1083 			{
1084 				L[0] = P[1];
1085 				L[1] = P[5];
1086 				L[2] = P[7];
1087 				L[3] = P[3];
1088 			}
1089 			else  // Down
1090 			{
1091 				L[0] = P[0];
1092 				L[1] = P[4];
1093 				L[2] = P[6];
1094 				L[3] = P[2];
1095 			}
1096 		}
1097 		else
1098 		{
1099 			if(dx > dy)  // Up
1100 			{
1101 				L[0] = P[0];
1102 				L[1] = P[2];
1103 				L[2] = P[6];
1104 				L[3] = P[4];
1105 			}
1106 			else  // Left
1107 			{
1108 				L[0] = P[1];
1109 				L[1] = P[3];
1110 				L[2] = P[7];
1111 				L[3] = P[5];
1112 			}
1113 		}
1114 
1115 		int C0 = Clipper::ComputeClipFlags(L[0]);
1116 		int C1 = Clipper::ComputeClipFlags(L[1]);
1117 		int C2 = Clipper::ComputeClipFlags(L[2]);
1118 		int C3 = Clipper::ComputeClipFlags(L[3]);
1119 
1120 		if((C0 & C1 & C2 & C3) == Clipper::CLIP_FINITE)
1121 		{
1122 			Polygon polygon(L, 4);
1123 
1124 			int clipFlagsOr = C0 | C1 | C2 | C3;
1125 
1126 			if(clipFlagsOr != Clipper::CLIP_FINITE)
1127 			{
1128 				if(!Clipper::Clip(polygon, clipFlagsOr, draw))
1129 				{
1130 					return false;
1131 				}
1132 			}
1133 
1134 			return draw.setupRoutine(&primitive, &triangle, &polygon, &data);
1135 		}
1136 	}
1137 
1138 	return false;
1139 }
1140 
setupPoint(Primitive & primitive,Triangle & triangle,const DrawCall & draw)1141 bool DrawCall::setupPoint(Primitive &primitive, Triangle &triangle, const DrawCall &draw)
1142 {
1143 	const DrawData &data = *draw.data;
1144 
1145 	Vertex &v = triangle.v0;
1146 
1147 	if(v.cullMask == 0)
1148 	{
1149 		return false;
1150 	}
1151 
1152 	float pSize = v.pointSize;
1153 
1154 	pSize = clamp(pSize, 1.0f, static_cast<float>(vk::MAX_POINT_SIZE));
1155 
1156 	float4 P[4];
1157 	int C[4];
1158 
1159 	P[0] = v.position;
1160 	P[1] = v.position;
1161 	P[2] = v.position;
1162 	P[3] = v.position;
1163 
1164 	const float X = pSize * P[0].w * data.halfPixelX[0];
1165 	const float Y = pSize * P[0].w * data.halfPixelY[0];
1166 
1167 	P[0].x -= X;
1168 	P[0].y += Y;
1169 	C[0] = Clipper::ComputeClipFlags(P[0]);
1170 
1171 	P[1].x += X;
1172 	P[1].y += Y;
1173 	C[1] = Clipper::ComputeClipFlags(P[1]);
1174 
1175 	P[2].x += X;
1176 	P[2].y -= Y;
1177 	C[2] = Clipper::ComputeClipFlags(P[2]);
1178 
1179 	P[3].x -= X;
1180 	P[3].y -= Y;
1181 	C[3] = Clipper::ComputeClipFlags(P[3]);
1182 
1183 	Polygon polygon(P, 4);
1184 
1185 	if((C[0] & C[1] & C[2] & C[3]) == Clipper::CLIP_FINITE)
1186 	{
1187 		int clipFlagsOr = C[0] | C[1] | C[2] | C[3];
1188 
1189 		if(clipFlagsOr != Clipper::CLIP_FINITE)
1190 		{
1191 			if(!Clipper::Clip(polygon, clipFlagsOr, draw))
1192 			{
1193 				return false;
1194 			}
1195 		}
1196 
1197 		primitive.pointSizeInv = 1.0f / pSize;
1198 
1199 		return draw.setupRoutine(&primitive, &triangle, &polygon, &data);
1200 	}
1201 
1202 	return false;
1203 }
1204 
addQuery(vk::Query * query)1205 void Renderer::addQuery(vk::Query *query)
1206 {
1207 	ASSERT(query->getType() == VK_QUERY_TYPE_OCCLUSION);
1208 	ASSERT(!occlusionQuery);
1209 
1210 	occlusionQuery = query;
1211 }
1212 
removeQuery(vk::Query * query)1213 void Renderer::removeQuery(vk::Query *query)
1214 {
1215 	ASSERT(query->getType() == VK_QUERY_TYPE_OCCLUSION);
1216 	ASSERT(occlusionQuery == query);
1217 
1218 	occlusionQuery = nullptr;
1219 }
1220 
1221 }  // namespace sw
1222