android-12.0.0_r34/s

/*
 * Copyright 2017 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can
 * be found in the LICENSE file.
 *
 */

//
//
//

// get rid of these
#include <stdio.h>
#include <stdlib.h>

//
//
//

#include "hs/cl/hs_cl.h"

#include "common/cl/assert_cl.h"

#include "context.h"
#include "grid.h"
#include "raster.h"
#include "extent_ring.h"
#include "raster_builder.h"

#include "tile.h"

#include "config_cl.h"
#include "runtime_cl_12.h"
#include "extent_cl_12.h"
#include "raster_builder_cl_12.h"

//
// RASTERIZATION SUB-PIPELINE
// --------------------------
//
// Phase 1: expand commands
//
// Phase 2: rasterize
//
// Phase 3: sort & segment || release paths
//
// Phase 4: prefix
//
// Phase 5: release rasters
//
//                                                      RASTER  COHORT
//                                                      ==============
//
//                      BUILDER                           RASTERIZER                              POST PROCESSING
//   <----------------------------------------------->  <------------>  <--------------------------------------------------------------------->
//
//   fill cmds  transforms  raster clips  path release  rasterize cmds  cohort map  raster release  TTSB  TTSK  cohort atomics  context atomics
//   ---------  ----------  ------------  ------------  --------------  ----------  --------------  ----  ----  --------------  ---------------
//      1,2        1,2           1,2           1,2             2            1-4         1,2,3,4       2-4   2-4       2-4            global
//
//
// NOTES: FINE-GRAINED SVM
// -----------------------
//
//   1) In a fine-grained system we know the exact number of
//      rasterize cmds per segment type before phase 1
//
//   2) A raster that's "under construction" shouldn't be rasterized
//      until it is complete.  This implies that a raster is not part
//      of a cohort until it is complete.  The raster builder must
//      handle raster promises being "forced" to completion -- this is
//      likely the result of composition construction and subsequent
//      rendering to a surface.
//
//   3) The raster cohort rasterizer state retains the fill cmd,
//      transform, raster clip and path release "ring" extents.
//
//   4) The rasterize cmd extent sizes (line, quad, cubic, rational
//      quad, rational cubic) are known ahead of time.
//
//   5) The raster cohort post processor is standalone and retains the
//      raster_map, cohort atomics, TTSK_RYX extent, and raster
//      references until complete.
//

//
// Notes:
//
// - Could have a pipeline stage before expansion count the exact
//   number of line/quad/cubic commands but the command buffers are
//   relatively small (64-bit commands * # of path segments).
//

//                          raster
//                          cohort atomics path_ids raster_ids transforms clips cmds_fill cmds_l/q/c ttsk_ryx
//
//
// BEGIN                      ^
//                            |
//   EXPAND                   |
//                            |
//   RASTERIZE                |
//                            |
//   SORT || RELEASE PATHS    |
//                            |
//   PREFIX                   |
//                            |
//   RELEASE RASTERS          |
//                            |
// END                        v
//
//
// BEGIN
//
//   EXPAND                   -- PRODUCES:   one or more extents of rasterization commands
//
//   RASTERIZE                -- DEPENDENCY: requires size of command extents before launching
//                            -- PRODUCES:   an extent of ttsk_ryx keys
//
//   SORT || RELEASE PATHS    -- DEPENDENCY: requires size of key extent before launching
//                            -- PRODUCES:   sorted array of keys
//
//   PREFIX                   -- DEPENDENCY: none -- can execute after SORT because grid size is number of rasters
//
//   RELEASE RASTERS          -- DEPENDENCY: none -- can execute after prefix
//
// END
//

// ------------------------
//
// DEPENDENCY is cleanly implemented with a host callback or device kernel launcher
//
// Can this hide resource acquisition?  Yes.  But there are two cases:
//
// 1. acqusition of resources occurs on the host thread and lack of
//    resources drains the host command queue until resources are
//    available (OpenCL 2.x)
//
// 2. the host commands lazily acquire resources (OpenCL 1.2)
//
// ------------------------
//
// How to express?
//
// Each substage launches its successors.  This supports both dependency models.
//
// If OpenCL 1.2 then the substage can't be launched until the prior
// stage's event is complete.  So this requires registering a callback
// to invoke the substage.
//
// ------------------------

//
// BUILD
//

struct skc_raster_builder_impl
{
  struct skc_raster_builder    * raster_builder;
  struct skc_runtime           * runtime;

  skc_grid_t                     cohort;

  // these are all durable/perm extents
  struct skc_extent_phrwg_thr1s  path_ids;    // read/write by host
  struct skc_extent_phw1g_tdrNs  transforms;  // write once by host + read by device
  struct skc_extent_phw1g_tdrNs  clips;       // write once by host + read by device
  struct skc_extent_phw1g_tdrNs  fill_cmds;   // write once by host + read by device
  struct skc_extent_phrwg_tdrNs  raster_ids;  // read/write by host + read by device

  struct {
    cl_kernel                    fills_expand;
    cl_kernel                    rasterize_all;
    cl_kernel                    segment;
    cl_kernel                    rasters_alloc;
    cl_kernel                    prefix;
  } kernels;
};

//
// RASTER COHORT
//
// This sub-pipeline snapshots the raster builder and then acquires
// and releases host and device resources as necessary (as late as
// possible).
//
// Note that the cohort extents are ephemeral and are only used by one
// or more stages of a the rasterization sub-pipeline.
//
// The pipeline implementation may vary between compute platforms.
//

struct skc_raster_cohort
{
  struct skc_raster_builder_impl    * impl;

  struct skc_extent_phrwg_thr1s_snap  path_ids;    // read/write by host
  struct skc_extent_phw1g_tdrNs_snap  transforms;  // write once by host + read by device
  struct skc_extent_phw1g_tdrNs_snap  clips;       // write once by host + read by device
  struct skc_extent_phw1g_tdrNs_snap  fill_cmds;   // write once by host + read by device
  struct skc_extent_phrwg_tdrNs_snap  raster_ids;  // read/write by host + read by device

  cl_command_queue                    cq;

  // sub-pipeline atomics
  struct skc_extent_thr_tdrw          atomics;

  // path primitives are expanded into line/quad/cubic/rational cmds
  struct skc_extent_tdrw              cmds;

  // rasterization output
  struct skc_extent_tdrw              keys;
  // struct skc_extent_thrw_tdrw      keys;

  // post-sort extent with metadata for each raster
  struct skc_extent_tdrw              metas;
  // struct skc_extent_thrw_tdrw      metas;

  // subbuf id
  skc_subbuf_id_t                     id;

  //
  // pipeline also uses the following global resources:
  //
  // - command queue from global factory
  // - global block pool and its atomics
  // - global path and raster host id map
  // - temporary host and device allocations
  //
};

//
// TTRK (64-BIT COMPARE)
//
//    0                                  63
//    | TTSB ID |   X  |   Y  | COHORT ID |
//    +---------+------+------+-----------+
//    |    27   |  12  |  12  |     13    |
//
//
// TTRK (32-BIT COMPARE)
//
//    0                                        63
//    | TTSB ID | N/A |   X  |   Y  | COHORT ID |
//    +---------+-----+------+------+-----------+
//    |    27   |  5  |  12  |  12  |     8     |
//

//
// TTRK is sortable intermediate key format for TTSK
//
// We're going to use the 32-bit comparison version for now
//

union skc_ttrk
{
  skc_ulong  u64;
  skc_uint2  u32v2;

  struct {
    skc_uint block    : SKC_TTXK_LO_BITS_ID;
    skc_uint na0      : SKC_TTRK_LO_BITS_NA;
    skc_uint x        : SKC_TTXK_HI_BITS_X;
    skc_uint y        : SKC_TTXK_HI_BITS_Y;
    skc_uint cohort   : SKC_TTRK_HI_BITS_COHORT;
  };

  struct {
    skc_uint na1;
    skc_uint yx       : SKC_TTXK_HI_BITS_YX;
    skc_uint na2      : SKC_TTRK_HI_BITS_COHORT;
  };

  struct {
    skc_uint na3;
    skc_uint na4      : SKC_TTXK_HI_BITS_X;
    skc_uint cohort_y : SKC_TTRK_HI_BITS_COHORT_Y;
  };
};

//
//
//

static
void
skc_raster_builder_pfn_release(struct skc_raster_builder_impl * const impl)
{
  // decrement reference count
  if (--impl->raster_builder->refcount != 0)
    return;

  //
  // otherwise, dispose of the the raster builder and its impl
  //
  struct skc_runtime * const runtime = impl->runtime;

  // free the raster builder
  skc_runtime_host_perm_free(runtime,impl->raster_builder);

  // free durable/perm extents
  skc_extent_phrwg_thr1s_free(runtime,&impl->path_ids);
  skc_extent_phw1g_tdrNs_free(runtime,&impl->transforms);
  skc_extent_phw1g_tdrNs_free(runtime,&impl->clips);
  skc_extent_phw1g_tdrNs_free(runtime,&impl->fill_cmds);
  skc_extent_phrwg_tdrNs_free(runtime,&impl->raster_ids);

  // release kernels
  cl(ReleaseKernel(impl->kernels.fills_expand));
  cl(ReleaseKernel(impl->kernels.rasterize_all));

#if 0
  cl(ReleaseKernel(impl->kernels.rasterize_lines));
  cl(ReleaseKernel(impl->kernels.rasterize_quads));
  cl(ReleaseKernel(impl->kernels.rasterize_cubics));
#endif

  cl(ReleaseKernel(impl->kernels.segment));
  cl(ReleaseKernel(impl->kernels.rasters_alloc));
  cl(ReleaseKernel(impl->kernels.prefix));

  // free the impl
  skc_runtime_host_perm_free(runtime,impl);
}

//
//
//

static
void
skc_raster_builder_rasters_release(struct skc_runtime * const runtime,
                                   skc_raster_t const * const rasters,
                                   skc_uint             const size,
                                   skc_uint             const from,
                                   skc_uint             const to)
{
  if (from <= to) // no wrap
    {
      skc_raster_t const * rasters_from = rasters + from;
      skc_uint             count_from   = to      - from;

      skc_grid_deps_unmap(runtime->deps,rasters_from,count_from);
      skc_runtime_raster_device_release(runtime,rasters_from,count_from);
    }
  else // from > to implies wrap
    {
      skc_raster_t const * rasters_lo = rasters + from;
      skc_uint             count_lo   = size    - from;

      skc_grid_deps_unmap(runtime->deps,rasters_lo,count_lo);
      skc_runtime_raster_device_release(runtime,rasters_lo,count_lo);

      skc_grid_deps_unmap(runtime->deps,rasters,to);
      skc_runtime_raster_device_release(runtime,rasters,to);
    }
}

static
void
skc_raster_builder_paths_release(struct skc_runtime                 * const runtime,
                                 struct skc_extent_phrwg_thr1s_snap * const snap)
{
  // release lo
  skc_runtime_path_device_release(runtime,snap->hr1.lo,snap->count.lo);

  // release hi
  if (snap->count.hi)
    skc_runtime_path_device_release(runtime,snap->hr1.hi,snap->count.hi);
}

static
void
skc_raster_builder_cohort_grid_pfn_dispose(skc_grid_t const grid)
{
  //
  // ALLOCATED RESOURCES
  //
  // path_ids          -
  // raster_ids        a
  // transforms        -
  // clips             -
  // fill_cmds         -
  // cq                a
  // cohort atomics    a
  // cmds              -
  // keys              a
  // meta              a
  //

  struct skc_raster_cohort       * const cohort  = skc_grid_get_data(grid);
  struct skc_raster_builder_impl * const impl    = cohort->impl;
  struct skc_runtime             * const runtime = impl->runtime;

  //
  // release paths -- FIXME -- Note that releasing paths can be
  // performed after rasterization is complete
  //

  // snap alloc the paths -- this host snap simply sets up pointers
  skc_extent_phrwg_thr1s_snap_alloc(runtime,&impl->path_ids,&cohort->path_ids);

  // unmap and release raster ids
  skc_raster_builder_paths_release(runtime,&cohort->path_ids);

  // release path ids
  skc_extent_phrwg_thr1s_snap_free(runtime,&cohort->path_ids);

  //
  // release rasters
  //
  skc_uint const size = cohort->raster_ids.snap->ring->size.pow2;
  skc_uint const from = skc_extent_ring_snap_from(cohort->raster_ids.snap);
  skc_uint const to   = skc_extent_ring_snap_to(cohort->raster_ids.snap);

  // unmap and release raster ids
  skc_raster_builder_rasters_release(runtime,impl->raster_ids.hrw,size,from,to);

  // release cohort's remaining allocated resources
  skc_extent_phrwg_tdrNs_snap_free(runtime,&cohort->raster_ids);
  skc_runtime_release_cq_in_order(runtime,cohort->cq);
  skc_extent_thr_tdrw_free(runtime,&cohort->atomics);
  skc_extent_tdrw_free(runtime,&cohort->keys);
  skc_extent_tdrw_free(runtime,&cohort->metas);
  // skc_extent_thrw_tdrw_free(runtime,&cohort->keys);
  // skc_extent_thrw_tdrw_free(runtime,&cohort->metas);
  skc_runtime_host_temp_free(runtime,cohort,cohort->id);

  // release the raster builder
  skc_raster_builder_pfn_release(impl);

  //
  // ALLOCATED RESOURCES
  //
  // path_ids          -
  // raster_ids        -
  // transforms        -
  // clips             -
  // fill_cmds         -
  // cq                -
  // cohort atomics    -
  // cmds              -
  // keys              -
  // meta              -
  //
}

//
//
//

static
void
skc_raster_cohort_prefix_release(skc_grid_t const grid)
{
  // FIXME -- note that pfn_dispose can be accomplished here

  // release the grid
  skc_grid_complete(grid);
}

static
void
skc_raster_cohort_prefix_cb(cl_event event, cl_int status, skc_grid_t const grid)
{
  SKC_CL_CB(status);

  struct skc_raster_cohort * const cohort    = skc_grid_get_data(grid);
  struct skc_scheduler     * const scheduler = cohort->impl->runtime->scheduler;

  // as quickly as possible, enqueue next stage in pipeline to context command scheduler
  SKC_SCHEDULER_SCHEDULE(scheduler,skc_raster_cohort_prefix_release,grid);
}

//
//
//

#if 0
static
int cmp64(const void * ptr_a, const void * ptr_b)
{
  skc_ulong const a = *(const skc_ulong *)ptr_a;
  skc_ulong const b = *(const skc_ulong *)ptr_b;

  if (a < b) return -1;
  if (a > b) return +1;
  else       return  0;
}
#endif

//
//
//

static
void
skc_raster_cohort_sort_prefix(skc_grid_t const grid)
{
  //
  // ALLOCATED RESOURCES
  //
  // path_ids          i
  // raster_ids        i
  // transforms        a
  // clips             a
  // fill_cmds         -
  // cq                a
  // cohort atomics    a
  // cmds              a
  // keys              a
  // meta              -
  //

  // use the backpointers
  struct skc_raster_cohort       * const cohort  = skc_grid_get_data(grid);
  struct skc_raster_builder_impl * const impl    = cohort->impl;
  struct skc_runtime             * const runtime = impl->runtime;

  // release transforms
  skc_extent_phw1g_tdrNs_snap_free(runtime,&cohort->transforms);

  // release clips
  skc_extent_phw1g_tdrNs_snap_free(runtime,&cohort->clips);

  // release expanded cmds
  skc_extent_tdrw_free(runtime,&cohort->cmds);

  // alloc the snapshost -- could be zero-sized
  skc_extent_phrwg_tdrNs_snap_alloc(runtime,
                                    &impl->raster_ids,
                                    &cohort->raster_ids,
                                    cohort->cq,NULL);

  // will never be zero
  skc_uint const rasters = skc_extent_ring_snap_count(cohort->raster_ids.snap);

  // acquire fixed-size device-side extent
  skc_extent_tdrw_alloc(runtime,
                        &cohort->metas,
                        sizeof(struct skc_raster_cohort_meta));

  // skc_extent_thrw_tdrw_alloc(runtime,
  //                            &cohort->metas,
  //                            sizeof(struct skc_raster_cohort_meta));

  // zero the metas
  skc_extent_tdrw_zero(&cohort->metas,cohort->cq,NULL);

  // get the read-only host copy of the device atomics
  struct skc_raster_cohort_atomic const * const atomics = cohort->atomics.hr;

  //
  // SORT
  //
  if (atomics->keys > 0)
    {
#ifndef NDEBUG
      fprintf(stderr,"raster cohort sort: %u\n",atomics->keys);
#endif

      //
      //
      //
      uint32_t keys_padded_in, keys_padded_out;

      hs_cl_pad(runtime->hs,atomics->keys,&keys_padded_in,&keys_padded_out);

      hs_cl_sort(runtime->hs,
                 cohort->cq,
                 0,NULL,NULL,
                 cohort->keys.drw,
                 NULL,
                 atomics->keys,
                 keys_padded_in,
                 keys_padded_out,
                 false);

      cl(SetKernelArg(impl->kernels.segment,0,SKC_CL_ARG(cohort->keys.drw)));
      cl(SetKernelArg(impl->kernels.segment,1,SKC_CL_ARG(cohort->metas.drw)));

#ifndef NDEBUG
      fprintf(stderr,"post-sort\n");
#endif

      // find start of each tile
      skc_device_enqueue_kernel(runtime->device,
                                SKC_DEVICE_KERNEL_ID_SEGMENT_TTRK,
                                cohort->cq,
                                impl->kernels.segment,
                                atomics->keys,
                                0,NULL,NULL);

#ifndef NDEBUG
      fprintf(stderr,"post-segment\n");
#endif

      //
      // DELETE ALL THIS WHEN READY
      //

#if 0
      //
      //
      //
      cl(Finish(cohort->cq));

      // map keys to host
      union skc_ttrk * const keys = skc_extent_thrw_tdrw_map(&cohort->keys,
                                                             cohort->cq,
                                                             NULL);
      // map meta to host
      struct skc_raster_cohort_meta * const metas = skc_extent_thrw_tdrw_map(&cohort->metas,
                                                                             cohort->cq,
                                                                             NULL);
      // block until done
      cl(Finish(cohort->cq));

      // sort keys
      qsort(keys,atomics->keys,sizeof(*keys),cmp64);

      // mask to determine if rk id is a new block
      skc_uint const subblock_mask = runtime->config->block.subblocks - 1;

      //
      // some counters
      //
      union skc_raster_cohort_meta_in meta_in = {
        .blocks = 0,
        .offset = 0,
        .pk     = 0,
        .rk     = 0
      };

      // get first key
      union skc_ttrk curr = keys[0];

      skc_uint ii=0, jj=0;

      // for all TTRK keys
      while (true)
        {
          // increment ttrk count
          meta_in.rk += 1;

          // was this a new block?
          if ((curr.u32v2.lo & subblock_mask) == 0)
            meta_in.blocks += 1;

          // break if we're out of keys
          if (++ii >= atomics->keys)
            break;

          // otherwise, process next key
          union skc_ttrk const next = keys[ii];

          // if new cohort then save curr meta and init next meta
          if (next.cohort != curr.cohort)
            {
              fprintf(stderr,"[ %u, %u, %u, %u ]\n",
                      meta_in.blocks,
                      meta_in.offset,
                      meta_in.pk,
                      meta_in.rk);

              // store back to buffer
              metas->inout[curr.cohort].in = meta_in;

              // update meta_in
              meta_in.blocks = 0;
              meta_in.offset = ii;
              meta_in.pk     = 0;
              meta_in.rk     = 0;
            }
          // otherwise, if same y but new x then increment TTPK count
          else if ((next.y == curr.y) && (next.x != curr.x))
            {
              meta_in.pk += 1;

#if 0
              fprintf(stderr,"%3u : %3u : ( %3u, %3u ) -> ( %3u )\n",
                      jj++,curr.cohort,curr.y,curr.x,next.x);
#endif
            }

#if 0
          fprintf(stderr,"( %3u, %3u )\n",next.y,next.x);
#endif

          curr = next;
        }

      fprintf(stderr,"[ %u, %u, %u, %u ]\n",
              meta_in.blocks,
              meta_in.offset,
              meta_in.pk,
              meta_in.rk);

      // store back to buffer
      metas->inout[curr.cohort].in = meta_in;


      // unmap
      skc_extent_thrw_tdrw_unmap(&cohort->keys,
                                 keys,
                                 cohort->cq,
                                 NULL);

      // unmap
      skc_extent_thrw_tdrw_unmap(&cohort->metas,
                                 metas,
                                 cohort->cq,
                                 NULL);
#endif
    }

#ifndef NDEBUG
  fprintf(stderr,"rasters_alloc: %u\n",rasters);
#endif

  //
  // RASTER ALLOC/INIT
  //
  cl(SetKernelArg(impl->kernels.rasters_alloc,0,SKC_CL_ARG(runtime->block_pool.atomics.drw)));
  cl(SetKernelArg(impl->kernels.rasters_alloc,1,SKC_CL_ARG(runtime->block_pool.ids.drw)));
  cl(SetKernelArg(impl->kernels.rasters_alloc,2,SKC_CL_ARG(runtime->block_pool.size->ring_mask)));
  cl(SetKernelArg(impl->kernels.rasters_alloc,3,SKC_CL_ARG(runtime->handle_pool.map.drw)));
  cl(SetKernelArg(impl->kernels.rasters_alloc,4,SKC_CL_ARG(cohort->metas.drw)));
  cl(SetKernelArg(impl->kernels.rasters_alloc,5,SKC_CL_ARG(cohort->raster_ids.drN)));
  cl(SetKernelArg(impl->kernels.rasters_alloc,6,SKC_CL_ARG(rasters)));

  skc_device_enqueue_kernel(runtime->device,
                            SKC_DEVICE_KERNEL_ID_RASTERS_ALLOC,
                            cohort->cq,
                            impl->kernels.rasters_alloc,
                            rasters,
                            0,NULL,NULL);

#ifndef NDEBUG
  fprintf(stderr,"post-alloc\n");
#endif

  //
  // PREFIX
  //
  cl(SetKernelArg(impl->kernels.prefix,0,SKC_CL_ARG(runtime->block_pool.atomics.drw)));
  cl(SetKernelArg(impl->kernels.prefix,1,SKC_CL_ARG(runtime->block_pool.ids.drw)));
  cl(SetKernelArg(impl->kernels.prefix,2,SKC_CL_ARG(runtime->block_pool.blocks.drw)));
  cl(SetKernelArg(impl->kernels.prefix,3,SKC_CL_ARG(runtime->block_pool.size->ring_mask)));

  cl(SetKernelArg(impl->kernels.prefix,4,SKC_CL_ARG(cohort->keys.drw)));
  cl(SetKernelArg(impl->kernels.prefix,5,SKC_CL_ARG(runtime->handle_pool.map.drw)));

  cl(SetKernelArg(impl->kernels.prefix,6,SKC_CL_ARG(cohort->metas.drw)));
  cl(SetKernelArg(impl->kernels.prefix,7,SKC_CL_ARG(rasters)));

  cl_event complete;

  skc_device_enqueue_kernel(runtime->device,
                            SKC_DEVICE_KERNEL_ID_PREFIX,
                            cohort->cq,
                            impl->kernels.prefix,
                            rasters,
                            0,NULL,
                            &complete);

  cl(SetEventCallback(complete,CL_COMPLETE,skc_raster_cohort_prefix_cb,grid));
  cl(ReleaseEvent(complete));

#ifndef NDEBUG
  fprintf(stderr,"post-prefix\n");
#endif

  // flush command queue
  cl(Flush(cohort->cq));

  //
  // ALLOCATED RESOURCES
  //
  // path_ids          a
  // raster_ids        a
  // transforms        -
  // clips             -
  // fill_cmds         -
  // cq                a
  // cohort atomics    a
  // cmds              -
  // keys              a
  // meta              a
  //
}

static
void
skc_raster_cohort_rasterize_cb(cl_event event, cl_int status, skc_grid_t const grid)
{
  SKC_CL_CB(status);

  struct skc_raster_cohort * const cohort = skc_grid_get_data(grid);

  // as quickly as possible, enqueue next stage in pipeline to context command scheduler
  SKC_SCHEDULER_SCHEDULE(cohort->impl->runtime->scheduler,skc_raster_cohort_sort_prefix,grid);
}

static
void
skc_raster_cohort_rasterize(skc_grid_t const grid)
{
  //
  // ALLOCATED RESOURCES
  //
  // path_ids          i
  // raster_ids        i
  // transforms        i
  // clips             i
  // fill_cmds         s
  // cq                a
  // cohort atomics    a
  // cmds              a
  // cmds_quad         a
  // cmds_cubic        a
  // keys              -
  // meta              -

  // use the backpointers
  struct skc_raster_cohort       * const cohort  = skc_grid_get_data(grid);
  struct skc_raster_builder_impl * const impl    = cohort->impl;
  struct skc_runtime             * const runtime = impl->runtime;

  //
  // RELEASED RESOURCES
  //
  // cmds       snap
  //

  // release the cmds extent and snap since it's only used by the expand stage
  skc_extent_phw1g_tdrNs_snap_free(runtime,&cohort->fill_cmds);

  //
  // NEW ALLOCATED RESOURCES
  //
  // transforms snap
  // clips snap
  // ttrk keys
  //
  skc_extent_phw1g_tdrNs_snap_alloc(runtime,
                                    &impl->transforms,
                                    &cohort->transforms,
                                    cohort->cq,NULL);

  skc_extent_phw1g_tdrNs_snap_alloc(runtime,
                                    &impl->clips,
                                    &cohort->clips,
                                    cohort->cq,NULL);

  // acquire device-side extent
  skc_extent_tdrw_alloc(runtime,
                        &cohort->keys,
                        sizeof(union skc_ttrk) * runtime->config->raster_cohort.rasterize.keys);

  // skc_extent_thrw_tdrw_alloc(runtime,
  //                            &cohort->keys,
  //                            sizeof(union skc_ttrk) * runtime->config->raster_cohort.rasterize.keys);

  //
  // acquire out-of-order command queue
  //
  // and launch up to 3 kernels
  //
  // for each kernel:
  //
  //   set runtime "global" kernel args:
  //
  //   - block pool atomics
  //   - block pool extent
  //
  //   set cohort "local" kernel args:
  //
  //   - atomics
  //   - cmds
  //
  // enqueue barrier
  // enqueue copy back of atomics on the command queue
  // set callback on copy back event
  // release command queue
  //
  struct skc_raster_cohort_atomic const * const atomics = cohort->atomics.hr;

  if (atomics->cmds > 0)
    {
      cl(SetKernelArg(impl->kernels.rasterize_all,0,SKC_CL_ARG(runtime->block_pool.atomics.drw)));
      cl(SetKernelArg(impl->kernels.rasterize_all,1,SKC_CL_ARG(runtime->block_pool.blocks.drw)));
      cl(SetKernelArg(impl->kernels.rasterize_all,2,SKC_CL_ARG(runtime->block_pool.ids.drw)));
      cl(SetKernelArg(impl->kernels.rasterize_all,3,SKC_CL_ARG(runtime->block_pool.size->ring_mask)));

      cl(SetKernelArg(impl->kernels.rasterize_all,4,SKC_CL_ARG(cohort->atomics.drw)));
      cl(SetKernelArg(impl->kernels.rasterize_all,5,SKC_CL_ARG(cohort->keys.drw)));

      cl(SetKernelArg(impl->kernels.rasterize_all,6,SKC_CL_ARG(cohort->transforms.drN)));
      cl(SetKernelArg(impl->kernels.rasterize_all,7,SKC_CL_ARG(cohort->clips.drN)));
      cl(SetKernelArg(impl->kernels.rasterize_all,8,SKC_CL_ARG(cohort->cmds.drw)));
      cl(SetKernelArg(impl->kernels.rasterize_all,9,SKC_CL_ARG(atomics->cmds)));

      skc_device_enqueue_kernel(runtime->device,
                                SKC_DEVICE_KERNEL_ID_RASTERIZE_ALL,
                                cohort->cq,
                                impl->kernels.rasterize_all,
                                atomics->cmds,
                                0,NULL,NULL);
    }

  //
  // copyback number of TTSK keys
  //
  cl_event complete;

  skc_extent_thr_tdrw_read(&cohort->atomics,cohort->cq,&complete);

  cl(SetEventCallback(complete,CL_COMPLETE,skc_raster_cohort_rasterize_cb,grid));
  cl(ReleaseEvent(complete));

  // flush command queue
  cl(Flush(cohort->cq));

  //
  // ALLOCATED RESOURCES
  //
  // path_ids          i
  // raster_ids        i
  // transforms        a
  // clips             a
  // fill_cmds         -
  // cq                a
  // cohort atomics    a
  // cmds              a
  // keys              a
  // meta              -
}

static
void
skc_raster_cohort_fills_expand_cb(cl_event event, cl_int status, skc_grid_t const grid)
{
  SKC_CL_CB(status);

  struct skc_raster_cohort * const cohort = skc_grid_get_data(grid);

  // as quickly as possible, enqueue next stage in pipeline to context command scheduler
  SKC_SCHEDULER_SCHEDULE(cohort->impl->runtime->scheduler,skc_raster_cohort_rasterize,grid);
}

static
void
skc_raster_builder_cohort_grid_pfn_execute(skc_grid_t const grid)
{
  //
  // ALLOCATED RESOURCES
  //
  // path_ids          i
  // raster_ids        i
  // transforms        i
  // clips             i
  // fill_cmds         i
  // cq                -
  // cohort atomics    -
  // cmds              -
  // keys              -
  // meta              -
  //

  // allocate the cohort
  struct skc_raster_cohort       * const cohort  = skc_grid_get_data(grid);

  // get impl
  struct skc_raster_builder_impl * const impl    = cohort->impl;
  struct skc_runtime             * const runtime = impl->runtime;

  // acquire in-order cq
  cohort->cq = skc_runtime_acquire_cq_in_order(runtime);

  // alloc the snapshot -- could be zero-sized
  skc_extent_phw1g_tdrNs_snap_alloc(runtime,
                                    &impl->fill_cmds,
                                    &cohort->fill_cmds,
                                    cohort->cq,NULL);

  // flush the cq to get the fill running
  // cl(Flush(cohort->cq));

  // create split atomics
  skc_extent_thr_tdrw_alloc(runtime,&cohort->atomics,sizeof(struct skc_raster_cohort_atomic));

  // zero the atomics
  skc_extent_thr_tdrw_zero(&cohort->atomics,cohort->cq,NULL);

  // get config
  struct skc_config const * const config = runtime->config;

  // acquire device-side extents
  skc_extent_tdrw_alloc(runtime,
                        &cohort->cmds,
                        sizeof(union skc_cmd_rasterize) * config->raster_cohort.expand.cmds);

  //
  // FILLS EXPAND
  //
  // need result of cmd counts before launching RASTERIZE grids
  //
  // - OpenCL 1.2: copy atomic counters back to host and launch RASTERIZE grids from host
  // - OpenCL 2.x: have a kernel size and launch RASTERIZE grids from device
  // - or launch a device-wide grid that feeds itself but that's unsatisfying
  //

  // how many commands?  could be zero
  skc_uint const work_size = skc_extent_ring_snap_count(cohort->fill_cmds.snap);

  if (work_size > 0)
    {
      cl(SetKernelArg(impl->kernels.fills_expand,0,SKC_CL_ARG(impl->runtime->block_pool.blocks.drw)));
      cl(SetKernelArg(impl->kernels.fills_expand,1,SKC_CL_ARG(cohort->atomics.drw)));
      cl(SetKernelArg(impl->kernels.fills_expand,2,SKC_CL_ARG(runtime->handle_pool.map.drw)));
      cl(SetKernelArg(impl->kernels.fills_expand,3,SKC_CL_ARG(cohort->fill_cmds.drN)));
      cl(SetKernelArg(impl->kernels.fills_expand,4,SKC_CL_ARG(cohort->cmds.drw)));

      skc_device_enqueue_kernel(runtime->device,
                                SKC_DEVICE_KERNEL_ID_FILLS_EXPAND,
                                cohort->cq,
                                impl->kernels.fills_expand,
                                work_size,
                                0,NULL,NULL);
    }

  //
  // copyback number of rasterization commands
  //
  cl_event complete;

  skc_extent_thr_tdrw_read(&cohort->atomics,cohort->cq,&complete);

  cl(SetEventCallback(complete,CL_COMPLETE,skc_raster_cohort_fills_expand_cb,grid));
  cl(ReleaseEvent(complete));

  // flush command queue
  cl(Flush(cohort->cq));

  //
  // ALLOCATED RESOURCES
  //
  // path_ids          i
  // raster_ids        i
  // transforms        i
  // clips             i
  // fill_cmds         s
  // cq                a
  // cohort atomics    a
  // cmds              a
  // keys              -
  // meta              -
  //
}

//
// move grid into waiting state
//
// this entails allocating a cohort from the temporary extent
//

static
void
skc_raster_builder_cohort_grid_pfn_waiting(skc_grid_t const grid)
{
  // get the impl
  struct skc_raster_builder_impl * const impl    = skc_grid_get_data(grid);
  struct skc_runtime             * const runtime = impl->runtime;

  // retain the raster builder
  impl->raster_builder->refcount += 1;

  // allocate the ephemeral/temp cohort
  skc_subbuf_id_t id;

  struct skc_raster_cohort * const cohort =
    skc_runtime_host_temp_alloc(runtime,
                                SKC_MEM_FLAGS_READ_WRITE,
                                sizeof(*cohort),
                                &id,
                                NULL);

  // save the id and backpointer
  cohort->id   = id;
  cohort->impl = impl;

  // set grid data -- replaces impl
  skc_grid_set_data(grid,cohort);

  //
  // ACQUIRE RESOURCES FOR THE COHORT
  //

  struct skc_raster_builder * const raster_builder = impl->raster_builder;

  // immediately take snapshots of all rings -- these are very inexpensive operations
  skc_extent_phrwg_thr1s_snap_init(runtime,&raster_builder->path_ids  .ring,&cohort->path_ids);
  skc_extent_phw1g_tdrNs_snap_init(runtime,&raster_builder->transforms.ring,&cohort->transforms);
  skc_extent_phw1g_tdrNs_snap_init(runtime,&raster_builder->clips     .ring,&cohort->clips);
  skc_extent_phw1g_tdrNs_snap_init(runtime,&raster_builder->fill_cmds .ring,&cohort->fill_cmds);
  skc_extent_phrwg_tdrNs_snap_init(runtime,&raster_builder->raster_ids.ring,&cohort->raster_ids);

  //
  // ALLOCATED RESOURCES
  //
  // path_ids          i
  // raster_ids        i
  // transforms        i
  // clips             i
  // fill_cmds         i
  // cq                -
  // cohort atomics    -
  // cmds              -
  // keys              -
  // meta              -
  //
}

//
//
//

static
void
skc_raster_builder_cohort_create(struct skc_raster_builder_impl * const impl)
{
  // attach a grid
  impl->cohort = SKC_GRID_DEPS_ATTACH(impl->runtime->deps,
                                      &impl->cohort,
                                      impl,
                                      skc_raster_builder_cohort_grid_pfn_waiting,
                                      skc_raster_builder_cohort_grid_pfn_execute,
                                      skc_raster_builder_cohort_grid_pfn_dispose);
}

//
//
//

static
skc_err
skc_raster_builder_pfn_add(struct skc_raster_builder_impl * const impl,
                           skc_path_t               const *       paths,
                           skc_uint                               count)
{
  // validate and retain the path
  skc_err err;

  err = skc_runtime_handle_device_validate_retain(impl->runtime,
                                                  SKC_TYPED_HANDLE_TYPE_IS_PATH,
                                                  paths,
                                                  count);

  if (err)
    return err;

  skc_runtime_handle_device_retain(impl->runtime,paths,count);

  // make sure there is a grid
  if (impl->cohort == NULL) {
    skc_raster_builder_cohort_create(impl);
  }

  // declare rasterization grid happens after path
  while (count-- > 0)
    skc_grid_happens_after_handle(impl->cohort,SKC_TYPED_HANDLE_TO_HANDLE(*paths++));

  return SKC_ERR_SUCCESS;
}

//
//
//

static
void
skc_raster_builder_pfn_end(struct skc_raster_builder_impl * const impl, skc_raster_t * const raster)
{
  //
  // acquire host-managed path raster handle and bump reference count
  // to 2 handles will be released (reduced to 1) once the rasters are
  // completely rasterized
  //
  *raster = skc_runtime_handle_device_acquire(impl->runtime);

  // make sure there is a grid
  if (impl->cohort == NULL) {
    skc_raster_builder_cohort_create(impl);
  }

  // map a handle to a grid
  skc_grid_map(impl->cohort,*raster);
}

//
// snapshot the ring and lazily start the grid
//
// FIXME -- might want to revisit this and settle on an even more
// opaque implementation.  Some options:
//
//  - never let the SKC API expose a forced grid start
//  - make snapshots kick off a forced grid start
//  - be lazy all the time everywhere
//

static
void
skc_raster_builder_pfn_start(struct skc_raster_builder_impl * const impl)
{
  skc_grid_t const cohort = impl->cohort;

  if (cohort != NULL) {
    skc_grid_start(cohort);
  }
}

//
// NOTE: THIS MIGHT BE REMOVED
//

static
void
skc_raster_builder_pfn_force(struct skc_raster_builder_impl * const impl)
{
  skc_grid_t const cohort = impl->cohort;

  if (cohort != NULL) {
    skc_grid_force(cohort);
  }
}

//
//
//

skc_err
skc_raster_builder_cl_12_create(struct skc_context          * const context,
                                struct skc_raster_builder * * const raster_builder)
{
  struct skc_runtime * const runtime = context->runtime;

  // allocate raster builder
  (*raster_builder) = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,sizeof(**raster_builder));

  // refcount
  (*raster_builder)->refcount = 1;

  // state
  SKC_ASSERT_STATE_INIT((*raster_builder),SKC_RASTER_BUILDER_STATE_READY);

  // allocate runtime raster builder
  struct skc_raster_builder_impl * const impl = skc_runtime_host_perm_alloc(runtime,SKC_MEM_FLAGS_READ_WRITE,sizeof(*impl));

  // save the impl
  (*raster_builder)->impl = impl;

  // intialize impl
  impl->raster_builder = (*raster_builder);
  impl->runtime        = runtime;
  impl->cohort         = NULL;

  // get config
  struct skc_config const * const config = runtime->config;

  skc_extent_phrwg_thr1s_alloc(runtime,&impl->path_ids  ,sizeof(skc_path_t         ) * config->raster_cohort.path_ids  .elem_count);
  skc_extent_phw1g_tdrNs_alloc(runtime,&impl->transforms,sizeof(union skc_transform) * config->raster_cohort.transforms.elem_count);
  skc_extent_phw1g_tdrNs_alloc(runtime,&impl->clips     ,sizeof(union skc_path_clip) * config->raster_cohort.clips     .elem_count);
  skc_extent_phw1g_tdrNs_alloc(runtime,&impl->fill_cmds ,sizeof(union skc_cmd_fill ) * config->raster_cohort.fill      .elem_count);
  skc_extent_phrwg_tdrNs_alloc(runtime,&impl->raster_ids,sizeof(skc_raster_t       ) * config->raster_cohort.raster_ids.elem_count);

  // retain the context
  //skc_context_retain(context);

  (*raster_builder)->context = context;

  (*raster_builder)->add     = skc_raster_builder_pfn_add;
  (*raster_builder)->end     = skc_raster_builder_pfn_end;
  (*raster_builder)->start   = skc_raster_builder_pfn_start;
  (*raster_builder)->force   = skc_raster_builder_pfn_force;
  (*raster_builder)->release = skc_raster_builder_pfn_release;

  // initialize raster builder with host-writable buffers
  (*raster_builder)->path_ids  .extent = impl->path_ids.hrw;
  (*raster_builder)->transforms.extent = impl->transforms.hw1;
  (*raster_builder)->clips     .extent = impl->clips.hw1;
  (*raster_builder)->fill_cmds .extent = impl->fill_cmds.hw1;
  (*raster_builder)->raster_ids.extent = impl->raster_ids.hrw;

  //
  // the rings perform bookkeeping on the extents
  //
  // the ring snapshotting and checkpointing are necessary because
  // another part of the API can _force_ the raster cohort to flush
  // its work-in-progress commands but only up to a checkpointed
  // boundary
  //
  skc_extent_ring_init(&(*raster_builder)->path_ids.ring,
                       config->raster_cohort.path_ids.elem_count,
                       config->raster_cohort.path_ids.snap_count,
                       sizeof(skc_path_t));

  skc_extent_ring_init(&(*raster_builder)->transforms.ring,
                       config->raster_cohort.transforms.elem_count,
                       config->raster_cohort.transforms.snap_count,
                       sizeof(union skc_transform));

  skc_extent_ring_init(&(*raster_builder)->clips.ring,
                       config->raster_cohort.clips.elem_count,
                       config->raster_cohort.clips.snap_count,
                       sizeof(union skc_path_clip));

  skc_extent_ring_init(&(*raster_builder)->fill_cmds.ring,
                       config->raster_cohort.fill.elem_count,
                       config->raster_cohort.fill.snap_count,
                       sizeof(union skc_cmd_fill));

  skc_extent_ring_init(&(*raster_builder)->raster_ids.ring,
                       config->raster_cohort.raster_ids.elem_count,
                       config->raster_cohort.raster_ids.snap_count,
                       sizeof(skc_raster_t));

  //
  // acquire kernels
  //
  impl->kernels.fills_expand     = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_FILLS_EXPAND);
  impl->kernels.rasterize_all    = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_RASTERIZE_ALL);

#if 0
  impl->kernels.rasterize_lines  = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_RASTERIZE_LINES);
  impl->kernels.rasterize_quads  = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_RASTERIZE_QUADS);
  impl->kernels.rasterize_cubics = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_RASTERIZE_CUBICS);
#endif

  impl->kernels.segment          = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_SEGMENT_TTRK);
  impl->kernels.rasters_alloc    = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_RASTERS_ALLOC);
  impl->kernels.prefix           = skc_device_acquire_kernel(runtime->device, SKC_DEVICE_KERNEL_ID_PREFIX);

  return SKC_ERR_SUCCESS;
}

//
//
//