android-13.0.0_r83/s

/*
 * Copyright (c) 2018-2019 Arm Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

/*
 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 *
 *          NOTE: Header to be included by implementation files only.
 *
 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 */

#include <algorithm>
#include <cstdint>
#include "depthwise.hpp"
#include "padding.hpp"
#include "utils.hpp"

#pragma once

#define MEMBERFN(TOUT) template <\
  unsigned int OutputTileRows, unsigned int OutputTileColumns,\
  unsigned int KernelRows, unsigned int KernelColumns,\
  unsigned int StrideRows, unsigned int StrideColumns,\
  typename TIn, typename TBias, typename TOut,\
  typename Derived\
> TOUT DepthwiseConvolutionBase<\
  OutputTileRows, OutputTileColumns,\
  KernelRows, KernelColumns,\
  StrideRows, StrideColumns,\
  TIn, TBias, TOut, Derived\
>

using namespace neon_convolution_kernels;

namespace depthwise
{

template <unsigned int KernelRows, unsigned int KernelColumns, size_t WeightSize, size_t BiasSize>
struct PackParameters
{
  static void execute(
    unsigned int n_channels,
    void *buffer,
    const void *weights,
    unsigned int weight_row_stride,
    unsigned int weight_col_stride,
    const void *biases
  );
};

const unsigned int CHANNEL_BLOCK = 16;

MEMBERFN(int)::get_output_size(
  const int dim_size, const unsigned int padding_before, const unsigned int padding_after
)
{
  return iceildiv(dim_size + padding_before + padding_after - KernelRows + 1, StrideRows);
}

MEMBERFN(int)::output_size(
  const int dim_size, const unsigned int padding_before, const unsigned int padding_after
) const
{
  return get_output_size(dim_size, padding_before, padding_after);
}

MEMBERFN()::DepthwiseConvolutionBase(
  const int n_batches,
  const int n_input_rows,
  const int n_input_cols,
  const int n_channels,
  ActivationFunction activation,
  const unsigned int padding_top,
  const unsigned int padding_left,
  const unsigned int padding_bottom,
  const unsigned int padding_right
) : DepthwiseConvolutionBase(
      n_batches, n_input_rows, n_input_cols, n_channels,
      get_output_size(n_input_rows, padding_top, padding_bottom),
      get_output_size(n_input_cols, padding_left, padding_right),
      activation,
      padding_top, padding_left, padding_bottom, padding_right
    )
{
}

MEMBERFN()::DepthwiseConvolutionBase(
  const int n_batches,
  const int n_input_rows,
  const int n_input_cols,
  const int n_channels,
  const int n_output_rows,
  const int n_output_cols,
  ActivationFunction activation,
  const unsigned int padding_top,
  const unsigned int padding_left,
  const unsigned int padding_bottom,
  const unsigned int padding_right
) : _input(nullptr), _output(nullptr),
    _packed_parameters(nullptr),
    _working_space(nullptr),
    _n_batches(n_batches),
    _n_input_rows(n_input_rows),
    _n_input_cols(n_input_cols),
    _n_channels(n_channels),
    _n_output_rows(n_output_rows),
    _n_output_cols(n_output_cols),
    _n_tile_rows(iceildiv(_n_output_rows, output_tile_rows)),
    _n_tile_cols(iceildiv(_n_output_cols, output_tile_cols)),
    _padding_top(padding_top),
    _padding_left(padding_left),
    _padding_bottom(padding_bottom),
    _padding_right(padding_right),
    _activation(activation),
    _input_col_stride(0), _input_row_stride(0), _input_batch_stride(0),
    _output_col_stride(0), _output_row_stride(0), _output_batch_stride(0)
{
}

MEMBERFN(void)::set_input(const void* const inptr)
{
  set_input(inptr, _n_channels);
}

MEMBERFN(void)::set_input(const void* const inptr, const int ld_col)
{
  set_input(inptr, _n_input_cols * ld_col, ld_col);
}

MEMBERFN(void)::set_input(const void* const inptr, const int ld_row, const int ld_col)
{
  set_input(inptr, _n_input_rows * ld_row, ld_row, ld_col);
}

MEMBERFN(void)::set_input(const void* const inptr, const int ld_batch, const int ld_row, const int ld_col)
{
  _input = static_cast<const TIn *>(inptr);
  _input_batch_stride = ld_batch;
  _input_row_stride = ld_row;
  _input_col_stride = ld_col;
}

MEMBERFN(void)::set_output(void* const outptr)
{
  set_output(outptr, _n_channels);
}

MEMBERFN(void)::set_output(void* const outptr, const int ld_col)
{
  set_output(outptr, _n_output_cols * ld_col, ld_col);
}

MEMBERFN(void)::set_output(void* const outptr, const int ld_row, const int ld_col)
{
  set_output(outptr, _n_output_rows * ld_row, ld_row, ld_col);
}

MEMBERFN(void)::set_output(void* const outptr, const int ld_batch, const int ld_row, const int ld_col)
{
  _output = static_cast<TOut *>(outptr);
  _output_batch_stride = ld_batch;
  _output_row_stride = ld_row;
  _output_col_stride = ld_col;
}

MEMBERFN(size_t)::get_packed_params_size(void) const
{
  return _n_channels * (sizeof(TIn)*KernelRows*KernelColumns + sizeof(TBias));
}

MEMBERFN(void)::set_packed_params_buffer(void *buffer)
{
  _packed_parameters = buffer;
}

MEMBERFN(void)::pack_params(const void *weights, const void *biases) const
{
  static_cast<const Derived *>(this)->pack_params(_packed_parameters, weights, biases);
}

MEMBERFN(void)::pack_params(void *buffer, const void *weights, const void *biases) const
{
  const unsigned int weight_col_stride = _n_channels;
  const unsigned int weight_row_stride = KernelColumns * weight_col_stride;
  static_cast<const Derived *>(this)->pack_params(
    buffer, weights, weight_row_stride, weight_col_stride, biases
  );
}

MEMBERFN(void)::pack_params(
  void * const buffer,
  const void * const weights,
  const unsigned int weight_row_stride,
  const unsigned int weight_col_stride,
  const void * const biases
) const
{
  static_cast<const Derived *>(this)->_pack_params(
    buffer, weights, weight_row_stride, weight_col_stride, biases
  );
}

MEMBERFN(void)::_pack_params(
  void * const buffer,
  const void * const weights,
  const unsigned int weight_row_stride,
  const unsigned int weight_col_stride,
  const void * const biases
) const
{
  // Default implementation
  PackParameters<KernelRows, KernelColumns, sizeof(TIn), sizeof(TOut)>::execute(
    _n_channels, buffer, weights, weight_row_stride, weight_col_stride, biases
  );
}

MEMBERFN(size_t)::get_working_space_size(const unsigned int nthreads) const
{
  return nthreads * (
    _get_input_working_space_size() + _get_output_working_space_size()
  );
}

MEMBERFN(void)::set_working_space(void *buffer)
{
  _working_space = buffer;
}

MEMBERFN(size_t)::_get_input_working_space_size(void) const
{
  return sizeof(TIn) * _n_channels;
}

MEMBERFN(size_t)::_get_output_working_space_size(void) const
{
  return sizeof(TOut) * _n_channels;
}

MEMBERFN(void *)::_get_input_working_space(const unsigned int threadid) const
{
  return static_cast<uint8_t*>(_working_space) + threadid * (
    _get_input_working_space_size() + _get_output_working_space_size()
  );
}

MEMBERFN(void *)::_get_output_working_space(const unsigned int threadid) const
{
  return static_cast<uint8_t*>(_get_input_working_space(threadid)) + _get_input_working_space_size();
}

MEMBERFN(unsigned int)::get_window() const
{
  // Parallelise over blocks of channels.
  return iceildiv(_n_channels, CHANNEL_BLOCK);
}

MEMBERFN(void)::run(
  const unsigned int start,
  const unsigned int stop,
  const unsigned int threadid
)
{
  // Clear the input padding buffer
  TIn *buf = static_cast<TIn *>(_get_input_working_space(threadid));
  const TIn pad_value = static_cast<Derived *>(this)->_input_padding_value();
  for (int n = 0; n < _n_channels; n++)
  {
    buf[n] = pad_value;
  }

  // Parallelise over blocks of channels
  const auto start_channel = CHANNEL_BLOCK * start;
  const auto stop_channel = std::min<unsigned int>(_n_channels, CHANNEL_BLOCK * stop);
  const auto params_size_per_channel = this->get_packed_params_size()/_n_channels;

  // Compute top and bottom padding for input and output
  const int input_pad_top = _padding_top;
  const int input_pad_left = _padding_left;
  constexpr int tile_overlap = kernel_rows - stride_rows;

  // Perform the convolution by calling `process_tile_row` for each tile row in
  // each batch.
  for (int batch = 0; batch < _n_batches; batch++)
  {
    const TIn* const inptr_batch = _input + batch*_input_batch_stride;
    TOut* const outptr_batch = _output + batch*_output_batch_stride;

    // Loop over rows of tiles
    for (int tile_i = 0; tile_i < _n_tile_rows; tile_i++)
    {
      // Pointer to the row
      const int input_row_offset = (tile_i == 0) ? 0 : input_pad_top;
      const TIn* const inptr_row = (inptr_batch + ((inner_tile_rows - tile_overlap)*tile_i - input_row_offset)*_input_row_stride);
      TOut* const outptr_row = outptr_batch + output_tile_rows * tile_i * _output_row_stride;

      // Input padding (top + bottom) for the row
      const int input_row_top = tile_i*(inner_tile_rows - tile_overlap) - input_pad_top;
      const int input_row_bottom = input_row_top + inner_tile_rows;
      const int input_row_pad_top = (tile_i == 0) ? input_pad_top : 0;
      const int input_row_pad_bottom = std::max(0, input_row_bottom - _n_input_rows);

      // Output padding (bottom) for the row
      const int output_row_bottom = (tile_i + 1)*output_tile_rows;
      const int output_row_pad_bottom = std::max(0, output_row_bottom - _n_output_rows);

      // Get the offset into the packed parameters
      const auto params_ptr = static_cast<const uint8_t*>(_packed_parameters) +
        start_channel*params_size_per_channel;

      // Process the row
      process_tile_row(
        threadid,
        stop_channel - start_channel,
        params_ptr,
        inptr_row + start_channel,
        outptr_row + start_channel,
        input_row_pad_top, input_pad_left, input_row_pad_bottom,
        output_row_pad_bottom,
        _n_tile_cols, _n_input_cols, _n_output_cols
      );
    }
  }
}

MEMBERFN(void)::process_tile_row(
  const unsigned int threadid,
  const int n_channels,
  const void* const packed_params,
  const TIn* const inptr,
  TOut* const outptr,
  const int row_pad_in_top,
  const int row_pad_in_left,
  const int row_pad_in_bottom,
  const int row_pad_out_bottom,
  const int n_tiles,
  const int n_input_cols,
  const int n_output_cols
)
{
  constexpr int tile_overlap = kernel_cols - stride_cols;

  // Loop over columns of tiles
  for (int tile_j = 0; tile_j < n_tiles; tile_j++)
  {
    // Input padding (left + right) for the tile
    const int t_pad_in_left = (tile_j == 0) ? row_pad_in_left : 0;
    const int t_in_start = tile_j*(inner_tile_cols - tile_overlap) - row_pad_in_left;
    const int t_in_end = t_in_start + inner_tile_cols;
    const int t_pad_in_right = std::max(0, t_in_end - n_input_cols);

    // Output padding (right) for the tile
    const int t_out_end = (tile_j + 1) * output_tile_cols;
    const int t_pad_out_right = std::max(0, t_out_end - n_output_cols);

    // Get pointers into the inputs and outputs
    const int col_offset = (tile_j == 0) ? 0 : row_pad_in_left;
    const TIn* const inptr_col = (inptr + ((inner_tile_cols - tile_overlap)*tile_j - col_offset)*_input_col_stride);
    TOut* const outptr_col = outptr + tile_j * output_tile_cols * _output_col_stride;

    // Process just this tile
    process_tile(
      threadid, n_channels, packed_params, inptr_col, outptr_col,
      row_pad_in_top, t_pad_in_left, row_pad_in_bottom, t_pad_in_right,  // Input paddings
      row_pad_out_bottom, t_pad_out_right  // Output paddings
    );
  }
}

MEMBERFN(TIn)::_input_padding_value(void) const
{
  return static_cast<TIn>(0);
}

MEMBERFN(void)::process_tile(
  const unsigned int threadid,
  const int n_channels,
  const void* const packed_params,
  const TIn* const inptr,
  TOut* const outptr,
  const int pad_in_top,
  const int pad_in_left,
  const int pad_in_bottom,
  const int pad_in_right,
  const int pad_out_bottom,
  const int pad_out_right
)
{
  Derived * dthis = static_cast<Derived *>(this);
  const bool pad_input = pad_in_top || pad_in_left || pad_in_bottom || pad_in_right;
  const bool pad_output = pad_out_bottom || pad_out_right;

  if (!pad_input && !pad_output)
  {
    switch(_activation)
    {
      case ActivationFunction::ReLU:
        dthis->template execute_tile<ActivationFunction::ReLU>(
          n_channels, packed_params,
          inptr, _input_row_stride, _input_col_stride,
          outptr, _output_row_stride, _output_col_stride
        );
        break;
      case ActivationFunction::ReLU6:
        dthis->template execute_tile<ActivationFunction::ReLU6>(
          n_channels, packed_params,
          inptr, _input_row_stride, _input_col_stride,
          outptr, _output_row_stride, _output_col_stride
        );
        break;
      default:
        dthis->template execute_tile<ActivationFunction::None>(
          n_channels, packed_params,
          inptr, _input_row_stride, _input_col_stride,
          outptr, _output_row_stride, _output_col_stride
        );
        break;
    }
  }
  else
  {
    // Create arrays of input and output pointers, pointing padded elements to
    // the working space padding buffers provided.
    const TIn *inptrs[inner_tile_rows][inner_tile_cols];
    for (int i = 0; i < inner_tile_rows; i++)
    {
      for (int j = 0; j < inner_tile_cols; j++)
      {
        if (i < pad_in_top || (inner_tile_rows - pad_in_bottom) <= i ||
            j < pad_in_left || (inner_tile_cols - pad_in_right) <= j)
        {
          // Padded input
          inptrs[i][j] = static_cast<const TIn *>(_get_input_working_space(threadid));
        }
        else
        {
          inptrs[i][j] = inptr + (i - pad_in_top)*_input_row_stride + (j - pad_in_left)*_input_col_stride;
        }
      }
    }

    TOut *outptrs[output_tile_rows][output_tile_cols];
    for (int i = 0; i < output_tile_rows; i++)
    {
      for (int j = 0; j < output_tile_cols; j++)
      {
        if (i < (output_tile_rows - pad_out_bottom) &&
            j < (output_tile_cols - pad_out_right))
        {
          outptrs[i][j] = outptr + i*_output_row_stride + j*_output_col_stride;
        }
        else
        {
          outptrs[i][j] = static_cast<TOut *>(_get_output_working_space(threadid));
        }
      }
    }

    switch(_activation)
    {
      case ActivationFunction::ReLU:
        dthis->template execute_tile<ActivationFunction::ReLU>(
          n_channels, packed_params, inptrs, outptrs
        );
        break;
      case ActivationFunction::ReLU6:
        dthis->template execute_tile<ActivationFunction::ReLU6>(
          n_channels, packed_params, inptrs, outptrs
        );
        break;
      default:
        dthis->template execute_tile<ActivationFunction::None>(
          n_channels, packed_params, inptrs, outptrs
        );
        break;
    }
  }
}

MEMBERFN(int)::n_channels(void) const
{
  return _n_channels;
}

}  // namespace depthwise