1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OPERATIONS_H_ 17 #define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OPERATIONS_H_ 18 19 #include <cstdint> 20 #include <set> 21 #include <string> 22 #include <vector> 23 24 #include "absl/types/variant.h" 25 #include "tensorflow/lite/delegates/gpu/common/data_type.h" 26 #include "tensorflow/lite/delegates/gpu/common/shape.h" 27 #include "tensorflow/lite/delegates/gpu/common/status.h" 28 #include "tensorflow/lite/delegates/gpu/common/tensor.h" 29 30 namespace tflite { 31 namespace gpu { 32 33 // Non exhaustive list of operations. 34 enum class OperationType { 35 UNKNOWN = 0, 36 ABS, 37 ADD, 38 BATCH_TO_SPACE, 39 BATCH_NORMALIZATION, 40 BATCHED_MATMUL, 41 CONCAT, 42 CONSTANT, 43 CONVOLUTION_2D, 44 CONVOLUTION_TRANSPOSED, 45 COPY, 46 COS, 47 DENSIFY, 48 DEPTHWISE_CONVOLUTION, 49 DEPTH_TO_SPACE, 50 DIV, 51 ELU, 52 EQUAL, 53 EXP, 54 FLOOR, 55 FLOOR_DIV, 56 FLOOR_MOD, 57 FULLY_CONNECTED, 58 FULLY_CONNECTED_INT8, 59 GATHER, 60 GREATER, 61 GREATER_EQUAL, 62 HARD_SWISH, 63 LESS, 64 LESS_EQUAL, 65 LOG, 66 LSTM, 67 MAXIMUM, 68 MAX_UNPOOLING_2D, 69 MEAN, 70 MEAN_STDDEV_NORMALIZATION, 71 MINIMUM, 72 MUL, 73 NEG, 74 NOT_EQUAL, 75 PAD, 76 POOLING_2D, 77 POW, 78 PRELU, 79 // Used to accurately run inference on quantized models. 80 QUANTIZE_AND_DEQUANTIZE, 81 REDUCE_MAXIMUM, 82 REDUCE_MINIMUM, 83 REDUCE_PRODUCT, 84 REDUCE_SUM, 85 RELU, 86 RESAMPLER, 87 RESHAPE, 88 RESIZE, 89 RSQRT, 90 SIGMOID, 91 SIN, 92 SLICE, 93 SOFTMAX, 94 SPACE_TO_BATCH, 95 SPACE_TO_DEPTH, 96 SPLIT, 97 SQRT, 98 SQUARE, 99 SQUARED_DIFF, 100 SUB, 101 TANH, 102 TILE, 103 TRANSPOSE, 104 }; 105 106 std::string ToString(enum OperationType op); 107 108 OperationType OperationTypeFromString(const std::string& name); 109 110 typedef absl::variant<absl::monostate, Tensor<HWC, DataType::FLOAT32>, 111 Tensor<Linear, DataType::FLOAT32>, float> 112 TensorOrScalar; 113 114 struct Padding2D { 115 Padding2D() = default; 116 Padding2D& operator=(const Padding2D& value); 117 bool operator==(const Padding2D& value); 118 bool operator!=(const Padding2D& value); 119 Padding2D& operator-(const Padding2D& value); 120 121 // Padding values for every axis (if needed), where 'prepended' defines 122 // padding for the beginning of each axis and 'appended' represents end part 123 // of the corresponding axis. 124 HW prepended = HW(-1, -1); 125 HW appended = HW(-1, -1); 126 }; 127 128 struct Padding3D { 129 Padding3D() = default; 130 Padding3D& operator=(const Padding3D& value); 131 bool operator==(const Padding3D& value); 132 bool operator!=(const Padding3D& value); 133 Padding3D& operator-(const Padding3D& value); 134 135 // Padding values for every axis (if needed), where 'prepended' defines 136 // padding for the beginning of each axis and 'appended' represents end part 137 // of the corresponding axis. 138 HWD prepended = HWD(0, 0, 0); 139 HWD appended = HWD(0, 0, 0); 140 }; 141 142 struct Crop2D : public Padding2D {}; 143 144 struct SpaceToBatchAttributes { 145 HW block; 146 Padding2D padding; 147 }; 148 149 struct BatchToSpaceAttributes { 150 HW block; 151 Crop2D crop; 152 }; 153 154 enum class PoolingType { 155 UNDEFINED = 0, 156 157 // average pooling 158 AVERAGE = 1, 159 160 // max pooling 161 MAX = 2, 162 }; 163 164 struct Pooling2DAttributes { 165 PoolingType type = PoolingType::UNDEFINED; 166 // Strides for every axis. 167 HW strides = HW(-1, -1); 168 HW kernel = HW(-1, -1); 169 Padding2D padding; 170 // NOTE(akulik): technically the number of outputs from Pooling node indicates 171 // whether indices are needed or not, but I decided to keep it inside 172 // attributes to simplify processing. 173 bool output_indices = false; 174 }; 175 176 struct Pooling3DAttributes { 177 PoolingType type = PoolingType::UNDEFINED; 178 // Strides for every axis. 179 HWD strides = HWD(0, 0, 0); 180 HWD kernel = HWD(0, 0, 0); 181 Padding3D padding; 182 // NOTE(akulik): technically the number of outputs from Pooling node indicates 183 // whether indices are needed or not, but I decided to keep it inside 184 // attributes to simplify processing. 185 bool output_indices = false; 186 }; 187 188 struct MaxUnpooling2DAttributes { 189 // Strides for every axis. 190 HW strides = HW(-1, -1); 191 HW kernel = HW(-1, -1); 192 Padding2D padding; 193 }; 194 195 struct MaxUnpooling3DAttributes { 196 // Strides for every axis. 197 HWD strides = HWD(0, 0, 0); 198 HWD kernel = HWD(0, 0, 0); 199 Padding3D padding; 200 }; 201 202 struct MeanAttributes { 203 // The vector of dimensions to calculate mean along. 204 std::set<Axis> dims; 205 }; 206 207 struct ConcatAttributes { 208 // Defines axis by which to concat on. 209 Axis axis = Axis::UNKNOWN; 210 }; 211 212 // @return shape of a tensor after MaxUnpooling2D operation is applied to 213 // the given input. 214 BHWC CalculateOutputShape(const BHWC& input, 215 const MaxUnpooling2DAttributes& attr); 216 217 // @return shape of a tensor after MaxUnpooling3D operation is applied to 218 // the given input. 219 BHWDC CalculateOutputShape(const BHWDC& input, 220 const MaxUnpooling3DAttributes& attr); 221 222 // @return shape of a tensor after Pooling2D operation is applied to the given 223 // input. 224 BHWC CalculateOutputShape(const BHWC& input, const Pooling2DAttributes& attr); 225 226 // @return shape of a tensor after Pooling3D operation is applied to the given 227 // input. 228 BHWDC CalculateOutputShape(const BHWDC& input, const Pooling3DAttributes& attr); 229 230 // @return shape of a tensor after Concat operation is applied to the given 231 // input. 232 absl::Status CalculateOutputShape(const std::vector<BHWC>& input, 233 const ConcatAttributes& attr, 234 BHWC* output_shape); 235 236 // @return shape of a tensor after Concat operation is applied to the given 237 // input. 238 absl::Status CalculateOutputShape(const std::vector<BHWDC>& input, 239 const ConcatAttributes& attr, 240 BHWDC* output_shape); 241 242 // @return padding for pooling operation to make sure output keep the same shape 243 // as the given input. 244 Padding2D CalculateSamePadding(const BHWC& input, 245 const Pooling2DAttributes& attr); 246 247 // @return padding for pooling operation to make sure output keep the same shape 248 // as the given input. 249 Padding3D CalculateSamePadding(const BHWDC& input, 250 const Pooling3DAttributes& attr); 251 252 // @return padding for max unpooling operation to make sure output keep the same 253 // shape as the given input. 254 Padding2D CalculateSamePadding(const BHWC& input, 255 const MaxUnpooling2DAttributes& attr); 256 257 // @return padding for max unpooling operation to make sure output keep the same 258 // shape as the given input. 259 Padding3D CalculateSamePadding(const BHWDC& input, 260 const MaxUnpooling3DAttributes& attr); 261 262 struct Convolution2DAttributes { 263 HW strides = HW(1, 1); // Along each axis. 264 HW dilations = HW(1, 1); // Along each axis. 265 Padding2D padding; 266 267 Tensor<OHWI, DataType::FLOAT32> weights; 268 Tensor<Linear, DataType::FLOAT32> bias; // optional 269 }; 270 271 struct Convolution3DAttributes { 272 HWD strides = HWD(0, 0, 0); // Along each axis. 273 HWD dilations = HWD(0, 0, 0); // Along each axis. 274 Padding3D padding; 275 276 Tensor<OHWDI, DataType::FLOAT32> weights; 277 Tensor<Linear, DataType::FLOAT32> bias; // optional 278 }; 279 280 // @return shape of a tensor after Convolution2D operation is applied to 281 // the given input. 282 BHWC CalculateOutputShape(const BHWC& input, 283 const Convolution2DAttributes& attr); 284 285 // @return shape of a tensor after Convolution3D operation is applied to 286 // the given input. 287 BHWDC CalculateOutputShape(const BHWDC& input, 288 const Convolution3DAttributes& attr); 289 290 // @return padding for convolution operation to make sure output keep the same 291 // shape as the given input. 292 Padding2D CalculateSamePadding(const BHWC& input, 293 const Convolution2DAttributes& attr); 294 295 // @return padding for convolution operation to make sure output keep the same 296 // shape as the given input. 297 Padding3D CalculateSamePadding(const BHWDC& input, 298 const Convolution3DAttributes& attr); 299 300 struct ConvolutionTransposedAttributes { 301 HW stride = HW(1, 1); // Along each axis. 302 HW adjacent; // TODO(sorokin): No op on Flow. 303 Padding2D padding; 304 305 Tensor<OHWI, DataType::FLOAT32> weights; 306 Tensor<Linear, DataType::FLOAT32> bias; // optional 307 }; 308 309 struct ConvolutionTransposed3DAttributes { 310 HWD stride = HWD(0, 0, 0); // Along each axis. 311 Padding3D padding; 312 313 Tensor<OHWDI, DataType::FLOAT32> weights; 314 Tensor<Linear, DataType::FLOAT32> bias; // optional 315 }; 316 317 Padding2D CalculateSamePadding(const BHWC& input, 318 const ConvolutionTransposedAttributes& attr); 319 320 Padding3D CalculateSamePadding(const BHWDC& input, 321 const ConvolutionTransposed3DAttributes& attr); 322 323 // @return shape of a tensor after ConvolutionTransposed operation is applied to 324 // the given input. 325 BHWC CalculateOutputShape(const BHWC& input, 326 const ConvolutionTransposedAttributes& attr); 327 328 // @return shape of a tensor after ConvolutionTransposed3D operation is applied 329 // to 330 // the given input. 331 BHWDC CalculateOutputShape(const BHWDC& input, 332 const ConvolutionTransposed3DAttributes& attr); 333 334 struct DepthwiseConvolution2DAttributes : public Convolution2DAttributes {}; 335 struct DepthwiseConvolution3DAttributes : public Convolution3DAttributes {}; 336 337 // @return shape of a tensor after DepthwiseConvolution2D operation is applied 338 // to the given input. 339 BHWC CalculateOutputShape(const BHWC& input, 340 const DepthwiseConvolution2DAttributes& attr); 341 342 // @return shape of a tensor after DepthwiseConvolution3D operation is applied 343 // to the given input. 344 BHWDC CalculateOutputShape(const BHWDC& input, 345 const DepthwiseConvolution3DAttributes& attr); 346 347 // @return padding for depthwise convolution operation to make sure output keep 348 // the same shape as the given input. 349 Padding2D CalculateSamePadding(const BHWC& input, 350 const DepthwiseConvolution2DAttributes& attr); 351 352 // @return padding for depthwise convolution operation to make sure output keep 353 // the same shape as the given input. 354 Padding3D CalculateSamePadding(const BHWDC& input, 355 const DepthwiseConvolution3DAttributes& attr); 356 357 // f(x):= { 358 // if x < 0 : x -> alpha * x 359 // if x >= 0 : x -> min(clip, x) 360 // } 361 // 362 // Examples: 363 // - ReLU: clip = 0, alpha = 0 364 // - ReLU6: clip = 6, alpha = 0 365 // - Leaky ReLU: clip = 0, alpha = a 366 struct ReLUAttributes { 367 // clip <= 0 mean it is not set. 368 float clip = 0; 369 370 float alpha = 0; 371 }; 372 373 struct PReLUAttributes { 374 // clip <= 0 mean it is not set. 375 float clip = 0; 376 377 // If alpha is linear, then it is sharded across CHANNELS axis, otherwise 378 // full shape alpha is required. 379 absl::variant<Tensor<Linear, DataType::FLOAT32>, 380 Tensor<HWC, DataType::FLOAT32>> 381 alpha; 382 }; 383 384 struct ReduceAttributes { 385 std::set<Axis> dims; 386 }; 387 388 struct SoftmaxAttributes { 389 Axis axis = Axis::UNKNOWN; 390 }; 391 392 enum LstmKernelType { 393 FULL = 0, 394 BASIC = 1, // Currently, only basic is supported. 395 }; 396 397 struct LstmAttributes { 398 LstmKernelType kernel_type = LstmKernelType::BASIC; 399 }; 400 401 enum class SamplingType { 402 UNKNOWN = 0, 403 NEAREST = 1, 404 BILINEAR = 2, 405 }; 406 407 struct Resize2DAttributes { 408 HW new_shape; 409 410 SamplingType type = SamplingType::UNKNOWN; 411 412 // If true, the centers of the 4 corner pixels of the input and output tensors 413 // are aligned, preserving the values at the corner pixels. Defaults to false. 414 bool align_corners = false; 415 416 bool half_pixel_centers = false; 417 }; 418 419 // TODO(b/147771327): rename to Resize3D 420 struct Resize3DAttributes { 421 HWD new_shape; 422 423 SamplingType type = SamplingType::NEAREST; 424 425 // If true, the centers of the 8 corner pixels of the input and output tensors 426 // are aligned, preserving the values at the corner pixels. Defaults to false. 427 bool align_corners = false; 428 429 bool half_pixel_centers = false; 430 }; 431 432 float CalculateResizeScale(int32_t input_size, int32_t output_size, 433 const Resize2DAttributes& attr); 434 435 float CalculateResizeScale(int32_t input_size, int32_t output_size, 436 const Resize3DAttributes& attr); 437 438 // @return shape of a tensor after scale operation is applied to the given 439 // input. 440 BHWC CalculateOutputShape(const BHWC& input, const Resize2DAttributes& attr); 441 442 // @return shape of a tensor after scale operation is applied to the given 443 // input. 444 BHWDC CalculateOutputShape(const BHWDC& input, const Resize3DAttributes& attr); 445 446 enum class PaddingContentType { 447 ZEROS = 0, 448 REFLECT = 1, 449 EDGE = 2, 450 }; 451 452 struct PadAttributes { 453 PaddingContentType type = PaddingContentType::ZEROS; 454 455 BHWC prepended; 456 BHWC appended; 457 }; 458 459 // @return shape of a tensor after Pad operation is applied to the given input. 460 BHWC CalculateOutputShape(const BHWC& input, const PadAttributes& attr); 461 462 struct Pad3DAttributes { 463 PaddingContentType type = PaddingContentType::ZEROS; 464 465 BHWDC prepended; 466 BHWDC appended; 467 }; 468 469 // @return shape of a tensor after Pad3D operation is applied to the given 470 // input. 471 BHWDC CalculateOutputShape(const BHWDC& input, const Pad3DAttributes& attr); 472 473 struct ConstTensorAttributes { 474 Tensor<BHWC, DataType::FLOAT32> tensor; 475 }; 476 477 struct DensifyAttributes { 478 Tensor<BHWC, DataType::FLOAT32> tensor; 479 }; 480 481 // Simple slicing without advanced support for shrinking, reverse slicing etc. 482 struct SliceAttributes { 483 // Specifies start and end dimensions for slicing. 484 BHWC starts; 485 BHWC ends; 486 487 // Stride should be >= 1. 488 BHWC strides; 489 }; 490 491 // @return shape of a tensor after Slice2D operation is applied to the given 492 // input. 493 BHWC CalculateOutputShape(const BHWC& input, const SliceAttributes& attr); 494 495 // Simple slicing without advanced support for shrinking, reverse slicing etc. 496 struct Slice3DAttributes { 497 // Specifies start and end dimensions for slicing. 498 BHWDC starts; 499 BHWDC ends; 500 501 // Stride should be >= 1. 502 BHWDC strides; 503 }; 504 505 // @return shape of a tensor after Slice3D operation is applied to the given 506 // input. 507 BHWDC CalculateOutputShape(const BHWDC& input, const Slice3DAttributes& attr); 508 509 struct FullyConnectedAttributes { 510 Tensor<OHWI, DataType::FLOAT32> weights; 511 Tensor<Linear, DataType::FLOAT32> bias; 512 }; 513 514 struct FullyConnectedInt8Attributes { 515 Tensor<OHWI, DataType::INT8> weights; 516 Tensor<Linear, DataType::FLOAT32> bias; 517 float scale; 518 int zero_point; 519 }; 520 521 FullyConnectedAttributes DequatizeFullyConnectedAttr( 522 const FullyConnectedInt8Attributes& attr); 523 524 // @return shape of a tensor after FullyConnected operation is applied to 525 // the given input. 526 BHWC CalculateOutputShape(const BHWC& input, 527 const FullyConnectedAttributes& attr); 528 529 // @return shape of a tensor after Mean operation is applied to the given input. 530 BHWC CalculateOutputShape(const BHWC& input, const MeanAttributes& attr); 531 532 // @return shape of a tensor after Mean operation is applied to the given input. 533 BHWDC CalculateOutputShape(const BHWDC& input, const MeanAttributes& attr); 534 535 struct ElementwiseAttributes { 536 TensorOrScalar param; 537 // For elementwise operation with 2 inputs op(A, B), runtime_tensor_is_second 538 // true when runtime tensor is B(on second position). this is important for 539 // ops that non commutative, for example substract. 540 bool runtime_tensor_is_second = false; 541 }; 542 543 struct ReshapeAttributes { 544 BHWC new_shape; 545 }; 546 547 struct Reshape3DAttributes { 548 BHWDC new_shape; 549 }; 550 551 struct TransposeAttributes { 552 // A permutation of the dimensions of input tensor 553 BHWC perm; 554 }; 555 556 // @return shape of a tensor after Transpose operation is applied to 557 // the given input. 558 BHWC CalculateOutputShape(const BHWC& input, const TransposeAttributes& attr); 559 560 struct Transpose3DAttributes { 561 // A permutation of the dimensions of input tensor 562 BHWDC perm; 563 }; 564 565 // @return shape of a tensor after Transpose3D operation is applied to 566 // the given input. 567 BHWDC CalculateOutputShape(const BHWDC& input, 568 const Transpose3DAttributes& attr); 569 570 struct SpaceToDepthAttributes { 571 int block_size; 572 }; 573 574 struct SplitAttributes { 575 // Defines axis by which to split. 576 Axis axis = Axis::UNKNOWN; 577 }; 578 579 // These help perform a combination of Quantize & Dequantize to adjust float 580 // values like quantized inference would. 581 struct QuantizeAndDequantizeAttributes { 582 float min = 0; 583 float max = 0; 584 float scale = 0; 585 }; 586 587 struct GatherAttributes { 588 Axis axis = Axis::UNKNOWN; 589 }; 590 591 } // namespace gpu 592 } // namespace tflite 593 594 #endif // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OPERATIONS_H_ 595