1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OPERATIONS_H_ 17 #define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OPERATIONS_H_ 18 19 #include <cstdint> 20 #include <set> 21 #include <string> 22 #include <variant> 23 #include <vector> 24 25 #include "absl/types/variant.h" 26 #include "tensorflow/lite/delegates/gpu/common/data_type.h" 27 #include "tensorflow/lite/delegates/gpu/common/shape.h" 28 #include "tensorflow/lite/delegates/gpu/common/status.h" 29 #include "tensorflow/lite/delegates/gpu/common/tensor.h" 30 31 namespace tflite { 32 namespace gpu { 33 34 // Non exhaustive list of operations. 35 enum class OperationType { 36 UNKNOWN = 0, 37 ABS, 38 ADD, 39 BATCH_TO_SPACE, 40 BATCH_NORMALIZATION, 41 BATCHED_MATMUL, 42 CAST, 43 CONCAT, 44 CONSTANT, 45 CONVOLUTION_2D, 46 CONVOLUTION_TRANSPOSED, 47 COPY, 48 COS, 49 CUMSUM, 50 DENSIFY, 51 DEPTHWISE_CONVOLUTION, 52 DEPTH_TO_SPACE, 53 DIV, 54 ELU, 55 EQUAL, 56 EXP, 57 FLOOR, 58 FLOOR_DIV, 59 FLOOR_MOD, 60 FULLY_CONNECTED, 61 FULLY_CONNECTED_INT8, 62 GATHER, 63 GREATER, 64 GREATER_EQUAL, 65 HARD_SWISH, 66 LESS, 67 LESS_EQUAL, 68 LOG, 69 LSTM, 70 MAXIMUM, 71 MAX_UNPOOLING_2D, 72 MEAN, 73 MEAN_STDDEV_NORMALIZATION, 74 MINIMUM, 75 MUL, 76 NEG, 77 NOT_EQUAL, 78 ONE_HOT, 79 PAD, 80 POOLING_2D, 81 POW, 82 PRELU, 83 // Used to accurately run inference on quantized models. 84 QUANTIZE_AND_DEQUANTIZE, 85 REDUCE_MAXIMUM, 86 REDUCE_MINIMUM, 87 REDUCE_PRODUCT, 88 REDUCE_SUM, 89 RELU, 90 RESAMPLER, 91 RESHAPE, 92 RESIZE, 93 RSQRT, 94 SELECT_V2, 95 SIGMOID, 96 SIN, 97 SLICE, 98 SOFTMAX, 99 SPACE_TO_BATCH, 100 SPACE_TO_DEPTH, 101 SPLIT, 102 SQRT, 103 SQUARE, 104 SQUARED_DIFF, 105 SUB, 106 TANH, 107 TILE, 108 TRANSPOSE, 109 }; 110 111 std::string ToString(enum OperationType op); 112 113 OperationType OperationTypeFromString(const std::string& name); 114 115 typedef absl::variant<absl::monostate, Tensor<HWC, DataType::FLOAT32>, 116 Tensor<Linear, DataType::FLOAT32>, float> 117 TensorOrScalar; 118 119 struct Padding2D { 120 Padding2D() = default; 121 Padding2D& operator=(const Padding2D& value); 122 bool operator==(const Padding2D& value); 123 bool operator!=(const Padding2D& value); 124 Padding2D& operator-(const Padding2D& value); 125 126 // Padding values for every axis (if needed), where 'prepended' defines 127 // padding for the beginning of each axis and 'appended' represents end part 128 // of the corresponding axis. 129 HW prepended = HW(-1, -1); 130 HW appended = HW(-1, -1); 131 }; 132 133 struct Padding3D { 134 Padding3D() = default; 135 Padding3D& operator=(const Padding3D& value); 136 bool operator==(const Padding3D& value); 137 bool operator!=(const Padding3D& value); 138 Padding3D& operator-(const Padding3D& value); 139 140 // Padding values for every axis (if needed), where 'prepended' defines 141 // padding for the beginning of each axis and 'appended' represents end part 142 // of the corresponding axis. 143 HWD prepended = HWD(0, 0, 0); 144 HWD appended = HWD(0, 0, 0); 145 }; 146 147 struct Crop2D : public Padding2D {}; 148 149 struct SpaceToBatchAttributes { 150 HW block; 151 Padding2D padding; 152 }; 153 154 struct BatchToSpaceAttributes { 155 HW block; 156 Crop2D crop; 157 }; 158 159 enum class PoolingType { 160 UNDEFINED = 0, 161 162 // average pooling 163 AVERAGE = 1, 164 165 // max pooling 166 MAX = 2, 167 }; 168 169 struct Pooling2DAttributes { 170 PoolingType type = PoolingType::UNDEFINED; 171 // Strides for every axis. 172 HW strides = HW(-1, -1); 173 HW kernel = HW(-1, -1); 174 Padding2D padding; 175 // NOTE(akulik): technically the number of outputs from Pooling node indicates 176 // whether indices are needed or not, but I decided to keep it inside 177 // attributes to simplify processing. 178 bool output_indices = false; 179 }; 180 181 struct Pooling3DAttributes { 182 PoolingType type = PoolingType::UNDEFINED; 183 // Strides for every axis. 184 HWD strides = HWD(0, 0, 0); 185 HWD kernel = HWD(0, 0, 0); 186 Padding3D padding; 187 // NOTE(akulik): technically the number of outputs from Pooling node indicates 188 // whether indices are needed or not, but I decided to keep it inside 189 // attributes to simplify processing. 190 bool output_indices = false; 191 }; 192 193 struct MaxUnpooling2DAttributes { 194 // Strides for every axis. 195 HW strides = HW(-1, -1); 196 HW kernel = HW(-1, -1); 197 Padding2D padding; 198 }; 199 200 struct MaxUnpooling3DAttributes { 201 // Strides for every axis. 202 HWD strides = HWD(0, 0, 0); 203 HWD kernel = HWD(0, 0, 0); 204 Padding3D padding; 205 }; 206 207 struct MeanAttributes { 208 // The vector of dimensions to calculate mean along. 209 std::set<Axis> dims; 210 }; 211 212 struct ConcatAttributes { 213 // Defines axis by which to concat on. 214 Axis axis = Axis::UNKNOWN; 215 }; 216 217 // @return shape of a tensor after MaxUnpooling2D operation is applied to 218 // the given input. 219 BHWC CalculateOutputShape(const BHWC& input, 220 const MaxUnpooling2DAttributes& attr); 221 222 // @return shape of a tensor after MaxUnpooling3D operation is applied to 223 // the given input. 224 BHWDC CalculateOutputShape(const BHWDC& input, 225 const MaxUnpooling3DAttributes& attr); 226 227 // @return shape of a tensor after Pooling2D operation is applied to the given 228 // input. 229 BHWC CalculateOutputShape(const BHWC& input, const Pooling2DAttributes& attr); 230 231 // @return shape of a tensor after Pooling3D operation is applied to the given 232 // input. 233 BHWDC CalculateOutputShape(const BHWDC& input, const Pooling3DAttributes& attr); 234 235 // @return shape of a tensor after Concat operation is applied to the given 236 // input. 237 absl::Status CalculateOutputShape(const std::vector<BHWC>& input, 238 const ConcatAttributes& attr, 239 BHWC* output_shape); 240 241 // @return shape of a tensor after Concat operation is applied to the given 242 // input. 243 absl::Status CalculateOutputShape(const std::vector<BHWDC>& input, 244 const ConcatAttributes& attr, 245 BHWDC* output_shape); 246 247 // @return padding for pooling operation to make sure output keep the same shape 248 // as the given input. 249 Padding2D CalculateSamePadding(const BHWC& input, 250 const Pooling2DAttributes& attr); 251 252 // @return padding for pooling operation to make sure output keep the same shape 253 // as the given input. 254 Padding3D CalculateSamePadding(const BHWDC& input, 255 const Pooling3DAttributes& attr); 256 257 // @return padding for max unpooling operation to make sure output keep the same 258 // shape as the given input. 259 Padding2D CalculateSamePadding(const BHWC& input, 260 const MaxUnpooling2DAttributes& attr); 261 262 // @return padding for max unpooling operation to make sure output keep the same 263 // shape as the given input. 264 Padding3D CalculateSamePadding(const BHWDC& input, 265 const MaxUnpooling3DAttributes& attr); 266 267 struct Convolution2DAttributes { 268 HW strides = HW(1, 1); // Along each axis. 269 HW dilations = HW(1, 1); // Along each axis. 270 Padding2D padding; 271 272 Tensor<OHWI, DataType::FLOAT32> weights; 273 Tensor<Linear, DataType::FLOAT32> bias; // optional 274 275 int groups = 1; // optional, split channels dimension on equal groups 276 // Restrictions: 277 // src.Channels() and dst.Channels() must be divisible by groups 278 // Restrictions for gpu delegates: 279 // src_group_channels = src.Channels() / groups; 280 // dst_group_channels = dst.Channels() / groups; 281 // src_group_channels and dst_group_channels must be divisible by 4 282 // if groups != 1, weights will have special format 283 // weights.o = group_weights.o * groups; 284 // weights.i = group_weights.i; 285 // weights.h = group_weights.h; 286 // weights.w = group_weights.w; 287 }; 288 289 struct Convolution3DAttributes { 290 HWD strides = HWD(0, 0, 0); // Along each axis. 291 HWD dilations = HWD(0, 0, 0); // Along each axis. 292 Padding3D padding; 293 294 Tensor<OHWDI, DataType::FLOAT32> weights; 295 Tensor<Linear, DataType::FLOAT32> bias; // optional 296 297 int groups = 1; // optional, split channels dimension on equal groups 298 // Restrictions: 299 // src.Channels() and dst.Channels() must be divisible by groups 300 // Restrictions for gpu delegates: 301 // src_group_channels = src.Channels() / groups; 302 // dst_group_channels = dst.Channels() / groups; 303 // src_group_channels and dst_group_channels must be divisible by 4 304 // if groups != 1, weights will have special format 305 // weights.o = group_weights.o * groups; 306 // weights.i = group_weights.i; 307 // weights.h = group_weights.h; 308 // weights.w = group_weights.w; 309 // weights.d = group_weights.d; 310 }; 311 312 // @return shape of a tensor after Convolution2D operation is applied to 313 // the given input. 314 BHWC CalculateOutputShape(const BHWC& input, 315 const Convolution2DAttributes& attr); 316 317 // @return shape of a tensor after Convolution3D operation is applied to 318 // the given input. 319 BHWDC CalculateOutputShape(const BHWDC& input, 320 const Convolution3DAttributes& attr); 321 322 // @return padding for convolution operation to make sure output keep the same 323 // shape as the given input. 324 Padding2D CalculateSamePadding(const BHWC& input, 325 const Convolution2DAttributes& attr); 326 327 // @return padding for convolution operation to make sure output keep the same 328 // shape as the given input. 329 Padding3D CalculateSamePadding(const BHWDC& input, 330 const Convolution3DAttributes& attr); 331 332 struct ConvolutionTransposedAttributes { 333 HW stride = HW(1, 1); // Along each axis. 334 HW adjacent; // TODO(sorokin): No op on Flow. 335 Padding2D padding; 336 337 Tensor<OHWI, DataType::FLOAT32> weights; 338 Tensor<Linear, DataType::FLOAT32> bias; // optional 339 }; 340 341 struct ConvolutionTransposed3DAttributes { 342 HWD stride = HWD(0, 0, 0); // Along each axis. 343 Padding3D padding; 344 345 Tensor<OHWDI, DataType::FLOAT32> weights; 346 Tensor<Linear, DataType::FLOAT32> bias; // optional 347 }; 348 349 Padding2D CalculateSamePadding(const BHWC& input, 350 const ConvolutionTransposedAttributes& attr); 351 352 Padding3D CalculateSamePadding(const BHWDC& input, 353 const ConvolutionTransposed3DAttributes& attr); 354 355 // @return shape of a tensor after ConvolutionTransposed operation is applied to 356 // the given input. 357 BHWC CalculateOutputShape(const BHWC& input, 358 const ConvolutionTransposedAttributes& attr); 359 360 // @return shape of a tensor after ConvolutionTransposed3D operation is applied 361 // to 362 // the given input. 363 BHWDC CalculateOutputShape(const BHWDC& input, 364 const ConvolutionTransposed3DAttributes& attr); 365 366 struct DepthwiseConvolution2DAttributes : public Convolution2DAttributes {}; 367 struct DepthwiseConvolution3DAttributes : public Convolution3DAttributes {}; 368 369 // @return shape of a tensor after DepthwiseConvolution2D operation is applied 370 // to the given input. 371 BHWC CalculateOutputShape(const BHWC& input, 372 const DepthwiseConvolution2DAttributes& attr); 373 374 // @return shape of a tensor after DepthwiseConvolution3D operation is applied 375 // to the given input. 376 BHWDC CalculateOutputShape(const BHWDC& input, 377 const DepthwiseConvolution3DAttributes& attr); 378 379 // @return padding for depthwise convolution operation to make sure output keep 380 // the same shape as the given input. 381 Padding2D CalculateSamePadding(const BHWC& input, 382 const DepthwiseConvolution2DAttributes& attr); 383 384 // @return padding for depthwise convolution operation to make sure output keep 385 // the same shape as the given input. 386 Padding3D CalculateSamePadding(const BHWDC& input, 387 const DepthwiseConvolution3DAttributes& attr); 388 389 // f(x):= { 390 // if x < 0 : x -> alpha * x 391 // if x >= 0 : x -> min(clip, x) 392 // } 393 // 394 // Examples: 395 // - ReLU: clip = 0, alpha = 0 396 // - ReLU6: clip = 6, alpha = 0 397 // - Leaky ReLU: clip = 0, alpha = a 398 struct ReLUAttributes { 399 // clip <= 0 mean it is not set. 400 float clip = 0; 401 402 float alpha = 0; 403 }; 404 405 struct PReLUAttributes { 406 // If alpha is linear, then it is sharded across CHANNELS axis, otherwise 407 // full shape alpha is required. 408 absl::variant<Tensor<Linear, DataType::FLOAT32>, 409 Tensor<HWC, DataType::FLOAT32>> 410 alpha; 411 }; 412 413 struct ReduceAttributes { 414 std::set<Axis> dims; 415 }; 416 417 struct SoftmaxAttributes { 418 Axis axis = Axis::UNKNOWN; 419 }; 420 421 enum LstmKernelType { 422 FULL = 0, 423 BASIC = 1, // Currently, only basic is supported. 424 }; 425 426 struct LstmAttributes { 427 LstmKernelType kernel_type = LstmKernelType::BASIC; 428 }; 429 430 enum class SamplingType { 431 UNKNOWN = 0, 432 NEAREST = 1, 433 BILINEAR = 2, 434 }; 435 436 struct Resize2DAttributes { 437 HW new_shape; 438 439 SamplingType type = SamplingType::UNKNOWN; 440 441 // If true, the centers of the 4 corner pixels of the input and output tensors 442 // are aligned, preserving the values at the corner pixels. Defaults to false. 443 bool align_corners = false; 444 445 bool half_pixel_centers = false; 446 }; 447 448 // TODO(b/147771327): rename to Resize3D 449 struct Resize3DAttributes { 450 HWD new_shape; 451 452 SamplingType type = SamplingType::NEAREST; 453 454 // If true, the centers of the 8 corner pixels of the input and output tensors 455 // are aligned, preserving the values at the corner pixels. Defaults to false. 456 bool align_corners = false; 457 458 bool half_pixel_centers = false; 459 }; 460 461 float CalculateResizeScale(int32_t input_size, int32_t output_size, 462 const Resize2DAttributes& attr); 463 464 float CalculateResizeScale(int32_t input_size, int32_t output_size, 465 const Resize3DAttributes& attr); 466 467 // @return shape of a tensor after scale operation is applied to the given 468 // input. 469 BHWC CalculateOutputShape(const BHWC& input, const Resize2DAttributes& attr); 470 471 // @return shape of a tensor after scale operation is applied to the given 472 // input. 473 BHWDC CalculateOutputShape(const BHWDC& input, const Resize3DAttributes& attr); 474 475 enum class PaddingContentType { 476 ZEROS = 0, 477 REFLECT = 1, 478 EDGE = 2, 479 }; 480 481 struct PadAttributes { 482 PaddingContentType type = PaddingContentType::ZEROS; 483 484 BHWC prepended; 485 BHWC appended; 486 }; 487 488 // @return shape of a tensor after Pad operation is applied to the given input. 489 BHWC CalculateOutputShape(const BHWC& input, const PadAttributes& attr); 490 491 struct Pad3DAttributes { 492 PaddingContentType type = PaddingContentType::ZEROS; 493 494 BHWDC prepended; 495 BHWDC appended; 496 }; 497 498 // @return shape of a tensor after Pad3D operation is applied to the given 499 // input. 500 BHWDC CalculateOutputShape(const BHWDC& input, const Pad3DAttributes& attr); 501 502 struct ConstTensorAttributes { 503 Tensor<BHWC, DataType::FLOAT32> tensor; 504 }; 505 506 struct DensifyAttributes { 507 Tensor<BHWC, DataType::FLOAT32> tensor; 508 }; 509 510 // Simple slicing without advanced support for shrinking, reverse slicing etc. 511 struct SliceAttributes { 512 // Specifies start and end dimensions for slicing. 513 BHWC starts; 514 BHWC ends; 515 516 // Stride should be >= 1. 517 BHWC strides; 518 }; 519 520 // @return shape of a tensor after Slice2D operation is applied to the given 521 // input. 522 BHWC CalculateOutputShape(const BHWC& input, const SliceAttributes& attr); 523 524 // Simple slicing without advanced support for shrinking, reverse slicing etc. 525 struct Slice3DAttributes { 526 // Specifies start and end dimensions for slicing. 527 BHWDC starts; 528 BHWDC ends; 529 530 // Stride should be >= 1. 531 BHWDC strides; 532 }; 533 534 // @return shape of a tensor after Slice3D operation is applied to the given 535 // input. 536 BHWDC CalculateOutputShape(const BHWDC& input, const Slice3DAttributes& attr); 537 538 struct FullyConnectedAttributes { 539 Tensor<OHWI, DataType::FLOAT32> weights; 540 Tensor<Linear, DataType::FLOAT32> bias; 541 }; 542 543 struct FullyConnectedInt8Attributes { 544 Tensor<OHWI, DataType::INT8> weights; 545 Tensor<Linear, DataType::FLOAT32> bias; 546 float scale; 547 int zero_point; 548 }; 549 550 FullyConnectedAttributes DequatizeFullyConnectedAttr( 551 const FullyConnectedInt8Attributes& attr); 552 553 // @return shape of a tensor after FullyConnected operation is applied to 554 // the given input. 555 BHWC CalculateOutputShape(const BHWC& input, 556 const FullyConnectedAttributes& attr); 557 558 // @return shape of a tensor after Mean operation is applied to the given input. 559 BHWC CalculateOutputShape(const BHWC& input, const MeanAttributes& attr); 560 561 // @return shape of a tensor after Mean operation is applied to the given input. 562 BHWDC CalculateOutputShape(const BHWDC& input, const MeanAttributes& attr); 563 564 struct ElementwiseAttributes { 565 TensorOrScalar param; 566 // For elementwise operation with 2 inputs op(A, B), runtime_tensor_is_second 567 // true when runtime tensor is B(on second position). this is important for 568 // ops that non commutative, for example subtract. 569 bool runtime_tensor_is_second = false; 570 }; 571 572 struct ReshapeAttributes { 573 BHWC new_shape; 574 }; 575 576 struct Reshape3DAttributes { 577 BHWDC new_shape; 578 }; 579 580 struct TransposeAttributes { 581 // A permutation of the dimensions of input tensor 582 BHWC perm; 583 }; 584 585 // @return shape of a tensor after Transpose operation is applied to 586 // the given input. 587 BHWC CalculateOutputShape(const BHWC& input, const TransposeAttributes& attr); 588 589 struct Transpose3DAttributes { 590 // A permutation of the dimensions of input tensor 591 BHWDC perm; 592 }; 593 594 // @return shape of a tensor after Transpose3D operation is applied to 595 // the given input. 596 BHWDC CalculateOutputShape(const BHWDC& input, 597 const Transpose3DAttributes& attr); 598 599 struct SpaceToDepthAttributes { 600 int block_size; 601 }; 602 603 struct SplitAttributes { 604 // Defines axis by which to split. 605 Axis axis = Axis::UNKNOWN; 606 }; 607 608 // These help perform a combination of Quantize & Dequantize to adjust float 609 // values like quantized inference would. 610 struct QuantizeAndDequantizeAttributes { 611 float min = 0; 612 float max = 0; 613 float scale = 0; 614 }; 615 616 struct GatherAttributes { 617 Axis axis = Axis::UNKNOWN; 618 }; 619 620 struct OneHotAttributes { 621 float on_value = 1; 622 float off_value = 0; 623 }; 624 625 struct SelectV2Attributes { 626 bool broadcast_true = false; 627 bool broadcast_false = false; 628 }; 629 630 struct CumsumAttributes { 631 Axis axis = Axis::UNKNOWN; 632 }; 633 634 } // namespace gpu 635 } // namespace tflite 636 637 #endif // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OPERATIONS_H_ 638