1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_LITE_SUPPORT_CC_TASK_VISION_UTILS_FRAME_BUFFER_UTILS_H_ 17 #define TENSORFLOW_LITE_SUPPORT_CC_TASK_VISION_UTILS_FRAME_BUFFER_UTILS_H_ 18 19 #include <memory> 20 #include <vector> 21 22 #include "absl/status/status.h" 23 #include "absl/types/optional.h" 24 #include "absl/types/variant.h" 25 #include "tensorflow_lite_support/cc/port/integral_types.h" 26 #include "tensorflow_lite_support/cc/task/vision/core/frame_buffer.h" 27 #include "tensorflow_lite_support/cc/task/vision/proto/bounding_box_proto_inc.h" 28 #include "tensorflow_lite_support/cc/task/vision/utils/frame_buffer_utils_interface.h" 29 30 namespace tflite { 31 namespace task { 32 namespace vision { 33 34 // Returns the minimal buffer size for a plane in bytes based on the given 35 // format and dimensions. 36 int GetBufferByteSize(FrameBuffer::Dimension dimension, 37 FrameBuffer::Format format); 38 39 // Rotates the `from_box` in `from_orientation` to `to_orientation` within an 40 // image of size `from_dimension`. 41 BoundingBox OrientBoundingBox(const BoundingBox& from_box, 42 FrameBuffer::Orientation from_orientation, 43 FrameBuffer::Orientation to_orientation, 44 FrameBuffer::Dimension from_dimension); 45 46 // Same as OrientBoundingBox but from normalized coordinates. 47 BoundingBox OrientAndDenormalizeBoundingBox( 48 float from_left, float from_top, float from_right, float from_bottom, 49 FrameBuffer::Orientation from_orientation, 50 FrameBuffer::Orientation to_orientation, 51 FrameBuffer::Dimension from_dimension); 52 53 // Rotates `(from_x, from_y)` coordinates from an image of dimension 54 // `from_dimension` and orientation `from_orientation` into `(to_x, to_y)` 55 // coordinates with orientation `to_orientation`. 56 void OrientCoordinates(int from_x, int from_y, 57 FrameBuffer::Orientation from_orientation, 58 FrameBuffer::Orientation to_orientation, 59 FrameBuffer::Dimension from_dimension, int* to_x, 60 int* to_y); 61 62 // Returns whether the conversion from from_orientation to to_orientation 63 // requires 90 or 270 degrees rotation. 64 bool RequireDimensionSwap(FrameBuffer::Orientation from_orientation, 65 FrameBuffer::Orientation to_orientation); 66 67 // Structure to express parameters needed to achieve orientation conversion. 68 struct OrientParams { 69 // Counterclockwise rotation angle in degrees. This is expressed as a 70 // multiple of 90 degrees. 71 int rotation_angle_deg; 72 // Flipping operation. It must come after the rotation. 73 enum class FlipType { kHorizontal, kVertical }; 74 absl::optional<FlipType> flip; 75 }; 76 77 // Returns rotation angle and the need for horizontal flipping or vertical 78 // flipping. 79 OrientParams GetOrientParams(FrameBuffer::Orientation from_orientation, 80 FrameBuffer::Orientation to_orientation); 81 82 // The parameters needed to crop / resize. 83 // 84 // The coordinate system has its origin at the upper left corner, and 85 // positive values extend down and to the right from it. 86 // 87 // After the operation, the `crop_origin` will become the new origin. 88 // `crop_width` and `crop_height` defines the desired cropping region. After 89 // cropping, a resize is performed based on the `resize_width` and 90 // `resize_height`. 91 // 92 // To perform just cropping, the `crop_width` and `crop_height` should be the 93 // same as `resize_width` `and resize_height`. 94 struct CropResizeOperation { CropResizeOperationCropResizeOperation95 CropResizeOperation(int crop_origin_x, int crop_origin_y, 96 FrameBuffer::Dimension crop_dimension, 97 FrameBuffer::Dimension resize_dimension) 98 : crop_origin_x(crop_origin_x), 99 crop_origin_y(crop_origin_y), 100 crop_dimension(crop_dimension), 101 resize_dimension(resize_dimension) {} 102 103 int crop_origin_x; 104 int crop_origin_y; 105 FrameBuffer::Dimension crop_dimension; 106 FrameBuffer::Dimension resize_dimension; 107 }; 108 109 // The parameters needed to convert to the specified format. 110 struct ConvertOperation { ConvertOperationConvertOperation111 explicit ConvertOperation(FrameBuffer::Format to_format) 112 : to_format(to_format) {} 113 FrameBuffer::Format to_format; 114 }; 115 116 // The parameters needed to change the orientation. 117 struct OrientOperation { OrientOperationOrientOperation118 explicit OrientOperation(FrameBuffer::Orientation to_orientation) 119 : to_orientation(to_orientation) {} 120 FrameBuffer::Orientation to_orientation; 121 }; 122 123 // A variant of the supported operations on FrameBuffers. Alias for user 124 // convenience. 125 using FrameBufferOperation = 126 absl::variant<CropResizeOperation, ConvertOperation, OrientOperation>; 127 128 // Image processing utility. This utility provides both basic image buffer 129 // manipulations (e.g. rotation, format conversion, resizing, etc) as well as 130 // capability for chaining pipeline executions. The actual buffer processing 131 // engine is configurable to allow optimization based on platforms. 132 // 133 // Examples: 134 // 135 // // Create an instance of FrameBufferUtils with Halide processing engine. 136 // std::unique_ptr<FrameBufferUtils> utils = FrameBufferUtils::Create(kHalide); 137 // 138 // // Perform single basic operation by each individual call. 139 // std::unique_ptr<FrameBuffer> input = FrameBuffer::Create(...); 140 // std::unique_ptr<FrameBuffer> output = FrameBuffer::Create(...); 141 // utils->Orient(*input, output.get()); 142 // utils->Resize(*input, output.get()); 143 // 144 // // Chaining processing operations. 145 // const std::vector<FrameBufferOperation> operations = { 146 // ConvertOperation(FrameBuffer::Format::kNV21), 147 // CropResizeOperation(/*crop_origin_x=*/20, /*crop_origin_y=*/20, 148 // /*crop_width=*/10, /*crop_height=*/10, 149 // /*resize_width=*/10, /*resize_height=*/10), 150 // OrientOperation(FrameBuffer::Orientation::kLeftTop)}; 151 // utils->Execute(*input, operations, output.get()); 152 class FrameBufferUtils { 153 public: 154 // Counter-clockwise rotation in degree. 155 enum class RotationDegree { k0 = 0, k90 = 1, k180 = 2, k270 = 3 }; 156 157 // Underlying process engine used for performing operations. 158 enum class ProcessEngine { 159 kLibyuv, 160 }; 161 162 // Factory method FrameBufferUtils instance. The processing engine is 163 // defined by `engine`. Create(ProcessEngine engine)164 static std::unique_ptr<FrameBufferUtils> Create(ProcessEngine engine) { 165 return absl::make_unique<FrameBufferUtils>(engine); 166 } 167 168 explicit FrameBufferUtils(ProcessEngine engine); 169 170 // Performs cropping operation. 171 // 172 // The coordinate system has its origin at the upper left corner, and 173 // positive values extend down and to the right from it. After cropping, 174 // (x0, y0) becomes (0, 0). The new width and height are 175 // (x1 - x0 + 1, y1 - y0 + 1). 176 // 177 // The `output_buffer` should have metadata populated and its backing buffer 178 // should be big enough to store the operation result. If the `output_buffer` 179 // size dimension does not match with crop dimension, then a resize is 180 // automatically performed. 181 absl::Status Crop(const FrameBuffer& buffer, int x0, int y0, int x1, int y1, 182 FrameBuffer* output_buffer); 183 184 // Performs resizing operation. 185 // 186 // The resize dimension is determined based on output_buffer's size metadata. 187 // 188 // The output_buffer should have metadata populated and its backing buffer 189 // should be big enough to store the operation result. 190 absl::Status Resize(const FrameBuffer& buffer, FrameBuffer* output_buffer); 191 192 // Performs rotation operation. 193 // 194 // The rotation is specified in counter-clockwise direction. 195 // 196 // The output_buffer should have metadata populated and its backing buffer 197 // should be big enough to store the operation result. 198 absl::Status Rotate(const FrameBuffer& buffer, RotationDegree rotation, 199 FrameBuffer* output_buffer); 200 201 // Performs horizontal flip operation. 202 // 203 // The `output_buffer` should have metadata populated and its backing buffer 204 // should be big enough to store the operation result. 205 absl::Status FlipHorizontally(const FrameBuffer& buffer, 206 FrameBuffer* output_buffer); 207 208 // Performs vertical flip operation. 209 // 210 // The `output_buffer` should have metadata populated and its backing buffer 211 // should be big enough to store the operation result. 212 absl::Status FlipVertically(const FrameBuffer& buffer, 213 FrameBuffer* output_buffer); 214 215 // Performs buffer format conversion. 216 // 217 // The `output_buffer` should have metadata populated and its backing buffer 218 // should be big enough to store the operation result. 219 absl::Status Convert(const FrameBuffer& buffer, FrameBuffer* output_buffer); 220 221 // Performs buffer orientation conversion. Depends on the orientations, this 222 // method may perform rotation and optional flipping operations. 223 // 224 // If `buffer` and `output_buffer` has the same orientation, then a copy 225 // operation will performed. 226 // 227 // The `output_buffer` should have metadata populated and its backing buffer 228 // should be big enough to store the operation result. 229 absl::Status Orient(const FrameBuffer& buffer, FrameBuffer* output_buffer); 230 231 // Performs the image processing operations specified, in that order. 232 // 233 // The `output_buffer` should have metadata populated and its backing buffer 234 // should be big enough to store the operation result. 235 absl::Status Execute(const FrameBuffer& buffer, 236 const std::vector<FrameBufferOperation>& operations, 237 FrameBuffer* output_buffer); 238 239 // Performs a chain of operations to convert `buffer` to desired metadata 240 // (width, height, format, orientation) defined by `output_buffer` and 241 // optional cropping (`bounding_box`). 242 // 243 // Internally, a chain of operations is constructed. For performance 244 // optimization, operations are performed in the following order: crop, 245 // resize, convert color space format, and rotate. 246 // 247 // The `output_buffer` should have metadata populated and its backing buffer 248 // should be big enough to store the operation result. Insufficient backing 249 // buffer size may cause garbage result or crash. Use `GetBufferByteSize` to 250 // calculate the minimal buffer size. 251 // 252 // If the `buffer` is already in desired format, then an extra copy will be 253 // performed. 254 // 255 // The input param `bounding_box` is defined in the `buffer` coordinate space. 256 absl::Status Preprocess(const FrameBuffer& buffer, 257 absl::optional<BoundingBox> bounding_box, 258 FrameBuffer* output_buffer); 259 260 private: 261 // Returns the new FrameBuffer size after the operation is applied. 262 FrameBuffer::Dimension GetSize(const FrameBuffer& buffer, 263 const FrameBufferOperation& operation); 264 265 // Returns the new FrameBuffer orientation after command is processed. 266 FrameBuffer::Orientation GetOrientation( 267 const FrameBuffer& buffer, const FrameBufferOperation& operation); 268 269 // Returns the new FrameBuffer format after command is processed. 270 FrameBuffer::Format GetFormat(const FrameBuffer& buffer, 271 const FrameBufferOperation& operation); 272 273 // Returns Plane struct based on one dimension buffer and its metadata. If 274 // an error occurred, it will return an empty vector. 275 std::vector<FrameBuffer::Plane> GetPlanes(const uint8* buffer, 276 FrameBuffer::Dimension dimension, 277 FrameBuffer::Format format); 278 279 // Executes command with params. 280 absl::Status Execute(const FrameBuffer& buffer, 281 const FrameBufferOperation& operation, 282 FrameBuffer* output_buffer); 283 284 // Execution engine conforms to FrameBufferUtilsInterface. 285 std::unique_ptr<FrameBufferUtilsInterface> utils_; 286 }; 287 288 } // namespace vision 289 } // namespace task 290 } // namespace tflite 291 292 #endif // TENSORFLOW_LITE_SUPPORT_CC_TASK_VISION_UTILS_FRAME_BUFFER_UTILS_H_ 293