• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow_lite_support/cc/task/vision/utils/libyuv_frame_buffer_utils.h"
17 
18 #include <stdint.h>
19 
20 #include <memory>
21 #include <string>
22 
23 #include "absl/status/status.h"
24 #include "absl/strings/str_cat.h"
25 #include "absl/strings/str_format.h"
26 #include "include/libyuv.h"
27 #include "tensorflow_lite_support/cc/common.h"
28 #include "tensorflow_lite_support/cc/port/integral_types.h"
29 #include "tensorflow_lite_support/cc/port/status_macros.h"
30 #include "tensorflow_lite_support/cc/port/statusor.h"
31 #include "tensorflow_lite_support/cc/task/vision/utils/frame_buffer_common_utils.h"
32 
33 namespace tflite {
34 namespace task {
35 namespace vision {
36 
37 using ::absl::StatusCode;
38 using ::tflite::support::CreateStatusWithPayload;
39 using ::tflite::support::TfLiteSupportStatus;
40 
41 namespace {
42 
43 // Converts NV12 `buffer` to the `output_buffer` of the target color space.
44 // Supported output format includes RGB24 and YV21.
ConvertFromNv12(const FrameBuffer & buffer,FrameBuffer * output_buffer)45 absl::Status ConvertFromNv12(const FrameBuffer& buffer,
46                              FrameBuffer* output_buffer) {
47   ASSIGN_OR_RETURN(FrameBuffer::YuvData yuv_data,
48                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
49   switch (output_buffer->format()) {
50     case FrameBuffer::Format::kRGB: {
51       // The RAW format of Libyuv represents the 8-bit interleaved RGB format in
52       // the big endian style with R being the first byte in memory.
53       int ret = libyuv::NV12ToRAW(
54           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
55           yuv_data.uv_row_stride,
56           const_cast<uint8*>(output_buffer->plane(0).buffer),
57           output_buffer->plane(0).stride.row_stride_bytes,
58           buffer.dimension().width, buffer.dimension().height);
59       if (ret != 0) {
60         return CreateStatusWithPayload(
61             StatusCode::kUnknown, "Libyuv NV12ToRAW operation failed.",
62             TfLiteSupportStatus::kImageProcessingBackendError);
63       }
64       break;
65     }
66     case FrameBuffer::Format::kRGBA: {
67       // The libyuv ABGR format is interleaved RGBA format in memory.
68       int ret = libyuv::NV12ToABGR(
69           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
70           yuv_data.uv_row_stride,
71           const_cast<uint8*>(output_buffer->plane(0).buffer),
72           output_buffer->plane(0).stride.row_stride_bytes,
73           buffer.dimension().width, buffer.dimension().height);
74       if (ret != 0) {
75         return CreateStatusWithPayload(
76             StatusCode::kUnknown, "Libyuv NV12ToABGR operation failed.",
77             TfLiteSupportStatus::kImageProcessingBackendError);
78       }
79       break;
80     }
81     case FrameBuffer::Format::kYV12:
82     case FrameBuffer::Format::kYV21: {
83       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
84                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
85       int ret = libyuv::NV12ToI420(
86           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
87           yuv_data.uv_row_stride, const_cast<uint8_t*>(output_data.y_buffer),
88           output_data.y_row_stride, const_cast<uint8_t*>(output_data.u_buffer),
89           output_data.uv_row_stride, const_cast<uint8_t*>(output_data.v_buffer),
90           output_data.uv_row_stride, output_buffer->dimension().width,
91           output_buffer->dimension().height);
92       if (ret != 0) {
93         return CreateStatusWithPayload(
94             StatusCode::kUnknown, "Libyuv NV12ToI420 operation failed.",
95             TfLiteSupportStatus::kImageProcessingBackendError);
96       }
97       break;
98     }
99     case FrameBuffer::Format::kNV21: {
100       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
101                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
102       libyuv::CopyPlane(yuv_data.y_buffer, yuv_data.y_row_stride,
103                         const_cast<uint8*>(output_data.y_buffer),
104                         output_data.y_row_stride, buffer.dimension().width,
105                         buffer.dimension().height);
106       ASSIGN_OR_RETURN(
107           const FrameBuffer::Dimension uv_plane_dimension,
108           GetUvPlaneDimension(buffer.dimension(), buffer.format()));
109       libyuv::SwapUVPlane(yuv_data.u_buffer, yuv_data.uv_row_stride,
110                           const_cast<uint8*>(output_data.v_buffer),
111                           output_data.uv_row_stride, uv_plane_dimension.width,
112                           uv_plane_dimension.height);
113       break;
114     }
115     case FrameBuffer::Format::kGRAY: {
116       libyuv::CopyPlane(yuv_data.y_buffer, yuv_data.y_row_stride,
117                         const_cast<uint8*>(output_buffer->plane(0).buffer),
118                         output_buffer->plane(0).stride.row_stride_bytes,
119                         output_buffer->dimension().width,
120                         output_buffer->dimension().height);
121       break;
122     }
123     default:
124       return absl::InternalError(absl::StrFormat("Format %i is not supported.",
125                                                  output_buffer->format()));
126   }
127   return absl::OkStatus();
128 }
129 
130 // Converts NV21 `buffer` into the `output_buffer` of the target color space.
131 // Supported output format includes RGB24 and YV21.
ConvertFromNv21(const FrameBuffer & buffer,FrameBuffer * output_buffer)132 absl::Status ConvertFromNv21(const FrameBuffer& buffer,
133                              FrameBuffer* output_buffer) {
134   ASSIGN_OR_RETURN(FrameBuffer::YuvData yuv_data,
135                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
136   switch (output_buffer->format()) {
137     case FrameBuffer::Format::kRGB: {
138       // The RAW format of Libyuv represents the 8-bit interleaved RGB format in
139       // the big endian style with R being the first byte in memory.
140       int ret = libyuv::NV21ToRAW(
141           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.v_buffer,
142           yuv_data.uv_pixel_stride,
143           const_cast<uint8*>(output_buffer->plane(0).buffer),
144           output_buffer->plane(0).stride.row_stride_bytes,
145           buffer.dimension().width, buffer.dimension().height);
146       if (ret != 0) {
147         return CreateStatusWithPayload(
148             StatusCode::kUnknown, "Libyuv NV21ToRAW operation failed.",
149             TfLiteSupportStatus::kImageProcessingBackendError);
150       }
151       break;
152     }
153     case FrameBuffer::Format::kRGBA: {
154       // The libyuv ABGR format is interleaved RGBA format in memory.
155       int ret = libyuv::NV21ToABGR(
156           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.v_buffer,
157           yuv_data.uv_pixel_stride,
158           const_cast<uint8*>(output_buffer->plane(0).buffer),
159           output_buffer->plane(0).stride.row_stride_bytes,
160           buffer.dimension().width, buffer.dimension().height);
161       if (ret != 0) {
162         return CreateStatusWithPayload(
163             StatusCode::kUnknown, "Libyuv NV21ToABGR operation failed.",
164             TfLiteSupportStatus::kImageProcessingBackendError);
165       }
166       break;
167     }
168     case FrameBuffer::Format::kYV12:
169     case FrameBuffer::Format::kYV21: {
170       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
171                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
172       int ret = libyuv::NV21ToI420(
173           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.v_buffer,
174           yuv_data.uv_row_stride, const_cast<uint8_t*>(output_data.y_buffer),
175           output_data.y_row_stride, const_cast<uint8_t*>(output_data.u_buffer),
176           output_data.uv_row_stride, const_cast<uint8_t*>(output_data.v_buffer),
177           output_data.uv_row_stride, output_buffer->dimension().width,
178           output_buffer->dimension().height);
179       if (ret != 0) {
180         return CreateStatusWithPayload(
181             StatusCode::kUnknown, "Libyuv NV21ToI420 operation failed.",
182             TfLiteSupportStatus::kImageProcessingBackendError);
183       }
184       break;
185     }
186     case FrameBuffer::Format::kNV12: {
187       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
188                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
189       libyuv::CopyPlane(yuv_data.y_buffer, yuv_data.y_row_stride,
190                         const_cast<uint8*>(output_data.y_buffer),
191                         output_data.y_row_stride, buffer.dimension().width,
192                         buffer.dimension().height);
193       ASSIGN_OR_RETURN(
194           const FrameBuffer::Dimension uv_plane_dimension,
195           GetUvPlaneDimension(buffer.dimension(), buffer.format()));
196       libyuv::SwapUVPlane(yuv_data.v_buffer, yuv_data.uv_row_stride,
197                           const_cast<uint8*>(output_data.u_buffer),
198                           output_data.uv_row_stride, uv_plane_dimension.width,
199                           uv_plane_dimension.height);
200       break;
201     }
202     case FrameBuffer::Format::kGRAY: {
203       libyuv::CopyPlane(yuv_data.y_buffer, yuv_data.y_row_stride,
204                         const_cast<uint8*>(output_buffer->plane(0).buffer),
205                         output_buffer->plane(0).stride.row_stride_bytes,
206                         output_buffer->dimension().width,
207                         output_buffer->dimension().height);
208       break;
209     }
210     default:
211       return CreateStatusWithPayload(
212           StatusCode::kInternal,
213           absl::StrFormat("Format %i is not supported.",
214                           output_buffer->format()),
215           TfLiteSupportStatus::kImageProcessingError);
216   }
217   return absl::OkStatus();
218 }
219 
220 // Converts YV12/YV21 `buffer` to the `output_buffer` of the target color space.
221 // Supported output format includes RGB24, NV12, and NV21.
ConvertFromYv(const FrameBuffer & buffer,FrameBuffer * output_buffer)222 absl::Status ConvertFromYv(const FrameBuffer& buffer,
223                            FrameBuffer* output_buffer) {
224   ASSIGN_OR_RETURN(FrameBuffer::YuvData yuv_data,
225                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
226   switch (output_buffer->format()) {
227     case FrameBuffer::Format::kRGB: {
228       // The RAW format of Libyuv represents the 8-bit interleaved RGB format in
229       // the big endian style with R being the first byte in memory.
230       int ret = libyuv::I420ToRAW(
231           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
232           yuv_data.uv_row_stride, yuv_data.v_buffer, yuv_data.uv_row_stride,
233           const_cast<uint8*>(output_buffer->plane(0).buffer),
234           output_buffer->plane(0).stride.row_stride_bytes,
235           buffer.dimension().width, buffer.dimension().height);
236       if (ret != 0) {
237         return CreateStatusWithPayload(
238             StatusCode::kUnknown, "Libyuv I420ToRAW operation failed.",
239             TfLiteSupportStatus::kImageProcessingBackendError);
240       }
241       break;
242     }
243     case FrameBuffer::Format::kRGBA: {
244       // The libyuv ABGR format is interleaved RGBA format in memory.
245       int ret = libyuv::I420ToABGR(
246           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
247           yuv_data.uv_row_stride, yuv_data.v_buffer, yuv_data.uv_row_stride,
248           const_cast<uint8*>(output_buffer->plane(0).buffer),
249           output_buffer->plane(0).stride.row_stride_bytes,
250           buffer.dimension().width, buffer.dimension().height);
251       if (ret != 0) {
252         return CreateStatusWithPayload(
253             StatusCode::kUnknown, "Libyuv I420ToABGR operation failed.",
254             TfLiteSupportStatus::kImageProcessingBackendError);
255       }
256       break;
257     }
258     case FrameBuffer::Format::kNV12: {
259       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
260                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
261       int ret = libyuv::I420ToNV12(
262           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
263           yuv_data.uv_row_stride, yuv_data.v_buffer, yuv_data.uv_row_stride,
264           const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
265           const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
266           output_buffer->dimension().width, output_buffer->dimension().height);
267       if (ret != 0) {
268         return CreateStatusWithPayload(
269             StatusCode::kUnknown, "Libyuv I420ToNV12 operation failed.",
270             TfLiteSupportStatus::kImageProcessingBackendError);
271       }
272       break;
273     }
274     case FrameBuffer::Format::kNV21: {
275       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
276                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
277       int ret = libyuv::I420ToNV21(
278           yuv_data.y_buffer, yuv_data.y_row_stride, yuv_data.u_buffer,
279           yuv_data.uv_row_stride, yuv_data.v_buffer, yuv_data.uv_row_stride,
280           const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
281           const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
282           output_buffer->dimension().width, output_buffer->dimension().height);
283       if (ret != 0) {
284         return CreateStatusWithPayload(
285             StatusCode::kUnknown, "Libyuv I420ToNV21 operation failed.",
286             TfLiteSupportStatus::kImageProcessingBackendError);
287       }
288       break;
289     }
290     case FrameBuffer::Format::kGRAY: {
291       libyuv::CopyPlane(yuv_data.y_buffer, yuv_data.y_row_stride,
292                         const_cast<uint8*>(output_buffer->plane(0).buffer),
293                         output_buffer->plane(0).stride.row_stride_bytes,
294                         output_buffer->dimension().width,
295                         output_buffer->dimension().height);
296       break;
297     }
298     case FrameBuffer::Format::kYV12:
299     case FrameBuffer::Format::kYV21: {
300       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_yuv_data,
301                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
302       ASSIGN_OR_RETURN(
303           const FrameBuffer::Dimension uv_plane_dimension,
304           GetUvPlaneDimension(buffer.dimension(), buffer.format()));
305       libyuv::CopyPlane(yuv_data.y_buffer, yuv_data.y_row_stride,
306                         const_cast<uint8*>(output_yuv_data.y_buffer),
307                         output_yuv_data.y_row_stride, buffer.dimension().width,
308                         buffer.dimension().height);
309       libyuv::CopyPlane(yuv_data.u_buffer, yuv_data.uv_row_stride,
310                         const_cast<uint8*>(output_yuv_data.u_buffer),
311                         output_yuv_data.uv_row_stride, uv_plane_dimension.width,
312                         uv_plane_dimension.height);
313       libyuv::CopyPlane(yuv_data.v_buffer, yuv_data.uv_row_stride,
314                         const_cast<uint8*>(output_yuv_data.v_buffer),
315                         output_yuv_data.uv_row_stride, uv_plane_dimension.width,
316                         uv_plane_dimension.height);
317       break;
318     }
319     default:
320       return CreateStatusWithPayload(
321           StatusCode::kInternal,
322           absl::StrFormat("Format %i is not supported.",
323                           output_buffer->format()),
324           TfLiteSupportStatus::kImageProcessingError);
325   }
326   return absl::OkStatus();
327 }
328 
329 // Resizes YV12/YV21 `buffer` to the target `output_buffer`.
ResizeYv(const FrameBuffer & buffer,FrameBuffer * output_buffer)330 absl::Status ResizeYv(const FrameBuffer& buffer, FrameBuffer* output_buffer) {
331   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
332                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
333   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
334                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
335   // TODO(b/151217096): Choose the optimal image resizing filter to optimize
336   // the model inference performance.
337   int ret = libyuv::I420Scale(
338       input_data.y_buffer, input_data.y_row_stride, input_data.u_buffer,
339       input_data.uv_row_stride, input_data.v_buffer, input_data.uv_row_stride,
340       buffer.dimension().width, buffer.dimension().height,
341       const_cast<uint8_t*>(output_data.y_buffer), output_data.y_row_stride,
342       const_cast<uint8_t*>(output_data.u_buffer), output_data.uv_row_stride,
343       const_cast<uint8_t*>(output_data.v_buffer), output_data.uv_row_stride,
344       output_buffer->dimension().width, output_buffer->dimension().height,
345       libyuv::FilterMode::kFilterBilinear);
346   if (ret != 0) {
347     return CreateStatusWithPayload(
348         StatusCode::kUnknown, "Libyuv I420Scale operation failed.",
349         TfLiteSupportStatus::kImageProcessingBackendError);
350   }
351   return absl::OkStatus();
352 }
353 
354 // Resizes NV12/NV21 `buffer` to the target `output_buffer`.
ResizeNv(const FrameBuffer & buffer,FrameBuffer * output_buffer)355 absl::Status ResizeNv(const FrameBuffer& buffer, FrameBuffer* output_buffer) {
356   const int buffer_size =
357       GetFrameBufferByteSize(buffer.dimension(), FrameBuffer::Format::kYV21);
358   auto yuv_raw_buffer = absl::make_unique<uint8[]>(buffer_size);
359   ASSIGN_OR_RETURN(
360       std::unique_ptr<FrameBuffer> yuv_buffer,
361       CreateFromRawBuffer(yuv_raw_buffer.get(), buffer.dimension(),
362                           FrameBuffer::Format::kYV21, buffer.orientation()));
363   // TODO(b/151375918): Current implementation is a workaround by converting
364   // input NV12/NV21 buffer to the YV12 formats, resizing the YV12 buffer, and
365   // converting the resized YV12 buffer back to the target format. Consider
366   // optimizes this by adding the support of NV12/NV21 resizing in Libyuv.
367   if (buffer.format() == FrameBuffer::Format::kNV12) {
368     RETURN_IF_ERROR(ConvertFromNv12(buffer, yuv_buffer.get()));
369   } else if (buffer.format() == FrameBuffer::Format::kNV21) {
370     RETURN_IF_ERROR(ConvertFromNv21(buffer, yuv_buffer.get()));
371   } else {
372     return CreateStatusWithPayload(
373         StatusCode::kInternal,
374         absl::StrFormat("Format %i is not supported.", buffer.format()),
375         TfLiteSupportStatus::kImageProcessingError);
376   }
377 
378   const int resized_buffer_size = GetFrameBufferByteSize(
379       output_buffer->dimension(), FrameBuffer::Format::kYV12);
380   auto resized_yuv_raw_buffer = absl::make_unique<uint8[]>(resized_buffer_size);
381   ASSIGN_OR_RETURN(std::unique_ptr<FrameBuffer> resized_yuv_buffer,
382                    CreateFromRawBuffer(resized_yuv_raw_buffer.get(),
383                                        output_buffer->dimension(),
384                                        FrameBuffer::Format::kYV12,
385                                        output_buffer->orientation()));
386   RETURN_IF_ERROR(ResizeYv(*yuv_buffer, resized_yuv_buffer.get()));
387 
388   RETURN_IF_ERROR(ConvertFromYv(*resized_yuv_buffer, output_buffer));
389   return absl::OkStatus();
390 }
391 
392 // Converts `buffer` to libyuv ARGB format and stores the conversion result
393 // in `dest_argb`.
ConvertRgbToArgb(const FrameBuffer & buffer,uint8 * dest_argb,int dest_stride_argb)394 absl::Status ConvertRgbToArgb(const FrameBuffer& buffer, uint8* dest_argb,
395                               int dest_stride_argb) {
396   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(buffer));
397   if (buffer.format() != FrameBuffer::Format::kRGB) {
398     return CreateStatusWithPayload(StatusCode::kInternal,
399                                    "RGB input format is expected.",
400                                    TfLiteSupportStatus::kImageProcessingError);
401   }
402 
403   if (dest_argb == nullptr || dest_stride_argb <= 0) {
404     return CreateStatusWithPayload(
405         StatusCode::kInternal,
406         "Invalid destination arguments for ConvertRgbToArgb.",
407         TfLiteSupportStatus::kImageProcessingError);
408   }
409 
410   if (buffer.plane_count() > 1) {
411     return CreateStatusWithPayload(
412         StatusCode::kInternal,
413         absl::StrFormat("Only single plane is supported for format %i.",
414                         buffer.format()),
415         TfLiteSupportStatus::kImageProcessingError);
416   }
417   int ret = libyuv::RGB24ToARGB(
418       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
419       dest_argb, dest_stride_argb, buffer.dimension().width,
420       buffer.dimension().height);
421   if (ret != 0) {
422     return CreateStatusWithPayload(
423         StatusCode::kUnknown, "Libyuv RGB24ToARGB operation failed.",
424         TfLiteSupportStatus::kImageProcessingBackendError);
425   }
426   return absl::OkStatus();
427 }
428 
429 // Converts `src_argb` in libyuv ARGB format to FrameBuffer::kRGB format and
430 // stores the conversion result in `output_buffer`.
ConvertArgbToRgb(uint8 * src_argb,int src_stride_argb,FrameBuffer * output_buffer)431 absl::Status ConvertArgbToRgb(uint8* src_argb, int src_stride_argb,
432                               FrameBuffer* output_buffer) {
433   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(*output_buffer));
434   if (output_buffer->format() != FrameBuffer::Format::kRGB) {
435     return absl::InternalError("RGB input format is expected.");
436   }
437 
438   if (src_argb == nullptr || src_stride_argb <= 0) {
439     return CreateStatusWithPayload(
440         StatusCode::kInternal, "Invalid source arguments for ConvertArgbToRgb.",
441         TfLiteSupportStatus::kImageProcessingError);
442   }
443 
444   if (output_buffer->plane_count() > 1) {
445     return CreateStatusWithPayload(
446         StatusCode::kInternal,
447         absl::StrFormat("Only single plane is supported for format %i.",
448                         output_buffer->format()),
449         TfLiteSupportStatus::kImageProcessingError);
450   }
451   int ret = libyuv::ARGBToRGB24(
452       src_argb, src_stride_argb,
453       const_cast<uint8*>(output_buffer->plane(0).buffer),
454       output_buffer->plane(0).stride.row_stride_bytes,
455       output_buffer->dimension().width, output_buffer->dimension().height);
456 
457   if (ret != 0) {
458     return CreateStatusWithPayload(
459         StatusCode::kUnknown, "Libyuv ARGBToRGB24 operation failed.",
460         TfLiteSupportStatus::kImageProcessingBackendError);
461   }
462   return absl::OkStatus();
463 }
464 
465 // Converts `buffer` in FrameBuffer::kRGBA format to libyuv ARGB (BGRA in
466 // memory) format and stores the conversion result in `dest_argb`.
ConvertRgbaToArgb(const FrameBuffer & buffer,uint8 * dest_argb,int dest_stride_argb)467 absl::Status ConvertRgbaToArgb(const FrameBuffer& buffer, uint8* dest_argb,
468                                int dest_stride_argb) {
469   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(buffer));
470   if (buffer.format() != FrameBuffer::Format::kRGBA) {
471     return CreateStatusWithPayload(
472         StatusCode::kInternal, "RGBA input format is expected.",
473         TfLiteSupportStatus::kImageProcessingBackendError);
474   }
475 
476   if (dest_argb == nullptr || dest_stride_argb <= 0) {
477     return CreateStatusWithPayload(
478         StatusCode::kInternal,
479         "Invalid source arguments for ConvertRgbaToArgb.",
480         TfLiteSupportStatus::kImageProcessingBackendError);
481   }
482 
483   if (buffer.plane_count() > 1) {
484     return CreateStatusWithPayload(
485         StatusCode::kInternal,
486         absl::StrFormat("Only single plane is supported for format %i.",
487                         buffer.format()),
488         TfLiteSupportStatus::kImageProcessingError);
489   }
490 
491   int ret = libyuv::ABGRToARGB(
492       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
493       dest_argb, dest_stride_argb, buffer.dimension().width,
494       buffer.dimension().height);
495   if (ret != 0) {
496     return CreateStatusWithPayload(
497         StatusCode::kInternal, "Libyuv ABGRToARGB operation failed.",
498         TfLiteSupportStatus::kImageProcessingBackendError);
499   }
500   return absl::OkStatus();
501 }
502 
503 // Converts kRGB `buffer` to the `output_buffer` of the target color space.
ConvertFromRgb(const FrameBuffer & buffer,FrameBuffer * output_buffer)504 absl::Status ConvertFromRgb(const FrameBuffer& buffer,
505                             FrameBuffer* output_buffer) {
506   if (output_buffer->format() == FrameBuffer::Format::kGRAY) {
507     int ret = libyuv::RAWToJ400(
508         buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
509         const_cast<uint8*>(output_buffer->plane(0).buffer),
510         output_buffer->plane(0).stride.row_stride_bytes,
511         buffer.dimension().width, buffer.dimension().height);
512     if (ret != 0) {
513       return CreateStatusWithPayload(
514           StatusCode::kInternal, "Libyuv RAWToJ400 operation failed.",
515           TfLiteSupportStatus::kImageProcessingBackendError);
516     }
517     return absl::OkStatus();
518   } else if (output_buffer->format() == FrameBuffer::Format::kYV12 ||
519              output_buffer->format() == FrameBuffer::Format::kYV21 ||
520              output_buffer->format() == FrameBuffer::Format::kNV12 ||
521              output_buffer->format() == FrameBuffer::Format::kNV21) {
522     // libyuv does not support conversion directly from kRGB to kNV12 / kNV21.
523     // For kNV12 / kNV21, the implementation converts the kRGB to I420,
524     // then converts I420 to kNV12 / kNV21.
525     // TODO(b/153000936): use libyuv::RawToNV12 / libyuv::RawToNV21 when they
526     // are ready.
527     FrameBuffer::YuvData yuv_data;
528     std::unique_ptr<uint8[]> tmp_yuv_buffer;
529     std::unique_ptr<FrameBuffer> yuv_frame_buffer;
530     if (output_buffer->format() == FrameBuffer::Format::kNV12 ||
531         output_buffer->format() == FrameBuffer::Format::kNV21) {
532       tmp_yuv_buffer = absl::make_unique<uint8[]>(
533           GetFrameBufferByteSize(buffer.dimension(), output_buffer->format()));
534       ASSIGN_OR_RETURN(
535           yuv_frame_buffer,
536           CreateFromRawBuffer(tmp_yuv_buffer.get(), buffer.dimension(),
537                               FrameBuffer::Format::kYV21,
538                               output_buffer->orientation()));
539       ASSIGN_OR_RETURN(
540           yuv_data, FrameBuffer::GetYuvDataFromFrameBuffer(*yuv_frame_buffer));
541     } else {
542       ASSIGN_OR_RETURN(yuv_data,
543                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
544     }
545     int ret = libyuv::RAWToI420(
546         buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
547         const_cast<uint8*>(yuv_data.y_buffer), yuv_data.y_row_stride,
548         const_cast<uint8*>(yuv_data.u_buffer), yuv_data.uv_row_stride,
549         const_cast<uint8*>(yuv_data.v_buffer), yuv_data.uv_row_stride,
550         buffer.dimension().width, buffer.dimension().height);
551     if (ret != 0) {
552       return CreateStatusWithPayload(
553           StatusCode::kInternal, "Libyuv RAWToI420 operation failed.",
554           TfLiteSupportStatus::kImageProcessingBackendError);
555     }
556     if (output_buffer->format() == FrameBuffer::Format::kNV12 ||
557         output_buffer->format() == FrameBuffer::Format::kNV21) {
558       return ConvertFromYv(*yuv_frame_buffer, output_buffer);
559     }
560     return absl::OkStatus();
561   }
562   return CreateStatusWithPayload(
563       StatusCode::kInternal,
564       absl::StrFormat("Format %i is not supported.", output_buffer->format()),
565       TfLiteSupportStatus::kImageProcessingError);
566 }
567 
568 // Converts kRGBA `buffer` to the `output_buffer` of the target color space.
ConvertFromRgba(const FrameBuffer & buffer,FrameBuffer * output_buffer)569 absl::Status ConvertFromRgba(const FrameBuffer& buffer,
570                              FrameBuffer* output_buffer) {
571   switch (output_buffer->format()) {
572     case FrameBuffer::Format::kGRAY: {
573       // libyuv does not support convert kRGBA (ABGR) foramat. In this method,
574       // the implementation converts kRGBA format to ARGB and use ARGB buffer
575       // for conversion.
576       // TODO(b/141181395): Use libyuv::ABGRToJ400 when it is ready.
577 
578       // Convert kRGBA to ARGB
579       int argb_buffer_size = GetFrameBufferByteSize(buffer.dimension(),
580                                                     FrameBuffer::Format::kRGBA);
581       auto argb_buffer = absl::make_unique<uint8[]>(argb_buffer_size);
582       const int argb_row_bytes = buffer.dimension().width * kRgbaPixelBytes;
583       RETURN_IF_ERROR(
584           ConvertRgbaToArgb(buffer, argb_buffer.get(), argb_row_bytes));
585 
586       // Convert ARGB to kGRAY
587       int ret = libyuv::ARGBToJ400(
588           argb_buffer.get(), argb_row_bytes,
589           const_cast<uint8*>(output_buffer->plane(0).buffer),
590           output_buffer->plane(0).stride.row_stride_bytes,
591           buffer.dimension().width, buffer.dimension().height);
592       if (ret != 0) {
593         return CreateStatusWithPayload(
594             StatusCode::kUnknown, "Libyuv ARGBToJ400 operation failed.",
595             TfLiteSupportStatus::kImageProcessingBackendError);
596       }
597       break;
598     }
599     case FrameBuffer::Format::kNV12: {
600       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
601                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
602       int ret = libyuv::ABGRToNV12(
603           buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
604           const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
605           const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
606           buffer.dimension().width, buffer.dimension().height);
607       if (ret != 0) {
608         return CreateStatusWithPayload(
609             StatusCode::kUnknown, "Libyuv ABGRToNV12 operation failed.",
610             TfLiteSupportStatus::kImageProcessingBackendError);
611       }
612       break;
613     }
614     case FrameBuffer::Format::kNV21: {
615       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
616                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
617       int ret = libyuv::ABGRToNV21(
618           buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
619           const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
620           const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
621           buffer.dimension().width, buffer.dimension().height);
622       if (ret != 0) {
623         return CreateStatusWithPayload(
624             StatusCode::kUnknown, "Libyuv ABGRToNV21 operation failed.",
625             TfLiteSupportStatus::kImageProcessingBackendError);
626       }
627       break;
628     }
629     case FrameBuffer::Format::kYV12:
630     case FrameBuffer::Format::kYV21: {
631       ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
632                        FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
633       int ret = libyuv::ABGRToI420(
634           buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
635           const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
636           const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
637           const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
638           buffer.dimension().width, buffer.dimension().height);
639       if (ret != 0) {
640         return CreateStatusWithPayload(
641             StatusCode::kUnknown, "Libyuv ABGRToI420 operation failed.",
642             TfLiteSupportStatus::kImageProcessingBackendError);
643       }
644       break;
645     }
646     case FrameBuffer::Format::kRGB: {
647       // ARGB is BGRA in memory and RGB24 is BGR in memory. The removal of the
648       // alpha channel will not impact the RGB ordering.
649       int ret = libyuv::ARGBToRGB24(
650           buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
651           const_cast<uint8*>(output_buffer->plane(0).buffer),
652           output_buffer->plane(0).stride.row_stride_bytes,
653           buffer.dimension().width, buffer.dimension().height);
654       if (ret != 0) {
655         return CreateStatusWithPayload(
656             StatusCode::kUnknown, "Libyuv ABGRToRGB24 operation failed.",
657             TfLiteSupportStatus::kImageProcessingBackendError);
658       }
659       break;
660     }
661     default:
662       return CreateStatusWithPayload(
663           StatusCode::kInternal,
664           absl::StrFormat("Convert Rgba to format %i is not supported.",
665                           output_buffer->format()),
666           TfLiteSupportStatus::kImageProcessingError);
667   }
668   return absl::OkStatus();
669 }
670 
671 // Returns libyuv rotation based on counter-clockwise angle_deg.
GetLibyuvRotationMode(int angle_deg)672 libyuv::RotationMode GetLibyuvRotationMode(int angle_deg) {
673   switch (angle_deg) {
674     case 90:
675       return libyuv::kRotate270;
676     case 270:
677       return libyuv::kRotate90;
678     case 180:
679       return libyuv::kRotate180;
680     default:
681       return libyuv::kRotate0;
682   }
683 }
684 
RotateRgba(const FrameBuffer & buffer,int angle_deg,FrameBuffer * output_buffer)685 absl::Status RotateRgba(const FrameBuffer& buffer, int angle_deg,
686                         FrameBuffer* output_buffer) {
687   if (buffer.plane_count() > 1) {
688     return CreateStatusWithPayload(
689         StatusCode::kInternal,
690         absl::StrFormat("Only single plane is supported for format %i.",
691                         buffer.format()),
692         TfLiteSupportStatus::kImageProcessingError);
693   }
694 
695   // libyuv::ARGBRotate assumes RGBA buffer is in the interleaved format.
696   int ret = libyuv::ARGBRotate(
697       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
698       const_cast<uint8*>(output_buffer->plane(0).buffer),
699       output_buffer->plane(0).stride.row_stride_bytes, buffer.dimension().width,
700       buffer.dimension().height, GetLibyuvRotationMode(angle_deg % 360));
701   if (ret != 0) {
702     return CreateStatusWithPayload(
703         StatusCode::kUnknown, "Libyuv ARGBRotate operation failed.",
704         TfLiteSupportStatus::kImageProcessingBackendError);
705   }
706   return absl::OkStatus();
707 }
708 
RotateRgb(const FrameBuffer & buffer,int angle_deg,FrameBuffer * output_buffer)709 absl::Status RotateRgb(const FrameBuffer& buffer, int angle_deg,
710                        FrameBuffer* output_buffer) {
711   // libyuv does not support rotate kRGB (RGB24) foramat. In this method, the
712   // implementation converts kRGB format to ARGB and use ARGB buffer for
713   // rotation. The result is then convert back to RGB.
714 
715   // Convert RGB to ARGB
716   int argb_buffer_size =
717       GetFrameBufferByteSize(buffer.dimension(), FrameBuffer::Format::kRGBA);
718   auto argb_buffer = absl::make_unique<uint8[]>(argb_buffer_size);
719   const int argb_row_bytes = buffer.dimension().width * kRgbaPixelBytes;
720   RETURN_IF_ERROR(ConvertRgbToArgb(buffer, argb_buffer.get(), argb_row_bytes));
721 
722   // Rotate ARGB
723   auto argb_rotated_buffer = absl::make_unique<uint8[]>(argb_buffer_size);
724   int rotated_row_bytes = output_buffer->dimension().width * kRgbaPixelBytes;
725   // TODO(b/151954340): Optimize the current implementation by utilizing
726   // ARGBMirror for 180 degree rotation.
727   int ret = libyuv::ARGBRotate(
728       argb_buffer.get(), argb_row_bytes, argb_rotated_buffer.get(),
729       rotated_row_bytes, buffer.dimension().width, buffer.dimension().height,
730       GetLibyuvRotationMode(angle_deg % 360));
731   if (ret != 0) {
732     return CreateStatusWithPayload(
733         StatusCode::kUnknown, "Libyuv ARGBRotate operation failed.",
734         TfLiteSupportStatus::kImageProcessingBackendError);
735   }
736 
737   // Convert ARGB to RGB
738   return ConvertArgbToRgb(argb_rotated_buffer.get(), rotated_row_bytes,
739                           output_buffer);
740 }
741 
RotateGray(const FrameBuffer & buffer,int angle_deg,FrameBuffer * output_buffer)742 absl::Status RotateGray(const FrameBuffer& buffer, int angle_deg,
743                         FrameBuffer* output_buffer) {
744   if (buffer.plane_count() > 1) {
745     return CreateStatusWithPayload(
746         StatusCode::kInternal,
747         absl::StrFormat("Only single plane is supported for format %i.",
748                         buffer.format()),
749         TfLiteSupportStatus::kImageProcessingError);
750   }
751   int ret = libyuv::RotatePlane(
752       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
753       const_cast<uint8*>(output_buffer->plane(0).buffer),
754       output_buffer->plane(0).stride.row_stride_bytes, buffer.dimension().width,
755       buffer.dimension().height, GetLibyuvRotationMode(angle_deg % 360));
756   if (ret != 0) {
757     return CreateStatusWithPayload(
758         StatusCode::kUnknown, "Libyuv RotatePlane operation failed.",
759         TfLiteSupportStatus::kImageProcessingBackendError);
760   }
761   return absl::OkStatus();
762 }
763 
764 // Rotates YV12/YV21 frame buffer.
RotateYv(const FrameBuffer & buffer,int angle_deg,FrameBuffer * output_buffer)765 absl::Status RotateYv(const FrameBuffer& buffer, int angle_deg,
766                       FrameBuffer* output_buffer) {
767   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
768                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
769   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
770                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
771   int ret = libyuv::I420Rotate(
772       input_data.y_buffer, input_data.y_row_stride, input_data.u_buffer,
773       input_data.uv_row_stride, input_data.v_buffer, input_data.uv_row_stride,
774       const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
775       const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
776       const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
777       buffer.dimension().width, buffer.dimension().height,
778       GetLibyuvRotationMode(angle_deg));
779   if (ret != 0) {
780     return CreateStatusWithPayload(
781         StatusCode::kUnknown, "Libyuv I420Rotate operation failed.",
782         TfLiteSupportStatus::kImageProcessingBackendError);
783   }
784   return absl::OkStatus();
785 }
786 
787 // Rotates NV12/NV21 frame buffer.
788 // TODO(b/152097364): Refactor NV12/NV21 rotation after libyuv explicitly
789 // support that.
RotateNv(const FrameBuffer & buffer,int angle_deg,FrameBuffer * output_buffer)790 absl::Status RotateNv(const FrameBuffer& buffer, int angle_deg,
791                       FrameBuffer* output_buffer) {
792   if (buffer.format() != FrameBuffer::Format::kNV12 &&
793       buffer.format() != FrameBuffer::Format::kNV21) {
794     return CreateStatusWithPayload(StatusCode::kInternal,
795                                    "kNV12 or kNV21 input formats are expected.",
796                                    TfLiteSupportStatus::kImageProcessingError);
797   }
798   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
799                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
800   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
801                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
802   const int rotated_buffer_size = GetFrameBufferByteSize(
803       output_buffer->dimension(), FrameBuffer::Format::kYV21);
804   auto rotated_yuv_raw_buffer = absl::make_unique<uint8[]>(rotated_buffer_size);
805   ASSIGN_OR_RETURN(std::unique_ptr<FrameBuffer> rotated_yuv_buffer,
806                    CreateFromRawBuffer(
807                        rotated_yuv_raw_buffer.get(), output_buffer->dimension(),
808                        /*target_format=*/FrameBuffer::Format::kYV21,
809                        output_buffer->orientation()));
810   ASSIGN_OR_RETURN(FrameBuffer::YuvData rotated_yuv_data,
811                    FrameBuffer::GetYuvDataFromFrameBuffer(*rotated_yuv_buffer));
812   // Get the first chroma plane and use it as the u plane. This is a workaround
813   // for optimizing NV21 rotation. For NV12, the implementation is logical
814   // correct. For NV21, use v plane as u plane will make the UV planes swapped
815   // in the intermediate rotated I420 frame. The output buffer is finally built
816   // by merging the swapped UV planes which produces V first interleaved UV
817   // buffer.
818   const uint8* chroma_buffer = buffer.format() == FrameBuffer::Format::kNV12
819                                    ? input_data.u_buffer
820                                    : input_data.v_buffer;
821   // Rotate the Y plane and store into the Y plane in `output_buffer`. Rotate
822   // the interleaved UV plane and store into the interleaved UV plane in
823   // `rotated_yuv_buffer`.
824   int ret = libyuv::NV12ToI420Rotate(
825       input_data.y_buffer, input_data.y_row_stride, chroma_buffer,
826       input_data.uv_row_stride, const_cast<uint8*>(output_data.y_buffer),
827       output_data.y_row_stride, const_cast<uint8*>(rotated_yuv_data.u_buffer),
828       rotated_yuv_data.uv_row_stride,
829       const_cast<uint8*>(rotated_yuv_data.v_buffer),
830       rotated_yuv_data.uv_row_stride, buffer.dimension().width,
831       buffer.dimension().height, GetLibyuvRotationMode(angle_deg % 360));
832   if (ret != 0) {
833     return CreateStatusWithPayload(
834         StatusCode::kUnknown, "Libyuv Nv12ToI420Rotate operation failed.",
835         TfLiteSupportStatus::kImageProcessingBackendError);
836   }
837   // Merge rotated UV planes into the output buffer. For NV21, the UV buffer of
838   // the intermediate I420 frame is swapped. MergeUVPlane builds the interleaved
839   // VU buffer for NV21 by putting the U plane in the I420 frame which is
840   // actually the V plane from the input buffer first.
841   const uint8* output_chroma_buffer =
842       buffer.format() == FrameBuffer::Format::kNV12 ? output_data.u_buffer
843                                                     : output_data.v_buffer;
844   // The width and height arguments of `libyuv::MergeUVPlane()` represent the
845   // width and height of the UV planes.
846   libyuv::MergeUVPlane(
847       rotated_yuv_data.u_buffer, rotated_yuv_data.uv_row_stride,
848       rotated_yuv_data.v_buffer, rotated_yuv_data.uv_row_stride,
849       const_cast<uint8*>(output_chroma_buffer), output_data.uv_row_stride,
850       (output_buffer->dimension().width + 1) / 2,
851       (output_buffer->dimension().height + 1) / 2);
852   return absl::OkStatus();
853 }
854 
855 // This method only supports kGRAY, kRGB, and kRGBA format.
FlipPlaneVertically(const FrameBuffer & buffer,FrameBuffer * output_buffer)856 absl::Status FlipPlaneVertically(const FrameBuffer& buffer,
857                                  FrameBuffer* output_buffer) {
858   if (buffer.plane_count() > 1) {
859     return CreateStatusWithPayload(
860         StatusCode::kInternal,
861         absl::StrFormat("Only single plane is supported for format %i.",
862                         buffer.format()),
863         TfLiteSupportStatus::kImageProcessingError);
864   }
865 
866   ASSIGN_OR_RETURN(int pixel_stride, GetPixelStrides(buffer.format()));
867 
868   // Flip vertically is achieved by passing in negative height.
869   libyuv::CopyPlane(buffer.plane(0).buffer,
870                     buffer.plane(0).stride.row_stride_bytes,
871                     const_cast<uint8*>(output_buffer->plane(0).buffer),
872                     output_buffer->plane(0).stride.row_stride_bytes,
873                     output_buffer->dimension().width * pixel_stride,
874                     -output_buffer->dimension().height);
875 
876   return absl::OkStatus();
877 }
878 
879 // This method only supports kGRAY, kRGBA, and kRGB formats.
CropPlane(const FrameBuffer & buffer,int x0,int y0,int x1,int y1,FrameBuffer * output_buffer)880 absl::Status CropPlane(const FrameBuffer& buffer, int x0, int y0, int x1,
881                        int y1, FrameBuffer* output_buffer) {
882   if (buffer.plane_count() > 1) {
883     return CreateStatusWithPayload(
884         StatusCode::kInternal,
885         absl::StrFormat("Only single plane is supported for format %i.",
886                         buffer.format()),
887         TfLiteSupportStatus::kImageProcessingError);
888   }
889 
890   ASSIGN_OR_RETURN(int pixel_stride, GetPixelStrides(buffer.format()));
891   FrameBuffer::Dimension crop_dimension = GetCropDimension(x0, x1, y0, y1);
892 
893   // Cropping is achieved by adjusting origin to (x0, y0).
894   int adjusted_offset =
895       buffer.plane(0).stride.row_stride_bytes * y0 + x0 * pixel_stride;
896 
897   libyuv::CopyPlane(buffer.plane(0).buffer + adjusted_offset,
898                     buffer.plane(0).stride.row_stride_bytes,
899                     const_cast<uint8*>(output_buffer->plane(0).buffer),
900                     output_buffer->plane(0).stride.row_stride_bytes,
901                     crop_dimension.width * pixel_stride, crop_dimension.height);
902 
903   return absl::OkStatus();
904 }
905 
906 // Crops NV12/NV21 FrameBuffer to the subregion defined by the top left pixel
907 // position (x0, y0) and the bottom right pixel position (x1, y1).
CropNv(const FrameBuffer & buffer,int x0,int y0,int x1,int y1,FrameBuffer * output_buffer)908 absl::Status CropNv(const FrameBuffer& buffer, int x0, int y0, int x1, int y1,
909                     FrameBuffer* output_buffer) {
910   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
911                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
912   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
913                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
914   // Crop Y plane by copying the buffer with the origin offset to (x0, y0).
915   int crop_offset_y = input_data.y_row_stride * y0 + x0;
916   int crop_width = x1 - x0 + 1;
917   int crop_height = y1 - y0 + 1;
918   libyuv::CopyPlane(input_data.y_buffer + crop_offset_y,
919                     input_data.y_row_stride,
920                     const_cast<uint8*>(output_data.y_buffer),
921                     output_data.y_row_stride, crop_width, crop_height);
922   // Crop chroma plane by copying the buffer with the origin offset to
923   // (x0 / 2, y0 / 2);
924   // TODO(b/152629712): Investigate the impact of color shifting caused by the
925   // bounding box with odd X or Y starting positions.
926   int crop_offset_chroma = input_data.uv_row_stride * (y0 / 2) +
927                            input_data.uv_pixel_stride * (x0 / 2);
928   ASSIGN_OR_RETURN(const uint8* input_chroma_buffer, GetUvRawBuffer(buffer));
929   ASSIGN_OR_RETURN(const uint8* output_chroma_buffer,
930                    GetUvRawBuffer(*output_buffer));
931   libyuv::CopyPlane(
932       input_chroma_buffer + crop_offset_chroma, input_data.uv_row_stride,
933       const_cast<uint8*>(output_chroma_buffer), output_data.uv_row_stride,
934       /*width=*/(crop_width + 1) / 2 * 2, /*height=*/(crop_height + 1) / 2);
935   return absl::OkStatus();
936 }
937 
938 // Crops YV12/YV21 FrameBuffer to the subregion defined by the top left pixel
939 // position (x0, y0) and the bottom right pixel position (x1, y1).
CropYv(const FrameBuffer & buffer,int x0,int y0,int x1,int y1,FrameBuffer * output_buffer)940 absl::Status CropYv(const FrameBuffer& buffer, int x0, int y0, int x1, int y1,
941                     FrameBuffer* output_buffer) {
942   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
943                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
944   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
945                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
946   // Crop Y plane by copying the buffer with the origin offset to (x0, y0).
947   int crop_offset_y = input_data.y_row_stride * y0 + x0;
948   FrameBuffer::Dimension crop_dimension = GetCropDimension(x0, x1, y0, y1);
949   libyuv::CopyPlane(
950       input_data.y_buffer + crop_offset_y, input_data.y_row_stride,
951       const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
952       crop_dimension.width, crop_dimension.height);
953   // Crop U plane by copying the buffer with the origin offset to
954   // (x0 / 2, y0 / 2).
955   ASSIGN_OR_RETURN(const FrameBuffer::Dimension crop_uv_dimension,
956                    GetUvPlaneDimension(crop_dimension, buffer.format()));
957   // TODO(b/152629712): Investigate the impact of color shifting caused by the
958   // bounding box with odd X or Y starting positions.
959   int crop_offset_chroma = input_data.uv_row_stride * (y0 / 2) +
960                            input_data.uv_pixel_stride * (x0 / 2);
961   libyuv::CopyPlane(
962       input_data.u_buffer + crop_offset_chroma, input_data.uv_row_stride,
963       const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
964       crop_uv_dimension.width, crop_uv_dimension.height);
965   // Crop V plane by copying the buffer with the origin offset to
966   // (x0 / 2, y0 / 2);
967   libyuv::CopyPlane(
968       input_data.v_buffer + crop_offset_chroma, input_data.uv_row_stride,
969       const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
970       /*width=*/(crop_dimension.width + 1) / 2,
971       /*height=*/(crop_dimension.height + 1) / 2);
972   return absl::OkStatus();
973 }
974 
CropResizeYuv(const FrameBuffer & buffer,int x0,int y0,int x1,int y1,FrameBuffer * output_buffer)975 absl::Status CropResizeYuv(const FrameBuffer& buffer, int x0, int y0, int x1,
976                            int y1, FrameBuffer* output_buffer) {
977   FrameBuffer::Dimension crop_dimension = GetCropDimension(x0, x1, y0, y1);
978   if (crop_dimension == output_buffer->dimension()) {
979     switch (buffer.format()) {
980       case FrameBuffer::Format::kNV12:
981       case FrameBuffer::Format::kNV21:
982         return CropNv(buffer, x0, y0, x1, y1, output_buffer);
983       case FrameBuffer::Format::kYV12:
984       case FrameBuffer::Format::kYV21:
985         return CropYv(buffer, x0, y0, x1, y1, output_buffer);
986       default:
987         return CreateStatusWithPayload(
988             StatusCode::kInternal,
989             absl::StrFormat("Format %i is not supported.", buffer.format()),
990             TfLiteSupportStatus::kImageProcessingError);
991     }
992   }
993   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
994                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
995   // Cropping YUV planes by offsetting the origins of each plane.
996   // TODO(b/152629712): Investigate the impact of color shifting caused by the
997   // bounding box with odd X or Y starting positions.
998   const int plane_y_offset = input_data.y_row_stride * y0 + x0;
999   const int plane_uv_offset = input_data.uv_row_stride * (y0 / 2) +
1000                               input_data.uv_pixel_stride * (x0 / 2);
1001   FrameBuffer::Plane cropped_plane_y = {
1002       /*buffer=*/input_data.y_buffer + plane_y_offset,
1003       /*stride=*/{input_data.y_row_stride, /*pixel_stride_bytes=*/1}};
1004   FrameBuffer::Plane cropped_plane_u = {
1005       /*buffer=*/input_data.u_buffer + plane_uv_offset,
1006       /*stride=*/{input_data.uv_row_stride, input_data.uv_pixel_stride}};
1007   FrameBuffer::Plane cropped_plane_v = {
1008       /*buffer=*/input_data.v_buffer + plane_uv_offset,
1009       /*stride=*/{input_data.uv_row_stride, input_data.uv_pixel_stride}};
1010 
1011   switch (buffer.format()) {
1012     case FrameBuffer::Format::kNV12: {
1013       std::unique_ptr<FrameBuffer> cropped_buffer = FrameBuffer::Create(
1014           {cropped_plane_y, cropped_plane_u, cropped_plane_v}, crop_dimension,
1015           buffer.format(), buffer.orientation());
1016       return ResizeNv(*cropped_buffer, output_buffer);
1017     }
1018     case FrameBuffer::Format::kNV21: {
1019       std::unique_ptr<FrameBuffer> cropped_buffer = FrameBuffer::Create(
1020           {cropped_plane_y, cropped_plane_v, cropped_plane_u}, crop_dimension,
1021           buffer.format(), buffer.orientation());
1022       return ResizeNv(*cropped_buffer, output_buffer);
1023     }
1024     case FrameBuffer::Format::kYV12: {
1025       std::unique_ptr<FrameBuffer> cropped_buffer = FrameBuffer::Create(
1026           {cropped_plane_y, cropped_plane_v, cropped_plane_u}, crop_dimension,
1027           buffer.format(), buffer.orientation());
1028       return ResizeYv(*cropped_buffer, output_buffer);
1029     }
1030     case FrameBuffer::Format::kYV21: {
1031       std::unique_ptr<FrameBuffer> cropped_buffer = FrameBuffer::Create(
1032           {cropped_plane_y, cropped_plane_u, cropped_plane_v}, crop_dimension,
1033           buffer.format(), buffer.orientation());
1034       return ResizeYv(*cropped_buffer, output_buffer);
1035     }
1036     default:
1037       return CreateStatusWithPayload(
1038           StatusCode::kInternal,
1039           absl::StrFormat("Format %i is not supported.", buffer.format()),
1040           TfLiteSupportStatus::kImageProcessingError);
1041   }
1042   return absl::OkStatus();
1043 }
1044 
FlipHorizontallyRgba(const FrameBuffer & buffer,FrameBuffer * output_buffer)1045 absl::Status FlipHorizontallyRgba(const FrameBuffer& buffer,
1046                                   FrameBuffer* output_buffer) {
1047   if (buffer.plane_count() > 1) {
1048     return CreateStatusWithPayload(
1049         StatusCode::kInternal,
1050         absl::StrFormat("Only single plane is supported for format %i.",
1051                         buffer.format()),
1052         TfLiteSupportStatus::kImageProcessingError);
1053   }
1054 
1055   int ret = libyuv::ARGBMirror(
1056       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
1057       const_cast<uint8*>(output_buffer->plane(0).buffer),
1058       output_buffer->plane(0).stride.row_stride_bytes,
1059       output_buffer->dimension().width, output_buffer->dimension().height);
1060 
1061   if (ret != 0) {
1062     return CreateStatusWithPayload(
1063         StatusCode::kUnknown, "Libyuv ARGBMirror operation failed.",
1064         TfLiteSupportStatus::kImageProcessingBackendError);
1065   }
1066 
1067   return absl::OkStatus();
1068 }
1069 
1070 // Flips `buffer` horizontally and store the result in `output_buffer`. This
1071 // method assumes all buffers have pixel stride equals to 1.
FlipHorizontallyPlane(const FrameBuffer & buffer,FrameBuffer * output_buffer)1072 absl::Status FlipHorizontallyPlane(const FrameBuffer& buffer,
1073                                    FrameBuffer* output_buffer) {
1074   if (buffer.plane_count() > 1) {
1075     return CreateStatusWithPayload(
1076         StatusCode::kInternal,
1077         absl::StrFormat("Only single plane is supported for format %i.",
1078                         buffer.format()),
1079         TfLiteSupportStatus::kImageProcessingError);
1080   }
1081   libyuv::MirrorPlane(
1082       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
1083       const_cast<uint8*>(output_buffer->plane(0).buffer),
1084       output_buffer->plane(0).stride.row_stride_bytes,
1085       output_buffer->dimension().width, output_buffer->dimension().height);
1086 
1087   return absl::OkStatus();
1088 }
1089 
ResizeRgb(const FrameBuffer & buffer,FrameBuffer * output_buffer)1090 absl::Status ResizeRgb(const FrameBuffer& buffer, FrameBuffer* output_buffer) {
1091   if (buffer.plane_count() > 1) {
1092     return CreateStatusWithPayload(
1093         StatusCode::kInternal,
1094         absl::StrFormat("Only single plane is supported for format %i.",
1095                         buffer.format()),
1096         TfLiteSupportStatus::kImageProcessingError);
1097   }
1098 
1099   // libyuv doesn't support scale kRGB (RGB24) foramat. In this method,
1100   // the implementation converts kRGB format to ARGB and use ARGB buffer for
1101   // scaling. The result is then convert back to RGB.
1102 
1103   // Convert RGB to ARGB
1104   int argb_buffer_size =
1105       GetFrameBufferByteSize(buffer.dimension(), FrameBuffer::Format::kRGBA);
1106   auto argb_buffer = absl::make_unique<uint8[]>(argb_buffer_size);
1107   const int argb_row_bytes = buffer.dimension().width * kRgbaPixelBytes;
1108   RETURN_IF_ERROR(ConvertRgbToArgb(buffer, argb_buffer.get(), argb_row_bytes));
1109 
1110   // Resize ARGB
1111   int resized_argb_buffer_size = GetFrameBufferByteSize(
1112       output_buffer->dimension(), FrameBuffer::Format::kRGBA);
1113   auto resized_argb_buffer =
1114       absl::make_unique<uint8[]>(resized_argb_buffer_size);
1115   int resized_argb_row_bytes =
1116       output_buffer->dimension().width * kRgbaPixelBytes;
1117   int ret = libyuv::ARGBScale(
1118       argb_buffer.get(), argb_row_bytes, buffer.dimension().width,
1119       buffer.dimension().height, resized_argb_buffer.get(),
1120       resized_argb_row_bytes, output_buffer->dimension().width,
1121       output_buffer->dimension().height, libyuv::FilterMode::kFilterBilinear);
1122   if (ret != 0) {
1123     return CreateStatusWithPayload(
1124         StatusCode::kUnknown, "Libyuv ARGBScale operation failed.",
1125         TfLiteSupportStatus::kImageProcessingBackendError);
1126   }
1127 
1128   // Convert ARGB to RGB
1129   return ConvertArgbToRgb(resized_argb_buffer.get(), resized_argb_row_bytes,
1130                           output_buffer);
1131 }
1132 
1133 // Horizontally flip `buffer` and store the result in `output_buffer`.
FlipHorizontallyRgb(const FrameBuffer & buffer,FrameBuffer * output_buffer)1134 absl::Status FlipHorizontallyRgb(const FrameBuffer& buffer,
1135                                  FrameBuffer* output_buffer) {
1136   if (buffer.plane_count() > 1) {
1137     return CreateStatusWithPayload(
1138         StatusCode::kInternal,
1139         absl::StrFormat("Only single plane is supported for format %i.",
1140                         buffer.format()),
1141         TfLiteSupportStatus::kImageProcessingError);
1142   }
1143 
1144 #if LIBYUV_VERSION >= 1747
1145   int ret = libyuv::RGB24Mirror(
1146       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
1147       const_cast<uint8*>(output_buffer->plane(0).buffer),
1148       output_buffer->plane(0).stride.row_stride_bytes, buffer.dimension().width,
1149       buffer.dimension().height);
1150   if (ret != 0) {
1151     return CreateStatusWithPayload(
1152         StatusCode::kUnknown, "Libyuv RGB24Mirror operation failed.",
1153         TfLiteSupportStatus::kImageProcessingBackendError);
1154   }
1155 
1156   return absl::OkStatus();
1157 #else
1158 #error LibyuvFrameBufferUtils requires LIBYUV_VERSION 1747 or above
1159 #endif  // LIBYUV_VERSION >= 1747
1160 }
1161 
ResizeRgba(const FrameBuffer & buffer,FrameBuffer * output_buffer)1162 absl::Status ResizeRgba(const FrameBuffer& buffer, FrameBuffer* output_buffer) {
1163   if (buffer.plane_count() > 1) {
1164     return CreateStatusWithPayload(
1165         StatusCode::kInternal,
1166         absl::StrFormat("Only single plane is supported for format %i.",
1167                         buffer.format()),
1168         TfLiteSupportStatus::kImageProcessingError);
1169   }
1170   int ret = libyuv::ARGBScale(
1171       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
1172       buffer.dimension().width, buffer.dimension().height,
1173       const_cast<uint8*>(output_buffer->plane(0).buffer),
1174       output_buffer->plane(0).stride.row_stride_bytes,
1175       output_buffer->dimension().width, output_buffer->dimension().height,
1176       libyuv::FilterMode::kFilterBilinear);
1177   if (ret != 0) {
1178     return CreateStatusWithPayload(
1179         StatusCode::kUnknown, "Libyuv ARGBScale operation failed.",
1180         TfLiteSupportStatus::kImageProcessingBackendError);
1181   }
1182   return absl::OkStatus();
1183 }
1184 
1185 // Flips NV12/NV21 FrameBuffer horizontally.
FlipHorizontallyNv(const FrameBuffer & buffer,FrameBuffer * output_buffer)1186 absl::Status FlipHorizontallyNv(const FrameBuffer& buffer,
1187                                 FrameBuffer* output_buffer) {
1188   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
1189                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
1190   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
1191                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
1192   ASSIGN_OR_RETURN(const uint8* input_chroma_buffer, GetUvRawBuffer(buffer));
1193   ASSIGN_OR_RETURN(const uint8* output_chroma_buffer,
1194                    GetUvRawBuffer(*output_buffer));
1195 
1196   int ret = libyuv::NV12Mirror(
1197       input_data.y_buffer, input_data.y_row_stride, input_chroma_buffer,
1198       input_data.uv_row_stride, const_cast<uint8*>(output_data.y_buffer),
1199       output_data.y_row_stride, const_cast<uint8*>(output_chroma_buffer),
1200       output_data.uv_row_stride, buffer.dimension().width,
1201       buffer.dimension().height);
1202 
1203   if (ret != 0) {
1204     return CreateStatusWithPayload(
1205         StatusCode::kUnknown, "Libyuv NV12Mirror operation failed.",
1206         TfLiteSupportStatus::kImageProcessingBackendError);
1207   }
1208 
1209   return absl::OkStatus();
1210 }
1211 
1212 // Flips YV12/YV21 FrameBuffer horizontally.
FlipHorizontallyYv(const FrameBuffer & buffer,FrameBuffer * output_buffer)1213 absl::Status FlipHorizontallyYv(const FrameBuffer& buffer,
1214                                 FrameBuffer* output_buffer) {
1215   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
1216                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
1217   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
1218                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
1219   int ret = libyuv::I420Mirror(
1220       input_data.y_buffer, input_data.y_row_stride, input_data.u_buffer,
1221       input_data.uv_row_stride, input_data.v_buffer, input_data.uv_row_stride,
1222       const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
1223       const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
1224       const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
1225       buffer.dimension().width, buffer.dimension().height);
1226   if (ret != 0) {
1227     return CreateStatusWithPayload(
1228         StatusCode::kUnknown, "Libyuv I420Mirror operation failed.",
1229         TfLiteSupportStatus::kImageProcessingBackendError);
1230   }
1231 
1232   return absl::OkStatus();
1233 }
1234 
1235 // Flips NV12/NV21 FrameBuffer vertically.
FlipVerticallyNv(const FrameBuffer & buffer,FrameBuffer * output_buffer)1236 absl::Status FlipVerticallyNv(const FrameBuffer& buffer,
1237                               FrameBuffer* output_buffer) {
1238   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
1239                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
1240   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
1241                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
1242   // Flip Y plane vertically by passing a negative height.
1243   libyuv::CopyPlane(input_data.y_buffer, input_data.y_row_stride,
1244                     const_cast<uint8*>(output_data.y_buffer),
1245                     output_data.y_row_stride, buffer.dimension().width,
1246                     -output_buffer->dimension().height);
1247   // Flip UV plane vertically by passing a negative height.
1248   ASSIGN_OR_RETURN(const uint8* input_chroma_buffer, GetUvRawBuffer(buffer));
1249   ASSIGN_OR_RETURN(const uint8* output_chroma_buffer,
1250                    GetUvRawBuffer(*output_buffer));
1251   ASSIGN_OR_RETURN(const FrameBuffer::Dimension uv_plane_dimension,
1252                    GetUvPlaneDimension(buffer.dimension(), buffer.format()));
1253   libyuv::CopyPlane(
1254       input_chroma_buffer, input_data.uv_row_stride,
1255       const_cast<uint8*>(output_chroma_buffer), output_data.uv_row_stride,
1256       /*width=*/uv_plane_dimension.width * 2, -uv_plane_dimension.height);
1257   return absl::OkStatus();
1258 }
1259 
1260 // Flips NV12/NV21 FrameBuffer vertically.
FlipVerticallyYv(const FrameBuffer & buffer,FrameBuffer * output_buffer)1261 absl::Status FlipVerticallyYv(const FrameBuffer& buffer,
1262                               FrameBuffer* output_buffer) {
1263   ASSIGN_OR_RETURN(FrameBuffer::YuvData input_data,
1264                    FrameBuffer::GetYuvDataFromFrameBuffer(buffer));
1265   ASSIGN_OR_RETURN(FrameBuffer::YuvData output_data,
1266                    FrameBuffer::GetYuvDataFromFrameBuffer(*output_buffer));
1267   // Flip buffer vertically by passing a negative height.
1268   int ret = libyuv::I420Copy(
1269       input_data.y_buffer, input_data.y_row_stride, input_data.u_buffer,
1270       input_data.uv_row_stride, input_data.v_buffer, input_data.uv_row_stride,
1271       const_cast<uint8*>(output_data.y_buffer), output_data.y_row_stride,
1272       const_cast<uint8*>(output_data.u_buffer), output_data.uv_row_stride,
1273       const_cast<uint8*>(output_data.v_buffer), output_data.uv_row_stride,
1274       buffer.dimension().width, -buffer.dimension().height);
1275   if (ret != 0) {
1276     return CreateStatusWithPayload(
1277         StatusCode::kUnknown, "Libyuv I420Copy operation failed.",
1278         TfLiteSupportStatus::kImageProcessingBackendError);
1279   }
1280   return absl::OkStatus();
1281 }
1282 
1283 // Resize `buffer` to metadata defined in `output_buffer`. This
1284 // method assumes buffer has pixel stride equals to 1 (grayscale equivalent).
ResizeGray(const FrameBuffer & buffer,FrameBuffer * output_buffer)1285 absl::Status ResizeGray(const FrameBuffer& buffer, FrameBuffer* output_buffer) {
1286   if (buffer.plane_count() > 1) {
1287     return CreateStatusWithPayload(
1288         StatusCode::kInternal,
1289         absl::StrFormat("Only single plane is supported for format %i.",
1290                         buffer.format()),
1291         TfLiteSupportStatus::kImageProcessingError);
1292   }
1293   libyuv::ScalePlane(
1294       buffer.plane(0).buffer, buffer.plane(0).stride.row_stride_bytes,
1295       buffer.dimension().width, buffer.dimension().height,
1296       const_cast<uint8*>(output_buffer->plane(0).buffer),
1297       output_buffer->plane(0).stride.row_stride_bytes,
1298       output_buffer->dimension().width, output_buffer->dimension().height,
1299       libyuv::FilterMode::kFilterBilinear);
1300   return absl::OkStatus();
1301 }
1302 
1303 // This method only supports kGRAY, kRGBA, and kRGB formats.
CropResize(const FrameBuffer & buffer,int x0,int y0,int x1,int y1,FrameBuffer * output_buffer)1304 absl::Status CropResize(const FrameBuffer& buffer, int x0, int y0, int x1,
1305                         int y1, FrameBuffer* output_buffer) {
1306   FrameBuffer::Dimension crop_dimension = GetCropDimension(x0, x1, y0, y1);
1307   if (crop_dimension == output_buffer->dimension()) {
1308     return CropPlane(buffer, x0, y0, x1, y1, output_buffer);
1309   }
1310 
1311   ASSIGN_OR_RETURN(int pixel_stride, GetPixelStrides(buffer.format()));
1312   // Cropping is achieved by adjusting origin to (x0, y0).
1313   int adjusted_offset =
1314       buffer.plane(0).stride.row_stride_bytes * y0 + x0 * pixel_stride;
1315   FrameBuffer::Plane plane = {
1316       /*buffer=*/buffer.plane(0).buffer + adjusted_offset,
1317       /*stride=*/{buffer.plane(0).stride.row_stride_bytes, pixel_stride}};
1318   auto adjusted_buffer =
1319       FrameBuffer::Create({plane}, crop_dimension, buffer.format(),
1320                           buffer.orientation(), buffer.timestamp());
1321 
1322   switch (buffer.format()) {
1323     case FrameBuffer::Format::kRGB:
1324       return ResizeRgb(*adjusted_buffer, output_buffer);
1325     case FrameBuffer::Format::kRGBA:
1326       return ResizeRgba(*adjusted_buffer, output_buffer);
1327     case FrameBuffer::Format::kGRAY:
1328       return ResizeGray(*adjusted_buffer, output_buffer);
1329     default:
1330       return CreateStatusWithPayload(
1331           StatusCode::kInternal,
1332           absl::StrFormat("Format %i is not supported.", buffer.format()),
1333           TfLiteSupportStatus::kImageProcessingError);
1334   }
1335 }
1336 }  // namespace
1337 
Crop(const FrameBuffer & buffer,int x0,int y0,int x1,int y1,FrameBuffer * output_buffer)1338 absl::Status LibyuvFrameBufferUtils::Crop(const FrameBuffer& buffer, int x0,
1339                                           int y0, int x1, int y1,
1340                                           FrameBuffer* output_buffer) {
1341   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(buffer));
1342   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(*output_buffer));
1343   RETURN_IF_ERROR(
1344       ValidateCropBufferInputs(buffer, *output_buffer, x0, y0, x1, y1));
1345   RETURN_IF_ERROR(ValidateBufferFormats(buffer, *output_buffer));
1346 
1347   switch (buffer.format()) {
1348     case FrameBuffer::Format::kRGBA:
1349     case FrameBuffer::Format::kRGB:
1350     case FrameBuffer::Format::kGRAY:
1351       return CropResize(buffer, x0, y0, x1, y1, output_buffer);
1352     case FrameBuffer::Format::kNV12:
1353     case FrameBuffer::Format::kNV21:
1354     case FrameBuffer::Format::kYV12:
1355     case FrameBuffer::Format::kYV21:
1356       return CropResizeYuv(buffer, x0, y0, x1, y1, output_buffer);
1357     default:
1358       return CreateStatusWithPayload(
1359           StatusCode::kInternal,
1360           absl::StrFormat("Format %i is not supported.", buffer.format()),
1361           TfLiteSupportStatus::kImageProcessingError);
1362   }
1363 }
1364 
Resize(const FrameBuffer & buffer,FrameBuffer * output_buffer)1365 absl::Status LibyuvFrameBufferUtils::Resize(const FrameBuffer& buffer,
1366                                             FrameBuffer* output_buffer) {
1367   RETURN_IF_ERROR(ValidateResizeBufferInputs(buffer, *output_buffer));
1368   switch (buffer.format()) {
1369     case FrameBuffer::Format::kYV12:
1370     case FrameBuffer::Format::kYV21:
1371       return ResizeYv(buffer, output_buffer);
1372     case FrameBuffer::Format::kNV12:
1373     case FrameBuffer::Format::kNV21:
1374       return ResizeNv(buffer, output_buffer);
1375     case FrameBuffer::Format::kRGB:
1376       return ResizeRgb(buffer, output_buffer);
1377     case FrameBuffer::Format::kRGBA:
1378       return ResizeRgba(buffer, output_buffer);
1379     case FrameBuffer::Format::kGRAY:
1380       return ResizeGray(buffer, output_buffer);
1381     default:
1382       return CreateStatusWithPayload(
1383           StatusCode::kInternal,
1384           absl::StrFormat("Format %i is not supported.", buffer.format()),
1385           TfLiteSupportStatus::kImageProcessingError);
1386   }
1387 }
1388 
Rotate(const FrameBuffer & buffer,int angle_deg,FrameBuffer * output_buffer)1389 absl::Status LibyuvFrameBufferUtils::Rotate(const FrameBuffer& buffer,
1390                                             int angle_deg,
1391                                             FrameBuffer* output_buffer) {
1392   RETURN_IF_ERROR(
1393       ValidateRotateBufferInputs(buffer, *output_buffer, angle_deg));
1394   RETURN_IF_ERROR(ValidateBufferFormats(buffer, *output_buffer));
1395   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(buffer));
1396   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(*output_buffer));
1397 
1398   switch (buffer.format()) {
1399     case FrameBuffer::Format::kGRAY:
1400       return RotateGray(buffer, angle_deg, output_buffer);
1401     case FrameBuffer::Format::kRGBA:
1402       return RotateRgba(buffer, angle_deg, output_buffer);
1403     case FrameBuffer::Format::kNV12:
1404     case FrameBuffer::Format::kNV21:
1405       return RotateNv(buffer, angle_deg, output_buffer);
1406     case FrameBuffer::Format::kYV12:
1407     case FrameBuffer::Format::kYV21:
1408       return RotateYv(buffer, angle_deg, output_buffer);
1409     case FrameBuffer::Format::kRGB:
1410       return RotateRgb(buffer, angle_deg, output_buffer);
1411     default:
1412       return CreateStatusWithPayload(
1413           StatusCode::kInternal,
1414           absl::StrFormat("Format %i is not supported.", buffer.format()),
1415           TfLiteSupportStatus::kImageProcessingError);
1416   }
1417 }
1418 
FlipHorizontally(const FrameBuffer & buffer,FrameBuffer * output_buffer)1419 absl::Status LibyuvFrameBufferUtils::FlipHorizontally(
1420     const FrameBuffer& buffer, FrameBuffer* output_buffer) {
1421   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(buffer));
1422   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(*output_buffer));
1423   RETURN_IF_ERROR(ValidateFlipBufferInputs(buffer, *output_buffer));
1424   RETURN_IF_ERROR(ValidateBufferFormats(buffer, *output_buffer));
1425 
1426   switch (buffer.format()) {
1427     case FrameBuffer::Format::kRGBA:
1428       return FlipHorizontallyRgba(buffer, output_buffer);
1429     case FrameBuffer::Format::kYV12:
1430     case FrameBuffer::Format::kYV21:
1431       return FlipHorizontallyYv(buffer, output_buffer);
1432     case FrameBuffer::Format::kNV12:
1433     case FrameBuffer::Format::kNV21:
1434       return FlipHorizontallyNv(buffer, output_buffer);
1435     case FrameBuffer::Format::kRGB:
1436       return FlipHorizontallyRgb(buffer, output_buffer);
1437     case FrameBuffer::Format::kGRAY:
1438       return FlipHorizontallyPlane(buffer, output_buffer);
1439     default:
1440       return CreateStatusWithPayload(
1441           StatusCode::kInternal,
1442           absl::StrFormat("Format %i is not supported.", buffer.format()),
1443           TfLiteSupportStatus::kImageProcessingError);
1444   }
1445 }
1446 
FlipVertically(const FrameBuffer & buffer,FrameBuffer * output_buffer)1447 absl::Status LibyuvFrameBufferUtils::FlipVertically(
1448     const FrameBuffer& buffer, FrameBuffer* output_buffer) {
1449   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(buffer));
1450   RETURN_IF_ERROR(ValidateBufferPlaneMetadata(*output_buffer));
1451   RETURN_IF_ERROR(ValidateFlipBufferInputs(buffer, *output_buffer));
1452   RETURN_IF_ERROR(ValidateBufferFormats(buffer, *output_buffer));
1453 
1454   switch (buffer.format()) {
1455     case FrameBuffer::Format::kRGBA:
1456     case FrameBuffer::Format::kRGB:
1457     case FrameBuffer::Format::kGRAY:
1458       return FlipPlaneVertically(buffer, output_buffer);
1459     case FrameBuffer::Format::kNV12:
1460     case FrameBuffer::Format::kNV21:
1461       return FlipVerticallyNv(buffer, output_buffer);
1462     case FrameBuffer::Format::kYV12:
1463     case FrameBuffer::Format::kYV21:
1464       return FlipVerticallyYv(buffer, output_buffer);
1465     default:
1466       return CreateStatusWithPayload(
1467           StatusCode::kInternal,
1468           absl::StrFormat("Format %i is not supported.", buffer.format()),
1469           TfLiteSupportStatus::kImageProcessingError);
1470   }
1471 }
1472 
Convert(const FrameBuffer & buffer,FrameBuffer * output_buffer)1473 absl::Status LibyuvFrameBufferUtils::Convert(const FrameBuffer& buffer,
1474                                              FrameBuffer* output_buffer) {
1475   RETURN_IF_ERROR(
1476       ValidateConvertFormats(buffer.format(), output_buffer->format()));
1477   switch (buffer.format()) {
1478     case FrameBuffer::Format::kNV12:
1479       return ConvertFromNv12(buffer, output_buffer);
1480     case FrameBuffer::Format::kNV21:
1481       return ConvertFromNv21(buffer, output_buffer);
1482     case FrameBuffer::Format::kYV12:
1483     case FrameBuffer::Format::kYV21:
1484       return ConvertFromYv(buffer, output_buffer);
1485     case FrameBuffer::Format::kRGB:
1486       return ConvertFromRgb(buffer, output_buffer);
1487     case FrameBuffer::Format::kRGBA:
1488       return ConvertFromRgba(buffer, output_buffer);
1489     default:
1490       return CreateStatusWithPayload(
1491           StatusCode::kInternal,
1492           absl::StrFormat("Format %i is not supported.", buffer.format()),
1493           TfLiteSupportStatus::kImageProcessingError);
1494   }
1495 }
1496 
1497 }  // namespace vision
1498 }  // namespace task
1499 }  // namespace tflite
1500