1 /*
2 * Android camera input device
3 *
4 * Copyright (C) 2017 Felix Matouschek
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #include <errno.h>
24 #include <pthread.h>
25 #include <stdatomic.h>
26 #include <stdbool.h>
27 #include <stdint.h>
28
29 #include <camera/NdkCameraDevice.h>
30 #include <camera/NdkCameraManager.h>
31 #include <media/NdkImage.h>
32 #include <media/NdkImageReader.h>
33
34 #include "libavformat/avformat.h"
35 #include "libavformat/internal.h"
36 #include "libavutil/avstring.h"
37 #include "libavutil/display.h"
38 #include "libavutil/imgutils.h"
39 #include "libavutil/log.h"
40 #include "libavutil/opt.h"
41 #include "libavutil/parseutils.h"
42 #include "libavutil/pixfmt.h"
43 #include "libavutil/threadmessage.h"
44 #include "libavutil/time.h"
45
46 /* This image format is available on all Android devices
47 * supporting the Camera2 API */
48 #define IMAGE_FORMAT_ANDROID AIMAGE_FORMAT_YUV_420_888
49
50 #define MAX_BUF_COUNT 2
51 #define VIDEO_STREAM_INDEX 0
52 #define VIDEO_TIMEBASE_ANDROID 1000000000
53
54 #define RETURN_CASE(x) case x: return AV_STRINGIFY(x);
55 #define RETURN_DEFAULT(x) default: return AV_STRINGIFY(x);
56
57 typedef struct AndroidCameraCtx {
58 const AVClass *class;
59
60 int requested_width;
61 int requested_height;
62 AVRational framerate;
63 int camera_index;
64 int input_queue_size;
65
66 uint8_t lens_facing;
67 int32_t sensor_orientation;
68 int width;
69 int height;
70 int32_t framerate_range[2];
71 int image_format;
72
73 ACameraManager *camera_mgr;
74 char *camera_id;
75 ACameraMetadata *camera_metadata;
76 ACameraDevice *camera_dev;
77 ACameraDevice_StateCallbacks camera_state_callbacks;
78 AImageReader *image_reader;
79 AImageReader_ImageListener image_listener;
80 ANativeWindow *image_reader_window;
81 ACaptureSessionOutputContainer *capture_session_output_container;
82 ACaptureSessionOutput *capture_session_output;
83 ACameraOutputTarget *camera_output_target;
84 ACaptureRequest *capture_request;
85 ACameraCaptureSession_stateCallbacks capture_session_state_callbacks;
86 ACameraCaptureSession *capture_session;
87
88 AVThreadMessageQueue *input_queue;
89 atomic_int exit;
90 atomic_int got_image_format;
91 } AndroidCameraCtx;
92
camera_status_string(camera_status_t val)93 static const char *camera_status_string(camera_status_t val)
94 {
95 switch(val) {
96 RETURN_CASE(ACAMERA_OK)
97 RETURN_CASE(ACAMERA_ERROR_UNKNOWN)
98 RETURN_CASE(ACAMERA_ERROR_INVALID_PARAMETER)
99 RETURN_CASE(ACAMERA_ERROR_CAMERA_DISCONNECTED)
100 RETURN_CASE(ACAMERA_ERROR_NOT_ENOUGH_MEMORY)
101 RETURN_CASE(ACAMERA_ERROR_METADATA_NOT_FOUND)
102 RETURN_CASE(ACAMERA_ERROR_CAMERA_DEVICE)
103 RETURN_CASE(ACAMERA_ERROR_CAMERA_SERVICE)
104 RETURN_CASE(ACAMERA_ERROR_SESSION_CLOSED)
105 RETURN_CASE(ACAMERA_ERROR_INVALID_OPERATION)
106 RETURN_CASE(ACAMERA_ERROR_STREAM_CONFIGURE_FAIL)
107 RETURN_CASE(ACAMERA_ERROR_CAMERA_IN_USE)
108 RETURN_CASE(ACAMERA_ERROR_MAX_CAMERA_IN_USE)
109 RETURN_CASE(ACAMERA_ERROR_CAMERA_DISABLED)
110 RETURN_CASE(ACAMERA_ERROR_PERMISSION_DENIED)
111 RETURN_DEFAULT(ACAMERA_ERROR_UNKNOWN)
112 }
113 }
114
media_status_string(media_status_t val)115 static const char *media_status_string(media_status_t val)
116 {
117 switch(val) {
118 RETURN_CASE(AMEDIA_OK)
119 RETURN_CASE(AMEDIA_ERROR_UNKNOWN)
120 RETURN_CASE(AMEDIA_ERROR_MALFORMED)
121 RETURN_CASE(AMEDIA_ERROR_UNSUPPORTED)
122 RETURN_CASE(AMEDIA_ERROR_INVALID_OBJECT)
123 RETURN_CASE(AMEDIA_ERROR_INVALID_PARAMETER)
124 RETURN_CASE(AMEDIA_ERROR_INVALID_OPERATION)
125 RETURN_CASE(AMEDIA_DRM_NOT_PROVISIONED)
126 RETURN_CASE(AMEDIA_DRM_RESOURCE_BUSY)
127 RETURN_CASE(AMEDIA_DRM_DEVICE_REVOKED)
128 RETURN_CASE(AMEDIA_DRM_SHORT_BUFFER)
129 RETURN_CASE(AMEDIA_DRM_SESSION_NOT_OPENED)
130 RETURN_CASE(AMEDIA_DRM_TAMPER_DETECTED)
131 RETURN_CASE(AMEDIA_DRM_VERIFY_FAILED)
132 RETURN_CASE(AMEDIA_DRM_NEED_KEY)
133 RETURN_CASE(AMEDIA_DRM_LICENSE_EXPIRED)
134 RETURN_CASE(AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE)
135 RETURN_CASE(AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED)
136 RETURN_CASE(AMEDIA_IMGREADER_CANNOT_LOCK_IMAGE)
137 RETURN_CASE(AMEDIA_IMGREADER_CANNOT_UNLOCK_IMAGE)
138 RETURN_CASE(AMEDIA_IMGREADER_IMAGE_NOT_LOCKED)
139 RETURN_DEFAULT(AMEDIA_ERROR_UNKNOWN)
140 }
141 }
142
error_state_callback_string(int val)143 static const char *error_state_callback_string(int val)
144 {
145 switch(val) {
146 RETURN_CASE(ERROR_CAMERA_IN_USE)
147 RETURN_CASE(ERROR_MAX_CAMERAS_IN_USE)
148 RETURN_CASE(ERROR_CAMERA_DISABLED)
149 RETURN_CASE(ERROR_CAMERA_DEVICE)
150 RETURN_CASE(ERROR_CAMERA_SERVICE)
151 default:
152 return "ERROR_CAMERA_UNKNOWN";
153 }
154 }
155
camera_dev_disconnected(void * context,ACameraDevice * device)156 static void camera_dev_disconnected(void *context, ACameraDevice *device)
157 {
158 AVFormatContext *avctx = context;
159 AndroidCameraCtx *ctx = avctx->priv_data;
160 atomic_store(&ctx->exit, 1);
161 av_log(avctx, AV_LOG_ERROR, "Camera with id %s disconnected.\n",
162 ACameraDevice_getId(device));
163 }
164
camera_dev_error(void * context,ACameraDevice * device,int error)165 static void camera_dev_error(void *context, ACameraDevice *device, int error)
166 {
167 AVFormatContext *avctx = context;
168 AndroidCameraCtx *ctx = avctx->priv_data;
169 atomic_store(&ctx->exit, 1);
170 av_log(avctx, AV_LOG_ERROR, "Error %s on camera with id %s.\n",
171 error_state_callback_string(error), ACameraDevice_getId(device));
172 }
173
open_camera(AVFormatContext * avctx)174 static int open_camera(AVFormatContext *avctx)
175 {
176 AndroidCameraCtx *ctx = avctx->priv_data;
177 camera_status_t ret;
178 ACameraIdList *camera_ids;
179
180 ret = ACameraManager_getCameraIdList(ctx->camera_mgr, &camera_ids);
181 if (ret != ACAMERA_OK) {
182 av_log(avctx, AV_LOG_ERROR, "Failed to get camera id list, error: %s.\n",
183 camera_status_string(ret));
184 return AVERROR_EXTERNAL;
185 }
186
187 if (ctx->camera_index < camera_ids->numCameras) {
188 ctx->camera_id = av_strdup(camera_ids->cameraIds[ctx->camera_index]);
189 if (!ctx->camera_id) {
190 av_log(avctx, AV_LOG_ERROR, "Failed to allocate memory for camera_id.\n");
191 return AVERROR(ENOMEM);
192 }
193 } else {
194 av_log(avctx, AV_LOG_ERROR, "No camera with index %d available.\n",
195 ctx->camera_index);
196 return AVERROR(ENXIO);
197 }
198
199 ACameraManager_deleteCameraIdList(camera_ids);
200
201 ret = ACameraManager_getCameraCharacteristics(ctx->camera_mgr,
202 ctx->camera_id, &ctx->camera_metadata);
203 if (ret != ACAMERA_OK) {
204 av_log(avctx, AV_LOG_ERROR, "Failed to get metadata for camera with id %s, error: %s.\n",
205 ctx->camera_id, camera_status_string(ret));
206 return AVERROR_EXTERNAL;
207 }
208
209 ctx->camera_state_callbacks.context = avctx;
210 ctx->camera_state_callbacks.onDisconnected = camera_dev_disconnected;
211 ctx->camera_state_callbacks.onError = camera_dev_error;
212
213 ret = ACameraManager_openCamera(ctx->camera_mgr, ctx->camera_id,
214 &ctx->camera_state_callbacks, &ctx->camera_dev);
215 if (ret != ACAMERA_OK) {
216 av_log(avctx, AV_LOG_ERROR, "Failed to open camera with id %s, error: %s.\n",
217 ctx->camera_id, camera_status_string(ret));
218 return AVERROR_EXTERNAL;
219 }
220
221 return 0;
222 }
223
get_sensor_orientation(AVFormatContext * avctx)224 static void get_sensor_orientation(AVFormatContext *avctx)
225 {
226 AndroidCameraCtx *ctx = avctx->priv_data;
227 ACameraMetadata_const_entry lens_facing;
228 ACameraMetadata_const_entry sensor_orientation;
229
230 ACameraMetadata_getConstEntry(ctx->camera_metadata,
231 ACAMERA_LENS_FACING, &lens_facing);
232 ACameraMetadata_getConstEntry(ctx->camera_metadata,
233 ACAMERA_SENSOR_ORIENTATION, &sensor_orientation);
234
235 ctx->lens_facing = lens_facing.data.u8[0];
236 ctx->sensor_orientation = sensor_orientation.data.i32[0];
237 }
238
match_video_size(AVFormatContext * avctx)239 static void match_video_size(AVFormatContext *avctx)
240 {
241 AndroidCameraCtx *ctx = avctx->priv_data;
242 ACameraMetadata_const_entry available_configs;
243 int found = 0;
244
245 ACameraMetadata_getConstEntry(ctx->camera_metadata,
246 ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
247 &available_configs);
248
249 for (int i = 0; i < available_configs.count; i++) {
250 int32_t input = available_configs.data.i32[i * 4 + 3];
251 int32_t format = available_configs.data.i32[i * 4 + 0];
252
253 if (input) {
254 continue;
255 }
256
257 if (format == IMAGE_FORMAT_ANDROID) {
258 int32_t width = available_configs.data.i32[i * 4 + 1];
259 int32_t height = available_configs.data.i32[i * 4 + 2];
260
261 //Same ratio
262 if ((ctx->requested_width == width && ctx->requested_height == height) ||
263 (ctx->requested_width == height && ctx->requested_height == width)) {
264 ctx->width = width;
265 ctx->height = height;
266 found = 1;
267 break;
268 }
269 }
270 }
271
272 if (!found || ctx->width == 0 || ctx->height == 0) {
273 ctx->width = available_configs.data.i32[1];
274 ctx->height = available_configs.data.i32[2];
275
276 av_log(avctx, AV_LOG_WARNING,
277 "Requested video_size %dx%d not available, falling back to %dx%d\n",
278 ctx->requested_width, ctx->requested_height, ctx->width, ctx->height);
279 }
280
281 return;
282 }
283
match_framerate(AVFormatContext * avctx)284 static void match_framerate(AVFormatContext *avctx)
285 {
286 AndroidCameraCtx *ctx = avctx->priv_data;
287 ACameraMetadata_const_entry available_framerates;
288 int found = 0;
289 int current_best_match = -1;
290 int requested_framerate = av_q2d(ctx->framerate);
291
292 ACameraMetadata_getConstEntry(ctx->camera_metadata,
293 ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES,
294 &available_framerates);
295
296 for (int i = 0; i < available_framerates.count; i++) {
297 int32_t min = available_framerates.data.i32[i * 2 + 0];
298 int32_t max = available_framerates.data.i32[i * 2 + 1];
299
300 if (requested_framerate == max) {
301 if (min == max) {
302 ctx->framerate_range[0] = min;
303 ctx->framerate_range[1] = max;
304 found = 1;
305 break;
306 } else if (current_best_match >= 0) {
307 int32_t current_best_match_min = available_framerates.data.i32[current_best_match * 2 + 0];
308 if (min > current_best_match_min) {
309 current_best_match = i;
310 }
311 } else {
312 current_best_match = i;
313 }
314 }
315 }
316
317 if (!found) {
318 if (current_best_match >= 0) {
319 ctx->framerate_range[0] = available_framerates.data.i32[current_best_match * 2 + 0];
320 ctx->framerate_range[1] = available_framerates.data.i32[current_best_match * 2 + 1];
321
322 } else {
323 ctx->framerate_range[0] = available_framerates.data.i32[0];
324 ctx->framerate_range[1] = available_framerates.data.i32[1];
325 }
326
327 av_log(avctx, AV_LOG_WARNING,
328 "Requested framerate %d not available, falling back to min: %d and max: %d fps\n",
329 requested_framerate, ctx->framerate_range[0], ctx->framerate_range[1]);
330 }
331
332 return;
333 }
334
get_image_format(AVFormatContext * avctx,AImage * image)335 static int get_image_format(AVFormatContext *avctx, AImage *image)
336 {
337 AndroidCameraCtx *ctx = avctx->priv_data;
338 int32_t image_pixelstrides[2];
339 uint8_t *image_plane_data[2];
340 int plane_data_length[2];
341
342 for (int i = 0; i < 2; i++) {
343 AImage_getPlanePixelStride(image, i + 1, &image_pixelstrides[i]);
344 AImage_getPlaneData(image, i + 1, &image_plane_data[i], &plane_data_length[i]);
345 }
346
347 if (image_pixelstrides[0] != image_pixelstrides[1]) {
348 av_log(avctx, AV_LOG_ERROR,
349 "Pixel strides of U and V plane should have been the same.\n");
350 return AVERROR_EXTERNAL;
351 }
352
353 switch (image_pixelstrides[0]) {
354 case 1:
355 ctx->image_format = AV_PIX_FMT_YUV420P;
356 break;
357 case 2:
358 if (image_plane_data[0] < image_plane_data[1]) {
359 ctx->image_format = AV_PIX_FMT_NV12;
360 } else {
361 ctx->image_format = AV_PIX_FMT_NV21;
362 }
363 break;
364 default:
365 av_log(avctx, AV_LOG_ERROR,
366 "Unknown pixel stride %d of U and V plane, cannot determine camera image format.\n",
367 image_pixelstrides[0]);
368 return AVERROR(ENOSYS);
369 }
370
371 return 0;
372 }
373
image_available(void * context,AImageReader * reader)374 static void image_available(void *context, AImageReader *reader)
375 {
376 AVFormatContext *avctx = context;
377 AndroidCameraCtx *ctx = avctx->priv_data;
378 media_status_t media_status;
379 int ret = 0;
380
381 AImage *image;
382 int64_t image_timestamp;
383 int32_t image_linestrides[4];
384 uint8_t *image_plane_data[4];
385 int plane_data_length[4];
386
387 AVPacket pkt;
388 int pkt_buffer_size = 0;
389
390 media_status = AImageReader_acquireLatestImage(reader, &image);
391 if (media_status != AMEDIA_OK) {
392 if (media_status == AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE) {
393 av_log(avctx, AV_LOG_WARNING,
394 "An image reader frame was discarded");
395 } else {
396 av_log(avctx, AV_LOG_ERROR,
397 "Failed to acquire latest image from image reader, error: %s.\n",
398 media_status_string(media_status));
399 ret = AVERROR_EXTERNAL;
400 }
401 goto error;
402 }
403
404 // Silently drop frames when exit is set
405 if (atomic_load(&ctx->exit)) {
406 goto error;
407 }
408
409 // Determine actual image format
410 if (!atomic_load(&ctx->got_image_format)) {
411 ret = get_image_format(avctx, image);
412 if (ret < 0) {
413 av_log(avctx, AV_LOG_ERROR,
414 "Could not get image format of camera.\n");
415 goto error;
416 } else {
417 atomic_store(&ctx->got_image_format, 1);
418 }
419 }
420
421 pkt_buffer_size = av_image_get_buffer_size(ctx->image_format, ctx->width, ctx->height, 32);
422 AImage_getTimestamp(image, &image_timestamp);
423
424 AImage_getPlaneRowStride(image, 0, &image_linestrides[0]);
425 AImage_getPlaneData(image, 0, &image_plane_data[0], &plane_data_length[0]);
426
427 switch (ctx->image_format) {
428 case AV_PIX_FMT_YUV420P:
429 AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
430 AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
431 AImage_getPlaneRowStride(image, 2, &image_linestrides[2]);
432 AImage_getPlaneData(image, 2, &image_plane_data[2], &plane_data_length[2]);
433 break;
434 case AV_PIX_FMT_NV12:
435 AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
436 AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
437 break;
438 case AV_PIX_FMT_NV21:
439 AImage_getPlaneRowStride(image, 2, &image_linestrides[1]);
440 AImage_getPlaneData(image, 2, &image_plane_data[1], &plane_data_length[1]);
441 break;
442 default:
443 av_log(avctx, AV_LOG_ERROR, "Unsupported camera image format.\n");
444 ret = AVERROR(ENOSYS);
445 goto error;
446 }
447
448 ret = av_new_packet(&pkt, pkt_buffer_size);
449 if (ret < 0) {
450 av_log(avctx, AV_LOG_ERROR,
451 "Failed to create new av packet, error: %s.\n", av_err2str(ret));
452 goto error;
453 }
454
455 pkt.stream_index = VIDEO_STREAM_INDEX;
456 pkt.pts = image_timestamp;
457 av_image_copy_to_buffer(pkt.data, pkt_buffer_size,
458 (const uint8_t * const *) image_plane_data,
459 image_linestrides, ctx->image_format,
460 ctx->width, ctx->height, 32);
461
462 ret = av_thread_message_queue_send(ctx->input_queue, &pkt, AV_THREAD_MESSAGE_NONBLOCK);
463
464 error:
465 if (ret < 0) {
466 if (ret != AVERROR(EAGAIN)) {
467 av_log(avctx, AV_LOG_ERROR,
468 "Error while processing new image, error: %s.\n", av_err2str(ret));
469 av_thread_message_queue_set_err_recv(ctx->input_queue, ret);
470 atomic_store(&ctx->exit, 1);
471 } else {
472 av_log(avctx, AV_LOG_WARNING,
473 "Input queue was full, dropping frame, consider raising the input_queue_size option (current value: %d)\n",
474 ctx->input_queue_size);
475 }
476 if (pkt_buffer_size) {
477 av_packet_unref(&pkt);
478 }
479 }
480
481 AImage_delete(image);
482
483 return;
484 }
485
create_image_reader(AVFormatContext * avctx)486 static int create_image_reader(AVFormatContext *avctx)
487 {
488 AndroidCameraCtx *ctx = avctx->priv_data;
489 media_status_t ret;
490
491 ret = AImageReader_new(ctx->width, ctx->height, IMAGE_FORMAT_ANDROID,
492 MAX_BUF_COUNT, &ctx->image_reader);
493 if (ret != AMEDIA_OK) {
494 av_log(avctx, AV_LOG_ERROR,
495 "Failed to create image reader, error: %s.\n", media_status_string(ret));
496 return AVERROR_EXTERNAL;
497 }
498
499 ctx->image_listener.context = avctx;
500 ctx->image_listener.onImageAvailable = image_available;
501
502 ret = AImageReader_setImageListener(ctx->image_reader, &ctx->image_listener);
503 if (ret != AMEDIA_OK) {
504 av_log(avctx, AV_LOG_ERROR,
505 "Failed to set image listener on image reader, error: %s.\n",
506 media_status_string(ret));
507 return AVERROR_EXTERNAL;
508 }
509
510 ret = AImageReader_getWindow(ctx->image_reader, &ctx->image_reader_window);
511 if (ret != AMEDIA_OK) {
512 av_log(avctx, AV_LOG_ERROR,
513 "Could not get image reader window, error: %s.\n",
514 media_status_string(ret));
515 return AVERROR_EXTERNAL;
516 }
517
518 return 0;
519 }
520
capture_session_closed(void * context,ACameraCaptureSession * session)521 static void capture_session_closed(void *context, ACameraCaptureSession *session)
522 {
523 av_log(context, AV_LOG_INFO, "Android camera capture session was closed.\n");
524 }
525
capture_session_ready(void * context,ACameraCaptureSession * session)526 static void capture_session_ready(void *context, ACameraCaptureSession *session)
527 {
528 av_log(context, AV_LOG_INFO, "Android camera capture session is ready.\n");
529 }
530
capture_session_active(void * context,ACameraCaptureSession * session)531 static void capture_session_active(void *context, ACameraCaptureSession *session)
532 {
533 av_log(context, AV_LOG_INFO, "Android camera capture session is active.\n");
534 }
535
create_capture_session(AVFormatContext * avctx)536 static int create_capture_session(AVFormatContext *avctx)
537 {
538 AndroidCameraCtx *ctx = avctx->priv_data;
539 camera_status_t ret;
540
541 ret = ACaptureSessionOutputContainer_create(&ctx->capture_session_output_container);
542 if (ret != ACAMERA_OK) {
543 av_log(avctx, AV_LOG_ERROR,
544 "Failed to create capture session output container, error: %s.\n",
545 camera_status_string(ret));
546 return AVERROR_EXTERNAL;
547 }
548
549 ANativeWindow_acquire(ctx->image_reader_window);
550
551 ret = ACaptureSessionOutput_create(ctx->image_reader_window, &ctx->capture_session_output);
552 if (ret != ACAMERA_OK) {
553 av_log(avctx, AV_LOG_ERROR,
554 "Failed to create capture session container, error: %s.\n",
555 camera_status_string(ret));
556 return AVERROR_EXTERNAL;
557 }
558
559 ret = ACaptureSessionOutputContainer_add(ctx->capture_session_output_container,
560 ctx->capture_session_output);
561 if (ret != ACAMERA_OK) {
562 av_log(avctx, AV_LOG_ERROR,
563 "Failed to add output to output container, error: %s.\n",
564 camera_status_string(ret));
565 return AVERROR_EXTERNAL;
566 }
567
568 ret = ACameraOutputTarget_create(ctx->image_reader_window, &ctx->camera_output_target);
569 if (ret != ACAMERA_OK) {
570 av_log(avctx, AV_LOG_ERROR,
571 "Failed to create camera output target, error: %s.\n",
572 camera_status_string(ret));
573 return AVERROR_EXTERNAL;
574 }
575
576 ret = ACameraDevice_createCaptureRequest(ctx->camera_dev, TEMPLATE_RECORD, &ctx->capture_request);
577 if (ret != ACAMERA_OK) {
578 av_log(avctx, AV_LOG_ERROR,
579 "Failed to create capture request, error: %s.\n",
580 camera_status_string(ret));
581 return AVERROR_EXTERNAL;
582 }
583
584 ret = ACaptureRequest_setEntry_i32(ctx->capture_request, ACAMERA_CONTROL_AE_TARGET_FPS_RANGE,
585 2, ctx->framerate_range);
586 if (ret != ACAMERA_OK) {
587 av_log(avctx, AV_LOG_ERROR,
588 "Failed to set target fps range in capture request, error: %s.\n",
589 camera_status_string(ret));
590 return AVERROR_EXTERNAL;
591 }
592
593 ret = ACaptureRequest_addTarget(ctx->capture_request, ctx->camera_output_target);
594 if (ret != ACAMERA_OK) {
595 av_log(avctx, AV_LOG_ERROR,
596 "Failed to add capture request capture request, error: %s.\n",
597 camera_status_string(ret));
598 return AVERROR_EXTERNAL;
599 }
600
601 ctx->capture_session_state_callbacks.context = avctx;
602 ctx->capture_session_state_callbacks.onClosed = capture_session_closed;
603 ctx->capture_session_state_callbacks.onReady = capture_session_ready;
604 ctx->capture_session_state_callbacks.onActive = capture_session_active;
605
606 ret = ACameraDevice_createCaptureSession(ctx->camera_dev, ctx->capture_session_output_container,
607 &ctx->capture_session_state_callbacks, &ctx->capture_session);
608 if (ret != ACAMERA_OK) {
609 av_log(avctx, AV_LOG_ERROR,
610 "Failed to create capture session, error: %s.\n",
611 camera_status_string(ret));
612 return AVERROR_EXTERNAL;
613 }
614
615 ret = ACameraCaptureSession_setRepeatingRequest(ctx->capture_session, NULL, 1, &ctx->capture_request, NULL);
616 if (ret != ACAMERA_OK) {
617 av_log(avctx, AV_LOG_ERROR,
618 "Failed to set repeating request on capture session, error: %s.\n",
619 camera_status_string(ret));
620 return AVERROR_EXTERNAL;
621 }
622
623 return 0;
624 }
625
wait_for_image_format(AVFormatContext * avctx)626 static int wait_for_image_format(AVFormatContext *avctx)
627 {
628 AndroidCameraCtx *ctx = avctx->priv_data;
629
630 while (!atomic_load(&ctx->got_image_format) && !atomic_load(&ctx->exit)) {
631 //Wait until first frame arrived and actual image format was determined
632 usleep(1000);
633 }
634
635 return atomic_load(&ctx->got_image_format);
636 }
637
add_display_matrix(AVFormatContext * avctx,AVStream * st)638 static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
639 {
640 AndroidCameraCtx *ctx = avctx->priv_data;
641 uint8_t *side_data;
642 int32_t display_matrix[9];
643
644 av_display_rotation_set(display_matrix, ctx->sensor_orientation);
645
646 if (ctx->lens_facing == ACAMERA_LENS_FACING_FRONT) {
647 av_display_matrix_flip(display_matrix, 1, 0);
648 }
649
650 side_data = av_stream_new_side_data(st,
651 AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
652
653 if (!side_data) {
654 return AVERROR(ENOMEM);
655 }
656
657 memcpy(side_data, display_matrix, sizeof(display_matrix));
658
659 return 0;
660 }
661
add_video_stream(AVFormatContext * avctx)662 static int add_video_stream(AVFormatContext *avctx)
663 {
664 AndroidCameraCtx *ctx = avctx->priv_data;
665 AVStream *st;
666 AVCodecParameters *codecpar;
667
668 st = avformat_new_stream(avctx, NULL);
669 if (!st) {
670 return AVERROR(ENOMEM);
671 }
672
673 st->id = VIDEO_STREAM_INDEX;
674 st->avg_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
675 st->r_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
676
677 if (!wait_for_image_format(avctx)) {
678 return AVERROR_EXTERNAL;
679 }
680
681 codecpar = st->codecpar;
682 codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
683 codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
684 codecpar->format = ctx->image_format;
685 codecpar->width = ctx->width;
686 codecpar->height = ctx->height;
687
688 avpriv_set_pts_info(st, 64, 1, VIDEO_TIMEBASE_ANDROID);
689
690 return add_display_matrix(avctx, st);
691 }
692
android_camera_read_close(AVFormatContext * avctx)693 static int android_camera_read_close(AVFormatContext *avctx)
694 {
695 AndroidCameraCtx *ctx = avctx->priv_data;
696
697 atomic_store(&ctx->exit, 1);
698
699 if (ctx->capture_session) {
700 ACameraCaptureSession_stopRepeating(ctx->capture_session);
701 // Following warning is emitted, after capture session closed callback is received:
702 // ACameraCaptureSession: Device is closed but session 0 is not notified
703 // Seems to be a bug in Android, we can ignore this
704 ACameraCaptureSession_close(ctx->capture_session);
705 ctx->capture_session = NULL;
706 }
707
708 if (ctx->capture_request) {
709 ACaptureRequest_removeTarget(ctx->capture_request, ctx->camera_output_target);
710 ACaptureRequest_free(ctx->capture_request);
711 ctx->capture_request = NULL;
712 }
713
714 if (ctx->camera_output_target) {
715 ACameraOutputTarget_free(ctx->camera_output_target);
716 ctx->camera_output_target = NULL;
717 }
718
719 if (ctx->capture_session_output) {
720 ACaptureSessionOutputContainer_remove(ctx->capture_session_output_container,
721 ctx->capture_session_output);
722 ACaptureSessionOutput_free(ctx->capture_session_output);
723 ctx->capture_session_output = NULL;
724 }
725
726 if (ctx->image_reader_window) {
727 ANativeWindow_release(ctx->image_reader_window);
728 ctx->image_reader_window = NULL;
729 }
730
731 if (ctx->capture_session_output_container) {
732 ACaptureSessionOutputContainer_free(ctx->capture_session_output_container);
733 ctx->capture_session_output_container = NULL;
734 }
735
736 if (ctx->camera_dev) {
737 ACameraDevice_close(ctx->camera_dev);
738 ctx->camera_dev = NULL;
739 }
740
741 if (ctx->image_reader) {
742 AImageReader_delete(ctx->image_reader);
743 ctx->image_reader = NULL;
744 }
745
746 if (ctx->camera_metadata) {
747 ACameraMetadata_free(ctx->camera_metadata);
748 ctx->camera_metadata = NULL;
749 }
750
751 av_freep(&ctx->camera_id);
752
753 if (ctx->camera_mgr) {
754 ACameraManager_delete(ctx->camera_mgr);
755 ctx->camera_mgr = NULL;
756 }
757
758 if (ctx->input_queue) {
759 AVPacket pkt;
760 av_thread_message_queue_set_err_send(ctx->input_queue, AVERROR_EOF);
761 while (av_thread_message_queue_recv(ctx->input_queue, &pkt, AV_THREAD_MESSAGE_NONBLOCK) >= 0) {
762 av_packet_unref(&pkt);
763 }
764 av_thread_message_queue_free(&ctx->input_queue);
765 }
766
767 return 0;
768 }
769
android_camera_read_header(AVFormatContext * avctx)770 static int android_camera_read_header(AVFormatContext *avctx)
771 {
772 AndroidCameraCtx *ctx = avctx->priv_data;
773 int ret;
774
775 atomic_init(&ctx->got_image_format, 0);
776 atomic_init(&ctx->exit, 0);
777
778 ret = av_thread_message_queue_alloc(&ctx->input_queue, ctx->input_queue_size, sizeof(AVPacket));
779 if (ret < 0) {
780 av_log(avctx, AV_LOG_ERROR,
781 "Failed to allocate input queue, error: %s.\n", av_err2str(ret));
782 goto error;
783 }
784
785 ctx->camera_mgr = ACameraManager_create();
786 if (!ctx->camera_mgr) {
787 av_log(avctx, AV_LOG_ERROR, "Failed to create Android camera manager.\n");
788 ret = AVERROR_EXTERNAL;
789 goto error;
790 }
791
792 ret = open_camera(avctx);
793 if (ret < 0) {
794 av_log(avctx, AV_LOG_ERROR, "Failed to open camera.\n");
795 goto error;
796 }
797
798 get_sensor_orientation(avctx);
799 match_video_size(avctx);
800 match_framerate(avctx);
801
802 ret = create_image_reader(avctx);
803 if (ret < 0) {
804 goto error;
805 }
806
807 ret = create_capture_session(avctx);
808 if (ret < 0) {
809 goto error;
810 }
811
812 ret = add_video_stream(avctx);
813
814 error:
815 if (ret < 0) {
816 android_camera_read_close(avctx);
817 av_log(avctx, AV_LOG_ERROR, "Failed to open android_camera.\n");
818 }
819
820 return ret;
821 }
822
android_camera_read_packet(AVFormatContext * avctx,AVPacket * pkt)823 static int android_camera_read_packet(AVFormatContext *avctx, AVPacket *pkt)
824 {
825 AndroidCameraCtx *ctx = avctx->priv_data;
826 int ret;
827
828 if (!atomic_load(&ctx->exit)) {
829 ret = av_thread_message_queue_recv(ctx->input_queue, pkt,
830 avctx->flags & AVFMT_FLAG_NONBLOCK ? AV_THREAD_MESSAGE_NONBLOCK : 0);
831 } else {
832 ret = AVERROR_EOF;
833 }
834
835 if (ret < 0) {
836 return ret;
837 } else {
838 return pkt->size;
839 }
840 }
841
842 #define OFFSET(x) offsetof(AndroidCameraCtx, x)
843 #define DEC AV_OPT_FLAG_DECODING_PARAM
844 static const AVOption options[] = {
845 { "video_size", "set video size given as a string such as 640x480 or hd720", OFFSET(requested_width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, DEC },
846 { "framerate", "set video frame rate", OFFSET(framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "30"}, 0, INT_MAX, DEC },
847 { "camera_index", "set index of camera to use", OFFSET(camera_index), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, DEC },
848 { "input_queue_size", "set maximum number of frames to buffer", OFFSET(input_queue_size), AV_OPT_TYPE_INT, {.i64 = 5}, 0, INT_MAX, DEC },
849 { NULL },
850 };
851
852 static const AVClass android_camera_class = {
853 .class_name = "android_camera indev",
854 .item_name = av_default_item_name,
855 .option = options,
856 .version = LIBAVUTIL_VERSION_INT,
857 .category = AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT,
858 };
859
860 const AVInputFormat ff_android_camera_demuxer = {
861 .name = "android_camera",
862 .long_name = NULL_IF_CONFIG_SMALL("Android camera input device"),
863 .priv_data_size = sizeof(AndroidCameraCtx),
864 .read_header = android_camera_read_header,
865 .read_packet = android_camera_read_packet,
866 .read_close = android_camera_read_close,
867 .flags = AVFMT_NOFILE,
868 .priv_class = &android_camera_class,
869 };
870