1 /*
2 * Android camera input device
3 *
4 * Copyright (C) 2017 Felix Matouschek
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #include <errno.h>
24 #include <pthread.h>
25 #include <stdatomic.h>
26 #include <stdbool.h>
27 #include <stdint.h>
28
29 #include <camera/NdkCameraDevice.h>
30 #include <camera/NdkCameraManager.h>
31 #include <media/NdkImage.h>
32 #include <media/NdkImageReader.h>
33
34 #include "libavformat/avformat.h"
35 #include "libavformat/internal.h"
36 #include "libavutil/avstring.h"
37 #include "libavutil/display.h"
38 #include "libavutil/imgutils.h"
39 #include "libavutil/log.h"
40 #include "libavutil/opt.h"
41 #include "libavutil/parseutils.h"
42 #include "libavutil/pixfmt.h"
43 #include "libavutil/threadmessage.h"
44 #include "libavutil/time.h"
45
46 #include "version.h"
47
48 /* This image format is available on all Android devices
49 * supporting the Camera2 API */
50 #define IMAGE_FORMAT_ANDROID AIMAGE_FORMAT_YUV_420_888
51
52 #define MAX_BUF_COUNT 2
53 #define VIDEO_STREAM_INDEX 0
54 #define VIDEO_TIMEBASE_ANDROID 1000000000
55
56 #define RETURN_CASE(x) case x: return AV_STRINGIFY(x);
57 #define RETURN_DEFAULT(x) default: return AV_STRINGIFY(x);
58
59 typedef struct AndroidCameraCtx {
60 const AVClass *class;
61
62 int requested_width;
63 int requested_height;
64 AVRational framerate;
65 int camera_index;
66 int input_queue_size;
67
68 uint8_t lens_facing;
69 int32_t sensor_orientation;
70 int width;
71 int height;
72 int32_t framerate_range[2];
73 int image_format;
74
75 ACameraManager *camera_mgr;
76 char *camera_id;
77 ACameraMetadata *camera_metadata;
78 ACameraDevice *camera_dev;
79 ACameraDevice_StateCallbacks camera_state_callbacks;
80 AImageReader *image_reader;
81 AImageReader_ImageListener image_listener;
82 ANativeWindow *image_reader_window;
83 ACaptureSessionOutputContainer *capture_session_output_container;
84 ACaptureSessionOutput *capture_session_output;
85 ACameraOutputTarget *camera_output_target;
86 ACaptureRequest *capture_request;
87 ACameraCaptureSession_stateCallbacks capture_session_state_callbacks;
88 ACameraCaptureSession *capture_session;
89
90 AVThreadMessageQueue *input_queue;
91 atomic_int exit;
92 atomic_int got_image_format;
93 } AndroidCameraCtx;
94
camera_status_string(camera_status_t val)95 static const char *camera_status_string(camera_status_t val)
96 {
97 switch(val) {
98 RETURN_CASE(ACAMERA_OK)
99 RETURN_CASE(ACAMERA_ERROR_UNKNOWN)
100 RETURN_CASE(ACAMERA_ERROR_INVALID_PARAMETER)
101 RETURN_CASE(ACAMERA_ERROR_CAMERA_DISCONNECTED)
102 RETURN_CASE(ACAMERA_ERROR_NOT_ENOUGH_MEMORY)
103 RETURN_CASE(ACAMERA_ERROR_METADATA_NOT_FOUND)
104 RETURN_CASE(ACAMERA_ERROR_CAMERA_DEVICE)
105 RETURN_CASE(ACAMERA_ERROR_CAMERA_SERVICE)
106 RETURN_CASE(ACAMERA_ERROR_SESSION_CLOSED)
107 RETURN_CASE(ACAMERA_ERROR_INVALID_OPERATION)
108 RETURN_CASE(ACAMERA_ERROR_STREAM_CONFIGURE_FAIL)
109 RETURN_CASE(ACAMERA_ERROR_CAMERA_IN_USE)
110 RETURN_CASE(ACAMERA_ERROR_MAX_CAMERA_IN_USE)
111 RETURN_CASE(ACAMERA_ERROR_CAMERA_DISABLED)
112 RETURN_CASE(ACAMERA_ERROR_PERMISSION_DENIED)
113 RETURN_DEFAULT(ACAMERA_ERROR_UNKNOWN)
114 }
115 }
116
media_status_string(media_status_t val)117 static const char *media_status_string(media_status_t val)
118 {
119 switch(val) {
120 RETURN_CASE(AMEDIA_OK)
121 RETURN_CASE(AMEDIA_ERROR_UNKNOWN)
122 RETURN_CASE(AMEDIA_ERROR_MALFORMED)
123 RETURN_CASE(AMEDIA_ERROR_UNSUPPORTED)
124 RETURN_CASE(AMEDIA_ERROR_INVALID_OBJECT)
125 RETURN_CASE(AMEDIA_ERROR_INVALID_PARAMETER)
126 RETURN_CASE(AMEDIA_ERROR_INVALID_OPERATION)
127 RETURN_CASE(AMEDIA_DRM_NOT_PROVISIONED)
128 RETURN_CASE(AMEDIA_DRM_RESOURCE_BUSY)
129 RETURN_CASE(AMEDIA_DRM_DEVICE_REVOKED)
130 RETURN_CASE(AMEDIA_DRM_SHORT_BUFFER)
131 RETURN_CASE(AMEDIA_DRM_SESSION_NOT_OPENED)
132 RETURN_CASE(AMEDIA_DRM_TAMPER_DETECTED)
133 RETURN_CASE(AMEDIA_DRM_VERIFY_FAILED)
134 RETURN_CASE(AMEDIA_DRM_NEED_KEY)
135 RETURN_CASE(AMEDIA_DRM_LICENSE_EXPIRED)
136 RETURN_CASE(AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE)
137 RETURN_CASE(AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED)
138 RETURN_CASE(AMEDIA_IMGREADER_CANNOT_LOCK_IMAGE)
139 RETURN_CASE(AMEDIA_IMGREADER_CANNOT_UNLOCK_IMAGE)
140 RETURN_CASE(AMEDIA_IMGREADER_IMAGE_NOT_LOCKED)
141 RETURN_DEFAULT(AMEDIA_ERROR_UNKNOWN)
142 }
143 }
144
error_state_callback_string(int val)145 static const char *error_state_callback_string(int val)
146 {
147 switch(val) {
148 RETURN_CASE(ERROR_CAMERA_IN_USE)
149 RETURN_CASE(ERROR_MAX_CAMERAS_IN_USE)
150 RETURN_CASE(ERROR_CAMERA_DISABLED)
151 RETURN_CASE(ERROR_CAMERA_DEVICE)
152 RETURN_CASE(ERROR_CAMERA_SERVICE)
153 default:
154 return "ERROR_CAMERA_UNKNOWN";
155 }
156 }
157
camera_dev_disconnected(void * context,ACameraDevice * device)158 static void camera_dev_disconnected(void *context, ACameraDevice *device)
159 {
160 AVFormatContext *avctx = context;
161 AndroidCameraCtx *ctx = avctx->priv_data;
162 atomic_store(&ctx->exit, 1);
163 av_log(avctx, AV_LOG_ERROR, "Camera with id %s disconnected.\n",
164 ACameraDevice_getId(device));
165 }
166
camera_dev_error(void * context,ACameraDevice * device,int error)167 static void camera_dev_error(void *context, ACameraDevice *device, int error)
168 {
169 AVFormatContext *avctx = context;
170 AndroidCameraCtx *ctx = avctx->priv_data;
171 atomic_store(&ctx->exit, 1);
172 av_log(avctx, AV_LOG_ERROR, "Error %s on camera with id %s.\n",
173 error_state_callback_string(error), ACameraDevice_getId(device));
174 }
175
open_camera(AVFormatContext * avctx)176 static int open_camera(AVFormatContext *avctx)
177 {
178 AndroidCameraCtx *ctx = avctx->priv_data;
179 camera_status_t ret;
180 ACameraIdList *camera_ids;
181
182 ret = ACameraManager_getCameraIdList(ctx->camera_mgr, &camera_ids);
183 if (ret != ACAMERA_OK) {
184 av_log(avctx, AV_LOG_ERROR, "Failed to get camera id list, error: %s.\n",
185 camera_status_string(ret));
186 return AVERROR_EXTERNAL;
187 }
188
189 if (ctx->camera_index < camera_ids->numCameras) {
190 ctx->camera_id = av_strdup(camera_ids->cameraIds[ctx->camera_index]);
191 if (!ctx->camera_id) {
192 av_log(avctx, AV_LOG_ERROR, "Failed to allocate memory for camera_id.\n");
193 return AVERROR(ENOMEM);
194 }
195 } else {
196 av_log(avctx, AV_LOG_ERROR, "No camera with index %d available.\n",
197 ctx->camera_index);
198 return AVERROR(ENXIO);
199 }
200
201 ACameraManager_deleteCameraIdList(camera_ids);
202
203 ret = ACameraManager_getCameraCharacteristics(ctx->camera_mgr,
204 ctx->camera_id, &ctx->camera_metadata);
205 if (ret != ACAMERA_OK) {
206 av_log(avctx, AV_LOG_ERROR, "Failed to get metadata for camera with id %s, error: %s.\n",
207 ctx->camera_id, camera_status_string(ret));
208 return AVERROR_EXTERNAL;
209 }
210
211 ctx->camera_state_callbacks.context = avctx;
212 ctx->camera_state_callbacks.onDisconnected = camera_dev_disconnected;
213 ctx->camera_state_callbacks.onError = camera_dev_error;
214
215 ret = ACameraManager_openCamera(ctx->camera_mgr, ctx->camera_id,
216 &ctx->camera_state_callbacks, &ctx->camera_dev);
217 if (ret != ACAMERA_OK) {
218 av_log(avctx, AV_LOG_ERROR, "Failed to open camera with id %s, error: %s.\n",
219 ctx->camera_id, camera_status_string(ret));
220 return AVERROR_EXTERNAL;
221 }
222
223 return 0;
224 }
225
get_sensor_orientation(AVFormatContext * avctx)226 static void get_sensor_orientation(AVFormatContext *avctx)
227 {
228 AndroidCameraCtx *ctx = avctx->priv_data;
229 ACameraMetadata_const_entry lens_facing;
230 ACameraMetadata_const_entry sensor_orientation;
231
232 ACameraMetadata_getConstEntry(ctx->camera_metadata,
233 ACAMERA_LENS_FACING, &lens_facing);
234 ACameraMetadata_getConstEntry(ctx->camera_metadata,
235 ACAMERA_SENSOR_ORIENTATION, &sensor_orientation);
236
237 ctx->lens_facing = lens_facing.data.u8[0];
238 ctx->sensor_orientation = sensor_orientation.data.i32[0];
239 }
240
match_video_size(AVFormatContext * avctx)241 static void match_video_size(AVFormatContext *avctx)
242 {
243 AndroidCameraCtx *ctx = avctx->priv_data;
244 ACameraMetadata_const_entry available_configs;
245 int found = 0;
246
247 ACameraMetadata_getConstEntry(ctx->camera_metadata,
248 ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
249 &available_configs);
250
251 for (int i = 0; i < available_configs.count; i++) {
252 int32_t input = available_configs.data.i32[i * 4 + 3];
253 int32_t format = available_configs.data.i32[i * 4 + 0];
254
255 if (input) {
256 continue;
257 }
258
259 if (format == IMAGE_FORMAT_ANDROID) {
260 int32_t width = available_configs.data.i32[i * 4 + 1];
261 int32_t height = available_configs.data.i32[i * 4 + 2];
262
263 //Same ratio
264 if ((ctx->requested_width == width && ctx->requested_height == height) ||
265 (ctx->requested_width == height && ctx->requested_height == width)) {
266 ctx->width = width;
267 ctx->height = height;
268 found = 1;
269 break;
270 }
271 }
272 }
273
274 if (!found || ctx->width == 0 || ctx->height == 0) {
275 ctx->width = available_configs.data.i32[1];
276 ctx->height = available_configs.data.i32[2];
277
278 av_log(avctx, AV_LOG_WARNING,
279 "Requested video_size %dx%d not available, falling back to %dx%d\n",
280 ctx->requested_width, ctx->requested_height, ctx->width, ctx->height);
281 }
282
283 return;
284 }
285
match_framerate(AVFormatContext * avctx)286 static void match_framerate(AVFormatContext *avctx)
287 {
288 AndroidCameraCtx *ctx = avctx->priv_data;
289 ACameraMetadata_const_entry available_framerates;
290 int found = 0;
291 int current_best_match = -1;
292 int requested_framerate = av_q2d(ctx->framerate);
293
294 ACameraMetadata_getConstEntry(ctx->camera_metadata,
295 ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES,
296 &available_framerates);
297
298 for (int i = 0; i < available_framerates.count; i++) {
299 int32_t min = available_framerates.data.i32[i * 2 + 0];
300 int32_t max = available_framerates.data.i32[i * 2 + 1];
301
302 if (requested_framerate == max) {
303 if (min == max) {
304 ctx->framerate_range[0] = min;
305 ctx->framerate_range[1] = max;
306 found = 1;
307 break;
308 } else if (current_best_match >= 0) {
309 int32_t current_best_match_min = available_framerates.data.i32[current_best_match * 2 + 0];
310 if (min > current_best_match_min) {
311 current_best_match = i;
312 }
313 } else {
314 current_best_match = i;
315 }
316 }
317 }
318
319 if (!found) {
320 if (current_best_match >= 0) {
321 ctx->framerate_range[0] = available_framerates.data.i32[current_best_match * 2 + 0];
322 ctx->framerate_range[1] = available_framerates.data.i32[current_best_match * 2 + 1];
323
324 } else {
325 ctx->framerate_range[0] = available_framerates.data.i32[0];
326 ctx->framerate_range[1] = available_framerates.data.i32[1];
327 }
328
329 av_log(avctx, AV_LOG_WARNING,
330 "Requested framerate %d not available, falling back to min: %d and max: %d fps\n",
331 requested_framerate, ctx->framerate_range[0], ctx->framerate_range[1]);
332 }
333
334 return;
335 }
336
get_image_format(AVFormatContext * avctx,AImage * image)337 static int get_image_format(AVFormatContext *avctx, AImage *image)
338 {
339 AndroidCameraCtx *ctx = avctx->priv_data;
340 int32_t image_pixelstrides[2];
341 uint8_t *image_plane_data[2];
342 int plane_data_length[2];
343
344 for (int i = 0; i < 2; i++) {
345 AImage_getPlanePixelStride(image, i + 1, &image_pixelstrides[i]);
346 AImage_getPlaneData(image, i + 1, &image_plane_data[i], &plane_data_length[i]);
347 }
348
349 if (image_pixelstrides[0] != image_pixelstrides[1]) {
350 av_log(avctx, AV_LOG_ERROR,
351 "Pixel strides of U and V plane should have been the same.\n");
352 return AVERROR_EXTERNAL;
353 }
354
355 switch (image_pixelstrides[0]) {
356 case 1:
357 ctx->image_format = AV_PIX_FMT_YUV420P;
358 break;
359 case 2:
360 if (image_plane_data[0] < image_plane_data[1]) {
361 ctx->image_format = AV_PIX_FMT_NV12;
362 } else {
363 ctx->image_format = AV_PIX_FMT_NV21;
364 }
365 break;
366 default:
367 av_log(avctx, AV_LOG_ERROR,
368 "Unknown pixel stride %d of U and V plane, cannot determine camera image format.\n",
369 image_pixelstrides[0]);
370 return AVERROR(ENOSYS);
371 }
372
373 return 0;
374 }
375
image_available(void * context,AImageReader * reader)376 static void image_available(void *context, AImageReader *reader)
377 {
378 AVFormatContext *avctx = context;
379 AndroidCameraCtx *ctx = avctx->priv_data;
380 media_status_t media_status;
381 int ret = 0;
382
383 AImage *image;
384 int64_t image_timestamp;
385 int32_t image_linestrides[4];
386 uint8_t *image_plane_data[4];
387 int plane_data_length[4];
388
389 AVPacket pkt;
390 int pkt_buffer_size = 0;
391
392 media_status = AImageReader_acquireLatestImage(reader, &image);
393 if (media_status != AMEDIA_OK) {
394 if (media_status == AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE) {
395 av_log(avctx, AV_LOG_WARNING,
396 "An image reader frame was discarded");
397 } else {
398 av_log(avctx, AV_LOG_ERROR,
399 "Failed to acquire latest image from image reader, error: %s.\n",
400 media_status_string(media_status));
401 ret = AVERROR_EXTERNAL;
402 }
403 goto error;
404 }
405
406 // Silently drop frames when exit is set
407 if (atomic_load(&ctx->exit)) {
408 goto error;
409 }
410
411 // Determine actual image format
412 if (!atomic_load(&ctx->got_image_format)) {
413 ret = get_image_format(avctx, image);
414 if (ret < 0) {
415 av_log(avctx, AV_LOG_ERROR,
416 "Could not get image format of camera.\n");
417 goto error;
418 } else {
419 atomic_store(&ctx->got_image_format, 1);
420 }
421 }
422
423 pkt_buffer_size = av_image_get_buffer_size(ctx->image_format, ctx->width, ctx->height, 32);
424 AImage_getTimestamp(image, &image_timestamp);
425
426 AImage_getPlaneRowStride(image, 0, &image_linestrides[0]);
427 AImage_getPlaneData(image, 0, &image_plane_data[0], &plane_data_length[0]);
428
429 switch (ctx->image_format) {
430 case AV_PIX_FMT_YUV420P:
431 AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
432 AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
433 AImage_getPlaneRowStride(image, 2, &image_linestrides[2]);
434 AImage_getPlaneData(image, 2, &image_plane_data[2], &plane_data_length[2]);
435 break;
436 case AV_PIX_FMT_NV12:
437 AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
438 AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
439 break;
440 case AV_PIX_FMT_NV21:
441 AImage_getPlaneRowStride(image, 2, &image_linestrides[1]);
442 AImage_getPlaneData(image, 2, &image_plane_data[1], &plane_data_length[1]);
443 break;
444 default:
445 av_log(avctx, AV_LOG_ERROR, "Unsupported camera image format.\n");
446 ret = AVERROR(ENOSYS);
447 goto error;
448 }
449
450 ret = av_new_packet(&pkt, pkt_buffer_size);
451 if (ret < 0) {
452 av_log(avctx, AV_LOG_ERROR,
453 "Failed to create new av packet, error: %s.\n", av_err2str(ret));
454 goto error;
455 }
456
457 pkt.stream_index = VIDEO_STREAM_INDEX;
458 pkt.pts = image_timestamp;
459 av_image_copy_to_buffer(pkt.data, pkt_buffer_size,
460 (const uint8_t * const *) image_plane_data,
461 image_linestrides, ctx->image_format,
462 ctx->width, ctx->height, 32);
463
464 ret = av_thread_message_queue_send(ctx->input_queue, &pkt, AV_THREAD_MESSAGE_NONBLOCK);
465
466 error:
467 if (ret < 0) {
468 if (ret != AVERROR(EAGAIN)) {
469 av_log(avctx, AV_LOG_ERROR,
470 "Error while processing new image, error: %s.\n", av_err2str(ret));
471 av_thread_message_queue_set_err_recv(ctx->input_queue, ret);
472 atomic_store(&ctx->exit, 1);
473 } else {
474 av_log(avctx, AV_LOG_WARNING,
475 "Input queue was full, dropping frame, consider raising the input_queue_size option (current value: %d)\n",
476 ctx->input_queue_size);
477 }
478 if (pkt_buffer_size) {
479 av_packet_unref(&pkt);
480 }
481 }
482
483 AImage_delete(image);
484
485 return;
486 }
487
create_image_reader(AVFormatContext * avctx)488 static int create_image_reader(AVFormatContext *avctx)
489 {
490 AndroidCameraCtx *ctx = avctx->priv_data;
491 media_status_t ret;
492
493 ret = AImageReader_new(ctx->width, ctx->height, IMAGE_FORMAT_ANDROID,
494 MAX_BUF_COUNT, &ctx->image_reader);
495 if (ret != AMEDIA_OK) {
496 av_log(avctx, AV_LOG_ERROR,
497 "Failed to create image reader, error: %s.\n", media_status_string(ret));
498 return AVERROR_EXTERNAL;
499 }
500
501 ctx->image_listener.context = avctx;
502 ctx->image_listener.onImageAvailable = image_available;
503
504 ret = AImageReader_setImageListener(ctx->image_reader, &ctx->image_listener);
505 if (ret != AMEDIA_OK) {
506 av_log(avctx, AV_LOG_ERROR,
507 "Failed to set image listener on image reader, error: %s.\n",
508 media_status_string(ret));
509 return AVERROR_EXTERNAL;
510 }
511
512 ret = AImageReader_getWindow(ctx->image_reader, &ctx->image_reader_window);
513 if (ret != AMEDIA_OK) {
514 av_log(avctx, AV_LOG_ERROR,
515 "Could not get image reader window, error: %s.\n",
516 media_status_string(ret));
517 return AVERROR_EXTERNAL;
518 }
519
520 return 0;
521 }
522
capture_session_closed(void * context,ACameraCaptureSession * session)523 static void capture_session_closed(void *context, ACameraCaptureSession *session)
524 {
525 av_log(context, AV_LOG_INFO, "Android camera capture session was closed.\n");
526 }
527
capture_session_ready(void * context,ACameraCaptureSession * session)528 static void capture_session_ready(void *context, ACameraCaptureSession *session)
529 {
530 av_log(context, AV_LOG_INFO, "Android camera capture session is ready.\n");
531 }
532
capture_session_active(void * context,ACameraCaptureSession * session)533 static void capture_session_active(void *context, ACameraCaptureSession *session)
534 {
535 av_log(context, AV_LOG_INFO, "Android camera capture session is active.\n");
536 }
537
create_capture_session(AVFormatContext * avctx)538 static int create_capture_session(AVFormatContext *avctx)
539 {
540 AndroidCameraCtx *ctx = avctx->priv_data;
541 camera_status_t ret;
542
543 ret = ACaptureSessionOutputContainer_create(&ctx->capture_session_output_container);
544 if (ret != ACAMERA_OK) {
545 av_log(avctx, AV_LOG_ERROR,
546 "Failed to create capture session output container, error: %s.\n",
547 camera_status_string(ret));
548 return AVERROR_EXTERNAL;
549 }
550
551 ANativeWindow_acquire(ctx->image_reader_window);
552
553 ret = ACaptureSessionOutput_create(ctx->image_reader_window, &ctx->capture_session_output);
554 if (ret != ACAMERA_OK) {
555 av_log(avctx, AV_LOG_ERROR,
556 "Failed to create capture session container, error: %s.\n",
557 camera_status_string(ret));
558 return AVERROR_EXTERNAL;
559 }
560
561 ret = ACaptureSessionOutputContainer_add(ctx->capture_session_output_container,
562 ctx->capture_session_output);
563 if (ret != ACAMERA_OK) {
564 av_log(avctx, AV_LOG_ERROR,
565 "Failed to add output to output container, error: %s.\n",
566 camera_status_string(ret));
567 return AVERROR_EXTERNAL;
568 }
569
570 ret = ACameraOutputTarget_create(ctx->image_reader_window, &ctx->camera_output_target);
571 if (ret != ACAMERA_OK) {
572 av_log(avctx, AV_LOG_ERROR,
573 "Failed to create camera output target, error: %s.\n",
574 camera_status_string(ret));
575 return AVERROR_EXTERNAL;
576 }
577
578 ret = ACameraDevice_createCaptureRequest(ctx->camera_dev, TEMPLATE_RECORD, &ctx->capture_request);
579 if (ret != ACAMERA_OK) {
580 av_log(avctx, AV_LOG_ERROR,
581 "Failed to create capture request, error: %s.\n",
582 camera_status_string(ret));
583 return AVERROR_EXTERNAL;
584 }
585
586 ret = ACaptureRequest_setEntry_i32(ctx->capture_request, ACAMERA_CONTROL_AE_TARGET_FPS_RANGE,
587 2, ctx->framerate_range);
588 if (ret != ACAMERA_OK) {
589 av_log(avctx, AV_LOG_ERROR,
590 "Failed to set target fps range in capture request, error: %s.\n",
591 camera_status_string(ret));
592 return AVERROR_EXTERNAL;
593 }
594
595 ret = ACaptureRequest_addTarget(ctx->capture_request, ctx->camera_output_target);
596 if (ret != ACAMERA_OK) {
597 av_log(avctx, AV_LOG_ERROR,
598 "Failed to add capture request capture request, error: %s.\n",
599 camera_status_string(ret));
600 return AVERROR_EXTERNAL;
601 }
602
603 ctx->capture_session_state_callbacks.context = avctx;
604 ctx->capture_session_state_callbacks.onClosed = capture_session_closed;
605 ctx->capture_session_state_callbacks.onReady = capture_session_ready;
606 ctx->capture_session_state_callbacks.onActive = capture_session_active;
607
608 ret = ACameraDevice_createCaptureSession(ctx->camera_dev, ctx->capture_session_output_container,
609 &ctx->capture_session_state_callbacks, &ctx->capture_session);
610 if (ret != ACAMERA_OK) {
611 av_log(avctx, AV_LOG_ERROR,
612 "Failed to create capture session, error: %s.\n",
613 camera_status_string(ret));
614 return AVERROR_EXTERNAL;
615 }
616
617 ret = ACameraCaptureSession_setRepeatingRequest(ctx->capture_session, NULL, 1, &ctx->capture_request, NULL);
618 if (ret != ACAMERA_OK) {
619 av_log(avctx, AV_LOG_ERROR,
620 "Failed to set repeating request on capture session, error: %s.\n",
621 camera_status_string(ret));
622 return AVERROR_EXTERNAL;
623 }
624
625 return 0;
626 }
627
wait_for_image_format(AVFormatContext * avctx)628 static int wait_for_image_format(AVFormatContext *avctx)
629 {
630 AndroidCameraCtx *ctx = avctx->priv_data;
631
632 while (!atomic_load(&ctx->got_image_format) && !atomic_load(&ctx->exit)) {
633 //Wait until first frame arrived and actual image format was determined
634 usleep(1000);
635 }
636
637 return atomic_load(&ctx->got_image_format);
638 }
639
add_display_matrix(AVFormatContext * avctx,AVStream * st)640 static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
641 {
642 AndroidCameraCtx *ctx = avctx->priv_data;
643 uint8_t *side_data;
644 int32_t display_matrix[9];
645
646 av_display_rotation_set(display_matrix, ctx->sensor_orientation);
647
648 if (ctx->lens_facing == ACAMERA_LENS_FACING_FRONT) {
649 av_display_matrix_flip(display_matrix, 1, 0);
650 }
651
652 side_data = av_stream_new_side_data(st,
653 AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
654
655 if (!side_data) {
656 return AVERROR(ENOMEM);
657 }
658
659 memcpy(side_data, display_matrix, sizeof(display_matrix));
660
661 return 0;
662 }
663
add_video_stream(AVFormatContext * avctx)664 static int add_video_stream(AVFormatContext *avctx)
665 {
666 AndroidCameraCtx *ctx = avctx->priv_data;
667 AVStream *st;
668 AVCodecParameters *codecpar;
669
670 st = avformat_new_stream(avctx, NULL);
671 if (!st) {
672 return AVERROR(ENOMEM);
673 }
674
675 st->id = VIDEO_STREAM_INDEX;
676 st->avg_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
677 st->r_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
678
679 if (!wait_for_image_format(avctx)) {
680 return AVERROR_EXTERNAL;
681 }
682
683 codecpar = st->codecpar;
684 codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
685 codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
686 codecpar->format = ctx->image_format;
687 codecpar->width = ctx->width;
688 codecpar->height = ctx->height;
689
690 avpriv_set_pts_info(st, 64, 1, VIDEO_TIMEBASE_ANDROID);
691
692 return add_display_matrix(avctx, st);
693 }
694
android_camera_read_close(AVFormatContext * avctx)695 static int android_camera_read_close(AVFormatContext *avctx)
696 {
697 AndroidCameraCtx *ctx = avctx->priv_data;
698
699 atomic_store(&ctx->exit, 1);
700
701 if (ctx->capture_session) {
702 ACameraCaptureSession_stopRepeating(ctx->capture_session);
703 // Following warning is emitted, after capture session closed callback is received:
704 // ACameraCaptureSession: Device is closed but session 0 is not notified
705 // Seems to be a bug in Android, we can ignore this
706 ACameraCaptureSession_close(ctx->capture_session);
707 ctx->capture_session = NULL;
708 }
709
710 if (ctx->capture_request) {
711 ACaptureRequest_removeTarget(ctx->capture_request, ctx->camera_output_target);
712 ACaptureRequest_free(ctx->capture_request);
713 ctx->capture_request = NULL;
714 }
715
716 if (ctx->camera_output_target) {
717 ACameraOutputTarget_free(ctx->camera_output_target);
718 ctx->camera_output_target = NULL;
719 }
720
721 if (ctx->capture_session_output) {
722 ACaptureSessionOutputContainer_remove(ctx->capture_session_output_container,
723 ctx->capture_session_output);
724 ACaptureSessionOutput_free(ctx->capture_session_output);
725 ctx->capture_session_output = NULL;
726 }
727
728 if (ctx->image_reader_window) {
729 ANativeWindow_release(ctx->image_reader_window);
730 ctx->image_reader_window = NULL;
731 }
732
733 if (ctx->capture_session_output_container) {
734 ACaptureSessionOutputContainer_free(ctx->capture_session_output_container);
735 ctx->capture_session_output_container = NULL;
736 }
737
738 if (ctx->camera_dev) {
739 ACameraDevice_close(ctx->camera_dev);
740 ctx->camera_dev = NULL;
741 }
742
743 if (ctx->image_reader) {
744 AImageReader_delete(ctx->image_reader);
745 ctx->image_reader = NULL;
746 }
747
748 if (ctx->camera_metadata) {
749 ACameraMetadata_free(ctx->camera_metadata);
750 ctx->camera_metadata = NULL;
751 }
752
753 av_freep(&ctx->camera_id);
754
755 if (ctx->camera_mgr) {
756 ACameraManager_delete(ctx->camera_mgr);
757 ctx->camera_mgr = NULL;
758 }
759
760 if (ctx->input_queue) {
761 AVPacket pkt;
762 av_thread_message_queue_set_err_send(ctx->input_queue, AVERROR_EOF);
763 while (av_thread_message_queue_recv(ctx->input_queue, &pkt, AV_THREAD_MESSAGE_NONBLOCK) >= 0) {
764 av_packet_unref(&pkt);
765 }
766 av_thread_message_queue_free(&ctx->input_queue);
767 }
768
769 return 0;
770 }
771
android_camera_read_header(AVFormatContext * avctx)772 static int android_camera_read_header(AVFormatContext *avctx)
773 {
774 AndroidCameraCtx *ctx = avctx->priv_data;
775 int ret;
776
777 atomic_init(&ctx->got_image_format, 0);
778 atomic_init(&ctx->exit, 0);
779
780 ret = av_thread_message_queue_alloc(&ctx->input_queue, ctx->input_queue_size, sizeof(AVPacket));
781 if (ret < 0) {
782 av_log(avctx, AV_LOG_ERROR,
783 "Failed to allocate input queue, error: %s.\n", av_err2str(ret));
784 goto error;
785 }
786
787 ctx->camera_mgr = ACameraManager_create();
788 if (!ctx->camera_mgr) {
789 av_log(avctx, AV_LOG_ERROR, "Failed to create Android camera manager.\n");
790 ret = AVERROR_EXTERNAL;
791 goto error;
792 }
793
794 ret = open_camera(avctx);
795 if (ret < 0) {
796 av_log(avctx, AV_LOG_ERROR, "Failed to open camera.\n");
797 goto error;
798 }
799
800 get_sensor_orientation(avctx);
801 match_video_size(avctx);
802 match_framerate(avctx);
803
804 ret = create_image_reader(avctx);
805 if (ret < 0) {
806 goto error;
807 }
808
809 ret = create_capture_session(avctx);
810 if (ret < 0) {
811 goto error;
812 }
813
814 ret = add_video_stream(avctx);
815
816 error:
817 if (ret < 0) {
818 android_camera_read_close(avctx);
819 av_log(avctx, AV_LOG_ERROR, "Failed to open android_camera.\n");
820 }
821
822 return ret;
823 }
824
android_camera_read_packet(AVFormatContext * avctx,AVPacket * pkt)825 static int android_camera_read_packet(AVFormatContext *avctx, AVPacket *pkt)
826 {
827 AndroidCameraCtx *ctx = avctx->priv_data;
828 int ret;
829
830 if (!atomic_load(&ctx->exit)) {
831 ret = av_thread_message_queue_recv(ctx->input_queue, pkt,
832 avctx->flags & AVFMT_FLAG_NONBLOCK ? AV_THREAD_MESSAGE_NONBLOCK : 0);
833 } else {
834 ret = AVERROR_EOF;
835 }
836
837 if (ret < 0) {
838 return ret;
839 } else {
840 return pkt->size;
841 }
842 }
843
844 #define OFFSET(x) offsetof(AndroidCameraCtx, x)
845 #define DEC AV_OPT_FLAG_DECODING_PARAM
846 static const AVOption options[] = {
847 { "video_size", "set video size given as a string such as 640x480 or hd720", OFFSET(requested_width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, DEC },
848 { "framerate", "set video frame rate", OFFSET(framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "30"}, 0, INT_MAX, DEC },
849 { "camera_index", "set index of camera to use", OFFSET(camera_index), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, DEC },
850 { "input_queue_size", "set maximum number of frames to buffer", OFFSET(input_queue_size), AV_OPT_TYPE_INT, {.i64 = 5}, 0, INT_MAX, DEC },
851 { NULL },
852 };
853
854 static const AVClass android_camera_class = {
855 .class_name = "android_camera indev",
856 .item_name = av_default_item_name,
857 .option = options,
858 .version = LIBAVUTIL_VERSION_INT,
859 .category = AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT,
860 };
861
862 AVInputFormat ff_android_camera_demuxer = {
863 .name = "android_camera",
864 .long_name = NULL_IF_CONFIG_SMALL("Android camera input device"),
865 .priv_data_size = sizeof(AndroidCameraCtx),
866 .read_header = android_camera_read_header,
867 .read_packet = android_camera_read_packet,
868 .read_close = android_camera_read_close,
869 .flags = AVFMT_NOFILE,
870 .priv_class = &android_camera_class,
871 };
872