1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3DEVICE_H 18 #define ANDROID_SERVERS_CAMERA3DEVICE_H 19 20 #include <utility> 21 #include <unordered_map> 22 #include <set> 23 #include <tuple> 24 25 #include <utils/Condition.h> 26 #include <utils/Errors.h> 27 #include <utils/List.h> 28 #include <utils/Mutex.h> 29 #include <utils/Thread.h> 30 #include <utils/KeyedVector.h> 31 #include <utils/Timers.h> 32 33 #include <camera/CaptureResult.h> 34 #include <gui/Flags.h> 35 36 #include "CameraServiceWatchdog.h" 37 #include <aidl/android/hardware/camera/device/CameraBlob.h> 38 39 #include "common/CameraDeviceBase.h" 40 #include "common/DepthPhotoProcessor.h" 41 #include "common/FrameProcessorBase.h" 42 #include "device3/BufferUtils.h" 43 #include "device3/StatusTracker.h" 44 #include "device3/Camera3BufferManager.h" 45 #include "device3/DistortionMapper.h" 46 #include "device3/ZoomRatioMapper.h" 47 #include "device3/RotateAndCropMapper.h" 48 #include "device3/UHRCropAndMeteringRegionMapper.h" 49 #include "device3/InFlightRequest.h" 50 #include "device3/Camera3OutputInterface.h" 51 #include "device3/Camera3OfflineSession.h" 52 #include "device3/Camera3StreamInterface.h" 53 #include "utils/AttributionAndPermissionUtils.h" 54 #include "utils/TagMonitor.h" 55 #include "utils/IPCTransport.h" 56 #include "utils/LatencyHistogram.h" 57 #include "utils/CameraServiceProxyWrapper.h" 58 #include <camera_metadata_hidden.h> 59 60 using android::camera3::camera_capture_request_t; 61 using android::camera3::camera_request_template; 62 using android::camera3::camera_stream_buffer_t; 63 using android::camera3::camera_stream_configuration_t; 64 using android::camera3::camera_stream_configuration_mode_t; 65 using android::camera3::CAMERA_TEMPLATE_COUNT; 66 using android::camera3::OutputStreamInfo; 67 using android::camera3::SurfaceHolder; 68 69 namespace android { 70 71 namespace camera3 { 72 73 class Camera3Stream; 74 class Camera3ZslStream; 75 class Camera3StreamInterface; 76 77 } // namespace camera3 78 79 /** 80 * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 or higher. 81 */ 82 class Camera3Device : 83 public CameraDeviceBase, 84 public camera3::SetErrorInterface, 85 public camera3::InflightRequestUpdateInterface, 86 public camera3::RequestBufferInterface, 87 public camera3::FlushBufferInterface, 88 public AttributionAndPermissionUtilsEncapsulator { 89 friend class HidlCamera3Device; 90 friend class AidlCamera3Device; 91 public: 92 93 explicit Camera3Device(std::shared_ptr<CameraServiceProxyWrapper>& cameraServiceProxyWrapper, 94 std::shared_ptr<AttributionAndPermissionUtils> attributionAndPermissionUtils, 95 const std::string& id, bool overrideForPerfClass, int rotationOverride, 96 bool isVendorClient, bool legacyClient = false); 97 98 virtual ~Camera3Device(); 99 // Delete and optionally close native handles and clear the input vector afterward 100 static void cleanupNativeHandles( 101 std::vector<native_handle_t*> *handles, bool closeFd = false); 102 getTransportType()103 virtual IPCTransport getTransportType() const override { 104 return mInterface->getTransportType(); 105 } 106 isHalBufferManagedStream(int32_t streamId)107 bool isHalBufferManagedStream(int32_t streamId) const { 108 return mInterface->isHalBufferManagedStream(streamId); 109 }; 110 111 /** 112 * CameraDeviceBase interface 113 */ 114 115 const std::string& getId() const override; 116 getVendorTagId()117 metadata_vendor_id_t getVendorTagId() const override { return mVendorTagId; } 118 119 // Watchdog thread 120 sp<CameraServiceWatchdog> mCameraServiceWatchdog; 121 122 // Transitions to idle state on success. 123 virtual status_t initialize(sp<CameraProviderManager> /*manager*/, 124 const std::string& /*monitorTags*/) = 0; 125 126 static constexpr int32_t METADATA_QUEUE_SIZE = 1 << 20; 127 128 template <typename FMQType> calculateFMQSize(const std::unique_ptr<FMQType> & fmq)129 static size_t calculateFMQSize(const std::unique_ptr<FMQType> &fmq) { 130 if (fmq == nullptr) { 131 ALOGE("%s: result metadata queue hasn't been initialized", __FUNCTION__); 132 return METADATA_QUEUE_SIZE; 133 } 134 size_t quantumSize = fmq->getQuantumSize(); 135 size_t quantumCount = fmq->getQuantumCount(); 136 if ((quantumSize == 0) || (quantumCount == 0) || 137 ((std::numeric_limits<size_t>::max() / quantumSize) < quantumCount)) { 138 ALOGE("%s: Error with FMQ quantum count / quantum size, quantum count %zu" 139 "quantum count %zu", __FUNCTION__, quantumSize, quantumCount); 140 return METADATA_QUEUE_SIZE; 141 } 142 return fmq->getQuantumSize() * fmq->getQuantumCount(); 143 } 144 status_t disconnect() override; 145 status_t dump(int fd, const Vector<String16> &args) override; 146 status_t startWatchingTags(const std::string &tags) override; 147 status_t stopWatchingTags() override; 148 status_t dumpWatchedEventsToVector(std::vector<std::string> &out) override; 149 const CameraMetadata& info() const override; 150 const CameraMetadata& infoPhysical(const std::string& physicalId) const override; isCompositeJpegRDisabled()151 bool isCompositeJpegRDisabled() const override { return mIsCompositeJpegRDisabled; }; isCompositeHeicDisabled()152 bool isCompositeHeicDisabled() const override { return mIsCompositeHeicDisabled; } isCompositeHeicUltraHDRDisabled()153 bool isCompositeHeicUltraHDRDisabled() const override { 154 return mIsCompositeHeicUltraHDRDisabled; 155 } 156 157 // Capture and setStreamingRequest will configure streams if currently in 158 // idle state 159 status_t capture(CameraMetadata &request, int64_t *lastFrameNumber = NULL) override; 160 status_t captureList(const List<const PhysicalCameraSettingsList> &requestsList, 161 const std::list<SurfaceMap> &surfaceMaps, 162 int64_t *lastFrameNumber = NULL) override; 163 status_t setStreamingRequest(const CameraMetadata &request, 164 int64_t *lastFrameNumber = NULL) override; 165 status_t setStreamingRequestList(const List<const PhysicalCameraSettingsList> &requestsList, 166 const std::list<SurfaceMap> &surfaceMaps, 167 int64_t *lastFrameNumber = NULL) override; 168 status_t clearStreamingRequest(int64_t *lastFrameNumber = NULL) override; 169 170 status_t waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) override; 171 172 // Actual stream creation/deletion is delayed until first request is submitted 173 // If adding streams while actively capturing, will pause device before adding 174 // stream, reconfiguring device, and unpausing. If the client create a stream 175 // with nullptr consumer surface, the client must then call setConsumers() 176 // and finish the stream configuration before starting output streaming. 177 status_t createStream(sp<Surface> consumer, 178 uint32_t width, uint32_t height, int format, 179 android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id, 180 const std::string& physicalCameraId, 181 const std::unordered_set<int32_t> &sensorPixelModesUsed, 182 std::vector<int> *surfaceIds = nullptr, 183 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 184 bool isShared = false, bool isMultiResolution = false, 185 uint64_t consumerUsage = 0, 186 int64_t dynamicRangeProfile = 187 ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 188 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 189 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 190 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 191 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 192 bool useReadoutTimestamp = false) 193 override; 194 195 status_t createStream(const std::vector<SurfaceHolder>& consumers, 196 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 197 android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id, 198 const std::string& physicalCameraId, 199 const std::unordered_set<int32_t> &sensorPixelModesUsed, 200 std::vector<int> *surfaceIds = nullptr, 201 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 202 bool isShared = false, bool isMultiResolution = false, 203 uint64_t consumerUsage = 0, 204 int64_t dynamicRangeProfile = 205 ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 206 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 207 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 208 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 209 bool useReadoutTimestamp = false) 210 override; 211 212 status_t createInputStream( 213 uint32_t width, uint32_t height, int format, bool isMultiResolution, 214 int *id) override; 215 216 status_t getStreamInfo(int id, StreamInfo *streamInfo) override; 217 status_t setStreamTransform(int id, int transform) override; 218 219 status_t deleteStream(int id) override; 220 beginConfigure()221 virtual status_t beginConfigure() override {return OK;}; 222 getSharedStreamId(const OutputStreamInfo &,int *)223 virtual status_t getSharedStreamId(const OutputStreamInfo& /*config*/, 224 int* /*streamId*/) override {return INVALID_OPERATION;}; 225 addSharedSurfaces(int,const std::vector<android::camera3::OutputStreamInfo> &,const std::vector<SurfaceHolder> &,std::vector<int> *)226 virtual status_t addSharedSurfaces(int /*streamId*/, 227 const std::vector<android::camera3::OutputStreamInfo>& /*outputInfo*/, 228 const std::vector<SurfaceHolder>& /*surfaces*/, 229 std::vector<int>* /*surfaceIds*/) override {return INVALID_OPERATION;}; 230 removeSharedSurfaces(int,const std::vector<size_t> &)231 virtual status_t removeSharedSurfaces(int /*streamId*/, 232 const std::vector<size_t>& /*surfaceIds*/) override {return INVALID_OPERATION;}; 233 setSharedStreamingRequest(const PhysicalCameraSettingsList &,const SurfaceMap &,int32_t *,int64_t *)234 virtual status_t setSharedStreamingRequest( 235 const PhysicalCameraSettingsList& /*request*/, const SurfaceMap& /*surfaceMap*/, 236 int32_t* /*sharedReqID*/, int64_t* /*lastFrameNumber = NULL*/) override { 237 return INVALID_OPERATION; 238 }; 239 clearSharedStreamingRequest(int64_t *)240 virtual status_t clearSharedStreamingRequest(int64_t* /*lastFrameNumber = NULL*/) override { 241 return INVALID_OPERATION; 242 }; 243 setSharedCaptureRequest(const PhysicalCameraSettingsList &,const SurfaceMap &,int32_t *,int64_t *)244 virtual status_t setSharedCaptureRequest(const PhysicalCameraSettingsList& /*request*/, 245 const SurfaceMap& /*surfaceMap*/, int32_t* /*sharedReqID*/, 246 int64_t* /*lastFrameNumber = NULL*/) override {return INVALID_OPERATION;}; 247 getSharedFrameProcessor()248 virtual sp<camera2::FrameProcessorBase> getSharedFrameProcessor() override {return nullptr;}; 249 startStreaming(const int32_t,const SurfaceMap &,int32_t *,int64_t *)250 virtual status_t startStreaming(const int32_t /*reqId*/, const SurfaceMap& /*surfaceMap*/, 251 int32_t* /*sharedReqID*/, int64_t* /*lastFrameNumber = NULL*/) 252 override {return INVALID_OPERATION;}; 253 status_t configureStreams(const CameraMetadata& sessionParams, 254 int operatingMode = 255 camera_stream_configuration_mode_t::CAMERA_STREAM_CONFIGURATION_NORMAL_MODE) override; 256 #if WB_CAMERA3_AND_PROCESSORS_WITH_DEPENDENCIES 257 status_t getInputSurface(sp<Surface> *surface) override; 258 #else 259 status_t getInputBufferProducer( 260 sp<IGraphicBufferProducer> *producer) override; 261 #endif 262 263 void getOfflineStreamIds(std::vector<int> *offlineStreamIds) override; 264 265 status_t createDefaultRequest(camera_request_template_t templateId, 266 CameraMetadata *request) override; 267 268 // Transitions to the idle state on success 269 status_t waitUntilDrained() override; 270 271 virtual status_t setNotifyCallback(wp<NotificationListener> listener) override; 272 bool willNotify3A() override; 273 status_t waitForNextFrame(nsecs_t timeout) override; 274 status_t getNextResult(CaptureResult *frame) override; 275 276 status_t triggerAutofocus(uint32_t id) override; 277 status_t triggerCancelAutofocus(uint32_t id) override; 278 status_t triggerPrecaptureMetering(uint32_t id) override; 279 280 status_t flush(int64_t *lastFrameNumber = NULL) override; 281 282 status_t prepare(int streamId) override; 283 284 status_t tearDown(int streamId) override; 285 286 status_t addBufferListenerForStream(int streamId, 287 wp<camera3::Camera3StreamBufferListener> listener) override; 288 289 status_t prepare(int maxCount, int streamId) override; 290 291 ssize_t getJpegBufferSize(const CameraMetadata &info, uint32_t width, 292 uint32_t height) const override; 293 ssize_t getPointCloudBufferSize(const CameraMetadata &info) const; 294 ssize_t getRawOpaqueBufferSize(const CameraMetadata &info, int32_t width, int32_t height, 295 bool maxResolution) const; 296 297 // Methods called by subclasses 298 void notifyStatus(bool idle); // updates from StatusTracker 299 300 /** 301 * Set the deferred consumer surfaces to the output stream and finish the deferred 302 * consumer configuration. 303 */ 304 status_t setConsumerSurfaces( 305 int streamId, const std::vector<SurfaceHolder>& consumers, 306 std::vector<int> *surfaceIds /*out*/) override; 307 308 /** 309 * Update a given stream. 310 */ 311 status_t updateStream(int streamId, const std::vector<SurfaceHolder> &newSurfaces, 312 const std::vector<OutputStreamInfo> &outputInfo, 313 const std::vector<size_t> &removedSurfaceIds, 314 KeyedVector<sp<Surface>, size_t> *outputMap/*out*/); 315 316 /** 317 * Drop buffers for stream of streamId if dropping is true. If dropping is false, do not 318 * drop buffers for stream of streamId. 319 */ 320 status_t dropStreamBuffers(bool dropping, int streamId) override; 321 322 nsecs_t getExpectedInFlightDuration() override; 323 switchToOffline(const std::vector<int32_t> &,sp<CameraOfflineSessionBase> *)324 virtual status_t switchToOffline(const std::vector<int32_t>& , 325 /*out*/ sp<CameraOfflineSessionBase>* ) override { 326 return INVALID_OPERATION; 327 }; 328 329 // RequestBufferInterface 330 bool startRequestBuffer() override; 331 void endRequestBuffer() override; 332 nsecs_t getWaitDuration() override; 333 334 // FlushBufferInterface 335 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out) override; 336 void getInflightRequestBufferKeys(std::vector<uint64_t>* out) override; 337 std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override; 338 339 /** 340 * Set the current behavior for the ROTATE_AND_CROP control when in AUTO. 341 * 342 * The value must be one of the ROTATE_AND_CROP_* values besides AUTO, 343 * and defaults to NONE. 344 */ 345 status_t setRotateAndCropAutoBehavior( 346 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue, bool fromHal); 347 348 /** 349 * Set the current behavior for the AUTOFRAMING control when in AUTO. 350 * 351 * The value must be one of the AUTOFRAMING_* values besides AUTO. 352 */ 353 status_t setAutoframingAutoBehavior( 354 camera_metadata_enum_android_control_autoframing_t autoframingValue); 355 356 /** 357 * Whether camera muting (producing black-only output) is supported. 358 * 359 * Calling setCameraMute(true) when this returns false will return an 360 * INVALID_OPERATION error. 361 */ 362 bool supportsCameraMute(); 363 364 /** 365 * Mute the camera. 366 * 367 * When muted, black image data is output on all output streams. 368 */ 369 status_t setCameraMute(bool enabled); 370 371 /** 372 * Mute the camera. 373 * 374 * When muted, black image data is output on all output streams. 375 * This method assumes the caller already acquired the 'mInterfaceLock' 376 * and 'mLock' locks. 377 */ 378 status_t setCameraMuteLocked(bool enabled); 379 380 /** 381 * Enables/disables camera service watchdog 382 */ 383 status_t setCameraServiceWatchdog(bool enabled); 384 385 // Set stream use case overrides 386 void setStreamUseCaseOverrides( 387 const std::vector<int64_t>& useCaseOverrides); 388 389 // Clear stream use case overrides 390 void clearStreamUseCaseOverrides(); 391 392 /** 393 * Whether the camera device supports zoom override. 394 */ 395 bool supportsZoomOverride(); 396 397 // Set/reset zoom override 398 status_t setZoomOverride(int32_t zoomOverride); 399 400 // Get the status trackeer for the camera device getStatusTracker()401 wp<camera3::StatusTracker> getStatusTracker() { return mStatusTracker; } 402 403 // Whether the device is in error state 404 bool hasDeviceError(); 405 406 /** 407 * The injection camera session to replace the internal camera 408 * session. 409 */ 410 status_t injectCamera(const std::string& injectedCamId, 411 sp<CameraProviderManager> manager); 412 413 /** 414 * Stop the injection camera and restore to internal camera session. 415 */ 416 status_t stopInjection(); 417 418 /** 419 * Inject session params into the current client. 420 */ 421 status_t injectSessionParams(const CameraMetadata& sessionParams); 422 423 protected: 424 status_t disconnectImpl(); 425 static status_t removeFwkOnlyKeys(CameraMetadata *request); 426 427 float getMaxPreviewFps(sp<camera3::Camera3OutputStreamInterface> stream); 428 429 static const size_t kDumpLockAttempts = 10; 430 static const size_t kDumpSleepDuration = 100000; // 0.10 sec 431 static const nsecs_t kActiveTimeout = 500000000; // 500 ms 432 static const nsecs_t kMinWarnInflightDuration = 5000000000; // 5 s 433 static const size_t kInFlightWarnLimit = 30; 434 static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 435 static const nsecs_t kMinInflightDuration = 5000000000; // 5 s 436 static const nsecs_t kBaseGetBufferWait = 3000000000; // 3 sec. 437 438 struct RequestTrigger; 439 // minimal jpeg buffer size: 256KB + blob header 440 static const ssize_t kMinJpegBufferSize = camera3::MIN_JPEG_BUFFER_SIZE + 441 sizeof(aidl::android::hardware::camera::device::CameraBlob); 442 // Constant to use for stream ID when one doesn't exist 443 static const int NO_STREAM = -1; 444 445 std::shared_ptr<CameraServiceProxyWrapper> mCameraServiceProxyWrapper; 446 447 // A lock to enforce serialization on the input/configure side 448 // of the public interface. 449 // Only locked by public methods inherited from CameraDeviceBase. 450 // Not locked by methods guarded by mOutputLock, since they may act 451 // concurrently to the input/configure side of the interface. 452 // Must be locked before mLock if both will be locked by a method 453 Mutex mInterfaceLock; 454 455 // The main lock on internal state 456 Mutex mLock; 457 458 // Camera device ID 459 const std::string mId; 460 461 // Legacy camera client flag 462 bool mLegacyClient; 463 464 // Current stream configuration mode; 465 int mOperatingMode; 466 // Current session wide parameters 467 hardware::camera2::impl::CameraMetadataNative mSessionParams; 468 // Constant to use for no set operating mode 469 static const int NO_MODE = -1; 470 471 // Flag indicating is the current active stream configuration is constrained high speed. 472 bool mIsConstrainedHighSpeedConfiguration; 473 474 /**** Scope for mLock ****/ 475 476 class HalInterface : public camera3::Camera3StreamBufferFreedListener, 477 public camera3::BufferRecordsInterface { 478 public: HalInterface(bool useHalBufManager,bool supportOfflineProcessing)479 HalInterface(bool useHalBufManager, bool supportOfflineProcessing) : 480 mUseHalBufManager(useHalBufManager), 481 mIsReconfigurationQuerySupported(true), 482 mSupportOfflineProcessing(supportOfflineProcessing) 483 {}; 484 HalInterface(const HalInterface &other); 485 HalInterface(); 486 487 virtual IPCTransport getTransportType() const = 0; 488 489 // Returns true if constructed with a valid device or session, and not yet cleared 490 virtual bool valid() = 0; 491 492 // Reset this HalInterface object (does not call close()) 493 virtual void clear() = 0; 494 495 // Calls into the HAL interface 496 497 // Caller takes ownership of requestTemplate 498 virtual status_t constructDefaultRequestSettings(camera_request_template templateId, 499 /*out*/ camera_metadata_t **requestTemplate) = 0; 500 501 virtual status_t configureStreams(const camera_metadata_t * sessionParams, 502 /*inout*/ camera_stream_configuration_t * config, 503 const std::vector<uint32_t>& bufferSizes, int64_t logId) = 0; 504 505 // The injection camera configures the streams to hal. 506 virtual status_t configureInjectedStreams( 507 const camera_metadata_t* sessionParams, 508 /*inout*/ camera_stream_configuration_t* config, 509 const std::vector<uint32_t>& bufferSizes, 510 const CameraMetadata& cameraCharacteristics) = 0; 511 512 // When the call succeeds, the ownership of acquire fences in requests is transferred to 513 // HalInterface. More specifically, the current implementation will send the fence to 514 // HAL process and close the FD in cameraserver process. When the call fails, the ownership 515 // of the acquire fence still belongs to the caller. 516 virtual status_t processBatchCaptureRequests( 517 std::vector<camera_capture_request_t*>& requests, 518 /*out*/uint32_t* numRequestProcessed) = 0; 519 520 virtual status_t flush() = 0; 521 522 virtual status_t dump(int fd) = 0; 523 524 virtual status_t close() = 0; 525 526 virtual void signalPipelineDrain(const std::vector<int>& streamIds) = 0; 527 528 virtual bool isReconfigurationRequired(CameraMetadata& oldSessionParams, 529 CameraMetadata& newSessionParams) = 0; 530 531 virtual status_t repeatingRequestEnd(uint32_t frameNumber, 532 const std::vector<int32_t> &streamIds) = 0; 533 534 ///////////////////////////////////////////////////////////////////// 535 // Implements BufferRecordsInterface 536 537 std::pair<bool, uint64_t> getBufferId( 538 const buffer_handle_t& buf, int streamId) override; 539 540 uint64_t removeOneBufferCache(int streamId, const native_handle_t* handle) override; 541 542 status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, 543 /*out*/ buffer_handle_t **buffer) override; 544 545 status_t pushInflightRequestBuffer( 546 uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override; 547 548 status_t popInflightRequestBuffer(uint64_t bufferId, 549 /*out*/ buffer_handle_t** buffer, 550 /*optional out*/ int32_t* streamId = nullptr) override; 551 552 ///////////////////////////////////////////////////////////////////// 553 554 //Check if a stream is hal buffer managed 555 bool isHalBufferManagedStream(int32_t streamId) const; 556 557 // Get a vector of (frameNumber, streamId) pair of currently inflight 558 // buffers 559 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out); 560 561 // Get a vector of bufferId of currently inflight buffers 562 void getInflightRequestBufferKeys(std::vector<uint64_t>* out); 563 564 void onStreamReConfigured(int streamId); 565 566 protected: 567 568 // Return true if the input caches match what we have; otherwise false 569 bool verifyBufferIds(int32_t streamId, std::vector<uint64_t>& inBufIds); 570 571 template <typename OfflineSessionInfoT> verifyBufferCaches(const OfflineSessionInfoT * offlineSessionInfo,camera3::BufferRecords * bufferRecords)572 status_t verifyBufferCaches( 573 const OfflineSessionInfoT *offlineSessionInfo, camera3::BufferRecords *bufferRecords) { 574 // Validate buffer caches 575 std::vector<int32_t> streams; 576 streams.reserve(offlineSessionInfo->offlineStreams.size()); 577 for (auto offlineStream : offlineSessionInfo->offlineStreams) { 578 int32_t id = offlineStream.id; 579 streams.push_back(id); 580 // Verify buffer caches 581 std::vector<uint64_t> bufIds(offlineStream.circulatingBufferIds.begin(), 582 offlineStream.circulatingBufferIds.end()); 583 { 584 // Due to timing it is possible that we may not have any remaining pending 585 // capture requests that can update the caches on Hal side. This can result in 586 // buffer cache mismatch between the service and the Hal and must be accounted 587 // for. 588 std::lock_guard<std::mutex> l(mFreedBuffersLock); 589 for (const auto& it : mFreedBuffers) { 590 if (it.first == id) { 591 ALOGV("%s: stream ID %d buffer id %" PRIu64 " cache removal still " 592 "pending", __FUNCTION__, id, it.second); 593 const auto& cachedEntry = std::find(bufIds.begin(), bufIds.end(), 594 it.second); 595 if (cachedEntry != bufIds.end()) { 596 bufIds.erase(cachedEntry); 597 } else { 598 ALOGE("%s: stream ID %d buffer id %" PRIu64 " cache removal still " 599 "pending however buffer is no longer in the offline stream " 600 "info!", __FUNCTION__, id, it.second); 601 } 602 } 603 } 604 } 605 if (!verifyBufferIds(id, bufIds)) { 606 ALOGE("%s: stream ID %d buffer cache records mismatch!", __FUNCTION__, id); 607 return UNKNOWN_ERROR; 608 } 609 } 610 611 // Move buffer records 612 bufferRecords->takeBufferCaches(mBufferRecords, streams); 613 bufferRecords->takeInflightBufferMap(mBufferRecords); 614 bufferRecords->takeRequestedBufferMap(mBufferRecords); 615 return OK; 616 } 617 618 virtual void onBufferFreed(int streamId, const native_handle_t* handle) override; 619 620 std::mutex mFreedBuffersLock; 621 std::vector<std::pair<int, uint64_t>> mFreedBuffers; 622 623 // Keep track of buffer cache and inflight buffer records 624 camera3::BufferRecords mBufferRecords; 625 626 uint32_t mNextStreamConfigCounter = 1; 627 628 // TODO: This can be removed after flags::session_hal_buf_manager is removed 629 bool mUseHalBufManager = false; 630 std::set<int32_t > mHalBufManagedStreamIds; 631 bool mIsReconfigurationQuerySupported; 632 633 const bool mSupportOfflineProcessing; 634 }; // class HalInterface 635 636 sp<HalInterface> mInterface; 637 638 CameraMetadata mDeviceInfo; 639 bool mSupportNativeZoomRatio; 640 bool mIsCompositeJpegRDisabled; 641 bool mIsCompositeHeicDisabled; 642 bool mIsCompositeHeicUltraHDRDisabled; 643 std::unordered_map<std::string, CameraMetadata> mPhysicalDeviceInfoMap; 644 645 CameraMetadata mRequestTemplateCache[CAMERA_TEMPLATE_COUNT]; 646 647 struct Size { 648 uint32_t width; 649 uint32_t height; widthSize650 explicit Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){} 651 }; 652 653 enum Status { 654 STATUS_ERROR, 655 STATUS_UNINITIALIZED, 656 STATUS_UNCONFIGURED, 657 STATUS_CONFIGURED, 658 STATUS_ACTIVE 659 } mStatus; 660 661 struct StatusInfo { 662 Status status; 663 bool isInternal; // status triggered by internal reconfigureCamera. 664 }; 665 666 bool mStatusIsInternal; 667 668 // Only clear mRecentStatusUpdates, mStatusWaiters from waitUntilStateThenRelock 669 Vector<StatusInfo> mRecentStatusUpdates; 670 int mStatusWaiters; 671 672 Condition mStatusChanged; 673 674 // Tracking cause of fatal errors when in STATUS_ERROR 675 std::string mErrorCause; 676 677 camera3::StreamSet mOutputStreams; 678 sp<camera3::Camera3Stream> mInputStream; 679 bool mIsInputStreamMultiResolution; 680 SessionStatsBuilder mSessionStatsBuilder; 681 // Map from stream group ID to physical cameras backing the stream group 682 std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap; 683 684 int mNextStreamId; 685 bool mNeedConfig; 686 687 int mFakeStreamId; 688 689 // Whether to send state updates upstream 690 // Pause when doing transparent reconfiguration 691 bool mPauseStateNotify; 692 693 // Need to hold on to stream references until configure completes. 694 Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams; 695 696 // Whether the HAL will send partial result 697 bool mUsePartialResult; 698 699 // Number of partial results that will be delivered by the HAL. 700 uint32_t mNumPartialResults; 701 702 /**** End scope for mLock ****/ 703 704 bool mDeviceTimeBaseIsRealtime; 705 // The offset converting from clock domain of other subsystem 706 // (video/hardware composer) to that of camera. Assumption is that this 707 // offset won't change during the life cycle of the camera device. In other 708 // words, camera device shouldn't be open during CPU suspend. 709 nsecs_t mTimestampOffset; 710 711 class CaptureRequest : public LightRefBase<CaptureRequest> { 712 public: 713 PhysicalCameraSettingsList mSettingsList; 714 sp<camera3::Camera3Stream> mInputStream; 715 camera_stream_buffer_t mInputBuffer; 716 camera3::Size mInputBufferSize; 717 Vector<sp<camera3::Camera3OutputStreamInterface> > 718 mOutputStreams; 719 SurfaceMap mOutputSurfaces; 720 CaptureResultExtras mResultExtras; 721 // The number of requests that should be submitted to HAL at a time. 722 // For example, if batch size is 8, this request and the following 7 723 // requests will be submitted to HAL at a time. The batch size for 724 // the following 7 requests will be ignored by the request thread. 725 int mBatchSize; 726 // Whether this request is from a repeating or repeating burst. 727 bool mRepeating; 728 // Whether this request has ROTATE_AND_CROP_AUTO set, so needs both 729 // overriding of ROTATE_AND_CROP value and adjustment of coordinates 730 // in several other controls in both the request and the result 731 bool mRotateAndCropAuto; 732 // Indicates that the ROTATE_AND_CROP value within 'mSettingsList' was modified 733 // irrespective of the original value. 734 bool mRotateAndCropChanged = false; 735 // Whether this request has AUTOFRAMING_AUTO set, so need to override the AUTOFRAMING value 736 // in the capture request. 737 bool mAutoframingAuto; 738 // Indicates that the auto framing value within 'mSettingsList' was modified 739 bool mAutoframingChanged = false; 740 // Indicates that the camera test pattern setting is modified 741 bool mTestPatternChanged = false; 742 743 // Whether this capture request has its zoom ratio set to 1.0x before 744 // the framework overrides it for camera HAL consumption. 745 bool mZoomRatioIs1x; 746 // The systemTime timestamp when the request is created. 747 nsecs_t mRequestTimeNs; 748 749 // Whether this capture request's distortion correction update has 750 // been done. 751 bool mDistortionCorrectionUpdated = false; 752 // Whether this capture request's rotation and crop update has been 753 // done. 754 bool mRotationAndCropUpdated = false; 755 // Whether this capture request's autoframing has been done. 756 bool mAutoframingUpdated = false; 757 // Whether this capture request's zoom ratio update has been done. 758 bool mZoomRatioUpdated = false; 759 // Whether this max resolution capture request's crop / metering region update has been 760 // done. 761 bool mUHRCropAndMeteringRegionsUpdated = false; 762 }; 763 typedef List<sp<CaptureRequest> > RequestList; 764 765 status_t checkStatusOkToCaptureLocked(); 766 767 status_t convertMetadataListToRequestListLocked( 768 const List<const PhysicalCameraSettingsList> &metadataList, 769 const std::list<SurfaceMap> &surfaceMaps, 770 bool repeating, nsecs_t requestTimeNs, 771 /*out*/ 772 RequestList *requestList); 773 774 void convertToRequestList(List<const PhysicalCameraSettingsList>& requestsList, 775 std::list<SurfaceMap>& surfaceMaps, 776 const CameraMetadata& request); 777 778 status_t submitRequestsHelper(const List<const PhysicalCameraSettingsList> &requestsList, 779 const std::list<SurfaceMap> &surfaceMaps, 780 bool repeating, 781 int64_t *lastFrameNumber = NULL); 782 783 // lock to ensure only one processCaptureResult is called at a time. 784 Mutex mProcessCaptureResultLock; 785 786 /** 787 * Common initialization code shared by both HAL paths 788 * 789 * Must be called with mLock and mInterfaceLock held. 790 */ 791 status_t initializeCommonLocked(sp<CameraProviderManager> manager); 792 793 /** 794 * Update capture request list so that each batch size honors the batch_size_max report from 795 * the HAL. Set the batch size to output stream for buffer operations. 796 * 797 * Must be called with mLock held. 798 */ 799 virtual void applyMaxBatchSizeLocked( 800 RequestList* requestList, const sp<camera3::Camera3OutputStreamInterface>& stream) = 0; 801 802 struct LatestRequestInfo { 803 CameraMetadata requestSettings; 804 std::unordered_map<std::string, CameraMetadata> physicalRequestSettings; 805 int32_t inputStreamId = -1; 806 std::set<int32_t> outputStreamIds; 807 }; 808 809 /** 810 * Get the first repeating request in the ongoing repeating request list. 811 */ 812 const sp<CaptureRequest> getOngoingRepeatingRequestLocked(); 813 814 /** 815 * Update the first repeating request in the ongoing repeating request list 816 * with the surface map provided. 817 */ 818 status_t updateOngoingRepeatingRequestLocked(const SurfaceMap& surfaceMap); 819 820 /** 821 * Get the repeating request last frame number. 822 */ 823 int64_t getRepeatingRequestLastFrameNumberLocked(); 824 825 /** 826 * Get the last request submitted to the hal by the request thread. 827 * 828 * Must be called with mLock held. 829 */ 830 virtual LatestRequestInfo getLatestRequestInfoLocked(); 831 832 virtual status_t injectionCameraInitialize(const std::string &injectCamId, 833 sp<CameraProviderManager> manager) = 0; 834 835 /** 836 * Update the current device status and wake all waiting threads. 837 * 838 * Must be called with mLock held. 839 */ 840 void internalUpdateStatusLocked(Status status); 841 842 /** 843 * Pause processing and flush everything, but don't tell the clients. 844 * This is for reconfiguring outputs transparently when according to the 845 * CameraDeviceBase interface we shouldn't need to. 846 * Must be called with mLock and mInterfaceLock both held. 847 */ 848 status_t internalPauseAndWaitLocked(nsecs_t maxExpectedDuration, 849 bool requestThreadInvocation); 850 851 /** 852 * Resume work after internalPauseAndWaitLocked() 853 * Must be called with mLock and mInterfaceLock both held. 854 */ 855 status_t internalResumeLocked(); 856 857 /** 858 * Wait until status tracker tells us we've transitioned to the target state 859 * set, which is either ACTIVE when active==true or IDLE (which is any 860 * non-ACTIVE state) when active==false. 861 * 862 * Needs to be called with mLock and mInterfaceLock held. This means there 863 * can ever only be one waiter at most. 864 * 865 * During the wait mLock is released. 866 * 867 */ 868 status_t waitUntilStateThenRelock(bool active, nsecs_t timeout, 869 bool requestThreadInvocation); 870 871 /** 872 * Implementation of waitUntilDrained. On success, will transition to IDLE state. 873 * 874 * Need to be called with mLock and mInterfaceLock held. 875 */ 876 status_t waitUntilDrainedLocked(nsecs_t maxExpectedDuration); 877 878 /** 879 * Do common work for setting up a streaming or single capture request. 880 * On success, will transition to ACTIVE if in IDLE. 881 */ 882 sp<CaptureRequest> setUpRequestLocked(const PhysicalCameraSettingsList &request, 883 const SurfaceMap &surfaceMap); 884 885 /** 886 * Build a CaptureRequest request from the CameraDeviceBase request 887 * settings. 888 */ 889 sp<CaptureRequest> createCaptureRequest(const PhysicalCameraSettingsList &request, 890 const SurfaceMap &surfaceMap); 891 892 /** 893 * Internally re-configure camera device using new session parameters. 894 * This will get triggered by the request thread. 895 */ 896 bool reconfigureCamera(const CameraMetadata& sessionParams, int clientStatusId); 897 898 /** 899 * Return true in case of any output or input abandoned streams, 900 * otherwise return false. 901 */ 902 bool checkAbandonedStreamsLocked(); 903 904 /** 905 * Filter stream session parameters and configure camera HAL. 906 */ 907 status_t filterParamsAndConfigureLocked(const CameraMetadata& sessionParams, 908 int operatingMode); 909 910 /** 911 * Take the currently-defined set of streams and configure the HAL to use 912 * them. This is a long-running operation (may be several hundered ms). 913 */ 914 status_t configureStreamsLocked(int operatingMode, 915 const CameraMetadata& sessionParams, bool notifyRequestThread = true); 916 917 /** 918 * Cancel stream configuration that did not finish successfully. 919 */ 920 void cancelStreamsConfigurationLocked(); 921 922 /** 923 * Add a fake stream to the current stream set as a workaround for 924 * not allowing 0 streams in the camera HAL spec. 925 */ 926 status_t addFakeStreamLocked(); 927 928 /** 929 * Remove a fake stream if the current config includes real streams. 930 */ 931 status_t tryRemoveFakeStreamLocked(); 932 933 /** 934 * Set device into an error state due to some fatal failure, and set an 935 * error message to indicate why. Only the first call's message will be 936 * used. The message is also sent to the log. 937 */ 938 void setErrorState(const char *fmt, ...) override; 939 void setErrorStateLocked(const char *fmt, ...) override; 940 void setErrorStateV(const char *fmt, va_list args); 941 void setErrorStateLockedV(const char *fmt, va_list args); 942 943 ///////////////////////////////////////////////////////////////////// 944 // Implements InflightRequestUpdateInterface 945 946 void onInflightEntryRemovedLocked(nsecs_t duration) override; 947 void checkInflightMapLengthLocked() override; 948 void onInflightMapFlushedLocked() override; 949 950 ///////////////////////////////////////////////////////////////////// 951 952 /** 953 * Debugging trylock/spin method 954 * Try to acquire a lock a few times with sleeps between before giving up. 955 */ 956 bool tryLockSpinRightRound(Mutex& lock); 957 958 /** 959 * Helper function to get the offset between MONOTONIC and BOOTTIME 960 * timestamp. 961 */ 962 static nsecs_t getMonoToBoottimeOffset(); 963 964 // Override rotate_and_crop control if needed 965 static bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/, 966 int rotationOverride, 967 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropOverride); 968 969 // Override auto framing control if needed 970 static bool overrideAutoframing(const sp<CaptureRequest> &request /*out*/, 971 camera_metadata_enum_android_control_autoframing_t autoframingOverride); 972 973 struct RequestTrigger { 974 // Metadata tag number, e.g. android.control.aePrecaptureTrigger 975 uint32_t metadataTag; 976 // Metadata value, e.g. 'START' or the trigger ID 977 int32_t entryValue; 978 979 // The last part of the fully qualified path, e.g. afTrigger getTagNameRequestTrigger980 const char *getTagName() const { 981 return get_camera_metadata_tag_name(metadataTag) ?: "NULL"; 982 } 983 984 // e.g. TYPE_BYTE, TYPE_INT32, etc. getTagTypeRequestTrigger985 int getTagType() const { 986 return get_camera_metadata_tag_type(metadataTag); 987 } 988 }; 989 990 /** 991 * Thread for managing capture request submission to HAL device. 992 */ 993 class RequestThread : public Thread { 994 995 public: 996 997 RequestThread(wp<Camera3Device> parent, 998 sp<camera3::StatusTracker> statusTracker, 999 sp<HalInterface> interface, 1000 const Vector<int32_t>& sessionParamKeys, 1001 bool useHalBufManager, 1002 bool supportCameraMute, 1003 int rotationOverride, 1004 bool supportSettingsOverride); 1005 ~RequestThread(); 1006 1007 void setNotificationListener(wp<NotificationListener> listener); 1008 1009 /** 1010 * Call after stream (re)-configuration is completed. 1011 */ 1012 void configurationComplete(bool isConstrainedHighSpeed, 1013 const CameraMetadata& sessionParams, 1014 const std::map<int32_t, std::set<std::string>>& groupIdPhysicalCameraMap); 1015 1016 /** 1017 * Set or clear the list of repeating requests. Does not block 1018 * on either. Use waitUntilPaused to wait until request queue 1019 * has emptied out. 1020 */ 1021 status_t setRepeatingRequests(const RequestList& requests, 1022 /*out*/ 1023 int64_t *lastFrameNumber = NULL); 1024 status_t clearRepeatingRequests(/*out*/ 1025 int64_t *lastFrameNumber = NULL); 1026 1027 status_t queueRequestList(List<sp<CaptureRequest> > &requests, 1028 /*out*/ 1029 int64_t *lastFrameNumber = NULL); 1030 1031 /** 1032 * Remove all queued and repeating requests, and pending triggers 1033 */ 1034 status_t clear(/*out*/int64_t *lastFrameNumber = NULL); 1035 1036 /** 1037 * Flush all pending requests in HAL. 1038 */ 1039 status_t flush(); 1040 1041 /** 1042 * Queue a trigger to be dispatched with the next outgoing 1043 * process_capture_request. The settings for that request only 1044 * will be temporarily rewritten to add the trigger tag/value. 1045 * Subsequent requests will not be rewritten (for this tag). 1046 */ 1047 status_t queueTrigger(RequestTrigger trigger[], size_t count); 1048 1049 /** 1050 * Pause/unpause the capture thread. Doesn't block, so use 1051 * waitUntilPaused to wait until the thread is paused. 1052 */ 1053 void setPaused(bool paused); 1054 1055 /** 1056 * Set Hal buffer managed streams 1057 * @param halBufferManagedStreams The streams for which hal buffer manager is enabled 1058 * 1059 */ 1060 void setHalBufferManagedStreams(const std::set<int32_t> &halBufferManagedStreams); 1061 1062 /** 1063 * Wait until thread processes the capture request with settings' 1064 * android.request.id == requestId. 1065 * 1066 * Returns TIMED_OUT in case the thread does not process the request 1067 * within the timeout. 1068 */ 1069 status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout); 1070 1071 /** 1072 * Shut down the thread. Shutdown is asynchronous, so thread may 1073 * still be running once this method returns. 1074 */ 1075 virtual void requestExit(); 1076 1077 /** 1078 * Get the latest request that was sent to the HAL 1079 * with process_capture_request. 1080 */ 1081 LatestRequestInfo getLatestRequestInfo() const; 1082 1083 /** 1084 * Returns true if the stream is a target of any queued or repeating 1085 * capture request 1086 */ 1087 bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream); 1088 1089 /** 1090 * Returns true if the surface is a target of any queued or repeating 1091 * capture request 1092 */ 1093 bool isOutputSurfacePending(int streamId, size_t surfaceId); 1094 1095 // dump processCaptureRequest latency dumpCaptureRequestLatency(int fd,const char * name)1096 void dumpCaptureRequestLatency(int fd, const char* name) { 1097 mRequestLatency.dump(fd, name); 1098 } 1099 1100 void signalPipelineDrain(const std::vector<int>& streamIds); 1101 void resetPipelineDrain(); 1102 1103 void clearPreviousRequest(); 1104 1105 status_t setRotateAndCropAutoBehavior( 1106 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue); 1107 1108 status_t setAutoframingAutoBehaviour( 1109 camera_metadata_enum_android_control_autoframing_t autoframingValue); 1110 1111 status_t setComposerSurface(bool composerSurfacePresent); 1112 1113 status_t setCameraMute(int32_t muteMode); 1114 1115 status_t setZoomOverride(int32_t zoomOverride); 1116 1117 status_t setHalInterface(sp<HalInterface> newHalInterface); 1118 1119 status_t setInjectedSessionParams(const CameraMetadata& sessionParams); 1120 1121 void injectSessionParams( 1122 const sp<CaptureRequest> &request, 1123 const CameraMetadata& injectedSessionParams); 1124 1125 /** 1126 * signal mLatestRequestmutex 1127 **/ 1128 void wakeupLatestRequest(bool latestRequestFailed, int32_t latestRequestId); 1129 1130 /** 1131 * Get the first repeating request in the ongoing repeating request list. 1132 */ 1133 const sp<CaptureRequest> getOngoingRepeatingRequest(); 1134 1135 /** 1136 * Update the first repeating request in the ongoing repeating request list 1137 * with the surface map provided. 1138 */ 1139 status_t updateOngoingRepeatingRequest(const SurfaceMap& surfaceMap); 1140 1141 // Get the repeating request last frame number. 1142 int64_t getRepeatingRequestLastFrameNumber(); 1143 1144 protected: 1145 1146 virtual bool threadLoop(); 1147 1148 static const std::string& getId(const wp<Camera3Device> &device); 1149 1150 status_t queueTriggerLocked(RequestTrigger trigger); 1151 // Mix-in queued triggers into this request 1152 int32_t insertTriggers(const sp<CaptureRequest> &request); 1153 // Purge the queued triggers from this request, 1154 // restoring the old field values for those tags. 1155 status_t removeTriggers(const sp<CaptureRequest> &request); 1156 1157 // HAL workaround: Make sure a trigger ID always exists if 1158 // a trigger does 1159 status_t addFakeTriggerIds(const sp<CaptureRequest> &request); 1160 1161 // Override rotate_and_crop control if needed; returns true if the current value was changed 1162 bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/); 1163 1164 // Override autoframing control if needed; returns true if the current value was changed 1165 bool overrideAutoframing(const sp<CaptureRequest> &request); 1166 1167 // Override test_pattern control if needed for camera mute; returns true 1168 // if the current value was changed 1169 bool overrideTestPattern(const sp<CaptureRequest> &request); 1170 1171 // Override settings override if needed for lower zoom latency; return 1172 // true if the current value was changed 1173 bool overrideSettingsOverride(const sp<CaptureRequest> &request); 1174 1175 static const nsecs_t kRequestTimeout = 50e6; // 50 ms 1176 1177 // TODO: does this need to be adjusted for long exposure requests? 1178 static const nsecs_t kRequestSubmitTimeout = 500e6; // 500 ms 1179 1180 // Used to prepare a batch of requests. 1181 struct NextRequest { 1182 sp<CaptureRequest> captureRequest; 1183 camera_capture_request_t halRequest; 1184 Vector<camera_stream_buffer_t> outputBuffers; 1185 bool submitted; 1186 }; 1187 1188 // Wait for the next batch of requests and put them in mNextRequests. mNextRequests will 1189 // be empty if it times out. 1190 void waitForNextRequestBatch(); 1191 1192 // Waits for a request, or returns NULL if times out. Must be called with mRequestLock hold. 1193 sp<CaptureRequest> waitForNextRequestLocked(); 1194 1195 // Prepare HAL requests and output buffers in mNextRequests. Return TIMED_OUT if getting any 1196 // output buffer timed out. If an error is returned, the caller should clean up the pending 1197 // request batch. 1198 status_t prepareHalRequests(); 1199 1200 // Return buffers, etc, for requests in mNextRequests that couldn't be fully constructed and 1201 // send request errors if sendRequestError is true. The buffers will be returned in the 1202 // ERROR state to mark them as not having valid data. mNextRequests will be cleared. 1203 void cleanUpFailedRequests(bool sendRequestError); 1204 1205 // Stop the repeating request if any of its output streams is abandoned. 1206 void checkAndStopRepeatingRequest(); 1207 1208 // Release physical camera settings and camera id resources. 1209 void cleanupPhysicalSettings(sp<CaptureRequest> request, 1210 /*out*/camera_capture_request_t *halRequest); 1211 1212 // Pause handling 1213 bool waitIfPaused(); 1214 void unpauseForNewRequests(); 1215 1216 // Relay error to parent device object setErrorState 1217 void setErrorState(const char *fmt, ...); 1218 1219 // If the input request is in mRepeatingRequests. Must be called with mRequestLock hold 1220 bool isRepeatingRequestLocked(const sp<CaptureRequest>&); 1221 1222 // Clear repeating requests. Must be called with mRequestLock held. 1223 status_t clearRepeatingRequestsLocked(/*out*/ int64_t *lastFrameNumber = NULL); 1224 1225 // send request in mNextRequests to HAL in a batch. Return true = sucssess 1226 bool sendRequestsBatch(); 1227 1228 // Calculate the expected (minimum, maximum, isFixedFps) duration info for a request 1229 struct ExpectedDurationInfo { 1230 nsecs_t minDuration; 1231 nsecs_t maxDuration; 1232 bool isFixedFps; 1233 }; 1234 ExpectedDurationInfo calculateExpectedDurationRange( 1235 const camera_metadata_t *request); 1236 1237 // Check and update latest session parameters based on the current request settings. 1238 bool updateSessionParameters(const CameraMetadata& settings); 1239 1240 // Check whether FPS range session parameter re-configuration is needed in constrained 1241 // high speed recording camera sessions. 1242 bool skipHFRTargetFPSUpdate(int32_t tag, const camera_metadata_ro_entry_t& newEntry, 1243 const camera_metadata_entry_t& currentEntry); 1244 1245 // Update next request sent to HAL 1246 void updateNextRequest(NextRequest& nextRequest); 1247 1248 wp<Camera3Device> mParent; 1249 wp<camera3::StatusTracker> mStatusTracker; 1250 sp<HalInterface> mInterface; 1251 1252 wp<NotificationListener> mListener; 1253 1254 const std::string mId; // The camera ID 1255 int mStatusId; // The RequestThread's component ID for 1256 // status tracking 1257 1258 Mutex mRequestLock; 1259 Condition mRequestSignal; 1260 bool mRequestClearing; 1261 1262 Condition mRequestSubmittedSignal; 1263 RequestList mRequestQueue; 1264 RequestList mRepeatingRequests; 1265 bool mFirstRepeating; 1266 // The next batch of requests being prepped for submission to the HAL, no longer 1267 // on the request queue. Read-only even with mRequestLock held, outside 1268 // of threadLoop 1269 Vector<NextRequest> mNextRequests; 1270 1271 // To protect flush() and sending a request batch to HAL. 1272 Mutex mFlushLock; 1273 1274 bool mReconfigured; 1275 1276 // Used by waitIfPaused, waitForNextRequest, waitUntilPaused, and signalPipelineDrain 1277 Mutex mPauseLock; 1278 bool mDoPause; 1279 Condition mDoPauseSignal; 1280 bool mPaused; 1281 bool mNotifyPipelineDrain; 1282 std::vector<int> mStreamIdsToBeDrained; 1283 1284 sp<CaptureRequest> mPrevRequest; 1285 int32_t mPrevTriggers; 1286 std::set<std::string> mPrevCameraIdsWithZoom; 1287 1288 uint32_t mFrameNumber; 1289 1290 mutable Mutex mLatestRequestMutex; 1291 Condition mLatestRequestSignal; 1292 // android.request.id for latest process_capture_request 1293 int32_t mLatestRequestId; 1294 int32_t mLatestFailedRequestId; 1295 LatestRequestInfo mLatestRequestInfo; 1296 1297 typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap; 1298 Mutex mTriggerMutex; 1299 TriggerMap mTriggerMap; 1300 TriggerMap mTriggerRemovedMap; 1301 TriggerMap mTriggerReplacedMap; 1302 uint32_t mCurrentAfTriggerId; 1303 uint32_t mCurrentPreCaptureTriggerId; 1304 camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride; 1305 camera_metadata_enum_android_control_autoframing_t mAutoframingOverride; 1306 bool mComposerOutput; 1307 int32_t mCameraMute; // 0 = no mute, otherwise the TEST_PATTERN_MODE to use 1308 int32_t mSettingsOverride; // -1 = use original, otherwise 1309 // the settings override to use. 1310 1311 int64_t mRepeatingLastFrameNumber; 1312 1313 // Flag indicating if we should prepare video stream for video requests. 1314 bool mPrepareVideoStream; 1315 1316 bool mConstrainedMode; 1317 1318 static const int32_t kRequestLatencyBinSize = 40; // in ms 1319 CameraLatencyHistogram mRequestLatency; 1320 1321 Vector<int32_t> mSessionParamKeys; 1322 CameraMetadata mLatestSessionParams; 1323 CameraMetadata mInjectedSessionParams; 1324 bool mForceNewRequestAfterReconfigure; 1325 1326 std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap; 1327 1328 bool mUseHalBufManager = false; 1329 std::set<int32_t > mHalBufManagedStreamIds; 1330 const bool mSupportCameraMute; 1331 const bool mRotationOverride; 1332 const bool mSupportSettingsOverride; 1333 int32_t mVndkVersion = -1; 1334 }; 1335 1336 virtual sp<RequestThread> createNewRequestThread(wp<Camera3Device> /*parent*/, 1337 sp<camera3::StatusTracker> /*statusTracker*/, 1338 sp<HalInterface> /*interface*/, 1339 const Vector<int32_t>& /*sessionParamKeys*/, 1340 bool /*useHalBufManager*/, 1341 bool /*supportCameraMute*/, 1342 int /*rotationOverride*/, 1343 bool /*supportSettingsOverride*/) = 0; 1344 1345 sp<RequestThread> mRequestThread; 1346 1347 /** 1348 * In-flight queue for tracking completion of capture requests. 1349 */ 1350 std::mutex mInFlightLock; 1351 camera3::InFlightRequestMap mInFlightMap; 1352 nsecs_t mExpectedInflightDuration = 0; 1353 int64_t mLastCompletedRegularFrameNumber = -1; 1354 int64_t mLastCompletedReprocessFrameNumber = -1; 1355 int64_t mLastCompletedZslFrameNumber = -1; 1356 // End of mInFlightLock protection scope 1357 1358 int mInFlightStatusId; // const after initialize 1359 1360 status_t registerInFlight(uint32_t frameNumber, 1361 int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, 1362 bool callback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration, 1363 bool isFixedFps, const std::set<std::set<std::string>>& physicalCameraIds, 1364 bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto, bool autoframingAuto, 1365 const std::set<std::string>& cameraIdsWithZoom, bool useZoomRatio, 1366 const SurfaceMap& outputSurfaces, nsecs_t requestTimeNs); 1367 1368 /** 1369 * Tracking for idle detection 1370 */ 1371 sp<camera3::StatusTracker> mStatusTracker; 1372 1373 /** 1374 * Graphic buffer manager for output streams. Each device has a buffer manager, which is used 1375 * by the output streams to get and return buffers if these streams are registered to this 1376 * buffer manager. 1377 */ 1378 sp<camera3::Camera3BufferManager> mBufferManager; 1379 1380 /** 1381 * Thread for preparing streams 1382 */ 1383 class PreparerThread : private Thread, public virtual RefBase { 1384 public: 1385 PreparerThread(); 1386 ~PreparerThread(); 1387 1388 void setNotificationListener(wp<NotificationListener> listener); 1389 1390 /** 1391 * Queue up a stream to be prepared. Streams are processed by a background thread in FIFO 1392 * order. Pre-allocate up to maxCount buffers for the stream, or the maximum number needed 1393 * for the pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 1394 */ 1395 status_t prepare(int maxCount, sp<camera3::Camera3StreamInterface>& stream); 1396 1397 /** 1398 * Cancel all current and pending stream preparation 1399 */ 1400 status_t clear(); 1401 1402 /** 1403 * Pause all preparation activities 1404 */ 1405 void pause(); 1406 1407 /** 1408 * Resume preparation activities 1409 */ 1410 status_t resume(); 1411 1412 private: 1413 Mutex mLock; 1414 Condition mThreadActiveSignal; 1415 1416 virtual bool threadLoop(); 1417 1418 // Guarded by mLock 1419 1420 wp<NotificationListener> mListener; 1421 std::list<std::tuple<int, sp<camera3::Camera3StreamInterface>>> mPendingStreams; 1422 bool mActive; 1423 bool mCancelNow; 1424 1425 // Only accessed by threadLoop and the destructor 1426 1427 sp<camera3::Camera3StreamInterface> mCurrentStream; 1428 int mCurrentMaxCount; 1429 bool mCurrentPrepareComplete; 1430 }; 1431 sp<PreparerThread> mPreparerThread; 1432 1433 /** 1434 * Output result queue and current HAL device 3A state 1435 */ 1436 1437 // Lock for output side of device 1438 std::mutex mOutputLock; 1439 1440 /**** Scope for mOutputLock ****/ 1441 // the minimal frame number of the next non-reprocess result 1442 uint32_t mNextResultFrameNumber; 1443 // the minimal frame number of the next reprocess result 1444 uint32_t mNextReprocessResultFrameNumber; 1445 // the minimal frame number of the next ZSL still capture result 1446 uint32_t mNextZslStillResultFrameNumber; 1447 // the minimal frame number of the next non-reprocess shutter 1448 uint32_t mNextShutterFrameNumber; 1449 // the minimal frame number of the next reprocess shutter 1450 uint32_t mNextReprocessShutterFrameNumber; 1451 // the minimal frame number of the next ZSL still capture shutter 1452 uint32_t mNextZslStillShutterFrameNumber; 1453 std::list<CaptureResult> mResultQueue; 1454 std::condition_variable mResultSignal; 1455 wp<NotificationListener> mListener; 1456 1457 /**** End scope for mOutputLock ****/ 1458 1459 /**** Scope for mInFlightLock ****/ 1460 1461 // Remove the in-flight map entry of the given index from mInFlightMap. 1462 // It must only be called with mInFlightLock held. 1463 void removeInFlightMapEntryLocked(int idx); 1464 1465 // Remove all in-flight requests and return all buffers. 1466 // This is used after HAL interface is closed to cleanup any request/buffers 1467 // not returned by HAL. 1468 void flushInflightRequests(); 1469 1470 /**** End scope for mInFlightLock ****/ 1471 1472 /** 1473 * Distortion correction support 1474 */ 1475 // Map from camera IDs to its corresponding distortion mapper. Only contains 1476 // 1 ID if the device isn't a logical multi-camera. Otherwise contains both 1477 // logical camera and its physical subcameras. 1478 std::unordered_map<std::string, camera3::DistortionMapper> mDistortionMappers; 1479 1480 /** 1481 * Zoom ratio mapper support 1482 */ 1483 std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers; 1484 1485 /** 1486 * UHR request crop / metering region mapper support 1487 */ 1488 std::unordered_map<std::string, camera3::UHRCropAndMeteringRegionMapper> 1489 mUHRCropAndMeteringRegionMappers; 1490 1491 /** 1492 * RotateAndCrop mapper support 1493 */ 1494 std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers; 1495 1496 // Debug tracker for metadata tag value changes 1497 // - Enabled with the -m <taglist> option to dumpsys, such as 1498 // dumpsys -m android.control.aeState,android.control.aeMode 1499 // - Disabled with -m off 1500 // - dumpsys -m 3a is a shortcut for ae/af/awbMode, State, and Triggers 1501 TagMonitor mTagMonitor; 1502 1503 void monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber, 1504 nsecs_t timestamp, const CameraMetadata& metadata, 1505 const std::unordered_map<std::string, CameraMetadata>& physicalMetadata, 1506 const camera_stream_buffer_t *outputBuffers, uint32_t numOutputBuffers, 1507 int32_t inputStreamId); 1508 1509 // Collect any statistics that are based on the stream of capture requests sent 1510 // to the HAL 1511 void collectRequestStats(int64_t frameNumber, const CameraMetadata& request); 1512 1513 metadata_vendor_id_t mVendorTagId; 1514 1515 // Cached last requested template id 1516 int mLastTemplateId; 1517 1518 // Synchronizes access to status tracker between inflight updates and disconnect. 1519 // b/79972865 1520 Mutex mTrackerLock; 1521 1522 // Whether HAL request buffers through requestStreamBuffers API 1523 bool mUseHalBufManager = false; 1524 std::set<int32_t > mHalBufManagedStreamIds; 1525 bool mSessionHalBufManager = false; 1526 // Lock to ensure requestStreamBuffers() callbacks are serialized 1527 std::mutex mRequestBufferInterfaceLock; 1528 1529 // The state machine to control when requestStreamBuffers should allow 1530 // HAL to request buffers. 1531 enum RequestBufferState { 1532 /** 1533 * This is the initial state. 1534 * requestStreamBuffers call will return FAILED_CONFIGURING in this state. 1535 * Will switch to RB_STATUS_READY after a successful configureStreams or 1536 * processCaptureRequest call. 1537 */ 1538 RB_STATUS_STOPPED, 1539 1540 /** 1541 * requestStreamBuffers call will proceed in this state. 1542 * When device is asked to stay idle via waitUntilStateThenRelock() call: 1543 * - Switch to RB_STATUS_STOPPED if there is no inflight requests and 1544 * request thread is paused. 1545 * - Switch to RB_STATUS_PENDING_STOP otherwise 1546 */ 1547 RB_STATUS_READY, 1548 1549 /** 1550 * requestStreamBuffers call will proceed in this state. 1551 * Switch to RB_STATUS_STOPPED when all inflight requests are fulfilled 1552 * and request thread is paused 1553 */ 1554 RB_STATUS_PENDING_STOP, 1555 }; 1556 1557 class RequestBufferStateMachine { 1558 public: 1559 status_t initialize(sp<camera3::StatusTracker> statusTracker); 1560 1561 status_t deInit(); 1562 1563 // Return if the state machine currently allows for requestBuffers 1564 // If the state allows for it, mRequestBufferOngoing will be set to true 1565 // and caller must call endRequestBuffer() later to unset the flag 1566 bool startRequestBuffer(); 1567 void endRequestBuffer(); 1568 1569 // Events triggered by application API call 1570 void onStreamsConfigured(); 1571 void onWaitUntilIdle(); 1572 1573 // Events usually triggered by hwBinder processCaptureResult callback thread 1574 // But can also be triggered on request thread for failed request, or on 1575 // hwbinder notify callback thread for shutter/error callbacks 1576 void onInflightMapEmpty(); 1577 1578 // Events triggered by RequestThread 1579 void onSubmittingRequest(); 1580 void onRequestThreadPaused(); 1581 1582 // Events triggered by successful switchToOffline call 1583 // Return true is there is no ongoing requestBuffer call. 1584 bool onSwitchToOfflineSuccess(); 1585 1586 private: 1587 void notifyTrackerLocked(bool active); 1588 1589 // Switch to STOPPED state and return true if all conditions allows for it. 1590 // Otherwise do nothing and return false. 1591 bool checkSwitchToStopLocked(); 1592 1593 std::mutex mLock; 1594 RequestBufferState mStatus = RB_STATUS_STOPPED; 1595 1596 bool mRequestThreadPaused = true; 1597 bool mInflightMapEmpty = true; 1598 bool mRequestBufferOngoing = false; 1599 bool mSwitchedToOffline = false; 1600 1601 wp<camera3::StatusTracker> mStatusTracker; 1602 int mRequestBufferStatusId; 1603 } mRequestBufferSM; 1604 1605 // Fix up result metadata for monochrome camera. 1606 bool mNeedFixupMonochromeTags; 1607 1608 // Whether HAL supports offline processing capability. 1609 bool mSupportOfflineProcessing = false; 1610 1611 // Whether the HAL supports camera muting via test pattern 1612 bool mSupportCameraMute = false; 1613 // Whether the HAL supports SOLID_COLOR or BLACK if mSupportCameraMute is true 1614 bool mSupportTestPatternSolidColor = false; 1615 // Whether the HAL supports zoom settings override 1616 bool mSupportZoomOverride = false; 1617 1618 // Whether the camera framework overrides the device characteristics for 1619 // performance class. 1620 bool mOverrideForPerfClass; 1621 1622 // Whether the camera framework overrides the device characteristics for 1623 // app compatibility reasons. 1624 int mRotationOverride; 1625 camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride; 1626 bool mComposerOutput; 1627 1628 // Auto framing override value 1629 camera_metadata_enum_android_control_autoframing mAutoframingOverride; 1630 1631 // Initial camera mute state stored before the request thread 1632 // is active. 1633 bool mCameraMuteInitial = false; 1634 1635 // Settings override value 1636 int32_t mSettingsOverride; // -1 = use original, otherwise 1637 // the settings override to use. 1638 1639 // Current active physical id of the logical multi-camera, if any 1640 std::string mActivePhysicalId; 1641 1642 // The current minimum expected frame duration based on AE_TARGET_FPS_RANGE 1643 nsecs_t mMinExpectedDuration = 0; 1644 // Whether the camera device runs at fixed frame rate based on AE_MODE and 1645 // AE_TARGET_FPS_RANGE 1646 bool mIsFixedFps = false; 1647 1648 // Flag to indicate that we shouldn't forward extension related metadata 1649 bool mSupportsExtensionKeys = false; 1650 1651 // If the client is a native client, either opened through vndk, or caling 1652 // Pid is a platform service. 1653 bool mIsNativeClient; 1654 1655 // Injection camera related methods. 1656 class Camera3DeviceInjectionMethods : public virtual RefBase { 1657 public: 1658 Camera3DeviceInjectionMethods(wp<Camera3Device> parent); 1659 1660 ~Camera3DeviceInjectionMethods(); 1661 1662 // Injection camera will replace the internal camera and configure streams 1663 // when device is IDLE and request thread is paused. 1664 status_t injectCamera( 1665 camera3::camera_stream_configuration& injectionConfig, 1666 const std::vector<uint32_t>& injectionBufferSizes); 1667 1668 // Stop the injection camera and switch back to backup hal interface. 1669 status_t stopInjection(); 1670 1671 bool isInjecting(); 1672 1673 bool isStreamConfigCompleteButNotInjected(); 1674 1675 const std::string& getInjectedCamId() const; 1676 1677 void getInjectionConfig(/*out*/ camera3::camera_stream_configuration* injectionConfig, 1678 /*out*/ std::vector<uint32_t>* injectionBufferSizes); 1679 1680 // When the streaming configuration is completed and the camera device is active, but the 1681 // injection camera has not yet been injected, the streaming configuration of the internal 1682 // camera will be stored first. 1683 void storeInjectionConfig( 1684 const camera3::camera_stream_configuration& injectionConfig, 1685 const std::vector<uint32_t>& injectionBufferSizes); 1686 1687 protected: 1688 // Configure the streams of injection camera, it need wait until the 1689 // output streams are created and configured to the original camera before 1690 // proceeding. 1691 status_t injectionConfigureStreams( 1692 camera3::camera_stream_configuration& injectionConfig, 1693 const std::vector<uint32_t>& injectionBufferSizes); 1694 1695 // Disconnect the injection camera and delete the hal interface. 1696 void injectionDisconnectImpl(); 1697 1698 // Use injection camera hal interface to replace and backup original 1699 // camera hal interface. 1700 virtual status_t replaceHalInterface(sp<HalInterface> /*newHalInterface*/, 1701 bool /*keepBackup*/) = 0; 1702 1703 wp<Camera3Device> mParent; 1704 1705 // Backup of the original camera hal interface. 1706 sp<HalInterface> mBackupHalInterface; 1707 1708 // Generated injection camera hal interface. 1709 sp<HalInterface> mInjectedCamHalInterface; 1710 1711 // The flag indicates that the stream configuration is complete, the camera device is 1712 // active, but the injection camera has not yet been injected. 1713 bool mIsStreamConfigCompleteButNotInjected = false; 1714 1715 // Copy the configuration of the internal camera. 1716 camera3::camera_stream_configuration mInjectionConfig; 1717 1718 // Copy the streams of the internal camera. 1719 Vector<camera3::camera_stream_t*> mInjectionStreams; 1720 1721 // Copy the bufferSizes of the output streams of the internal camera. 1722 std::vector<uint32_t> mInjectionBufferSizes; 1723 1724 // Synchronizes access to injection camera between initialize and 1725 // disconnect. 1726 Mutex mInjectionLock; 1727 1728 // The injection camera ID. 1729 std::string mInjectedCamId; 1730 }; 1731 1732 virtual sp<Camera3DeviceInjectionMethods> 1733 createCamera3DeviceInjectionMethods(wp<Camera3Device>) = 0; 1734 1735 sp<Camera3DeviceInjectionMethods> mInjectionMethods; 1736 1737 void overrideStreamUseCaseLocked(); 1738 status_t deriveAndSetTransformLocked(camera3::Camera3OutputStreamInterface& stream, 1739 int mirrorMode, int surfaceId); 1740 1741 1742 }; // class Camera3Device 1743 1744 }; // namespace android 1745 1746 #endif 1747