• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Camera2-ZslProcessor"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20 //#define LOG_NNDEBUG 0
21 
22 #ifdef LOG_NNDEBUG
23 #define ALOGVV(...) ALOGV(__VA_ARGS__)
24 #else
25 #define ALOGVV(...) ((void)0)
26 #endif
27 
28 #include <inttypes.h>
29 
30 #include <utils/Log.h>
31 #include <utils/Trace.h>
32 #include <gui/Surface.h>
33 
34 #include "common/CameraDeviceBase.h"
35 #include "api1/Camera2Client.h"
36 #include "api1/client2/CaptureSequencer.h"
37 #include "api1/client2/ZslProcessor.h"
38 
39 namespace android {
40 namespace camera2 {
41 
ZslProcessor(sp<Camera2Client> client,wp<CaptureSequencer> sequencer)42 ZslProcessor::ZslProcessor(
43     sp<Camera2Client> client,
44     wp<CaptureSequencer> sequencer):
45         Thread(false),
46         mState(RUNNING),
47         mClient(client),
48         mDevice(client->getCameraDevice()),
49         mSequencer(sequencer),
50         mId(client->getCameraId()),
51         mDeleted(false),
52         mZslBufferAvailable(false),
53         mZslStreamId(NO_STREAM),
54         mZslReprocessStreamId(NO_STREAM),
55         mFrameListHead(0),
56         mZslQueueHead(0),
57         mZslQueueTail(0) {
58     mZslQueue.insertAt(0, kZslBufferDepth);
59     mFrameList.insertAt(0, kFrameListDepth);
60     sp<CaptureSequencer> captureSequencer = mSequencer.promote();
61     if (captureSequencer != 0) captureSequencer->setZslProcessor(this);
62 }
63 
~ZslProcessor()64 ZslProcessor::~ZslProcessor() {
65     ALOGV("%s: Exit", __FUNCTION__);
66     disconnect();
67 }
68 
onFrameAvailable()69 void ZslProcessor::onFrameAvailable() {
70     Mutex::Autolock l(mInputMutex);
71     if (!mZslBufferAvailable) {
72         mZslBufferAvailable = true;
73         mZslBufferAvailableSignal.signal();
74     }
75 }
76 
onResultAvailable(const CaptureResult & result)77 void ZslProcessor::onResultAvailable(const CaptureResult &result) {
78     ATRACE_CALL();
79     ALOGV("%s:", __FUNCTION__);
80     Mutex::Autolock l(mInputMutex);
81     camera_metadata_ro_entry_t entry;
82     entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
83     nsecs_t timestamp = entry.data.i64[0];
84     (void)timestamp;
85     ALOGVV("Got preview frame for timestamp %" PRId64, timestamp);
86 
87     if (mState != RUNNING) return;
88 
89     mFrameList.editItemAt(mFrameListHead) = result.mMetadata;
90     mFrameListHead = (mFrameListHead + 1) % kFrameListDepth;
91 
92     findMatchesLocked();
93 }
94 
onBufferReleased(buffer_handle_t * handle)95 void ZslProcessor::onBufferReleased(buffer_handle_t *handle) {
96     Mutex::Autolock l(mInputMutex);
97 
98     // Verify that the buffer is in our queue
99     size_t i = 0;
100     for (; i < mZslQueue.size(); i++) {
101         if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break;
102     }
103     if (i == mZslQueue.size()) {
104         ALOGW("%s: Released buffer %p not found in queue",
105                 __FUNCTION__, handle);
106     }
107 
108     // Erase entire ZSL queue since we've now completed the capture and preview
109     // is stopped.
110     clearZslQueueLocked();
111 
112     mState = RUNNING;
113 }
114 
updateStream(const Parameters & params)115 status_t ZslProcessor::updateStream(const Parameters &params) {
116     ATRACE_CALL();
117     ALOGV("%s: Configuring ZSL streams", __FUNCTION__);
118     status_t res;
119 
120     Mutex::Autolock l(mInputMutex);
121 
122     sp<Camera2Client> client = mClient.promote();
123     if (client == 0) {
124         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
125         return INVALID_OPERATION;
126     }
127     sp<CameraDeviceBase> device = mDevice.promote();
128     if (device == 0) {
129         ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
130         return INVALID_OPERATION;
131     }
132 
133     if (mZslConsumer == 0) {
134         // Create CPU buffer queue endpoint
135         sp<IGraphicBufferProducer> producer;
136         sp<IGraphicBufferConsumer> consumer;
137         BufferQueue::createBufferQueue(&producer, &consumer);
138         mZslConsumer = new BufferItemConsumer(consumer,
139             GRALLOC_USAGE_HW_CAMERA_ZSL,
140             kZslBufferDepth);
141         mZslConsumer->setFrameAvailableListener(this);
142         mZslConsumer->setName(String8("Camera2Client::ZslConsumer"));
143         mZslWindow = new Surface(producer);
144     }
145 
146     if (mZslStreamId != NO_STREAM) {
147         // Check if stream parameters have to change
148         uint32_t currentWidth, currentHeight;
149         res = device->getStreamInfo(mZslStreamId,
150                 &currentWidth, &currentHeight, 0);
151         if (res != OK) {
152             ALOGE("%s: Camera %d: Error querying capture output stream info: "
153                     "%s (%d)", __FUNCTION__,
154                     mId, strerror(-res), res);
155             return res;
156         }
157         if (currentWidth != (uint32_t)params.fastInfo.arrayWidth ||
158                 currentHeight != (uint32_t)params.fastInfo.arrayHeight) {
159             res = device->deleteReprocessStream(mZslReprocessStreamId);
160             if (res != OK) {
161                 ALOGE("%s: Camera %d: Unable to delete old reprocess stream "
162                         "for ZSL: %s (%d)", __FUNCTION__,
163                         mId, strerror(-res), res);
164                 return res;
165             }
166             ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed",
167                 __FUNCTION__, mId, mZslStreamId);
168             res = device->deleteStream(mZslStreamId);
169             if (res != OK) {
170                 ALOGE("%s: Camera %d: Unable to delete old output stream "
171                         "for ZSL: %s (%d)", __FUNCTION__,
172                         mId, strerror(-res), res);
173                 return res;
174             }
175             mZslStreamId = NO_STREAM;
176         }
177     }
178 
179     mDeleted = false;
180 
181     if (mZslStreamId == NO_STREAM) {
182         // Create stream for HAL production
183         // TODO: Sort out better way to select resolution for ZSL
184         int streamType = params.quirks.useZslFormat ?
185                 (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL :
186                 (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
187         res = device->createStream(mZslWindow,
188                 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight,
189                 streamType, &mZslStreamId);
190         if (res != OK) {
191             ALOGE("%s: Camera %d: Can't create output stream for ZSL: "
192                     "%s (%d)", __FUNCTION__, mId,
193                     strerror(-res), res);
194             return res;
195         }
196         res = device->createReprocessStreamFromStream(mZslStreamId,
197                 &mZslReprocessStreamId);
198         if (res != OK) {
199             ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: "
200                     "%s (%d)", __FUNCTION__, mId,
201                     strerror(-res), res);
202             return res;
203         }
204     }
205     client->registerFrameListener(Camera2Client::kPreviewRequestIdStart,
206             Camera2Client::kPreviewRequestIdEnd,
207             this,
208             /*sendPartials*/false);
209 
210     return OK;
211 }
212 
deleteStream()213 status_t ZslProcessor::deleteStream() {
214     ATRACE_CALL();
215     Mutex::Autolock l(mInputMutex);
216     // WAR(b/15408128): do not delete stream unless client is being disconnected.
217     mDeleted = true;
218     return OK;
219 }
220 
disconnect()221 status_t ZslProcessor::disconnect() {
222     ATRACE_CALL();
223     status_t res;
224 
225     Mutex::Autolock l(mInputMutex);
226 
227     if (mZslStreamId != NO_STREAM) {
228         sp<CameraDeviceBase> device = mDevice.promote();
229         if (device == 0) {
230             ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
231             return INVALID_OPERATION;
232         }
233 
234         clearZslQueueLocked();
235 
236         res = device->deleteReprocessStream(mZslReprocessStreamId);
237         if (res != OK) {
238             ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: "
239                     "%s (%d)", __FUNCTION__, mId,
240                     mZslReprocessStreamId, strerror(-res), res);
241             return res;
242         }
243 
244         mZslReprocessStreamId = NO_STREAM;
245         res = device->deleteStream(mZslStreamId);
246         if (res != OK) {
247             ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: "
248                     "%s (%d)", __FUNCTION__, mId,
249                     mZslStreamId, strerror(-res), res);
250             return res;
251         }
252 
253         mZslWindow.clear();
254         mZslConsumer.clear();
255 
256         mZslStreamId = NO_STREAM;
257     }
258     return OK;
259 }
260 
getStreamId() const261 int ZslProcessor::getStreamId() const {
262     Mutex::Autolock l(mInputMutex);
263     return mZslStreamId;
264 }
265 
pushToReprocess(int32_t requestId)266 status_t ZslProcessor::pushToReprocess(int32_t requestId) {
267     ALOGV("%s: Send in reprocess request with id %d",
268             __FUNCTION__, requestId);
269     Mutex::Autolock l(mInputMutex);
270     status_t res;
271     sp<Camera2Client> client = mClient.promote();
272 
273     if (client == 0) {
274         ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId);
275         return INVALID_OPERATION;
276     }
277 
278     IF_ALOGV() {
279         dumpZslQueue(-1);
280     }
281 
282     if (mZslQueueTail != mZslQueueHead) {
283         CameraMetadata request;
284         size_t index = mZslQueueTail;
285         while (index != mZslQueueHead) {
286             if (!mZslQueue[index].frame.isEmpty()) {
287                 request = mZslQueue[index].frame;
288                 break;
289             }
290             index = (index + 1) % kZslBufferDepth;
291         }
292         if (index == mZslQueueHead) {
293             ALOGV("%s: ZSL queue has no valid frames to send yet.",
294                   __FUNCTION__);
295             return NOT_ENOUGH_DATA;
296         }
297         // Verify that the frame is reasonable for reprocessing
298 
299         camera_metadata_entry_t entry;
300         entry = request.find(ANDROID_CONTROL_AE_STATE);
301         if (entry.count == 0) {
302             ALOGE("%s: ZSL queue frame has no AE state field!",
303                     __FUNCTION__);
304             return BAD_VALUE;
305         }
306         if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED &&
307                 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) {
308             ALOGV("%s: ZSL queue frame AE state is %d, need full capture",
309                     __FUNCTION__, entry.data.u8[0]);
310             return NOT_ENOUGH_DATA;
311         }
312 
313         buffer_handle_t *handle =
314             &(mZslQueue[index].buffer.mGraphicBuffer->handle);
315 
316         uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS;
317         res = request.update(ANDROID_REQUEST_TYPE,
318                 &requestType, 1);
319         int32_t inputStreams[1] =
320                 { mZslReprocessStreamId };
321         if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS,
322                 inputStreams, 1);
323         int32_t outputStreams[1] =
324                 { client->getCaptureStreamId() };
325         if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS,
326                 outputStreams, 1);
327         res = request.update(ANDROID_REQUEST_ID,
328                 &requestId, 1);
329 
330         if (res != OK ) {
331             ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__);
332             return INVALID_OPERATION;
333         }
334 
335         res = client->stopStream();
336         if (res != OK) {
337             ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: "
338                 "%s (%d)",
339                 __FUNCTION__, mId, strerror(-res), res);
340             return INVALID_OPERATION;
341         }
342         // TODO: have push-and-clear be atomic
343         res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId,
344                 handle, this);
345         if (res != OK) {
346             ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)",
347                     __FUNCTION__, strerror(-res), res);
348             return res;
349         }
350 
351         // Update JPEG settings
352         {
353             SharedParameters::Lock l(client->getParameters());
354             res = l.mParameters.updateRequestJpeg(&request);
355             if (res != OK) {
356                 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL "
357                         "capture request: %s (%d)", __FUNCTION__,
358                         mId,
359                         strerror(-res), res);
360                 return res;
361             }
362         }
363 
364         mLatestCapturedRequest = request;
365         res = client->getCameraDevice()->capture(request);
366         if (res != OK ) {
367             ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)",
368                     __FUNCTION__, strerror(-res), res);
369             return res;
370         }
371 
372         mState = LOCKED;
373     } else {
374         ALOGV("%s: No ZSL buffers yet", __FUNCTION__);
375         return NOT_ENOUGH_DATA;
376     }
377     return OK;
378 }
379 
clearZslQueue()380 status_t ZslProcessor::clearZslQueue() {
381     Mutex::Autolock l(mInputMutex);
382     // If in middle of capture, can't clear out queue
383     if (mState == LOCKED) return OK;
384 
385     return clearZslQueueLocked();
386 }
387 
clearZslQueueLocked()388 status_t ZslProcessor::clearZslQueueLocked() {
389     for (size_t i = 0; i < mZslQueue.size(); i++) {
390         if (mZslQueue[i].buffer.mTimestamp != 0) {
391             mZslConsumer->releaseBuffer(mZslQueue[i].buffer);
392         }
393         mZslQueue.replaceAt(i);
394     }
395     mZslQueueHead = 0;
396     mZslQueueTail = 0;
397     return OK;
398 }
399 
dump(int fd,const Vector<String16> &) const400 void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const {
401     Mutex::Autolock l(mInputMutex);
402     if (!mLatestCapturedRequest.isEmpty()) {
403         String8 result("    Latest ZSL capture request:\n");
404         write(fd, result.string(), result.size());
405         mLatestCapturedRequest.dump(fd, 2, 6);
406     } else {
407         String8 result("    Latest ZSL capture request: none yet\n");
408         write(fd, result.string(), result.size());
409     }
410     dumpZslQueue(fd);
411 }
412 
threadLoop()413 bool ZslProcessor::threadLoop() {
414     status_t res;
415 
416     {
417         Mutex::Autolock l(mInputMutex);
418         while (!mZslBufferAvailable) {
419             res = mZslBufferAvailableSignal.waitRelative(mInputMutex,
420                     kWaitDuration);
421             if (res == TIMED_OUT) return true;
422         }
423         mZslBufferAvailable = false;
424     }
425 
426     do {
427         res = processNewZslBuffer();
428     } while (res == OK);
429 
430     return true;
431 }
432 
processNewZslBuffer()433 status_t ZslProcessor::processNewZslBuffer() {
434     ATRACE_CALL();
435     status_t res;
436     sp<BufferItemConsumer> zslConsumer;
437     {
438         Mutex::Autolock l(mInputMutex);
439         if (mZslConsumer == 0) return OK;
440         zslConsumer = mZslConsumer;
441     }
442     ALOGVV("Trying to get next buffer");
443     BufferItemConsumer::BufferItem item;
444     res = zslConsumer->acquireBuffer(&item, 0);
445     if (res != OK) {
446         if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) {
447             ALOGE("%s: Camera %d: Error receiving ZSL image buffer: "
448                     "%s (%d)", __FUNCTION__,
449                     mId, strerror(-res), res);
450         } else {
451             ALOGVV("  No buffer");
452         }
453         return res;
454     }
455 
456     Mutex::Autolock l(mInputMutex);
457 
458     if (mState == LOCKED) {
459         ALOGVV("In capture, discarding new ZSL buffers");
460         zslConsumer->releaseBuffer(item);
461         return OK;
462     }
463 
464     ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail);
465 
466     if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) {
467         ALOGVV("Releasing oldest buffer");
468         zslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer);
469         mZslQueue.replaceAt(mZslQueueTail);
470         mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth;
471     }
472 
473     ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead);
474 
475     queueHead.buffer = item;
476     queueHead.frame.release();
477 
478     mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth;
479 
480     ALOGVV("  Acquired buffer, timestamp %" PRId64, queueHead.buffer.mTimestamp);
481 
482     findMatchesLocked();
483 
484     return OK;
485 }
486 
findMatchesLocked()487 void ZslProcessor::findMatchesLocked() {
488     ALOGVV("Scanning");
489     for (size_t i = 0; i < mZslQueue.size(); i++) {
490         ZslPair &queueEntry = mZslQueue.editItemAt(i);
491         nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
492         IF_ALOGV() {
493             camera_metadata_entry_t entry;
494             nsecs_t frameTimestamp = 0;
495             if (!queueEntry.frame.isEmpty()) {
496                 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
497                 frameTimestamp = entry.data.i64[0];
498             }
499             ALOGVV("   %d: b: %" PRId64 "\tf: %" PRId64, i,
500                     bufferTimestamp, frameTimestamp );
501         }
502         if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) {
503             // Have buffer, no matching frame. Look for one
504             for (size_t j = 0; j < mFrameList.size(); j++) {
505                 bool match = false;
506                 CameraMetadata &frame = mFrameList.editItemAt(j);
507                 if (!frame.isEmpty()) {
508                     camera_metadata_entry_t entry;
509                     entry = frame.find(ANDROID_SENSOR_TIMESTAMP);
510                     if (entry.count == 0) {
511                         ALOGE("%s: Can't find timestamp in frame!",
512                                 __FUNCTION__);
513                         continue;
514                     }
515                     nsecs_t frameTimestamp = entry.data.i64[0];
516                     if (bufferTimestamp == frameTimestamp) {
517                         ALOGVV("%s: Found match %" PRId64, __FUNCTION__,
518                                 frameTimestamp);
519                         match = true;
520                     } else {
521                         int64_t delta = abs(bufferTimestamp - frameTimestamp);
522                         if ( delta < 1000000) {
523                             ALOGVV("%s: Found close match %" PRId64 " (delta %" PRId64 ")",
524                                     __FUNCTION__, bufferTimestamp, delta);
525                             match = true;
526                         }
527                     }
528                 }
529                 if (match) {
530                     queueEntry.frame.acquire(frame);
531                     break;
532                 }
533             }
534         }
535     }
536 }
537 
dumpZslQueue(int fd) const538 void ZslProcessor::dumpZslQueue(int fd) const {
539     String8 header("ZSL queue contents:");
540     String8 indent("    ");
541     ALOGV("%s", header.string());
542     if (fd != -1) {
543         header = indent + header + "\n";
544         write(fd, header.string(), header.size());
545     }
546     for (size_t i = 0; i < mZslQueue.size(); i++) {
547         const ZslPair &queueEntry = mZslQueue[i];
548         nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp;
549         camera_metadata_ro_entry_t entry;
550         nsecs_t frameTimestamp = 0;
551         int frameAeState = -1;
552         if (!queueEntry.frame.isEmpty()) {
553             entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP);
554             if (entry.count > 0) frameTimestamp = entry.data.i64[0];
555             entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE);
556             if (entry.count > 0) frameAeState = entry.data.u8[0];
557         }
558         String8 result =
559                 String8::format("   %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i,
560                         bufferTimestamp, frameTimestamp, frameAeState);
561         ALOGV("%s", result.string());
562         if (fd != -1) {
563             result = indent + result + "\n";
564             write(fd, result.string(), result.size());
565         }
566 
567     }
568 }
569 
570 }; // namespace camera2
571 }; // namespace android
572