1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "EvsStateControl.h"
17
18 #include "FormatConvert.h"
19 #include "RenderDirectView.h"
20 #include "RenderPixelCopy.h"
21 #include "RenderTopView.h"
22
23 #include <aidl/android/hardware/automotive/vehicle/VehicleGear.h>
24 #include <aidl/android/hardware/automotive/vehicle/VehicleProperty.h>
25 #include <aidl/android/hardware/automotive/vehicle/VehiclePropertyType.h>
26 #include <aidl/android/hardware/automotive/vehicle/VehicleTurnSignal.h>
27 #include <android-base/logging.h>
28 #include <android/binder_manager.h>
29 #include <utils/SystemClock.h>
30
31 #include <inttypes.h>
32 #include <stdio.h>
33 #include <string.h>
34
35 using ::aidl::android::hardware::automotive::vehicle::StatusCode;
36 using ::aidl::android::hardware::automotive::vehicle::VehicleGear;
37 using ::aidl::android::hardware::automotive::vehicle::VehicleProperty;
38 using ::aidl::android::hardware::automotive::vehicle::VehiclePropertyType;
39 using ::aidl::android::hardware::automotive::vehicle::VehiclePropValue;
40 using ::aidl::android::hardware::automotive::vehicle::VehicleTurnSignal;
41 using ::android::base::Result;
42 using ::android::frameworks::automotive::vhal::IHalPropValue;
43 using ::android::frameworks::automotive::vhal::IVhalClient;
44 using ::android::hardware::automotive::evs::V1_0::EvsResult;
45 using ::android::hardware::automotive::vehicle::VhalResult;
46 using EvsDisplayState = ::android::hardware::automotive::evs::V1_0::DisplayState;
47 using BufferDesc_1_0 = ::android::hardware::automotive::evs::V1_0::BufferDesc;
48 using BufferDesc_1_1 = ::android::hardware::automotive::evs::V1_1::BufferDesc;
49
isSfReady()50 static bool isSfReady() {
51 return ::ndk::SpAIBinder(::AServiceManager_getService("SurfaceFlinger")).get() != nullptr;
52 }
53
54 // TODO: Seems like it'd be nice if the Vehicle HAL provided such helpers (but how & where?)
getPropType(VehicleProperty prop)55 inline constexpr VehiclePropertyType getPropType(VehicleProperty prop) {
56 return static_cast<VehiclePropertyType>(
57 static_cast<int32_t>(prop)
58 & static_cast<int32_t>(VehiclePropertyType::MASK));
59 }
60
EvsStateControl(std::shared_ptr<IVhalClient> pVnet,android::sp<IEvsEnumerator> pEvs,android::sp<IEvsDisplay> pDisplay,const ConfigManager & config)61 EvsStateControl::EvsStateControl(std::shared_ptr<IVhalClient> pVnet,
62 android::sp<IEvsEnumerator> pEvs,
63 android::sp<IEvsDisplay> pDisplay, const ConfigManager& config) :
64 mVehicle(pVnet),
65 mEvs(pEvs),
66 mDisplay(pDisplay),
67 mConfig(config),
68 mCurrentState(OFF),
69 mEvsStats(EvsStats::build()) {
70 // Initialize the property value containers we'll be updating (they'll be zeroed by default)
71 static_assert(getPropType(VehicleProperty::GEAR_SELECTION) == VehiclePropertyType::INT32,
72 "Unexpected type for GEAR_SELECTION property");
73 static_assert(getPropType(VehicleProperty::TURN_SIGNAL_STATE) == VehiclePropertyType::INT32,
74 "Unexpected type for TURN_SIGNAL_STATE property");
75
76 mGearValue.prop = static_cast<int32_t>(VehicleProperty::GEAR_SELECTION);
77 mTurnSignalValue.prop = static_cast<int32_t>(VehicleProperty::TURN_SIGNAL_STATE);
78
79 // This way we only ever deal with cameras which exist in the system
80 // Build our set of cameras for the states we support
81 LOG(DEBUG) << "Requesting camera list";
82 mEvs->getCameraList_1_1(
83 [this, &config](hidl_vec<CameraDesc> cameraList) {
84 LOG(INFO) << "Camera list callback received " << cameraList.size() << "cameras.";
85 for (auto&& cam: cameraList) {
86 LOG(DEBUG) << "Found camera " << cam.v1.cameraId;
87 bool cameraConfigFound = false;
88
89 // Check our configuration for information about this camera
90 // Note that a camera can have a compound function string
91 // such that a camera can be "right/reverse" and be used for both.
92 // If more than one camera is listed for a given function, we'll
93 // list all of them and let the UX/rendering logic use one, some
94 // or all of them as appropriate.
95 for (auto&& info: config.getCameras()) {
96 if (cam.v1.cameraId == info.cameraId) {
97 // We found a match!
98 if (info.function.find("reverse") != std::string::npos) {
99 mCameraList[State::REVERSE].emplace_back(info);
100 mCameraDescList[State::REVERSE].emplace_back(cam);
101 }
102 if (info.function.find("right") != std::string::npos) {
103 mCameraList[State::RIGHT].emplace_back(info);
104 mCameraDescList[State::RIGHT].emplace_back(cam);
105 }
106 if (info.function.find("left") != std::string::npos) {
107 mCameraList[State::LEFT].emplace_back(info);
108 mCameraDescList[State::LEFT].emplace_back(cam);
109 }
110 if (info.function.find("park") != std::string::npos) {
111 mCameraList[State::PARKING].emplace_back(info);
112 mCameraDescList[State::PARKING].emplace_back(cam);
113 }
114 cameraConfigFound = true;
115 break;
116 }
117 }
118 if (!cameraConfigFound) {
119 LOG(WARNING) << "No config information for hardware camera "
120 << cam.v1.cameraId;
121 }
122 }
123 }
124 );
125
126 LOG(DEBUG) << "State controller ready";
127 }
128
startUpdateLoop()129 bool EvsStateControl::startUpdateLoop() {
130 // Create the thread and report success if it gets started
131 mRenderThread = std::thread([this](){ updateLoop(); });
132 return mRenderThread.joinable();
133 }
134
135
terminateUpdateLoop()136 void EvsStateControl::terminateUpdateLoop() {
137 if (mRenderThread.get_id() == std::this_thread::get_id()) {
138 // We should not join by ourselves
139 mRenderThread.detach();
140 } else if (mRenderThread.joinable()) {
141 // Join a rendering thread
142 mRenderThread.join();
143 }
144 }
145
146
postCommand(const Command & cmd,bool clear)147 void EvsStateControl::postCommand(const Command& cmd, bool clear) {
148 // Push the command onto the queue watched by updateLoop
149 mLock.lock();
150 if (clear) {
151 std::queue<Command> emptyQueue;
152 std::swap(emptyQueue, mCommandQueue);
153 }
154
155 mCommandQueue.push(cmd);
156 mLock.unlock();
157
158 // Send a signal to wake updateLoop in case it is asleep
159 mWakeSignal.notify_all();
160 }
161
162
updateLoop()163 void EvsStateControl::updateLoop() {
164 LOG(DEBUG) << "Starting EvsStateControl update loop";
165
166 bool run = true;
167 while (run) {
168 // Process incoming commands
169 sp<IEvsDisplay> displayHandle;
170 {
171 std::lock_guard <std::mutex> lock(mLock);
172 while (!mCommandQueue.empty()) {
173 const Command& cmd = mCommandQueue.front();
174 switch (cmd.operation) {
175 case Op::EXIT:
176 run = false;
177 break;
178 case Op::CHECK_VEHICLE_STATE:
179 // Just running selectStateForCurrentConditions below will take care of this
180 break;
181 case Op::TOUCH_EVENT:
182 // Implement this given the x/y location of the touch event
183 break;
184 }
185 mCommandQueue.pop();
186 }
187
188 displayHandle = mDisplay.promote();
189 }
190
191 if (!displayHandle) {
192 LOG(ERROR) << "We've lost the display";
193 break;
194 }
195
196 // Review vehicle state and choose an appropriate renderer
197 if (!selectStateForCurrentConditions()) {
198 LOG(ERROR) << "selectStateForCurrentConditions failed so we're going to die";
199 break;
200 }
201
202 // If we have an active renderer, give it a chance to draw
203 if (mCurrentRenderer) {
204 // Get the output buffer we'll use to display the imagery
205 BufferDesc_1_0 tgtBuffer = {};
206 displayHandle->getTargetBuffer([&tgtBuffer](const BufferDesc_1_0& buff) {
207 tgtBuffer = buff;
208 }
209 );
210
211 if (tgtBuffer.memHandle == nullptr) {
212 LOG(ERROR) << "Didn't get requested output buffer -- skipping this frame.";
213 run = false;
214 } else {
215 // Generate our output image
216 if (!mCurrentRenderer->drawFrame(convertBufferDesc(tgtBuffer))) {
217 // If drawing failed, we want to exit quickly so an app restart can happen
218 run = false;
219 }
220
221 // Send the finished image back for display
222 displayHandle->returnTargetBufferForDisplay(tgtBuffer);
223
224 if (!mFirstFrameIsDisplayed) {
225 mFirstFrameIsDisplayed = true;
226 // returnTargetBufferForDisplay() is finished, the frame should be displayed
227 mEvsStats.finishComputingFirstFrameLatency(android::uptimeMillis());
228 }
229 }
230 } else if (run) {
231 // No active renderer, so sleep until somebody wakes us with another command
232 // or exit if we received EXIT command
233 std::unique_lock<std::mutex> lock(mLock);
234 mWakeSignal.wait(lock);
235 }
236 }
237
238 LOG(WARNING) << "EvsStateControl update loop ending";
239
240 if (mCurrentRenderer) {
241 // Deactive the renderer
242 mCurrentRenderer->deactivate();
243 }
244
245 // If `ICarTelemetry` service was not ready before, we need to try sending data again.
246 mEvsStats.sendCollectedDataBlocking();
247
248 printf("Shutting down app due to state control loop ending\n");
249 LOG(ERROR) << "Shutting down app due to state control loop ending";
250 }
251
252
selectStateForCurrentConditions()253 bool EvsStateControl::selectStateForCurrentConditions() {
254 static int32_t sMockGear = mConfig.getMockGearSignal();
255 static int32_t sMockSignal = int32_t(VehicleTurnSignal::NONE);
256
257 if (mVehicle != nullptr) {
258 // Query the car state
259 if (invokeGet(&mGearValue) != StatusCode::OK) {
260 LOG(ERROR) << "GEAR_SELECTION not available from vehicle. Exiting.";
261 return false;
262 }
263 if ((mTurnSignalValue.prop == 0) || (invokeGet(&mTurnSignalValue) != StatusCode::OK)) {
264 // Silently treat missing turn signal state as no turn signal active
265 mTurnSignalValue.value.int32Values = {sMockSignal};
266 mTurnSignalValue.prop = 0;
267 }
268 } else {
269 // While testing without a vehicle, behave as if we're in reverse for the first 20 seconds
270 static const int kShowTime = 20; // seconds
271
272 // See if it's time to turn off the default reverse camera
273 static std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
274 std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
275 if (std::chrono::duration_cast<std::chrono::seconds>(now - start).count() > kShowTime) {
276 // Switch to drive (which should turn off the reverse camera)
277 sMockGear = int32_t(VehicleGear::GEAR_DRIVE);
278 }
279
280 // Build the placeholder vehicle state values (treating single values as 1 element vectors)
281 mGearValue.value.int32Values = {sMockGear};
282 mTurnSignalValue.value.int32Values = {sMockSignal};
283 }
284
285 // Choose our desired EVS state based on the current car state
286 // TODO: Update this logic, and consider user input when choosing if a view should be presented
287 State desiredState = OFF;
288 if (mGearValue.value.int32Values[0] == int32_t(VehicleGear::GEAR_REVERSE)) {
289 desiredState = REVERSE;
290 } else if (mTurnSignalValue.value.int32Values[0] == int32_t(VehicleTurnSignal::RIGHT)) {
291 desiredState = RIGHT;
292 } else if (mTurnSignalValue.value.int32Values[0] == int32_t(VehicleTurnSignal::LEFT)) {
293 desiredState = LEFT;
294 } else if (mGearValue.value.int32Values[0] == int32_t(VehicleGear::GEAR_PARK)) {
295 desiredState = PARKING;
296 }
297
298 // Apply the desire state
299 return configureEvsPipeline(desiredState);
300 }
301
invokeGet(VehiclePropValue * pRequestedPropValue)302 StatusCode EvsStateControl::invokeGet(VehiclePropValue* pRequestedPropValue) {
303 auto halPropValue = mVehicle->createHalPropValue(pRequestedPropValue->prop);
304 // We are only setting int32Values.
305 halPropValue->setInt32Values(pRequestedPropValue->value.int32Values);
306
307 VhalResult<std::unique_ptr<IHalPropValue>> result = mVehicle->getValueSync(*halPropValue);
308
309 if (!result.ok()) {
310 return static_cast<StatusCode>(result.error().code());
311 }
312 pRequestedPropValue->value.int32Values = result.value()->getInt32Values();
313 pRequestedPropValue->timestamp = result.value()->getTimestamp();
314 return StatusCode::OK;
315 }
316
configureEvsPipeline(State desiredState)317 bool EvsStateControl::configureEvsPipeline(State desiredState) {
318 static bool isGlReady = false;
319
320 if (mCurrentState == desiredState) {
321 // Nothing to do here...
322 return true;
323 }
324
325 // Used by CarStats to accurately compute stats, it needs to be close to the beginning.
326 auto desiredStateTimeMillis = android::uptimeMillis();
327
328 LOG(DEBUG) << "Switching to state " << desiredState;
329 LOG(DEBUG) << " Current state " << mCurrentState
330 << " has " << mCameraList[mCurrentState].size() << " cameras";
331 LOG(DEBUG) << " Desired state " << desiredState
332 << " has " << mCameraList[desiredState].size() << " cameras";
333
334 if (!isGlReady && !isSfReady()) {
335 // Graphics is not ready yet; using CPU renderer.
336 if (mCameraList[desiredState].size() >= 1) {
337 mDesiredRenderer = std::make_unique<RenderPixelCopy>(mEvs,
338 mCameraList[desiredState][0]);
339 if (!mDesiredRenderer) {
340 LOG(ERROR) << "Failed to construct Pixel Copy renderer. Skipping state change.";
341 return false;
342 }
343 } else {
344 LOG(DEBUG) << "Unsupported, desiredState " << desiredState
345 << " has " << mCameraList[desiredState].size() << " cameras.";
346 }
347 } else {
348 // Assumes that SurfaceFlinger is available always after being launched.
349
350 // Do we need a new direct view renderer?
351 if (mCameraList[desiredState].size() == 1) {
352 // We have a camera assigned to this state for direct view.
353 mDesiredRenderer = std::make_unique<RenderDirectView>(mEvs,
354 mCameraDescList[desiredState][0],
355 mConfig);
356 if (!mDesiredRenderer) {
357 LOG(ERROR) << "Failed to construct direct renderer. Skipping state change.";
358 return false;
359 }
360 } else if (mCameraList[desiredState].size() > 1 ||
361 (mCameraList[desiredState].size() > 0 && desiredState == PARKING)) {
362 //TODO(b/140668179): RenderTopView needs to be updated to use new
363 // ConfigManager.
364 mDesiredRenderer = std::make_unique<RenderTopView>(mEvs,
365 mCameraList[desiredState],
366 mConfig);
367 if (!mDesiredRenderer) {
368 LOG(ERROR) << "Failed to construct top view renderer. Skipping state change.";
369 return false;
370 }
371 } else {
372 LOG(DEBUG) << "Unsupported, desiredState " << desiredState
373 << " has " << mCameraList[desiredState].size() << " cameras.";
374 }
375
376 // GL renderer is now ready.
377 isGlReady = true;
378 }
379
380 // Since we're changing states, shut down the current renderer
381 if (mCurrentRenderer != nullptr) {
382 mCurrentRenderer->deactivate();
383 mCurrentRenderer = nullptr; // It's a smart pointer, so destructs on assignment to null
384 }
385
386 // Now set the display state based on whether we have a video feed to show
387 sp<IEvsDisplay> displayHandle = mDisplay.promote();
388 if (!displayHandle) {
389 return false;
390 }
391
392 if (mDesiredRenderer == nullptr) {
393 LOG(DEBUG) << "Turning off the display";
394 displayHandle->setDisplayState(EvsDisplayState::NOT_VISIBLE);
395 } else {
396 mCurrentRenderer = std::move(mDesiredRenderer);
397
398 // Start the camera stream
399 LOG(DEBUG) << "EvsStartCameraStreamTiming start time: "
400 << android::elapsedRealtime() << " ms.";
401 if (!mCurrentRenderer->activate()) {
402 LOG(ERROR) << "New renderer failed to activate";
403 return false;
404 }
405
406 // Activate the display
407 LOG(DEBUG) << "EvsActivateDisplayTiming start time: "
408 << android::elapsedRealtime() << " ms.";
409 Return<EvsResult> result = displayHandle->setDisplayState(
410 EvsDisplayState::VISIBLE_ON_NEXT_FRAME);
411 if (result != EvsResult::OK) {
412 LOG(ERROR) << "setDisplayState returned an error "
413 << result.description();
414 return false;
415 }
416 }
417
418 // Record our current state
419 LOG(INFO) << "Activated state " << desiredState;
420 mCurrentState = desiredState;
421
422 mFirstFrameIsDisplayed = false; // Got a new renderer, mark first frame is not displayed.
423
424 if (mCurrentRenderer != nullptr && desiredState == State::REVERSE) {
425 // Start computing the latency when the evs state changes.
426 mEvsStats.startComputingFirstFrameLatency(desiredStateTimeMillis);
427 }
428
429 return true;
430 }
431