1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.voiceinteraction; 18 19 import static android.Manifest.permission.CAMERA; 20 import static android.Manifest.permission.RECORD_AUDIO; 21 import static android.app.AppOpsManager.OP_CAMERA; 22 import static android.app.AppOpsManager.OP_RECORD_AUDIO; 23 import static android.service.voice.VisualQueryDetectionServiceFailure.ERROR_CODE_ILLEGAL_ATTENTION_STATE; 24 import static android.service.voice.VisualQueryDetectionServiceFailure.ERROR_CODE_ILLEGAL_STREAMING_STATE; 25 26 import android.annotation.NonNull; 27 import android.annotation.Nullable; 28 import android.content.Context; 29 import android.media.AudioFormat; 30 import android.media.permission.Identity; 31 import android.os.Binder; 32 import android.os.IBinder; 33 import android.os.ParcelFileDescriptor; 34 import android.os.PersistableBundle; 35 import android.os.RemoteException; 36 import android.os.SharedMemory; 37 import android.provider.Settings; 38 import android.service.voice.IDetectorSessionVisualQueryDetectionCallback; 39 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; 40 import android.service.voice.ISandboxedDetectionService; 41 import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback; 42 import android.service.voice.VisualQueryAttentionResult; 43 import android.service.voice.VisualQueryDetectedResult; 44 import android.service.voice.VisualQueryDetectionServiceFailure; 45 import android.util.Slog; 46 47 import com.android.internal.app.IHotwordRecognitionStatusCallback; 48 import com.android.internal.app.IVisualQueryDetectionAttentionListener; 49 import com.android.server.voiceinteraction.VoiceInteractionManagerServiceImpl.DetectorRemoteExceptionListener; 50 51 import java.io.PrintWriter; 52 import java.util.Objects; 53 import java.util.concurrent.ScheduledExecutorService; 54 55 /** 56 * A class that provides visual query detector to communicate with the {@link 57 * android.service.voice.VisualQueryDetectionService}. 58 * 59 * This class can handle the visual query detection whose detector is created by using 60 * {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle 61 * ,SharedMemory, HotwordDetector.Callback)}. 62 */ 63 final class VisualQueryDetectorSession extends DetectorSession { 64 65 private static final String TAG = "VisualQueryDetectorSession"; 66 67 private static final String VISUAL_QUERY_DETECTION_AUDIO_OP_MESSAGE = 68 "Providing query detection result from VisualQueryDetectionService to " 69 + "VoiceInteractionService"; 70 71 private static final String VISUAL_QUERY_DETECTION_CAMERA_OP_MESSAGE = 72 "Providing query detection result from VisualQueryDetectionService to " 73 + "VoiceInteractionService"; 74 private IVisualQueryDetectionAttentionListener mAttentionListener; 75 private boolean mEgressingData; 76 private boolean mQueryStreaming; 77 private boolean mEnableAccessibilityDataEgress; 78 79 //TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc. VisualQueryDetectorSession( @onNull HotwordDetectionConnection.ServiceConnection remoteService, @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, @NonNull DetectorRemoteExceptionListener listener, int userId)80 VisualQueryDetectorSession( 81 @NonNull HotwordDetectionConnection.ServiceConnection remoteService, 82 @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, 83 @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, 84 Identity voiceInteractorIdentity, 85 @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, 86 @NonNull DetectorRemoteExceptionListener listener, int userId) { 87 super(remoteService, lock, context, token, callback, 88 voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, 89 logging, listener, userId); 90 mEgressingData = false; 91 mQueryStreaming = false; 92 mAttentionListener = null; 93 mEnableAccessibilityDataEgress = Settings.Secure.getIntForUser( 94 mContext.getContentResolver(), 95 Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED, 0, 96 mUserId) == 1; 97 // TODO: handle notify RemoteException to client 98 } 99 100 @Override 101 @SuppressWarnings("GuardedBy") informRestartProcessLocked()102 void informRestartProcessLocked() { 103 Slog.v(TAG, "informRestartProcessLocked"); 104 mUpdateStateAfterStartFinished.set(false); 105 try { 106 mCallback.onProcessRestarted(); 107 } catch (RemoteException e) { 108 Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); 109 notifyOnDetectorRemoteException(); 110 } 111 } 112 setVisualQueryDetectionAttentionListenerLocked( @ullable IVisualQueryDetectionAttentionListener listener)113 void setVisualQueryDetectionAttentionListenerLocked( 114 @Nullable IVisualQueryDetectionAttentionListener listener) { 115 mAttentionListener = listener; 116 } 117 118 @SuppressWarnings("GuardedBy") startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback)119 boolean startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) { 120 if (DEBUG) { 121 Slog.d(TAG, "startPerceivingLocked"); 122 } 123 124 IDetectorSessionVisualQueryDetectionCallback internalCallback = 125 new IDetectorSessionVisualQueryDetectionCallback.Stub(){ 126 127 @Override 128 public void onAttentionGained(VisualQueryAttentionResult attentionResult) { 129 Slog.v(TAG, "BinderCallback#onAttentionGained"); 130 synchronized (mLock) { 131 mEgressingData = true; 132 if (mAttentionListener == null) { 133 return; 134 } 135 try { 136 mAttentionListener.onAttentionGained(attentionResult); 137 } catch (RemoteException e) { 138 Slog.e(TAG, "Error delivering attention gained event.", e); 139 try { 140 callback.onVisualQueryDetectionServiceFailure( 141 new VisualQueryDetectionServiceFailure( 142 ERROR_CODE_ILLEGAL_ATTENTION_STATE, 143 "Attention listener fails to switch to GAINED state.")); 144 } catch (RemoteException ex) { 145 Slog.v(TAG, "Fail to call onVisualQueryDetectionServiceFailure"); 146 } 147 } 148 } 149 } 150 151 @Override 152 public void onAttentionLost(int interactionIntention) { 153 Slog.v(TAG, "BinderCallback#onAttentionLost"); 154 synchronized (mLock) { 155 mEgressingData = false; 156 if (mAttentionListener == null) { 157 return; 158 } 159 try { 160 mAttentionListener.onAttentionLost(interactionIntention); 161 } catch (RemoteException e) { 162 Slog.e(TAG, "Error delivering attention lost event.", e); 163 try { 164 callback.onVisualQueryDetectionServiceFailure( 165 new VisualQueryDetectionServiceFailure( 166 ERROR_CODE_ILLEGAL_ATTENTION_STATE, 167 "Attention listener fails to switch to LOST state.")); 168 } catch (RemoteException ex) { 169 Slog.v(TAG, "Fail to call onVisualQueryDetectionServiceFailure"); 170 } 171 } 172 } 173 } 174 175 @Override 176 public void onQueryDetected(@NonNull String partialQuery) throws RemoteException { 177 Slog.v(TAG, "BinderCallback#onQueryDetected"); 178 synchronized (mLock) { 179 Objects.requireNonNull(partialQuery); 180 if (!mEgressingData) { 181 Slog.v(TAG, "Query should not be egressed within the unattention state."); 182 callback.onVisualQueryDetectionServiceFailure( 183 new VisualQueryDetectionServiceFailure( 184 ERROR_CODE_ILLEGAL_STREAMING_STATE, 185 "Cannot stream queries without attention signals.")); 186 return; 187 } 188 try { 189 enforcePermissionsForVisualQueryDelivery(RECORD_AUDIO, OP_RECORD_AUDIO, 190 VISUAL_QUERY_DETECTION_AUDIO_OP_MESSAGE); 191 } catch (SecurityException e) { 192 Slog.w(TAG, "Ignoring #onQueryDetected due to a SecurityException", e); 193 try { 194 callback.onVisualQueryDetectionServiceFailure( 195 new VisualQueryDetectionServiceFailure( 196 ERROR_CODE_ILLEGAL_STREAMING_STATE, 197 "Cannot stream queries without audio permission.")); 198 } catch (RemoteException e1) { 199 notifyOnDetectorRemoteException(); 200 throw e1; 201 } 202 return; 203 } 204 mQueryStreaming = true; 205 callback.onQueryDetected(partialQuery); 206 Slog.i(TAG, "Egressed from visual query detection process."); 207 } 208 } 209 210 @Override 211 public void onResultDetected(@NonNull VisualQueryDetectedResult partialResult) 212 throws RemoteException { 213 Slog.v(TAG, "BinderCallback#onResultDetected"); 214 synchronized (mLock) { 215 Objects.requireNonNull(partialResult); 216 if (!mEgressingData) { 217 Slog.v(TAG, "Result should not be egressed within the unattention state."); 218 callback.onVisualQueryDetectionServiceFailure( 219 new VisualQueryDetectionServiceFailure( 220 ERROR_CODE_ILLEGAL_STREAMING_STATE, 221 "Cannot stream results without attention signals.")); 222 return; 223 } 224 if (!checkDetectedResultDataLocked(partialResult)) { 225 Slog.v(TAG, "Accessibility data can be egressed only when the " 226 + "isAccessibilityDetectionEnabled() is true."); 227 callback.onVisualQueryDetectionServiceFailure( 228 new VisualQueryDetectionServiceFailure( 229 ERROR_CODE_ILLEGAL_STREAMING_STATE, 230 "Cannot stream accessibility data without " 231 + "enabling the setting.")); 232 return; 233 } 234 235 // Show camera icon if visual only accessibility data egresses 236 if (partialResult.getAccessibilityDetectionData() != null) { 237 try { 238 enforcePermissionsForVisualQueryDelivery(CAMERA, OP_CAMERA, 239 VISUAL_QUERY_DETECTION_CAMERA_OP_MESSAGE); 240 } catch (SecurityException e) { 241 Slog.w(TAG, "Ignoring #onQueryDetected due to a SecurityException", e); 242 try { 243 callback.onVisualQueryDetectionServiceFailure( 244 new VisualQueryDetectionServiceFailure( 245 ERROR_CODE_ILLEGAL_STREAMING_STATE, 246 "Cannot stream visual only accessibility data " 247 + "without camera permission.")); 248 } catch (RemoteException e1) { 249 notifyOnDetectorRemoteException(); 250 throw e1; 251 } 252 return; 253 } 254 } 255 256 // Show microphone icon if text query egresses 257 if (!partialResult.getPartialQuery().isEmpty()) { 258 try { 259 enforcePermissionsForVisualQueryDelivery(RECORD_AUDIO, OP_RECORD_AUDIO, 260 VISUAL_QUERY_DETECTION_AUDIO_OP_MESSAGE); 261 } catch (SecurityException e) { 262 Slog.w(TAG, "Ignoring #onQueryDetected due to a SecurityException", e); 263 try { 264 callback.onVisualQueryDetectionServiceFailure( 265 new VisualQueryDetectionServiceFailure( 266 ERROR_CODE_ILLEGAL_STREAMING_STATE, 267 "Cannot stream queries without audio permission.")); 268 } catch (RemoteException e1) { 269 notifyOnDetectorRemoteException(); 270 throw e1; 271 } 272 return; 273 } 274 } 275 276 mQueryStreaming = true; 277 callback.onResultDetected(partialResult); 278 Slog.i(TAG, "Egressed from visual query detection process."); 279 } 280 } 281 282 @Override 283 public void onQueryFinished() throws RemoteException { 284 Slog.v(TAG, "BinderCallback#onQueryFinished"); 285 synchronized (mLock) { 286 if (!mQueryStreaming) { 287 Slog.v(TAG, "Query streaming state signal FINISHED is block since there is" 288 + " no active query being streamed."); 289 callback.onVisualQueryDetectionServiceFailure( 290 new VisualQueryDetectionServiceFailure( 291 ERROR_CODE_ILLEGAL_STREAMING_STATE, 292 "Cannot send FINISHED signal with no query streamed.")); 293 return; 294 } 295 callback.onQueryFinished(); 296 mQueryStreaming = false; 297 } 298 } 299 300 @Override 301 public void onQueryRejected() throws RemoteException { 302 Slog.v(TAG, "BinderCallback#onQueryRejected"); 303 synchronized (mLock) { 304 if (!mQueryStreaming) { 305 Slog.v(TAG, "Query streaming state signal REJECTED is block since there is" 306 + " no active query being streamed."); 307 callback.onVisualQueryDetectionServiceFailure( 308 new VisualQueryDetectionServiceFailure( 309 ERROR_CODE_ILLEGAL_STREAMING_STATE, 310 "Cannot send REJECTED signal with no query streamed.")); 311 return; 312 } 313 callback.onQueryRejected(); 314 mQueryStreaming = false; 315 } 316 } 317 318 @SuppressWarnings("GuardedBy") 319 private boolean checkDetectedResultDataLocked(VisualQueryDetectedResult result) { 320 return result.getAccessibilityDetectionData() == null 321 || mEnableAccessibilityDataEgress; 322 } 323 }; 324 return mRemoteDetectionService.run( 325 service -> service.detectWithVisualSignals(internalCallback)); 326 } 327 328 @SuppressWarnings("GuardedBy") stopPerceivingLocked()329 boolean stopPerceivingLocked() { 330 if (DEBUG) { 331 Slog.d(TAG, "stopPerceivingLocked"); 332 } 333 return mRemoteDetectionService.run(ISandboxedDetectionService::stopDetection); 334 } 335 336 @Override startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback)337 void startListeningFromExternalSourceLocked( 338 ParcelFileDescriptor audioStream, 339 AudioFormat audioFormat, 340 @Nullable PersistableBundle options, 341 IMicrophoneHotwordDetectionVoiceInteractionCallback callback) 342 throws UnsupportedOperationException { 343 throw new UnsupportedOperationException("HotwordDetectionService method" 344 + " should not be called from VisualQueryDetectorSession."); 345 } 346 updateAccessibilityEgressStateLocked(boolean enable)347 void updateAccessibilityEgressStateLocked(boolean enable) { 348 if (DEBUG) { 349 Slog.d(TAG, "updateAccessibilityEgressStateLocked"); 350 } 351 mEnableAccessibilityDataEgress = enable; 352 } 353 enforcePermissionsForVisualQueryDelivery(String permission, int op, String msg)354 void enforcePermissionsForVisualQueryDelivery(String permission, int op, String msg) { 355 Binder.withCleanCallingIdentity(() -> { 356 synchronized (mLock) { 357 enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, 358 permission, msg); 359 mAppOpsManager.noteOpNoThrow( 360 op, mVoiceInteractorIdentity.uid, 361 mVoiceInteractorIdentity.packageName, 362 mVoiceInteractorIdentity.attributionTag, 363 msg); 364 } 365 }); 366 } 367 368 @SuppressWarnings("GuardedBy") dumpLocked(String prefix, PrintWriter pw)369 public void dumpLocked(String prefix, PrintWriter pw) { 370 super.dumpLocked(prefix, pw); 371 pw.print(prefix); 372 } 373 } 374 375 376