1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.voiceinteraction; 18 19 import static android.app.AppOpsManager.MODE_ALLOWED; 20 import static android.service.voice.HotwordAudioStream.KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES; 21 22 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__CLOSE_ERROR_FROM_SYSTEM; 23 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__EMPTY_AUDIO_STREAM_LIST; 24 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__ENDED; 25 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__ILLEGAL_COPY_BUFFER_SIZE; 26 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__INTERRUPTED_EXCEPTION; 27 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__NO_PERMISSION; 28 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__STARTED; 29 import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG; 30 31 import android.annotation.NonNull; 32 import android.app.AppOpsManager; 33 import android.os.ParcelFileDescriptor; 34 import android.os.PersistableBundle; 35 import android.service.voice.HotwordAudioStream; 36 import android.service.voice.HotwordDetectedResult; 37 import android.util.Slog; 38 39 import com.android.internal.annotations.VisibleForTesting; 40 41 import java.io.IOException; 42 import java.io.InputStream; 43 import java.io.OutputStream; 44 import java.util.ArrayList; 45 import java.util.Arrays; 46 import java.util.List; 47 import java.util.concurrent.Callable; 48 import java.util.concurrent.ExecutorService; 49 import java.util.concurrent.Executors; 50 51 /** 52 * Copies the audio streams in {@link HotwordDetectedResult}s. This allows the system to manage the 53 * lifetime of the {@link ParcelFileDescriptor}s and ensures that the flow of data is in the right 54 * direction from the {@link android.service.voice.HotwordDetectionService} to the client (i.e., the 55 * voice interactor). 56 * 57 * @hide 58 */ 59 final class HotwordAudioStreamCopier { 60 61 private static final String TAG = "HotwordAudioStreamCopier"; 62 private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService"; 63 private static final String TASK_ID_PREFIX = "HotwordDetectedResult@"; 64 private static final String THREAD_NAME_PREFIX = "Copy-"; 65 @VisibleForTesting 66 static final int DEFAULT_COPY_BUFFER_LENGTH_BYTES = 32_768; 67 68 // Corresponds to the OS pipe capacity in bytes 69 @VisibleForTesting 70 static final int MAX_COPY_BUFFER_LENGTH_BYTES = 65_536; 71 72 private final AppOpsManager mAppOpsManager; 73 private final int mDetectorType; 74 private final int mVoiceInteractorUid; 75 private final String mVoiceInteractorPackageName; 76 private final String mVoiceInteractorAttributionTag; 77 private final ExecutorService mExecutorService = Executors.newCachedThreadPool(); 78 HotwordAudioStreamCopier(@onNull AppOpsManager appOpsManager, int detectorType, int voiceInteractorUid, @NonNull String voiceInteractorPackageName, @NonNull String voiceInteractorAttributionTag)79 HotwordAudioStreamCopier(@NonNull AppOpsManager appOpsManager, int detectorType, 80 int voiceInteractorUid, @NonNull String voiceInteractorPackageName, 81 @NonNull String voiceInteractorAttributionTag) { 82 mAppOpsManager = appOpsManager; 83 mDetectorType = detectorType; 84 mVoiceInteractorUid = voiceInteractorUid; 85 mVoiceInteractorPackageName = voiceInteractorPackageName; 86 mVoiceInteractorAttributionTag = voiceInteractorAttributionTag; 87 } 88 89 /** 90 * Starts copying the audio streams in the given {@link HotwordDetectedResult}. 91 * <p> 92 * The returned {@link HotwordDetectedResult} is identical the one that was passed in, except 93 * that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()} 94 * are replaced with descriptors from pipes managed by {@link HotwordAudioStreamCopier}. The 95 * returned value should be passed on to the client (i.e., the voice interactor). 96 * </p> 97 * 98 * @throws IOException If there was an error creating the managed pipe. 99 */ 100 @NonNull startCopyingAudioStreams(@onNull HotwordDetectedResult result)101 public HotwordDetectedResult startCopyingAudioStreams(@NonNull HotwordDetectedResult result) 102 throws IOException { 103 List<HotwordAudioStream> audioStreams = result.getAudioStreams(); 104 if (audioStreams.isEmpty()) { 105 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 106 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__EMPTY_AUDIO_STREAM_LIST, 107 mVoiceInteractorUid, /* streamSizeBytes= */ 0, /* bundleSizeBytes= */ 0, 108 /* streamCount= */ 0); 109 return result; 110 } 111 112 final int audioStreamCount = audioStreams.size(); 113 List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size()); 114 List<CopyTaskInfo> copyTaskInfos = new ArrayList<>(audioStreams.size()); 115 int totalMetadataBundleSizeBytes = 0; 116 int totalInitialAudioSizeBytes = 0; 117 for (HotwordAudioStream audioStream : audioStreams) { 118 ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe(); 119 ParcelFileDescriptor clientAudioSource = clientPipe[0]; 120 ParcelFileDescriptor clientAudioSink = clientPipe[1]; 121 HotwordAudioStream newAudioStream = 122 audioStream.buildUpon().setAudioStreamParcelFileDescriptor( 123 clientAudioSource).build(); 124 newAudioStreams.add(newAudioStream); 125 126 int copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES; 127 PersistableBundle metadata = audioStream.getMetadata(); 128 totalMetadataBundleSizeBytes += HotwordDetectedResult.getParcelableSize(metadata); 129 if (metadata.containsKey(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES)) { 130 copyBufferLength = metadata.getInt(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES, -1); 131 if (copyBufferLength < 1 || copyBufferLength > MAX_COPY_BUFFER_LENGTH_BYTES) { 132 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 133 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__ILLEGAL_COPY_BUFFER_SIZE, 134 mVoiceInteractorUid, /* streamSizeBytes= */ 0, /* bundleSizeBytes= */ 0, 135 audioStreamCount); 136 Slog.w(TAG, "Attempted to set an invalid copy buffer length (" 137 + copyBufferLength + ") for: " + audioStream); 138 copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES; 139 } else if (DEBUG) { 140 Slog.i(TAG, "Copy buffer length set to " + copyBufferLength + " for: " 141 + audioStream); 142 } 143 } 144 145 // We are including the non-streamed initial audio 146 // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics. 147 totalInitialAudioSizeBytes += audioStream.getInitialAudio().length; 148 149 ParcelFileDescriptor serviceAudioSource = 150 audioStream.getAudioStreamParcelFileDescriptor(); 151 copyTaskInfos.add(new CopyTaskInfo(serviceAudioSource, clientAudioSink, 152 copyBufferLength)); 153 } 154 155 String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result); 156 mExecutorService.execute( 157 new HotwordDetectedResultCopyTask(resultTaskId, copyTaskInfos, 158 totalMetadataBundleSizeBytes, totalInitialAudioSizeBytes)); 159 160 return result.buildUpon().setAudioStreams(newAudioStreams).build(); 161 } 162 163 private static class CopyTaskInfo { 164 private final ParcelFileDescriptor mSource; 165 private final ParcelFileDescriptor mSink; 166 private final int mCopyBufferLength; 167 CopyTaskInfo(ParcelFileDescriptor source, ParcelFileDescriptor sink, int copyBufferLength)168 CopyTaskInfo(ParcelFileDescriptor source, ParcelFileDescriptor sink, int copyBufferLength) { 169 mSource = source; 170 mSink = sink; 171 mCopyBufferLength = copyBufferLength; 172 } 173 } 174 175 private class HotwordDetectedResultCopyTask implements Runnable { 176 private final String mResultTaskId; 177 private final List<CopyTaskInfo> mCopyTaskInfos; 178 private final int mTotalMetadataSizeBytes; 179 private final int mTotalInitialAudioSizeBytes; 180 private final ExecutorService mExecutorService = Executors.newCachedThreadPool(); 181 HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos, int totalMetadataSizeBytes, int totalInitialAudioSizeBytes)182 HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos, 183 int totalMetadataSizeBytes, int totalInitialAudioSizeBytes) { 184 mResultTaskId = resultTaskId; 185 mCopyTaskInfos = copyTaskInfos; 186 mTotalMetadataSizeBytes = totalMetadataSizeBytes; 187 mTotalInitialAudioSizeBytes = totalInitialAudioSizeBytes; 188 } 189 190 @Override run()191 public void run() { 192 Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId); 193 int size = mCopyTaskInfos.size(); 194 List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size); 195 for (int i = 0; i < size; i++) { 196 CopyTaskInfo copyTaskInfo = mCopyTaskInfos.get(i); 197 String streamTaskId = mResultTaskId + "@" + i; 198 tasks.add(new SingleAudioStreamCopyTask(streamTaskId, copyTaskInfo.mSource, 199 copyTaskInfo.mSink, copyTaskInfo.mCopyBufferLength, mDetectorType, 200 mVoiceInteractorUid)); 201 } 202 203 if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD, 204 mVoiceInteractorUid, mVoiceInteractorPackageName, 205 mVoiceInteractorAttributionTag, OP_MESSAGE) == MODE_ALLOWED) { 206 try { 207 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 208 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__STARTED, 209 mVoiceInteractorUid, mTotalInitialAudioSizeBytes, 210 mTotalMetadataSizeBytes, size); 211 // TODO(b/244599891): Set timeout, close after inactivity 212 mExecutorService.invokeAll(tasks); 213 214 // We are including the non-streamed initial audio 215 // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics. 216 int totalStreamSizeBytes = mTotalInitialAudioSizeBytes; 217 for (SingleAudioStreamCopyTask task : tasks) { 218 totalStreamSizeBytes += task.mTotalCopiedBytes; 219 } 220 221 Slog.i(TAG, mResultTaskId + ": Task was completed. Total bytes egressed: " 222 + totalStreamSizeBytes + " (including " + mTotalInitialAudioSizeBytes 223 + " bytes NOT streamed), total metadata bundle size bytes: " 224 + mTotalMetadataSizeBytes); 225 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 226 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__ENDED, 227 mVoiceInteractorUid, totalStreamSizeBytes, mTotalMetadataSizeBytes, 228 size); 229 } catch (InterruptedException e) { 230 // We are including the non-streamed initial audio 231 // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics. 232 int totalStreamSizeBytes = mTotalInitialAudioSizeBytes; 233 for (SingleAudioStreamCopyTask task : tasks) { 234 totalStreamSizeBytes += task.mTotalCopiedBytes; 235 } 236 237 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 238 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__INTERRUPTED_EXCEPTION, 239 mVoiceInteractorUid, totalStreamSizeBytes, mTotalMetadataSizeBytes, 240 size); 241 Slog.i(TAG, mResultTaskId + ": Task was interrupted. Total bytes egressed: " 242 + totalStreamSizeBytes + " (including " + mTotalInitialAudioSizeBytes 243 + " bytes NOT streamed), total metadata bundle size bytes: " 244 + mTotalMetadataSizeBytes); 245 bestEffortPropagateError(e.getMessage()); 246 } finally { 247 mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD, 248 mVoiceInteractorUid, mVoiceInteractorPackageName, 249 mVoiceInteractorAttributionTag); 250 } 251 } else { 252 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 253 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__NO_PERMISSION, 254 mVoiceInteractorUid, /* streamSizeBytes= */ 0, /* bundleSizeBytes= */ 0, 255 size); 256 bestEffortPropagateError( 257 "Failed to obtain RECORD_AUDIO_HOTWORD permission for voice interactor with" 258 + " uid=" + mVoiceInteractorUid 259 + " packageName=" + mVoiceInteractorPackageName 260 + " attributionTag=" + mVoiceInteractorAttributionTag); 261 } 262 } 263 bestEffortPropagateError(@onNull String errorMessage)264 private void bestEffortPropagateError(@NonNull String errorMessage) { 265 try { 266 for (CopyTaskInfo copyTaskInfo : mCopyTaskInfos) { 267 copyTaskInfo.mSource.closeWithError(errorMessage); 268 copyTaskInfo.mSink.closeWithError(errorMessage); 269 } 270 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 271 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__CLOSE_ERROR_FROM_SYSTEM, 272 mVoiceInteractorUid, /* streamSizeBytes= */ 0, /* bundleSizeBytes= */ 0, 273 mCopyTaskInfos.size()); 274 } catch (IOException e) { 275 Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e); 276 } 277 } 278 } 279 280 private static class SingleAudioStreamCopyTask implements Callable<Void> { 281 private final String mStreamTaskId; 282 private final ParcelFileDescriptor mAudioSource; 283 private final ParcelFileDescriptor mAudioSink; 284 private final int mCopyBufferLength; 285 286 private final int mDetectorType; 287 private final int mUid; 288 289 private volatile int mTotalCopiedBytes = 0; 290 SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource, ParcelFileDescriptor audioSink, int copyBufferLength, int detectorType, int uid)291 SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource, 292 ParcelFileDescriptor audioSink, int copyBufferLength, int detectorType, int uid) { 293 mStreamTaskId = streamTaskId; 294 mAudioSource = audioSource; 295 mAudioSink = audioSink; 296 mCopyBufferLength = copyBufferLength; 297 mDetectorType = detectorType; 298 mUid = uid; 299 } 300 301 @Override call()302 public Void call() throws Exception { 303 Thread.currentThread().setName(THREAD_NAME_PREFIX + mStreamTaskId); 304 305 // Note: We are intentionally NOT using try-with-resources here. If we did, 306 // the ParcelFileDescriptors will be automatically closed WITHOUT errors before we go 307 // into the IOException-catch block. We want to propagate the error while closing the 308 // PFDs. 309 InputStream fis = null; 310 OutputStream fos = null; 311 try { 312 fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource); 313 fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink); 314 byte[] buffer = new byte[mCopyBufferLength]; 315 while (true) { 316 if (Thread.interrupted()) { 317 Slog.e(TAG, 318 mStreamTaskId + ": SingleAudioStreamCopyTask task was interrupted"); 319 break; 320 } 321 322 int bytesRead = fis.read(buffer); 323 if (bytesRead < 0) { 324 Slog.i(TAG, mStreamTaskId + ": Reached end of audio stream"); 325 break; 326 } 327 if (bytesRead > 0) { 328 if (DEBUG) { 329 // TODO(b/244599440): Add proper logging 330 Slog.d(TAG, mStreamTaskId + ": Copied " + bytesRead 331 + " bytes from audio stream. First 20 bytes=" + Arrays.toString( 332 Arrays.copyOfRange(buffer, 0, 20))); 333 } 334 fos.write(buffer, 0, bytesRead); 335 mTotalCopiedBytes += bytesRead; 336 } 337 // TODO(b/244599891): Close PFDs after inactivity 338 } 339 } catch (IOException e) { 340 mAudioSource.closeWithError(e.getMessage()); 341 mAudioSink.closeWithError(e.getMessage()); 342 // This is expected when VIS closes the read side of the pipe on their end, 343 // so when the HotwordAudioStreamCopier tries to write, we will get that broken 344 // pipe error. HDS is also closing the write side of the pipe (the system is on the 345 // read end of that pipe). 346 Slog.i(TAG, mStreamTaskId + ": Failed to copy audio stream", e); 347 HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType, 348 HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__CLOSE_ERROR_FROM_SYSTEM, 349 mUid, /* streamSizeBytes= */ 0, /* bundleSizeBytes= */ 0, 350 /* streamCount= */ 0); 351 } finally { 352 if (fis != null) { 353 fis.close(); 354 } 355 if (fos != null) { 356 fos.close(); 357 } 358 } 359 360 return null; 361 } 362 } 363 364 } 365