1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 package com.example.android.tflitecamerademo; 17 18 import android.app.Activity; 19 import android.app.AlertDialog; 20 import android.app.Dialog; 21 import android.app.DialogFragment; 22 import android.app.Fragment; 23 import android.content.Context; 24 import android.content.DialogInterface; 25 import android.content.pm.PackageInfo; 26 import android.content.pm.PackageManager; 27 import android.content.res.Configuration; 28 import android.graphics.Bitmap; 29 import android.graphics.ImageFormat; 30 import android.graphics.Matrix; 31 import android.graphics.Point; 32 import android.graphics.RectF; 33 import android.graphics.SurfaceTexture; 34 import android.hardware.camera2.CameraAccessException; 35 import android.hardware.camera2.CameraCaptureSession; 36 import android.hardware.camera2.CameraCharacteristics; 37 import android.hardware.camera2.CameraDevice; 38 import android.hardware.camera2.CameraManager; 39 import android.hardware.camera2.CaptureRequest; 40 import android.hardware.camera2.CaptureResult; 41 import android.hardware.camera2.TotalCaptureResult; 42 import android.hardware.camera2.params.StreamConfigurationMap; 43 import android.media.ImageReader; 44 import android.os.Bundle; 45 import android.os.Handler; 46 import android.os.HandlerThread; 47 import android.os.Process; 48 import android.text.SpannableString; 49 import android.text.SpannableStringBuilder; 50 import android.util.Log; 51 import android.util.Size; 52 import android.view.LayoutInflater; 53 import android.view.Surface; 54 import android.view.TextureView; 55 import android.view.View; 56 import android.view.ViewGroup; 57 import android.widget.AdapterView; 58 import android.widget.ArrayAdapter; 59 import android.widget.ListView; 60 import android.widget.NumberPicker; 61 import android.widget.TextView; 62 import android.widget.Toast; 63 import androidx.annotation.NonNull; 64 import android.support.v13.app.FragmentCompat; 65 import java.io.IOException; 66 import java.util.ArrayList; 67 import java.util.Arrays; 68 import java.util.Collections; 69 import java.util.Comparator; 70 import java.util.List; 71 import java.util.concurrent.Semaphore; 72 import java.util.concurrent.TimeUnit; 73 74 /** Basic fragments for the Camera. */ 75 public class Camera2BasicFragment extends Fragment 76 implements FragmentCompat.OnRequestPermissionsResultCallback { 77 78 /** Tag for the {@link Log}. */ 79 private static final String TAG = "TfLiteCameraDemo"; 80 81 private static final String FRAGMENT_DIALOG = "dialog"; 82 83 private static final String HANDLE_THREAD_NAME = "CameraBackground"; 84 85 private static final int PERMISSIONS_REQUEST_CODE = 1; 86 87 private final Object lock = new Object(); 88 private boolean runClassifier = false; 89 private boolean checkedPermissions = false; 90 private TextView textView; 91 private NumberPicker np; 92 private ImageClassifier classifier; 93 private ListView deviceView; 94 private ListView modelView; 95 96 97 /** Max preview width that is guaranteed by Camera2 API */ 98 private static final int MAX_PREVIEW_WIDTH = 1920; 99 100 /** Max preview height that is guaranteed by Camera2 API */ 101 private static final int MAX_PREVIEW_HEIGHT = 1080; 102 103 /** 104 * {@link TextureView.SurfaceTextureListener} handles several lifecycle events on a {@link 105 * TextureView}. 106 */ 107 private final TextureView.SurfaceTextureListener surfaceTextureListener = 108 new TextureView.SurfaceTextureListener() { 109 110 @Override 111 public void onSurfaceTextureAvailable(SurfaceTexture texture, int width, int height) { 112 openCamera(width, height); 113 } 114 115 @Override 116 public void onSurfaceTextureSizeChanged(SurfaceTexture texture, int width, int height) { 117 configureTransform(width, height); 118 } 119 120 @Override 121 public boolean onSurfaceTextureDestroyed(SurfaceTexture texture) { 122 return true; 123 } 124 125 @Override 126 public void onSurfaceTextureUpdated(SurfaceTexture texture) {} 127 }; 128 129 // Model parameter constants. 130 private String gpu; 131 private String cpu; 132 private String nnApi; 133 private String mobilenetV1Quant; 134 private String mobilenetV1Float; 135 136 137 138 /** ID of the current {@link CameraDevice}. */ 139 private String cameraId; 140 141 /** An {@link AutoFitTextureView} for camera preview. */ 142 private AutoFitTextureView textureView; 143 144 /** A {@link CameraCaptureSession } for camera preview. */ 145 private CameraCaptureSession captureSession; 146 147 /** A reference to the opened {@link CameraDevice}. */ 148 private CameraDevice cameraDevice; 149 150 /** The {@link android.util.Size} of camera preview. */ 151 private Size previewSize; 152 153 /** {@link CameraDevice.StateCallback} is called when {@link CameraDevice} changes its state. */ 154 private final CameraDevice.StateCallback stateCallback = 155 new CameraDevice.StateCallback() { 156 157 @Override 158 public void onOpened(@NonNull CameraDevice currentCameraDevice) { 159 // This method is called when the camera is opened. We start camera preview here. 160 cameraOpenCloseLock.release(); 161 cameraDevice = currentCameraDevice; 162 createCameraPreviewSession(); 163 } 164 165 @Override 166 public void onDisconnected(@NonNull CameraDevice currentCameraDevice) { 167 cameraOpenCloseLock.release(); 168 currentCameraDevice.close(); 169 cameraDevice = null; 170 } 171 172 @Override 173 public void onError(@NonNull CameraDevice currentCameraDevice, int error) { 174 cameraOpenCloseLock.release(); 175 currentCameraDevice.close(); 176 cameraDevice = null; 177 Activity activity = getActivity(); 178 if (null != activity) { 179 activity.finish(); 180 } 181 } 182 }; 183 184 private ArrayList<String> deviceStrings = new ArrayList<String>(); 185 private ArrayList<String> modelStrings = new ArrayList<String>(); 186 187 /** Current indices of device and model. */ 188 int currentDevice = -1; 189 190 int currentModel = -1; 191 192 int currentNumThreads = -1; 193 194 /** An additional thread for running tasks that shouldn't block the UI. */ 195 private HandlerThread backgroundThread; 196 197 /** A {@link Handler} for running tasks in the background. */ 198 private Handler backgroundHandler; 199 200 /** An {@link ImageReader} that handles image capture. */ 201 private ImageReader imageReader; 202 203 /** {@link CaptureRequest.Builder} for the camera preview */ 204 private CaptureRequest.Builder previewRequestBuilder; 205 206 /** {@link CaptureRequest} generated by {@link #previewRequestBuilder} */ 207 private CaptureRequest previewRequest; 208 209 /** A {@link Semaphore} to prevent the app from exiting before closing the camera. */ 210 private Semaphore cameraOpenCloseLock = new Semaphore(1); 211 212 /** A {@link CameraCaptureSession.CaptureCallback} that handles events related to capture. */ 213 private CameraCaptureSession.CaptureCallback captureCallback = 214 new CameraCaptureSession.CaptureCallback() { 215 216 @Override 217 public void onCaptureProgressed( 218 @NonNull CameraCaptureSession session, 219 @NonNull CaptureRequest request, 220 @NonNull CaptureResult partialResult) {} 221 222 @Override 223 public void onCaptureCompleted( 224 @NonNull CameraCaptureSession session, 225 @NonNull CaptureRequest request, 226 @NonNull TotalCaptureResult result) {} 227 }; 228 229 /** 230 * Shows a {@link Toast} on the UI thread for the classification results. 231 * 232 * @param text The message to show 233 */ showToast(String s)234 private void showToast(String s) { 235 SpannableStringBuilder builder = new SpannableStringBuilder(); 236 SpannableString str1 = new SpannableString(s); 237 builder.append(str1); 238 showToast(builder); 239 } 240 showToast(SpannableStringBuilder builder)241 private void showToast(SpannableStringBuilder builder) { 242 final Activity activity = getActivity(); 243 if (activity != null) { 244 activity.runOnUiThread( 245 new Runnable() { 246 @Override 247 public void run() { 248 textView.setText(builder, TextView.BufferType.SPANNABLE); 249 } 250 }); 251 } 252 } 253 254 /** 255 * Resizes image. 256 * 257 * Attempting to use too large a preview size could exceed the camera bus' bandwidth limitation, 258 * resulting in gorgeous previews but the storage of garbage capture data. 259 * 260 * Given {@code choices} of {@code Size}s supported by a camera, choose the smallest one that is 261 * at least as large as the respective texture view size, and that is at most as large as the 262 * respective max size, and whose aspect ratio matches with the specified value. If such size 263 * doesn't exist, choose the largest one that is at most as large as the respective max size, and 264 * whose aspect ratio matches with the specified value. 265 * 266 * @param choices The list of sizes that the camera supports for the intended output class 267 * @param textureViewWidth The width of the texture view relative to sensor coordinate 268 * @param textureViewHeight The height of the texture view relative to sensor coordinate 269 * @param maxWidth The maximum width that can be chosen 270 * @param maxHeight The maximum height that can be chosen 271 * @param aspectRatio The aspect ratio 272 * @return The optimal {@code Size}, or an arbitrary one if none were big enough 273 */ chooseOptimalSize( Size[] choices, int textureViewWidth, int textureViewHeight, int maxWidth, int maxHeight, Size aspectRatio)274 private static Size chooseOptimalSize( 275 Size[] choices, 276 int textureViewWidth, 277 int textureViewHeight, 278 int maxWidth, 279 int maxHeight, 280 Size aspectRatio) { 281 282 // Collect the supported resolutions that are at least as big as the preview Surface 283 List<Size> bigEnough = new ArrayList<>(); 284 // Collect the supported resolutions that are smaller than the preview Surface 285 List<Size> notBigEnough = new ArrayList<>(); 286 int w = aspectRatio.getWidth(); 287 int h = aspectRatio.getHeight(); 288 for (Size option : choices) { 289 if (option.getWidth() <= maxWidth 290 && option.getHeight() <= maxHeight 291 && option.getHeight() == option.getWidth() * h / w) { 292 if (option.getWidth() >= textureViewWidth && option.getHeight() >= textureViewHeight) { 293 bigEnough.add(option); 294 } else { 295 notBigEnough.add(option); 296 } 297 } 298 } 299 300 // Pick the smallest of those big enough. If there is no one big enough, pick the 301 // largest of those not big enough. 302 if (bigEnough.size() > 0) { 303 return Collections.min(bigEnough, new CompareSizesByArea()); 304 } else if (notBigEnough.size() > 0) { 305 return Collections.max(notBigEnough, new CompareSizesByArea()); 306 } else { 307 Log.e(TAG, "Couldn't find any suitable preview size"); 308 return choices[0]; 309 } 310 } 311 newInstance()312 public static Camera2BasicFragment newInstance() { 313 return new Camera2BasicFragment(); 314 } 315 316 /** Layout the preview and buttons. */ 317 @Override onCreateView( LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState)318 public View onCreateView( 319 LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { 320 return inflater.inflate(R.layout.fragment_camera2_basic, container, false); 321 } 322 updateActiveModel()323 private void updateActiveModel() { 324 // Get UI information before delegating to background 325 final int modelIndex = modelView.getCheckedItemPosition(); 326 final int deviceIndex = deviceView.getCheckedItemPosition(); 327 final int numThreads = np.getValue(); 328 329 backgroundHandler.post( 330 () -> { 331 if (modelIndex == currentModel 332 && deviceIndex == currentDevice 333 && numThreads == currentNumThreads) { 334 return; 335 } 336 currentModel = modelIndex; 337 currentDevice = deviceIndex; 338 currentNumThreads = numThreads; 339 340 // Disable classifier while updating 341 if (classifier != null) { 342 classifier.close(); 343 classifier = null; 344 } 345 346 // Lookup names of parameters. 347 String model = modelStrings.get(modelIndex); 348 String device = deviceStrings.get(deviceIndex); 349 350 Log.i(TAG, "Changing model to " + model + " device " + device); 351 352 // Try to load model. 353 try { 354 if (model.equals(mobilenetV1Quant)) { 355 classifier = new ImageClassifierQuantizedMobileNet(getActivity()); 356 } else if (model.equals(mobilenetV1Float)) { 357 classifier = new ImageClassifierFloatMobileNet(getActivity()); 358 } else { 359 showToast("Failed to load model"); 360 } 361 } catch (IOException e) { 362 Log.d(TAG, "Failed to load", e); 363 classifier = null; 364 } 365 366 // Customize the interpreter to the type of device we want to use. 367 if (classifier == null) { 368 return; 369 } 370 classifier.setNumThreads(numThreads); 371 if (device.equals(cpu)) { 372 } else if (device.equals(gpu)) { 373 classifier.useGpu(); 374 } else if (device.equals(nnApi)) { 375 classifier.useNNAPI(); 376 } 377 }); 378 } 379 380 /** Connect the buttons to their event handler. */ 381 @Override onViewCreated(final View view, Bundle savedInstanceState)382 public void onViewCreated(final View view, Bundle savedInstanceState) { 383 gpu = getString(R.string.gpu); 384 cpu = getString(R.string.cpu); 385 nnApi = getString(R.string.nnapi); 386 mobilenetV1Quant = getString(R.string.mobilenetV1Quant); 387 mobilenetV1Float = getString(R.string.mobilenetV1Float); 388 389 // Get references to widgets. 390 textureView = (AutoFitTextureView) view.findViewById(R.id.texture); 391 textView = (TextView) view.findViewById(R.id.text); 392 deviceView = (ListView) view.findViewById(R.id.device); 393 modelView = (ListView) view.findViewById(R.id.model); 394 395 // Build list of models 396 modelStrings.add(mobilenetV1Quant); 397 modelStrings.add(mobilenetV1Float); 398 399 // Build list of devices 400 int defaultModelIndex = 0; 401 deviceStrings.add(cpu); 402 deviceStrings.add(gpu); 403 deviceStrings.add(nnApi); 404 405 deviceView.setAdapter( 406 new ArrayAdapter<String>( 407 getContext(), R.layout.listview_row, R.id.listview_row_text, deviceStrings)); 408 deviceView.setChoiceMode(ListView.CHOICE_MODE_SINGLE); 409 deviceView.setOnItemClickListener( 410 new AdapterView.OnItemClickListener() { 411 @Override 412 public void onItemClick(AdapterView<?> parent, View view, int position, long id) { 413 updateActiveModel(); 414 } 415 }); 416 deviceView.setItemChecked(0, true); 417 418 modelView.setChoiceMode(ListView.CHOICE_MODE_SINGLE); 419 ArrayAdapter<String> modelAdapter = 420 new ArrayAdapter<>( 421 getContext(), R.layout.listview_row, R.id.listview_row_text, modelStrings); 422 modelView.setAdapter(modelAdapter); 423 modelView.setItemChecked(defaultModelIndex, true); 424 modelView.setOnItemClickListener( 425 new AdapterView.OnItemClickListener() { 426 @Override 427 public void onItemClick(AdapterView<?> parent, View view, int position, long id) { 428 updateActiveModel(); 429 } 430 }); 431 432 np = (NumberPicker) view.findViewById(R.id.np); 433 np.setMinValue(1); 434 np.setMaxValue(10); 435 np.setWrapSelectorWheel(true); 436 np.setOnValueChangedListener( 437 new NumberPicker.OnValueChangeListener() { 438 @Override 439 public void onValueChange(NumberPicker picker, int oldVal, int newVal) { 440 updateActiveModel(); 441 } 442 }); 443 444 // Start initial model. 445 } 446 447 /** Load the model and labels. */ 448 @Override onActivityCreated(Bundle savedInstanceState)449 public void onActivityCreated(Bundle savedInstanceState) { 450 super.onActivityCreated(savedInstanceState); 451 startBackgroundThread(); 452 } 453 454 @Override onResume()455 public void onResume() { 456 super.onResume(); 457 startBackgroundThread(); 458 459 // When the screen is turned off and turned back on, the SurfaceTexture is already 460 // available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open 461 // a camera and start preview from here (otherwise, we wait until the surface is ready in 462 // the SurfaceTextureListener). 463 if (textureView.isAvailable()) { 464 openCamera(textureView.getWidth(), textureView.getHeight()); 465 } else { 466 textureView.setSurfaceTextureListener(surfaceTextureListener); 467 } 468 } 469 470 @Override onPause()471 public void onPause() { 472 closeCamera(); 473 stopBackgroundThread(); 474 super.onPause(); 475 } 476 477 @Override onDestroy()478 public void onDestroy() { 479 if (classifier != null) { 480 classifier.close(); 481 } 482 super.onDestroy(); 483 } 484 485 /** 486 * Sets up member variables related to camera. 487 * 488 * @param width The width of available size for camera preview 489 * @param height The height of available size for camera preview 490 */ setUpCameraOutputs(int width, int height)491 private void setUpCameraOutputs(int width, int height) { 492 Activity activity = getActivity(); 493 CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); 494 try { 495 for (String cameraId : manager.getCameraIdList()) { 496 CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); 497 498 // We don't use a front facing camera in this sample. 499 Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING); 500 if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) { 501 continue; 502 } 503 504 StreamConfigurationMap map = 505 characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 506 if (map == null) { 507 continue; 508 } 509 510 // // For still image captures, we use the largest available size. 511 Size largest = 512 Collections.max( 513 Arrays.asList(map.getOutputSizes(ImageFormat.JPEG)), new CompareSizesByArea()); 514 imageReader = 515 ImageReader.newInstance( 516 largest.getWidth(), largest.getHeight(), ImageFormat.JPEG, /*maxImages*/ 2); 517 518 // Find out if we need to swap dimension to get the preview size relative to sensor 519 // coordinate. 520 int displayRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); 521 // noinspection ConstantConditions 522 /* Orientation of the camera sensor */ 523 int sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); 524 boolean swappedDimensions = false; 525 switch (displayRotation) { 526 case Surface.ROTATION_0: 527 case Surface.ROTATION_180: 528 if (sensorOrientation == 90 || sensorOrientation == 270) { 529 swappedDimensions = true; 530 } 531 break; 532 case Surface.ROTATION_90: 533 case Surface.ROTATION_270: 534 if (sensorOrientation == 0 || sensorOrientation == 180) { 535 swappedDimensions = true; 536 } 537 break; 538 default: 539 Log.e(TAG, "Display rotation is invalid: " + displayRotation); 540 } 541 542 Point displaySize = new Point(); 543 activity.getWindowManager().getDefaultDisplay().getSize(displaySize); 544 int rotatedPreviewWidth = width; 545 int rotatedPreviewHeight = height; 546 int maxPreviewWidth = displaySize.x; 547 int maxPreviewHeight = displaySize.y; 548 549 if (swappedDimensions) { 550 rotatedPreviewWidth = height; 551 rotatedPreviewHeight = width; 552 maxPreviewWidth = displaySize.y; 553 maxPreviewHeight = displaySize.x; 554 } 555 556 if (maxPreviewWidth > MAX_PREVIEW_WIDTH) { 557 maxPreviewWidth = MAX_PREVIEW_WIDTH; 558 } 559 560 if (maxPreviewHeight > MAX_PREVIEW_HEIGHT) { 561 maxPreviewHeight = MAX_PREVIEW_HEIGHT; 562 } 563 564 previewSize = 565 chooseOptimalSize( 566 map.getOutputSizes(SurfaceTexture.class), 567 rotatedPreviewWidth, 568 rotatedPreviewHeight, 569 maxPreviewWidth, 570 maxPreviewHeight, 571 largest); 572 573 // We fit the aspect ratio of TextureView to the size of preview we picked. 574 int orientation = getResources().getConfiguration().orientation; 575 if (orientation == Configuration.ORIENTATION_LANDSCAPE) { 576 textureView.setAspectRatio(previewSize.getWidth(), previewSize.getHeight()); 577 } else { 578 textureView.setAspectRatio(previewSize.getHeight(), previewSize.getWidth()); 579 } 580 581 this.cameraId = cameraId; 582 return; 583 } 584 } catch (CameraAccessException e) { 585 Log.e(TAG, "Failed to access Camera", e); 586 } catch (NullPointerException e) { 587 // Currently an NPE is thrown when the Camera2API is used but not supported on the 588 // device this code runs. 589 ErrorDialog.newInstance(getString(R.string.camera_error)) 590 .show(getChildFragmentManager(), FRAGMENT_DIALOG); 591 } 592 } 593 getRequiredPermissions()594 private String[] getRequiredPermissions() { 595 Activity activity = getActivity(); 596 try { 597 PackageInfo info = 598 activity 599 .getPackageManager() 600 .getPackageInfo(activity.getPackageName(), PackageManager.GET_PERMISSIONS); 601 String[] ps = info.requestedPermissions; 602 if (ps != null && ps.length > 0) { 603 return ps; 604 } else { 605 return new String[0]; 606 } 607 } catch (Exception e) { 608 return new String[0]; 609 } 610 } 611 612 /** Opens the camera specified by {@link Camera2BasicFragment#cameraId}. */ openCamera(int width, int height)613 private void openCamera(int width, int height) { 614 if (!checkedPermissions && !allPermissionsGranted()) { 615 FragmentCompat.requestPermissions(this, getRequiredPermissions(), PERMISSIONS_REQUEST_CODE); 616 return; 617 } else { 618 checkedPermissions = true; 619 } 620 setUpCameraOutputs(width, height); 621 configureTransform(width, height); 622 Activity activity = getActivity(); 623 CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); 624 try { 625 if (!cameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) { 626 throw new RuntimeException("Time out waiting to lock camera opening."); 627 } 628 manager.openCamera(cameraId, stateCallback, backgroundHandler); 629 } catch (CameraAccessException e) { 630 Log.e(TAG, "Failed to open Camera", e); 631 } catch (InterruptedException e) { 632 throw new RuntimeException("Interrupted while trying to lock camera opening.", e); 633 } 634 } 635 allPermissionsGranted()636 private boolean allPermissionsGranted() { 637 for (String permission : getRequiredPermissions()) { 638 if (getActivity().checkPermission(permission, Process.myPid(), Process.myUid()) 639 != PackageManager.PERMISSION_GRANTED) { 640 return false; 641 } 642 } 643 return true; 644 } 645 646 @Override onRequestPermissionsResult( int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults)647 public void onRequestPermissionsResult( 648 int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { 649 super.onRequestPermissionsResult(requestCode, permissions, grantResults); 650 } 651 652 /** Closes the current {@link CameraDevice}. */ closeCamera()653 private void closeCamera() { 654 try { 655 cameraOpenCloseLock.acquire(); 656 if (null != captureSession) { 657 captureSession.close(); 658 captureSession = null; 659 } 660 if (null != cameraDevice) { 661 cameraDevice.close(); 662 cameraDevice = null; 663 } 664 if (null != imageReader) { 665 imageReader.close(); 666 imageReader = null; 667 } 668 } catch (InterruptedException e) { 669 throw new RuntimeException("Interrupted while trying to lock camera closing.", e); 670 } finally { 671 cameraOpenCloseLock.release(); 672 } 673 } 674 675 /** Starts a background thread and its {@link Handler}. */ startBackgroundThread()676 private void startBackgroundThread() { 677 backgroundThread = new HandlerThread(HANDLE_THREAD_NAME); 678 backgroundThread.start(); 679 backgroundHandler = new Handler(backgroundThread.getLooper()); 680 // Start the classification train & load an initial model. 681 synchronized (lock) { 682 runClassifier = true; 683 } 684 backgroundHandler.post(periodicClassify); 685 updateActiveModel(); 686 } 687 688 /** Stops the background thread and its {@link Handler}. */ stopBackgroundThread()689 private void stopBackgroundThread() { 690 backgroundThread.quitSafely(); 691 try { 692 backgroundThread.join(); 693 backgroundThread = null; 694 backgroundHandler = null; 695 synchronized (lock) { 696 runClassifier = false; 697 } 698 } catch (InterruptedException e) { 699 Log.e(TAG, "Interrupted when stopping background thread", e); 700 } 701 } 702 703 /** Takes photos and classify them periodically. */ 704 private Runnable periodicClassify = 705 new Runnable() { 706 @Override 707 public void run() { 708 synchronized (lock) { 709 if (runClassifier) { 710 classifyFrame(); 711 } 712 } 713 backgroundHandler.post(periodicClassify); 714 } 715 }; 716 717 /** Creates a new {@link CameraCaptureSession} for camera preview. */ createCameraPreviewSession()718 private void createCameraPreviewSession() { 719 try { 720 SurfaceTexture texture = textureView.getSurfaceTexture(); 721 assert texture != null; 722 723 // We configure the size of default buffer to be the size of camera preview we want. 724 texture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); 725 726 // This is the output Surface we need to start preview. 727 Surface surface = new Surface(texture); 728 729 // We set up a CaptureRequest.Builder with the output Surface. 730 previewRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); 731 previewRequestBuilder.addTarget(surface); 732 733 // Here, we create a CameraCaptureSession for camera preview. 734 cameraDevice.createCaptureSession( 735 Arrays.asList(surface), 736 new CameraCaptureSession.StateCallback() { 737 738 @Override 739 public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) { 740 // The camera is already closed 741 if (null == cameraDevice) { 742 return; 743 } 744 745 // When the session is ready, we start displaying the preview. 746 captureSession = cameraCaptureSession; 747 try { 748 // Auto focus should be continuous for camera preview. 749 previewRequestBuilder.set( 750 CaptureRequest.CONTROL_AF_MODE, 751 CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE); 752 753 // Finally, we start displaying the camera preview. 754 previewRequest = previewRequestBuilder.build(); 755 captureSession.setRepeatingRequest( 756 previewRequest, captureCallback, backgroundHandler); 757 } catch (CameraAccessException e) { 758 Log.e(TAG, "Failed to set up config to capture Camera", e); 759 } 760 } 761 762 @Override 763 public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession) { 764 showToast("Failed"); 765 } 766 }, 767 null); 768 } catch (CameraAccessException e) { 769 Log.e(TAG, "Failed to preview Camera", e); 770 } 771 } 772 773 /** 774 * Configures the necessary {@link android.graphics.Matrix} transformation to `textureView`. This 775 * method should be called after the camera preview size is determined in setUpCameraOutputs and 776 * also the size of `textureView` is fixed. 777 * 778 * @param viewWidth The width of `textureView` 779 * @param viewHeight The height of `textureView` 780 */ configureTransform(int viewWidth, int viewHeight)781 private void configureTransform(int viewWidth, int viewHeight) { 782 Activity activity = getActivity(); 783 if (null == textureView || null == previewSize || null == activity) { 784 return; 785 } 786 int rotation = activity.getWindowManager().getDefaultDisplay().getRotation(); 787 Matrix matrix = new Matrix(); 788 RectF viewRect = new RectF(0, 0, viewWidth, viewHeight); 789 RectF bufferRect = new RectF(0, 0, previewSize.getHeight(), previewSize.getWidth()); 790 float centerX = viewRect.centerX(); 791 float centerY = viewRect.centerY(); 792 if (Surface.ROTATION_90 == rotation || Surface.ROTATION_270 == rotation) { 793 bufferRect.offset(centerX - bufferRect.centerX(), centerY - bufferRect.centerY()); 794 matrix.setRectToRect(viewRect, bufferRect, Matrix.ScaleToFit.FILL); 795 float scale = 796 Math.max( 797 (float) viewHeight / previewSize.getHeight(), 798 (float) viewWidth / previewSize.getWidth()); 799 matrix.postScale(scale, scale, centerX, centerY); 800 matrix.postRotate(90 * (rotation - 2), centerX, centerY); 801 } else if (Surface.ROTATION_180 == rotation) { 802 matrix.postRotate(180, centerX, centerY); 803 } 804 textureView.setTransform(matrix); 805 } 806 807 /** Classifies a frame from the preview stream. */ classifyFrame()808 private void classifyFrame() { 809 if (classifier == null || getActivity() == null || cameraDevice == null) { 810 // It's important to not call showToast every frame, or else the app will starve and 811 // hang. updateActiveModel() already puts an error message up with showToast. 812 // showToast("Uninitialized Classifier or invalid context."); 813 return; 814 } 815 SpannableStringBuilder textToShow = new SpannableStringBuilder(); 816 Bitmap bitmap = textureView.getBitmap(classifier.getImageSizeX(), classifier.getImageSizeY()); 817 classifier.classifyFrame(bitmap, textToShow); 818 bitmap.recycle(); 819 showToast(textToShow); 820 } 821 822 /** Compares two {@code Size}s based on their areas. */ 823 private static class CompareSizesByArea implements Comparator<Size> { 824 825 @Override compare(Size lhs, Size rhs)826 public int compare(Size lhs, Size rhs) { 827 // We cast here to ensure the multiplications won't overflow 828 return Long.signum( 829 (long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight()); 830 } 831 } 832 833 /** Shows an error message dialog. */ 834 public static class ErrorDialog extends DialogFragment { 835 836 private static final String ARG_MESSAGE = "message"; 837 newInstance(String message)838 public static ErrorDialog newInstance(String message) { 839 ErrorDialog dialog = new ErrorDialog(); 840 Bundle args = new Bundle(); 841 args.putString(ARG_MESSAGE, message); 842 dialog.setArguments(args); 843 return dialog; 844 } 845 846 @Override onCreateDialog(Bundle savedInstanceState)847 public Dialog onCreateDialog(Bundle savedInstanceState) { 848 final Activity activity = getActivity(); 849 return new AlertDialog.Builder(activity) 850 .setMessage(getArguments().getString(ARG_MESSAGE)) 851 .setPositiveButton( 852 android.R.string.ok, 853 new DialogInterface.OnClickListener() { 854 @Override 855 public void onClick(DialogInterface dialogInterface, int i) { 856 activity.finish(); 857 } 858 }) 859 .create(); 860 } 861 } 862 } 863