1 /*
<lambda>null2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.integration.diagnose
18 
19 import android.Manifest
20 import android.annotation.SuppressLint
21 import android.content.ContentValues
22 import android.content.pm.PackageManager
23 import android.os.Build
24 import android.os.Bundle
25 import android.provider.MediaStore
26 import android.util.Log
27 import android.util.Size
28 import android.view.View
29 import android.widget.Button
30 import android.widget.Toast
31 import androidx.appcompat.app.AppCompatActivity
32 import androidx.camera.core.ImageAnalysis
33 import androidx.camera.core.ImageCapture
34 import androidx.camera.core.ImageCaptureException
35 import androidx.camera.mlkit.vision.MlKitAnalyzer
36 import androidx.camera.video.MediaStoreOutputOptions
37 import androidx.camera.video.Recording
38 import androidx.camera.video.VideoRecordEvent
39 import androidx.camera.view.CameraController.IMAGE_ANALYSIS
40 import androidx.camera.view.CameraController.IMAGE_CAPTURE
41 import androidx.camera.view.CameraController.VIDEO_CAPTURE
42 import androidx.camera.view.LifecycleCameraController
43 import androidx.camera.view.PreviewView
44 import androidx.camera.view.video.AudioConfig
45 import androidx.core.app.ActivityCompat
46 import androidx.core.content.ContextCompat
47 import androidx.core.util.Preconditions
48 import androidx.lifecycle.lifecycleScope
49 import com.google.android.material.tabs.TabLayout
50 import com.google.mlkit.vision.barcode.BarcodeScanner
51 import com.google.mlkit.vision.barcode.BarcodeScanning
52 import java.io.IOException
53 import java.text.SimpleDateFormat
54 import java.util.Locale
55 import java.util.concurrent.Executor
56 import java.util.concurrent.ExecutorService
57 import java.util.concurrent.Executors
58 import kotlinx.coroutines.ExecutorCoroutineDispatcher
59 import kotlinx.coroutines.asCoroutineDispatcher
60 import kotlinx.coroutines.launch
61 import kotlinx.coroutines.withContext
62 
63 @SuppressLint("NullAnnotationGroup", "MissingPermission")
64 class MainActivity : AppCompatActivity() {
65 
66     private lateinit var cameraController: LifecycleCameraController
67     private lateinit var activeRecording: Recording
68     private lateinit var previewView: PreviewView
69     private lateinit var overlayView: OverlayView
70     private lateinit var executor: Executor
71     private lateinit var tabLayout: TabLayout
72     private lateinit var diagnosis: Diagnosis
73     private lateinit var barcodeScanner: BarcodeScanner
74     private lateinit var analyzer: MlKitAnalyzer
75     private lateinit var diagnoseBtn: Button
76     private lateinit var imageCaptureBtn: Button
77     private lateinit var videoCaptureBtn: Button
78     private lateinit var calibrationExecutor: ExecutorService
79     private var calibrationThreadId: Long = -1
80     private lateinit var diagnosisDispatcher: ExecutorCoroutineDispatcher
81 
82     override fun onCreate(savedInstanceState: Bundle?) {
83         super.onCreate(savedInstanceState)
84         setContentView(R.layout.activity_main)
85         previewView = findViewById(R.id.preview_view)
86         overlayView = findViewById(R.id.overlay_view)
87         overlayView.visibility = View.INVISIBLE
88         cameraController = LifecycleCameraController(this)
89         previewView.controller = cameraController
90         executor = ContextCompat.getMainExecutor(this)
91         tabLayout = findViewById(R.id.tabLayout_view)
92         diagnosis = Diagnosis()
93         barcodeScanner = BarcodeScanning.getClient()
94         diagnoseBtn = findViewById(R.id.diagnose_btn)
95         imageCaptureBtn = findViewById(R.id.image_capture_btn)
96         imageCaptureBtn.visibility = View.INVISIBLE
97         videoCaptureBtn = findViewById(R.id.video_capture_btn)
98         videoCaptureBtn.visibility = View.INVISIBLE
99         calibrationExecutor =
100             Executors.newSingleThreadExecutor() { runnable ->
101                 val thread = Executors.defaultThreadFactory().newThread(runnable)
102                 thread.name = "CalibrationThread"
103                 calibrationThreadId = thread.id
104                 return@newSingleThreadExecutor thread
105             }
106         diagnosisDispatcher = Executors.newFixedThreadPool(1).asCoroutineDispatcher()
107 
108         // Request CAMERA permission and fail gracefully if not granted.
109         if (allPermissionsGranted()) {
110             startCamera()
111         } else {
112             ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
113         }
114 
115         // Setting up Tabs
116         val photoTab = tabLayout.newTab().setText("Photo")
117         photoTab.view.id = R.id.image_capture
118         tabLayout.addTab(photoTab)
119         val videoTab = tabLayout.newTab().setText("Video")
120         videoTab.view.id = R.id.video_capture
121         tabLayout.addTab(videoTab)
122         val diagnoseTab = tabLayout.newTab().setText("Diagnose")
123         diagnoseTab.view.id = R.id.diagnose
124         tabLayout.addTab(diagnoseTab)
125 
126         // Setup UI events
127         tabLayout.addOnTabSelectedListener(
128             object : TabLayout.OnTabSelectedListener {
129                 override fun onTabSelected(tab: TabLayout.Tab?) {
130                     Log.d(TAG, "tab selected id:${tab?.view?.id}")
131                     selectMode(tab?.view?.id)
132                 }
133 
134                 override fun onTabReselected(tab: TabLayout.Tab?) {
135                     Log.d(TAG, "tab reselected:${tab?.view?.id}")
136                     selectMode(tab?.view?.id)
137                 }
138 
139                 override fun onTabUnselected(tab: TabLayout.Tab?) {
140                     Log.d(TAG, "tab unselected:${tab?.view?.id}")
141                     deselectMode(tab?.view?.id)
142                 }
143             }
144         )
145 
146         imageCaptureBtn.setOnClickListener {
147             val name =
148                 SimpleDateFormat(FILENAME_FORMAT, Locale.US).format(System.currentTimeMillis())
149             val contentValues =
150                 ContentValues().apply {
151                     put(MediaStore.MediaColumns.DISPLAY_NAME, name)
152                     put(MediaStore.MediaColumns.MIME_TYPE, "image/jpeg")
153                     if (Build.VERSION.SDK_INT > Build.VERSION_CODES.P) {
154                         put(MediaStore.Images.Media.RELATIVE_PATH, "Pictures/CameraX-Image")
155                     }
156                 }
157 
158             // Create output options object which contains file + metadata
159             val outputOptions =
160                 ImageCapture.OutputFileOptions.Builder(
161                         contentResolver,
162                         MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
163                         contentValues
164                     )
165                     .build()
166 
167             // Set up image capture listener, which is triggered after photo has
168             // been taken
169             cameraController.takePicture(
170                 outputOptions,
171                 executor,
172                 object : ImageCapture.OnImageSavedCallback {
173                     override fun onError(exc: ImageCaptureException) {
174                         val msg = "Photo capture failed: ${exc.message}"
175                         showToast(msg)
176                     }
177 
178                     override fun onImageSaved(output: ImageCapture.OutputFileResults) {
179                         val msg = "Photo capture succeeded: ${output.savedUri}"
180                         showToast(msg)
181                     }
182                 }
183             )
184         }
185 
186         videoCaptureBtn.setOnClickListener {
187             // determine whether the onclick is to start recording or stop recording
188             if (cameraController.isRecording) {
189                 activeRecording.stop()
190                 videoCaptureBtn.setText(R.string.start_video_capture)
191                 val msg = "video stopped recording"
192                 showToast(msg)
193             } else {
194                 // building file output
195                 val name =
196                     SimpleDateFormat(FILENAME_FORMAT, Locale.US).format(System.currentTimeMillis())
197                 val contentValues =
198                     ContentValues().apply {
199                         put(MediaStore.MediaColumns.DISPLAY_NAME, name)
200                         put(MediaStore.MediaColumns.MIME_TYPE, "video/mp4")
201                         if (Build.VERSION.SDK_INT > Build.VERSION_CODES.P) {
202                             put(MediaStore.Video.Media.RELATIVE_PATH, "Movies/CameraX-Video")
203                         }
204                     }
205                 val outputOptions =
206                     MediaStoreOutputOptions.Builder(
207                             contentResolver,
208                             MediaStore.Video.Media.EXTERNAL_CONTENT_URI
209                         )
210                         .setContentValues(contentValues)
211                         .build()
212                 Log.d(TAG, "finished composing video name")
213 
214                 val audioConfig = AudioConfig.create(true)
215 
216                 // start recording
217                 try {
218                     activeRecording =
219                         cameraController.startRecording(outputOptions, audioConfig, executor) {
220                             event ->
221                             if (event is VideoRecordEvent.Finalize) {
222                                 val uri = event.outputResults.outputUri
223                                 if (event.error == VideoRecordEvent.Finalize.ERROR_NONE) {
224                                     val msg = "Video record succeeded: $uri"
225                                     showToast(msg)
226                                 } else {
227                                     Log.e(TAG, "Video saving failed: ${event.cause}")
228                                 }
229                             }
230                         }
231                     videoCaptureBtn.setText(R.string.stop_video_capture)
232                     val msg = "video recording"
233                     showToast(msg)
234                 } catch (exception: RuntimeException) {
235                     Log.e(TAG, "Video failed to record: " + exception.message)
236                 }
237             }
238         }
239 
240         diagnoseBtn.setOnClickListener {
241             lifecycleScope.launch {
242                 try {
243                     val reportFile =
244                         withContext(diagnosisDispatcher) {
245                             // creating tasks to diagnose
246                             val taskList = mutableListOf<DiagnosisTask>()
247                             taskList.add(CollectDeviceInfoTask())
248                             taskList.add(ImageCaptureTask())
249                             val isAggregated = true
250                             Log.i(TAG, "dispatcher: ${Thread.currentThread().name}")
251                             diagnosis.diagnose(
252                                 baseContext,
253                                 taskList,
254                                 cameraController,
255                                 isAggregated
256                             )
257                         }
258                     val msg: String =
259                         if (reportFile != null) {
260                             "Successfully collected diagnosis to ${reportFile.path}"
261                         } else {
262                             "Diagnosis failed: No file"
263                         }
264                     showToast(msg)
265                 } catch (e: IOException) {
266                     val msg = "Failed to collect information"
267                     showToast(msg)
268                     Log.e(TAG, "IOException caught: ${e.message}")
269                 }
270             }
271         }
272     }
273 
274     override fun onRequestPermissionsResult(
275         requestCode: Int,
276         permissions: Array<String>,
277         grantResults: IntArray
278     ) {
279         super.onRequestPermissionsResult(requestCode, permissions, grantResults)
280         if (requestCode == REQUEST_CODE_PERMISSIONS) {
281             if (allPermissionsGranted()) {
282                 startCamera()
283             } else {
284                 val msg = "Permissions not granted by the user"
285                 showToast(msg)
286                 // TODO: fail gracefully
287                 finish()
288             }
289         }
290     }
291 
292     private fun startCamera() {
293         // Setup CameraX
294         cameraController.bindToLifecycle(this)
295         Log.d(TAG, "started camera")
296     }
297 
298     private fun selectMode(id: Int?) {
299         when (id) {
300             R.id.image_capture -> photoMode()
301             R.id.video_capture -> videoMode()
302             R.id.diagnose -> diagnose()
303         }
304     }
305 
306     private fun deselectMode(id: Int?) {
307         when (id) {
308             R.id.image_capture -> imageCaptureBtn.visibility = View.INVISIBLE
309             R.id.video_capture -> videoCaptureBtn.visibility = View.INVISIBLE
310             R.id.diagnose -> {
311                 // disable overlay
312                 overlayView.visibility = View.INVISIBLE
313                 // unbind MLKit analyzer
314                 cameraController.clearImageAnalysisAnalyzer()
315             }
316         }
317     }
318 
319     private fun photoMode() {
320         cameraController.setEnabledUseCases(IMAGE_CAPTURE)
321         imageCaptureBtn.visibility = View.VISIBLE
322     }
323 
324     private fun videoMode() {
325         cameraController.setEnabledUseCases(VIDEO_CAPTURE)
326         videoCaptureBtn.visibility = View.VISIBLE
327     }
328 
329     private fun diagnose() {
330         // enable overlay and diagnose button
331         overlayView.visibility = View.VISIBLE
332         // enable image analysis use case
333         cameraController.setEnabledUseCases(IMAGE_ANALYSIS)
334 
335         val calibrate = Calibration(Size(previewView.width, previewView.height))
336 
337         analyzer =
338             MlKitAnalyzer(
339                 listOf(barcodeScanner),
340                 ImageAnalysis.COORDINATE_SYSTEM_VIEW_REFERENCED,
341                 calibrationExecutor
342             ) { result ->
343                 // validating thread
344                 checkCalibrationThread()
345                 val barcodes = result.getValue(barcodeScanner)
346                 if (barcodes != null && barcodes.size > 0) {
347                     calibrate.analyze(barcodes)
348                     // run UI on main thread
349                     lifecycleScope.launch {
350                         // gives overlayView access to Calibration
351                         overlayView.setCalibrationResult(calibrate)
352                         // enable diagnose button when alignment is successful
353                         diagnoseBtn.isEnabled = calibrate.isAligned
354                         overlayView.invalidate()
355                     }
356                 }
357             }
358         cameraController.setImageAnalysisAnalyzer(calibrationExecutor, analyzer)
359     }
360 
361     private fun allPermissionsGranted() =
362         REQUIRED_PERMISSIONS.all {
363             ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED
364         }
365 
366     private fun checkCalibrationThread() {
367         Preconditions.checkState(
368             calibrationThreadId == Thread.currentThread().id,
369             "Not working on Calibration Thread"
370         )
371     }
372 
373     private fun showToast(msg: String) {
374         Toast.makeText(baseContext, msg, Toast.LENGTH_SHORT).show()
375         Log.d(TAG, msg)
376     }
377 
378     override fun onDestroy() {
379         super.onDestroy()
380         calibrationExecutor.shutdown()
381         diagnosisDispatcher.close()
382     }
383 
384     companion object {
385         private const val TAG = "DiagnoseApp"
386         private const val FILENAME_FORMAT = "yyyy-MM-dd-HH-mm-ss-SSS"
387         private const val REQUEST_CODE_PERMISSIONS = 10
388         private val REQUIRED_PERMISSIONS =
389             mutableListOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO)
390                 .apply {
391                     if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) {
392                         add(Manifest.permission.WRITE_EXTERNAL_STORAGE)
393                     }
394                 }
395                 .toTypedArray()
396     }
397 }
398