1 /*
<lambda>null2  * Copyright 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.compose.ui.autofill
18 
19 import android.graphics.Rect
20 import android.os.Build
21 import android.util.Log
22 import android.util.SparseArray
23 import android.view.View
24 import android.view.ViewStructure
25 import android.view.autofill.AutofillId
26 import android.view.autofill.AutofillValue
27 import androidx.annotation.RequiresApi
28 import androidx.collection.MutableIntSet
29 import androidx.collection.mutableObjectListOf
30 import androidx.compose.ui.ComposeUiFlags
31 import androidx.compose.ui.ExperimentalComposeUiApi
32 import androidx.compose.ui.focus.FocusListener
33 import androidx.compose.ui.focus.FocusTargetModifierNode
34 import androidx.compose.ui.internal.checkPreconditionNotNull
35 import androidx.compose.ui.node.requireSemanticsInfo
36 import androidx.compose.ui.platform.coreshims.ViewCompatShims
37 import androidx.compose.ui.semantics.SemanticsActions
38 import androidx.compose.ui.semantics.SemanticsConfiguration
39 import androidx.compose.ui.semantics.SemanticsInfo
40 import androidx.compose.ui.semantics.SemanticsListener
41 import androidx.compose.ui.semantics.SemanticsOwner
42 import androidx.compose.ui.semantics.SemanticsProperties
43 import androidx.compose.ui.semantics.getOrNull
44 import androidx.compose.ui.spatial.RectManager
45 import androidx.compose.ui.text.AnnotatedString
46 import androidx.compose.ui.util.fastForEach
47 
48 private const val logTag = "ComposeAutofillManager"
49 
50 /**
51  * Semantic autofill implementation for Android.
52  *
53  * @param view The parent compose view.
54  */
55 @RequiresApi(Build.VERSION_CODES.O)
56 internal class AndroidAutofillManager(
57     var platformAutofillManager: PlatformAutofillManager,
58     private val semanticsOwner: SemanticsOwner,
59     private val view: View,
60     private val rectManager: RectManager,
61     private val packageName: String,
62 ) : AutofillManager(), SemanticsListener, FocusListener {
63     private var reusableRect = Rect()
64     private var rootAutofillId: AutofillId
65 
66     init {
67         view.importantForAutofill = View.IMPORTANT_FOR_AUTOFILL_YES
68         rootAutofillId =
69             checkPreconditionNotNull(ViewCompatShims.getAutofillId(view)?.toAutofillId())
70     }
71 
72     override fun commit() {
73         platformAutofillManager.commit()
74     }
75 
76     override fun cancel() {
77         platformAutofillManager.cancel()
78     }
79 
80     override fun onFocusChanged(
81         previous: FocusTargetModifierNode?,
82         current: FocusTargetModifierNode?
83     ) {
84         previous?.requireSemanticsInfo()?.let {
85             if (it.semanticsConfiguration?.isAutofillable() == true) {
86                 platformAutofillManager.notifyViewExited(view, it.semanticsId)
87             }
88         }
89         current?.requireSemanticsInfo()?.let {
90             if (it.semanticsConfiguration?.isAutofillable() == true) {
91                 val semanticsId = it.semanticsId
92                 rectManager.rects.withRect(semanticsId) { l, t, r, b ->
93                     platformAutofillManager.notifyViewEntered(view, semanticsId, Rect(l, t, r, b))
94                 }
95             }
96         }
97     }
98 
99     /** Send events to the autofill service in response to semantics changes. */
100     override fun onSemanticsChanged(
101         semanticsInfo: SemanticsInfo,
102         previousSemanticsConfiguration: SemanticsConfiguration?
103     ) {
104         val config = semanticsInfo.semanticsConfiguration
105         val prevConfig = previousSemanticsConfiguration
106         val semanticsId = semanticsInfo.semanticsId
107 
108         // Check Input Text.
109         val previousText = prevConfig?.getOrNull(SemanticsProperties.InputText)?.text
110         val newText = config?.getOrNull(SemanticsProperties.InputText)?.text
111         if (previousText !== newText) {
112             when {
113                 previousText == null ->
114                     platformAutofillManager.notifyViewVisibilityChanged(view, semanticsId, true)
115                 newText == null ->
116                     platformAutofillManager.notifyViewVisibilityChanged(view, semanticsId, false)
117                 else -> {
118                     val contentDataType = config.getOrNull(SemanticsProperties.ContentDataType)
119                     if (contentDataType == ContentDataType.Text) {
120                         platformAutofillManager.notifyValueChanged(
121                             view,
122                             semanticsId,
123                             AutofillApi26Helper.getAutofillTextValue(newText.toString())
124                         )
125                     }
126                 }
127             }
128         }
129 
130         // Check Focus.
131         if (@OptIn(ExperimentalComposeUiApi::class) !ComposeUiFlags.isTrackFocusEnabled) {
132             val previousFocus = prevConfig?.getOrNull(SemanticsProperties.Focused)
133             val currFocus = config?.getOrNull(SemanticsProperties.Focused)
134             if (previousFocus != true && currFocus == true && config.isAutofillable()) {
135                 rectManager.rects.withRect(semanticsId) { l, t, r, b ->
136                     platformAutofillManager.notifyViewEntered(view, semanticsId, Rect(l, t, r, b))
137                 }
138             }
139             if (previousFocus == true && currFocus != true && prevConfig.isAutofillable()) {
140                 platformAutofillManager.notifyViewExited(view, semanticsId)
141             }
142         }
143 
144         // Update currentlyDisplayedIDs if relevance to Autocommit has changed.
145         val prevRelatedToAutoCommit = prevConfig?.isRelatedToAutoCommit() == true
146         val currRelatedToAutoCommit = config?.isRelatedToAutoCommit() == true
147         if (prevRelatedToAutoCommit != currRelatedToAutoCommit) {
148             if (currRelatedToAutoCommit) {
149                 currentlyDisplayedIDs.add(semanticsId)
150             } else {
151                 currentlyDisplayedIDs.remove(semanticsId)
152             }
153         }
154     }
155 
156     /** Populate the structure of the entire view hierarchy when the framework requests it. */
157     fun populateViewStructure(rootViewStructure: ViewStructure) {
158         val autofillApi = AutofillApi26Helper
159         val rootSemanticInfo = semanticsOwner.rootInfo
160 
161         // Populate view structure for the root.
162         rootViewStructure.populate(rootSemanticInfo, rootAutofillId, packageName, rectManager)
163 
164         // We save the semanticInfo and viewStructure of the item in a list. These are always stored
165         // as pairs, and we need to cast them back to the required types when we extract them.
166         val populateChildren = mutableObjectListOf<Any>(rootSemanticInfo, rootViewStructure)
167 
168         @Suppress("Range") // isNotEmpty ensures removeAt is not called with -1.
169         while (populateChildren.isNotEmpty()) {
170 
171             val parentStructure =
172                 populateChildren.removeAt(populateChildren.lastIndex) as ViewStructure
173             val parentInfo = populateChildren.removeAt(populateChildren.lastIndex) as SemanticsInfo
174 
175             parentInfo.childrenInfo.fastForEach { childInfo ->
176                 if (childInfo.isDeactivated || !childInfo.isAttached || !childInfo.isPlaced) {
177                     return@fastForEach
178                 }
179 
180                 // TODO(b/378160001): For now we only populate nodes that are relevant for autofill.
181                 //  Populate the structure for all nodes in the future.
182                 val semanticsConfigurationChild = childInfo.semanticsConfiguration
183                 if (semanticsConfigurationChild?.isRelatedToAutofill() != true) {
184                     populateChildren.add(childInfo)
185                     populateChildren.add(parentStructure)
186                     return@fastForEach
187                 }
188 
189                 val childIndex = autofillApi.addChildCount(parentStructure, 1)
190                 val childStructure = autofillApi.newChild(parentStructure, childIndex)
191                 childStructure.populate(childInfo, rootAutofillId, packageName, rectManager)
192                 populateChildren.add(childInfo)
193                 populateChildren.add(childStructure)
194             }
195         }
196     }
197 
198     /** When the autofill service provides data, perform autofill using semantic actions. */
199     fun performAutofill(values: SparseArray<AutofillValue>) {
200         for (index in 0 until values.size()) {
201             val itemId = values.keyAt(index)
202             val value = values[itemId]
203             when {
204                 AutofillApi26Helper.isText(value) ->
205                     semanticsOwner[itemId]
206                         ?.semanticsConfiguration
207                         ?.getOrNull(SemanticsActions.OnAutofillText)
208                         ?.action
209                         ?.invoke(AnnotatedString(AutofillApi26Helper.textValue(value).toString()))
210 
211                 // TODO(b/138604541): Add Autofill support for date fields.
212                 AutofillApi26Helper.isDate(value) ->
213                     Log.w(logTag, "Auto filling Date fields is not yet supported.")
214 
215                 // TODO(b/138604541): Add Autofill support for dropdown lists.
216                 AutofillApi26Helper.isList(value) ->
217                     Log.w(logTag, "Auto filling dropdown lists is not yet supported.")
218 
219                 // TODO(b/138604541): Add Autofill support for toggle fields.
220                 AutofillApi26Helper.isToggle(value) ->
221                     Log.w(logTag, "Auto filling toggle fields are not yet supported.")
222             }
223         }
224     }
225 
226     // Consider moving the currently displayed IDs to a separate VisibilityManager class. This might
227     // be needed by ContentCapture and Accessibility.
228     private var currentlyDisplayedIDs = MutableIntSet()
229 
230     internal fun requestAutofill(semanticsInfo: SemanticsInfo) {
231         rectManager.rects.withRect(semanticsInfo.semanticsId) { left, top, right, bottom ->
232             reusableRect.set(left, top, right, bottom)
233             platformAutofillManager.requestAutofill(view, semanticsInfo.semanticsId, reusableRect)
234         }
235     }
236 
237     internal fun onPostAttach(semanticsInfo: SemanticsInfo) {
238         if (semanticsInfo.semanticsConfiguration?.isRelatedToAutoCommit() == true) {
239             currentlyDisplayedIDs.add(semanticsInfo.semanticsId)
240             // `notifyVisibilityChanged` is called when nodes appear onscreen (and become visible).
241             platformAutofillManager.notifyViewVisibilityChanged(
242                 view,
243                 semanticsInfo.semanticsId,
244                 true
245             )
246         }
247     }
248 
249     internal fun onPostLayoutNodeReused(semanticsInfo: SemanticsInfo, previousSemanticsId: Int) {
250         if (currentlyDisplayedIDs.remove(previousSemanticsId)) {
251             platformAutofillManager.notifyViewVisibilityChanged(view, previousSemanticsId, false)
252         }
253         if (semanticsInfo.semanticsConfiguration?.isRelatedToAutoCommit() == true) {
254             currentlyDisplayedIDs.add(semanticsInfo.semanticsId)
255             platformAutofillManager.notifyViewVisibilityChanged(
256                 view,
257                 semanticsInfo.semanticsId,
258                 true
259             )
260         }
261     }
262 
263     internal fun onLayoutNodeDeactivated(semanticsInfo: SemanticsInfo) {
264         if (currentlyDisplayedIDs.remove(semanticsInfo.semanticsId)) {
265             platformAutofillManager.notifyViewVisibilityChanged(
266                 view,
267                 semanticsInfo.semanticsId,
268                 false
269             )
270         }
271     }
272 
273     internal fun onDetach(semanticsInfo: SemanticsInfo) {
274         if (currentlyDisplayedIDs.remove(semanticsInfo.semanticsId)) {
275             // `notifyVisibilityChanged` is called when nodes go offscreen (and become invisible
276             // to the user).
277             platformAutofillManager.notifyViewVisibilityChanged(
278                 view,
279                 semanticsInfo.semanticsId,
280                 false
281             )
282         }
283     }
284 
285     private var pendingAutofillCommit = false
286 
287     internal fun onEndApplyChanges() {
288         if (currentlyDisplayedIDs.isEmpty() && pendingAutofillCommit) {
289             // We call AutofillManager.commit() when no more autofillable components are
290             // onscreen.
291             platformAutofillManager.commit()
292             pendingAutofillCommit = false
293         }
294         if (currentlyDisplayedIDs.isNotEmpty()) {
295             pendingAutofillCommit = true
296         }
297     }
298 }
299 
SemanticsConfigurationnull300 private fun SemanticsConfiguration.isAutofillable(): Boolean {
301     // TODO add more actions once we add support for Toggle, List, Date etc.
302     return props.contains(SemanticsActions.OnAutofillText)
303 }
304 
isRelatedToAutoCommitnull305 private fun SemanticsConfiguration.isRelatedToAutoCommit(): Boolean {
306     return props.contains(SemanticsProperties.ContentType)
307 }
308 
isRelatedToAutofillnull309 private fun SemanticsConfiguration.isRelatedToAutofill(): Boolean {
310     return props.contains(SemanticsActions.OnAutofillText) ||
311         props.contains(SemanticsProperties.ContentType) ||
312         props.contains(SemanticsProperties.ContentDataType)
313 }
314