android/accessibilityservice/TouchInteractionController.java

/*
 * Copyright (C) 2021 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.accessibilityservice;

import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.os.RemoteException;
import android.util.ArrayMap;
import android.view.MotionEvent;
import android.view.accessibility.AccessibilityInteractionClient;

import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.Executor;

/**
 * This class allows a service to handle touch exploration and the detection of specialized
 * accessibility gestures. The service receives motion events and can match those motion events
 * against the gestures it supports. The service can also request the framework enter three other
 * states of operation for the duration of this interaction. Upon entering any of these states the
 * framework will take over and the service will not receive motion events until the start of a new
 * interaction. The states are as follows:
 *
 * <ul>
 *   <li>The service can tell the framework that this interaction is touch exploration. The user is
 *       trying to explore the screen rather than manipulate it. The framework will then convert the
 *       motion events to hover events to support touch exploration.
 *   <li>The service can tell the framework that this interaction is a dragging interaction where
 *       two fingers are used to execute a one-finger gesture such as scrolling the screen. The
 *       service must specify which of the two fingers should be passed through to rest of the input
 *       pipeline.
 *   <li>Finally, the service can request that the framework delegate this interaction, meaning pass
 *       it through to the rest of the input pipeline as-is.
 * </ul>
 *
 * When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is enabled, this
 * controller will receive all motion events received by the framework for the specified display
 * when not touch-exploring or delegating. If the service classifies this interaction as touch
 * exploration or delegating the framework will stop sending motion events to the service for the
 * duration of this interaction. If the service classifies this interaction as a dragging
 * interaction the framework will send motion events to the service to allow the service to
 * determine if the interaction still qualifies as dragging or if it has become a delegating
 * interaction. If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled
 * this controller will not receive any motion events because touch interactions are being passed
 * through to the input pipeline unaltered.
 * Note that {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE }
 * requires setting {@link android.R.attr#canRequestTouchExplorationMode} as well.
 */
public final class TouchInteractionController {
    /** The state where the user is not touching the screen. */
    public static final int STATE_CLEAR = 0;
    /**
     * The state where the user is touching the screen and the service is receiving motion events.
     */
    public static final int STATE_TOUCH_INTERACTING = 1;
    /**
     * The state where the user is explicitly exploring the screen. The service is not receiving
     * motion events.
     */
    public static final int STATE_TOUCH_EXPLORING = 2;
    /**
     * The state where the user is dragging with two fingers. The service is not receiving motion
     * events. The selected finger is being dispatched to the rest of the input pipeline to execute
     * the drag.
     */
    public static final int STATE_DRAGGING = 3;
    /**
     * The user is performing a gesture which is being passed through to the input pipeline as-is.
     * The service is not receiving motion events.
     */
    public static final int STATE_DELEGATING = 4;

    @IntDef({
        STATE_CLEAR,
        STATE_TOUCH_INTERACTING,
        STATE_TOUCH_EXPLORING,
        STATE_DRAGGING,
        STATE_DELEGATING
    })
    @Retention(RetentionPolicy.SOURCE)
    private @interface State {}

    // The maximum number of pointers that can be touching the screen at once. (See MAX_POINTER_ID
    // in frameworks/native/include/input/Input.h)
    private static final int MAX_POINTER_COUNT = 32;

    private final AccessibilityService mService;
    private final Object mLock;
    private final int mDisplayId;
    private boolean mServiceDetectsGestures;
    /** Map of callbacks to executors. Lazily created when adding the first callback. */
    private ArrayMap<Callback, Executor> mCallbacks;
    // A list of motion events that should be queued until a pending transition has taken place.
    private Queue<MotionEvent> mQueuedMotionEvents = new LinkedList<>();
    // Whether this controller is waiting for a state transition.
    // Motion events will be queued and sent to listeners after the transition has taken place.
    private boolean mStateChangeRequested = false;

    // The current state of the display.
    private int mState = STATE_CLEAR;

    TouchInteractionController(
            @NonNull AccessibilityService service, @NonNull Object lock, int displayId) {
        mDisplayId = displayId;
        mLock = lock;
        mService = service;
    }

    /**
     * Adds the specified callback to the list of callbacks. The callback will
     * run using on the specified {@link Executor}', or on the service's main thread if the
     * Executor is {@code null}.
     * @param callback the callback to add, must be non-null
     * @param executor the executor for this callback, or {@code null} to execute on the service's
     *     main thread
     */
    public void registerCallback(@Nullable Executor executor, @NonNull Callback callback) {
        synchronized (mLock) {
            if (mCallbacks == null) {
                mCallbacks = new ArrayMap<>();
            }
            mCallbacks.put(callback, executor);
            if (mCallbacks.size() == 1) {
                setServiceDetectsGestures(true);
            }
        }
    }

    /**
     * Unregisters the specified callback.
     *
     * @param callback the callback to remove, must be non-null
     * @return {@code true} if the callback was removed, {@code false} otherwise
     */
    public boolean unregisterCallback(@NonNull Callback callback) {
        if (mCallbacks == null) {
            return false;
        }
        synchronized (mLock) {
            boolean result = mCallbacks.remove(callback) != null;
            if (result && mCallbacks.size() == 0) {
                setServiceDetectsGestures(false);
            }
            return result;
        }
    }

    /**
     * Removes all callbacks and returns control of touch interactions to the framework.
     */
    public void unregisterAllCallbacks() {
        if (mCallbacks != null) {
            synchronized (mLock) {
                mCallbacks.clear();
                setServiceDetectsGestures(false);
            }
        }
    }

    /**
     * Dispatches motion events to any registered callbacks. This should be called on the service's
     * main thread.
     */
    void onMotionEvent(MotionEvent event) {
        if (mStateChangeRequested) {
            mQueuedMotionEvents.add(event);
        } else {
            sendEventToAllListeners(event);
        }
    }

    private void sendEventToAllListeners(MotionEvent event) {
        final ArrayMap<Callback, Executor> entries;
        synchronized (mLock) {
            // callbacks may remove themselves. Perform a shallow copy to avoid concurrent
            // modification.
            entries = new ArrayMap<>(mCallbacks);
        }
        for (int i = 0, count = entries.size(); i < count; i++) {
            final Callback callback = entries.keyAt(i);
            final Executor executor = entries.valueAt(i);
            if (executor != null) {
                executor.execute(() -> callback.onMotionEvent(event));
            } else {
                // We're already on the main thread, just run the callback.
                callback.onMotionEvent(event);
            }
        }
    }

    /**
     * Dispatches motion events to any registered callbacks. This should be called on the service's
     * main thread.
     */
    void onStateChanged(@State int state) {
        mState = state;
        final ArrayMap<Callback, Executor> entries;
        synchronized (mLock) {
            // callbacks may remove themselves. Perform a shallow copy to avoid concurrent
            // modification.
            entries = new ArrayMap<>(mCallbacks);
        }
        for (int i = 0, count = entries.size(); i < count; i++) {
            final Callback callback = entries.keyAt(i);
            final Executor executor = entries.valueAt(i);
            if (executor != null) {
                executor.execute(() -> callback.onStateChanged(state));
            } else {
                // We're already on the main thread, just run the callback.
                callback.onStateChanged(state);
            }
        }
        mStateChangeRequested = false;
        while (mQueuedMotionEvents.size() > 0) {
            sendEventToAllListeners(mQueuedMotionEvents.poll());
        }
    }

    /**
     * When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled, this
     * controller will receive all motion events received by the framework for the specified display
     * when not touch-exploring, delegating, or dragging. This allows the service to detect its own
     * gestures, and use its own logic to judge when the framework should start touch-exploring,
     * delegating, or dragging. If {@link
     * AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled this flag has no
     * effect.
     *
     * @see AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE
     */
    private void setServiceDetectsGestures(boolean mode) {
        final IAccessibilityServiceConnection connection =
                AccessibilityInteractionClient.getInstance()
                        .getConnection(mService.getConnectionId());
        if (connection != null) {
            try {
                connection.setServiceDetectsGesturesEnabled(mDisplayId, mode);
                mServiceDetectsGestures = mode;
            } catch (RemoteException re) {
                throw new RuntimeException(re);
            }
        }
    }

    /**
     * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at
     * least one callback has been added for this display this function tells the framework to
     * initiate touch exploration. Touch exploration will continue for the duration of this
     * interaction.
     */
    public void requestTouchExploration() {
        validateTransitionRequest();
        final IAccessibilityServiceConnection connection =
                AccessibilityInteractionClient.getInstance()
                        .getConnection(mService.getConnectionId());
        if (connection != null) {
            try {
                connection.requestTouchExploration(mDisplayId);
            } catch (RemoteException re) {
                throw new RuntimeException(re);
            }
            mStateChangeRequested = true;
        }
    }

    /**
     * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
     * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
     * one callback has been added, this function tells the framework to initiate a dragging
     * interaction using the specified pointer. The pointer's movements will be passed through to
     * the rest of the input pipeline. Dragging is often used to perform two-finger scrolling.
     *
     * @param pointerId the pointer to be passed through to the rest of the input pipeline. If the
     *            pointer id is valid but not actually present on the screen it will be ignored.
     * @throws IllegalArgumentException if the pointer id is outside of the allowed range.
     */
    public void requestDragging(int pointerId) {
        validateTransitionRequest();
        if (pointerId < 0 || pointerId > MAX_POINTER_COUNT) {
            throw new IllegalArgumentException("Invalid pointer id: " + pointerId);
        }
        final IAccessibilityServiceConnection connection =
                AccessibilityInteractionClient.getInstance()
                        .getConnection(mService.getConnectionId());
        if (connection != null) {
            try {
                connection.requestDragging(mDisplayId, pointerId);
            } catch (RemoteException re) {
                throw new RuntimeException(re);
            }
            mStateChangeRequested = true;
        }
    }

    /**
     * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
     * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
     * one callback has been added, this function tells the framework to initiate a delegating
     * interaction. Motion events will be passed through as-is to the rest of the input pipeline for
     * the duration of this interaction.
     */
    public void requestDelegating() {
        validateTransitionRequest();
        final IAccessibilityServiceConnection connection =
                AccessibilityInteractionClient.getInstance()
                        .getConnection(mService.getConnectionId());
        if (connection != null) {
            try {
                connection.requestDelegating(mDisplayId);
            } catch (RemoteException re) {
                throw new RuntimeException(re);
            }
            mStateChangeRequested = true;
        }
    }

    /**
     * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
     * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
     * one callback has been added, this function tells the framework to perform a click.
     * The framework will first try to perform
     * {@link AccessibilityNodeInfo.AccessibilityAction#ACTION_CLICK} on the item with
     * accessibility focus. If that fails, the framework will simulate a click using motion events
     * on the last location to have accessibility focus.
     */
    public void performClick() {
        final IAccessibilityServiceConnection connection =
                AccessibilityInteractionClient.getInstance()
                        .getConnection(mService.getConnectionId());
        if (connection != null) {
            try {
                connection.onDoubleTap(mDisplayId);
            } catch (RemoteException re) {
                throw new RuntimeException(re);
            }
        }
    }

    /**
     * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
     * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
     * one callback has been added, this function tells the framework to perform a long click.
     * The framework will simulate a long click using motion events on the last location with
     * accessibility focus and will delegate any movements to the rest of the input pipeline. This
     * allows a user to double-tap and hold to trigger a drag and then execute that drag by moving
     * their finger.
     */
    public void performLongClickAndStartDrag() {
        final IAccessibilityServiceConnection connection =
                AccessibilityInteractionClient.getInstance()
                        .getConnection(mService.getConnectionId());
        if (connection != null) {
            try {
                connection.onDoubleTapAndHold(mDisplayId);
            } catch (RemoteException re) {
                throw new RuntimeException(re);
            }
        }
    }

    private void validateTransitionRequest() {
        if (!mServiceDetectsGestures || mCallbacks.size() == 0) {
            throw new IllegalStateException(
                    "State transitions are not allowed without first adding a callback.");
        }
        if ((mState == STATE_DELEGATING || mState == STATE_TOUCH_EXPLORING)) {
            throw new IllegalStateException(
                    "State transition requests are not allowed in " + stateToString(mState));
        }
    }

    /** @return the maximum number of pointers that this display will accept. */
    public int getMaxPointerCount() {
        return MAX_POINTER_COUNT;
    }

    /** @return the display id associated with this controller. */
    public int getDisplayId() {
        return mDisplayId;
    }

    /**
     * @return the current state of this controller.
     * @see TouchInteractionController#STATE_CLEAR
     * @see TouchInteractionController#STATE_DELEGATING
     * @see TouchInteractionController#STATE_DRAGGING
     * @see TouchInteractionController#STATE_TOUCH_EXPLORING
     */
    public int getState() {
        synchronized (mLock) {
            return mState;
        }
    }

    /** Returns a string representation of the specified state. */
    @NonNull
    public static String stateToString(int state) {
        switch (state) {
            case STATE_CLEAR:
                return "STATE_CLEAR";
            case STATE_TOUCH_INTERACTING:
                return "STATE_TOUCH_INTERACTING";
            case STATE_TOUCH_EXPLORING:
                return "STATE_TOUCH_EXPLORING";
            case STATE_DRAGGING:
                return "STATE_DRAGGING";
            case STATE_DELEGATING:
                return "STATE_DELEGATING";
            default:
                return "Unknown state: " + state;
        }
    }

    /** callbacks allow services to receive motion events and state change updates. */
    public interface Callback {
        /**
         * Called when the framework has sent a motion event to the service.
         *
         * @param event the event being passed to the service.
         */
        void onMotionEvent(@NonNull MotionEvent event);

        /**
         * Called when the state of motion event dispatch for this display has changed.
         *
         * @param state the new state of motion event dispatch.
         * @see TouchInteractionController#STATE_CLEAR
         * @see TouchInteractionController#STATE_DELEGATING
         * @see TouchInteractionController#STATE_DRAGGING
         * @see TouchInteractionController#STATE_TOUCH_EXPLORING
         */
        void onStateChanged(@State int state);
    }
}