QtAccessibilityDelegate.java

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 
package org.qtproject.qt.android;
 
import android.app.Activity;
import android.content.Context;
import android.graphics.Rect;
import android.os.Build;
import android.os.Bundle;
import android.system.Os;
import android.text.TextUtils;
import android.util.Log;
import android.view.MotionEvent;
import android.view.View;
import android.view.ViewGroup;
import android.view.ViewParent;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityManager;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityNodeInfo.CollectionInfo;
import android.view.accessibility.AccessibilityNodeProvider;
 
class QtAccessibilityDelegate extends View.AccessibilityDelegate
{
    private static final String TAG = "Qt A11Y";
 
    // Qt uses the upper half of the unsigned integers
    // all low positive ints should be fine.
    public static final int INVALID_ID = 333; // half evil
 
    // The platform might ask for the class implementing the "view".
    // Pretend to be an inner class of the QtSurface.
    private static final String DEFAULT_CLASS_NAME = "$VirtualChild";
 
    private View m_view = null;
    private final AccessibilityManager m_manager;
    private final QtLayout m_layout;
 
    // The accessible object that currently has the "accessibility focus"
    // usually indicated by a yellow rectangle on screen.
    private int m_focusedVirtualViewId = INVALID_ID;
    // When exploring the screen by touch, the item "hovered" by the finger.
    private int m_hoveredVirtualViewId = INVALID_ID;
 
    // Cache coordinates of the view to know the global offset
    // this is because the Android platform window does not take
    // the offset of the view on screen into account (eg status bar on top)
    private final int[] m_globalOffset = new int[2];
    private int m_oldOffsetX = 0;
    private int m_oldOffsetY = 0;
 
    private class HoverEventListener implements View.OnHoverListener
    {
        @Override
        public boolean onHover(View v, MotionEvent event)
        {
            return dispatchHoverEvent(event);
        }
    }
    // TODO do we want to have one QtAccessibilityDelegate for the whole app (QtRootLayout) or
    // e.g. one per window?
    public QtAccessibilityDelegate(QtLayout layout)
    {
        m_layout = layout;
 
        m_manager = (AccessibilityManager) m_layout.getContext().getSystemService(Context.ACCESSIBILITY_SERVICE);
        if (m_manager != null) {
            AccessibilityManagerListener accServiceListener = new AccessibilityManagerListener();
            if (!m_manager.addAccessibilityStateChangeListener(accServiceListener))
                Log.w("Qt A11y", "Could not register a11y state change listener");
            if (m_manager.isEnabled())
                accServiceListener.onAccessibilityStateChanged(true);
        }
    }
 
    private class AccessibilityManagerListener implements AccessibilityManager.AccessibilityStateChangeListener
    {
        @Override
        public void onAccessibilityStateChanged(boolean enabled)
        {
            if (Os.getenv("QT_ANDROID_DISABLE_ACCESSIBILITY") != null)
                return;
            if (enabled) {
                try {
                        View view = m_view;
                        if (view == null) {
                            view = new View(m_layout.getContext());
                            view.setId(View.NO_ID);
                        }
 
                        // ### Keep this for debugging for a while. It allows us to visually see that our View
                        // ### is on top of the surface(s)
                        //noinspection CommentedOutCode
                        {
                            // ColorDrawable color = new ColorDrawable(0x80ff8080);    //0xAARRGGBB
                            // view.setBackground(color);
                        }
                        view.setAccessibilityDelegate(QtAccessibilityDelegate.this);
 
                        // if all is fine, add it to the layout
                        if (m_view == null) {
                            //m_layout.addAccessibilityView(view);
                            m_layout.addView(view, m_layout.getChildCount(),
                                             new ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT));
                        }
                        m_view = view;
 
                        m_view.setOnHoverListener(new HoverEventListener());
                    } catch (Exception e) {
                        // Unknown exception means something went wrong.
                        Log.w("Qt A11y", "Unknown exception: " + e);
                    }
            } else {
                if (m_view != null) {
                    m_layout.removeView(m_view);
                    m_view = null;
                }
            }
 
            QtNativeAccessibility.setActive(enabled);
        }
    }
 
 
    @Override
    public AccessibilityNodeProvider getAccessibilityNodeProvider(View host)
    {
        return m_nodeProvider;
    }
 
    // For "explore by touch" we need all movement events here first
    // (user moves finger over screen to discover items on screen).
    private boolean dispatchHoverEvent(MotionEvent event)
    {
        if (!m_manager.isTouchExplorationEnabled()) {
            return false;
        }
 
        int virtualViewId = QtNativeAccessibility.hitTest(event.getX(), event.getY());
        if (virtualViewId == INVALID_ID) {
            virtualViewId = View.NO_ID;
        }
 
        switch (event.getAction()) {
            case MotionEvent.ACTION_HOVER_ENTER:
            case MotionEvent.ACTION_HOVER_MOVE:
            case MotionEvent.ACTION_HOVER_EXIT:
                setHoveredVirtualViewId(virtualViewId);
                break;
        }
 
        return true;
    }
 
    public void notifyScrolledEvent(int viewId)
    {
        QtNative.runAction(() -> sendEventForVirtualViewId(viewId,
                AccessibilityEvent.TYPE_VIEW_SCROLLED));
    }
 
    public void notifyLocationChange(int viewId)
    {
        QtNative.runAction(() -> {
            if (m_focusedVirtualViewId == viewId)
                invalidateVirtualViewId(m_focusedVirtualViewId);
        });
    }
 
    public void notifyObjectHide(int viewId, int parentId)
    {
        QtNative.runAction(() -> {
            // If the object had accessibility focus, we need to clear it.
            // Note: This code is mostly copied from
            // AccessibilityNodeProvider::performAction, but we remove the
            // focus only if the focused view id matches the one that was hidden.
            if (m_focusedVirtualViewId == viewId) {
                m_focusedVirtualViewId = INVALID_ID;
                m_view.invalidate();
                sendEventForVirtualViewId(viewId,
                        AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
            }
            // When the object is hidden, we need to notify its parent about
            // content change, not the hidden object itself
            invalidateVirtualViewId(parentId);
        });
    }
 
    public void notifyObjectShow(int parentId)
    {
        QtNative.runAction(() -> {
            // When the object is shown, we need to notify its parent about
            // content change, not the shown object itself
            invalidateVirtualViewId(parentId);
        });
    }
 
    public void notifyObjectFocus(int viewId)
    {
        QtNative.runAction(() -> {
            if (m_view == null)
                return;
            m_focusedVirtualViewId = viewId;
            m_view.invalidate();
            sendEventForVirtualViewId(viewId,
                    AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
        });
    }
 
    public void notifyValueChanged(int viewId, String value)
    {
        QtNative.runAction(() -> {
            // Send a TYPE_ANNOUNCEMENT event with the new value
 
            if ((viewId == INVALID_ID) || !m_manager.isEnabled()) {
                Log.w(TAG, "notifyValueChanged() for invalid view");
                return;
            }
 
            final ViewGroup group = (ViewGroup) m_view.getParent();
            if (group == null) {
                Log.w(TAG, "Could not announce value because ViewGroup was null.");
                return;
            }
 
            final AccessibilityEvent event =
                    AccessibilityEvent.obtain(AccessibilityEvent.TYPE_ANNOUNCEMENT);
 
            event.setEnabled(true);
            event.setClassName(m_view.getClass().getName() + DEFAULT_CLASS_NAME);
 
            event.setContentDescription(value);
 
            if (event.getText().isEmpty() && TextUtils.isEmpty(event.getContentDescription())) {
                Log.w(TAG, "No value to announce for " + event.getClassName());
                return;
            }
 
            event.setPackageName(m_view.getContext().getPackageName());
            event.setSource(m_view, viewId);
 
            if (!group.requestSendAccessibilityEvent(m_view, event))
                Log.w(TAG, "Failed to send value change announcement for " + event.getClassName());
        });
    }
 
    public void sendEventForVirtualViewId(int virtualViewId, int eventType)
    {
        final AccessibilityEvent event = getEventForVirtualViewId(virtualViewId, eventType);
        sendAccessibilityEvent(event);
    }
 
    public void sendAccessibilityEvent(AccessibilityEvent event)
    {
        if (event == null)
            return;
 
        final ViewGroup group = (ViewGroup) m_view.getParent();
        if (group == null) {
            Log.w(TAG, "Could not send AccessibilityEvent because group was null. This should really not happen.");
            return;
        }
 
        group.requestSendAccessibilityEvent(m_view, event);
    }
 
    public void invalidateVirtualViewId(int virtualViewId)
    {
        final AccessibilityEvent event = getEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
 
        if (event == null)
            return;
 
        event.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE);
        sendAccessibilityEvent(event);
    }
 
    private void setHoveredVirtualViewId(int virtualViewId)
    {
        if (m_hoveredVirtualViewId == virtualViewId) {
            return;
        }
 
        final int previousVirtualViewId = m_hoveredVirtualViewId;
        m_hoveredVirtualViewId = virtualViewId;
        sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER);
        sendEventForVirtualViewId(previousVirtualViewId, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
    }
 
    private AccessibilityEvent getEventForVirtualViewId(int virtualViewId, int eventType)
    {
        if ((virtualViewId == INVALID_ID) || !m_manager.isEnabled()) {
            Log.w(TAG, "getEventForVirtualViewId for invalid view");
            return null;
        }
 
        if (m_layout.getChildCount() == 0)
            return null;
 
        final AccessibilityEvent event = AccessibilityEvent.obtain(eventType);
 
        event.setEnabled(true);
        event.setClassName(m_view.getClass().getName() + DEFAULT_CLASS_NAME);
 
        event.setContentDescription(QtNativeAccessibility.descriptionForAccessibleObject(virtualViewId));
        if (event.getText().isEmpty() && TextUtils.isEmpty(event.getContentDescription()))
            Log.w(TAG, "AccessibilityEvent with empty description");
 
        event.setPackageName(m_view.getContext().getPackageName());
        event.setSource(m_view, virtualViewId);
        return event;
    }
 
    // This can be used for debug by performActionForVirtualViewId()
    private void dumpNodes(int parentId)
    {
        Log.i(TAG, "A11Y hierarchy: " + parentId + " parent: " + QtNativeAccessibility.parentId(parentId));
        Log.i(TAG, "    desc: " + QtNativeAccessibility.descriptionForAccessibleObject(parentId) + " rect: " + QtNativeAccessibility.screenRect(parentId));
        Log.i(TAG, " NODE: " + getNodeForVirtualViewId(parentId));
        int[] ids = QtNativeAccessibility.childIdListForAccessibleObject(parentId);
        for (int id : ids) {
            Log.i(TAG, parentId + " has child: " + id);
            dumpNodes(id);
        }
    }
 
    private AccessibilityNodeInfo getNodeForView()
    {
        // Since we don't want the parent to be focusable, but we can't remove
        // actions from a node, copy over the necessary fields.
        final AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(m_view);
        final AccessibilityNodeInfo source = AccessibilityNodeInfo.obtain(m_view);
        m_view.onInitializeAccessibilityNodeInfo(source);
 
        // Get the actual position on screen, taking the status bar into account.
        m_view.getLocationOnScreen(m_globalOffset);
        final int offsetX = m_globalOffset[0];
        final int offsetY = m_globalOffset[1];
 
        // Copy over parent and screen bounds.
        final Rect m_tempParentRect = new Rect();
        source.getBoundsInParent(m_tempParentRect);
        result.setBoundsInParent(m_tempParentRect);
 
        final Rect m_tempScreenRect = new Rect();
        source.getBoundsInScreen(m_tempScreenRect);
        m_tempScreenRect.offset(offsetX, offsetY);
        result.setBoundsInScreen(m_tempScreenRect);
 
        // Set up the parent view, if applicable.
        final ViewParent parent = m_view.getParent();
        if (parent instanceof View) {
            result.setParent((View) parent);
        }
 
        result.setVisibleToUser(source.isVisibleToUser());
        result.setPackageName(source.getPackageName());
        result.setClassName(source.getClassName());
 
        // Spit out the entire hierarchy for debugging purposes
        // dumpNodes(-1);
 
        if (m_layout.getChildCount() != 0) {
            int[] ids = QtNativeAccessibility.childIdListForAccessibleObject(-1);
            for (int id : ids)
                result.addChild(m_view, id);
        }
 
        // The offset values have changed, so we need to re-focus the
        // currently focused item, otherwise it will have an incorrect
        // focus frame
        if ((m_oldOffsetX != offsetX) || (m_oldOffsetY != offsetY)) {
            m_oldOffsetX = offsetX;
            m_oldOffsetY = offsetY;
            if (m_focusedVirtualViewId != INVALID_ID) {
                m_nodeProvider.performAction(m_focusedVirtualViewId,
                                             AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS,
                                             new Bundle());
                m_nodeProvider.performAction(m_focusedVirtualViewId,
                                             AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS,
                                             new Bundle());
            }
        }
 
        return result;
    }
 
    private AccessibilityNodeInfo getNodeForVirtualViewId(int virtualViewId)
    {
        final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain();
 
        node.setClassName(m_view.getClass().getName() + DEFAULT_CLASS_NAME);
        node.setPackageName(m_view.getContext().getPackageName());
 
        if (m_layout.getChildCount() == 0 || !QtNativeAccessibility.populateNode(virtualViewId, node)) {
            return node;
        }
 
        // set only if valid, otherwise we return a node that is invalid and will crash when accessed
        node.setSource(m_view, virtualViewId);
 
        if (TextUtils.isEmpty(node.getText()) && TextUtils.isEmpty(node.getContentDescription()))
            Log.w(TAG, "AccessibilityNodeInfo with empty contentDescription: " + virtualViewId);
 
        int parentId = QtNativeAccessibility.parentId(virtualViewId);
        node.setParent(m_view, parentId);
 
        Rect screenRect = QtNativeAccessibility.screenRect(virtualViewId);
        final int offsetX = m_globalOffset[0];
        final int offsetY = m_globalOffset[1];
        screenRect.offset(offsetX, offsetY);
        node.setBoundsInScreen(screenRect);
 
        Rect parentScreenRect = QtNativeAccessibility.screenRect(parentId);
        screenRect.offset(-parentScreenRect.left, -parentScreenRect.top);
        node.setBoundsInParent(screenRect);
 
        // Manage internal accessibility focus state.
        if (m_focusedVirtualViewId == virtualViewId) {
            node.setAccessibilityFocused(true);
            node.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
        } else {
            node.setAccessibilityFocused(false);
            node.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS);
        }
 
        int[] ids = QtNativeAccessibility.childIdListForAccessibleObject(virtualViewId);
        for (int id : ids)
            node.addChild(m_view, id);
        if (node.isScrollable()) {
            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
                node.setCollectionInfo(new CollectionInfo(ids.length, 1, false));
            } else {
                node.setCollectionInfo(CollectionInfo.obtain(ids.length, 1, false));
            }
        }
 
        return node;
    }
 
    private final AccessibilityNodeProvider m_nodeProvider = new AccessibilityNodeProvider()
    {
        @Override
        public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId)
        {
            if (virtualViewId == View.NO_ID || m_layout.getChildCount() == 0) {
                return getNodeForView();
            }
            return getNodeForVirtualViewId(virtualViewId);
        }
 
        @Override
        public boolean performAction(int virtualViewId, int action, Bundle arguments)
        {
            boolean handled = false;
            //Log.i(TAG, "PERFORM ACTION: " + action + " on " + virtualViewId);
            switch (action) {
                case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
                    // Only handle the FOCUS action if it's placing focus on
                    // a different view that was previously focused.
                    if (m_focusedVirtualViewId != virtualViewId) {
                        m_focusedVirtualViewId = virtualViewId;
                        m_view.invalidate();
                        sendEventForVirtualViewId(virtualViewId,
                                AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
                        handled = true;
                    }
                    break;
                case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
                    if (m_focusedVirtualViewId == virtualViewId) {
                        m_focusedVirtualViewId = INVALID_ID;
                    }
                    // Since we're managing focus at the parent level, we are
                    // likely to receive a FOCUS action before a CLEAR_FOCUS
                    // action. We'll give the benefit of the doubt to the
                    // framework and always handle FOCUS_CLEARED.
                    m_view.invalidate();
                    sendEventForVirtualViewId(virtualViewId,
                            AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
                    handled = true;
                    break;
                default:
                    // Let the node provider handle focus for the view node.
                    if (virtualViewId == View.NO_ID) {
                        return m_view.performAccessibilityAction(action, arguments);
                    }
            }
            handled |= performActionForVirtualViewId(virtualViewId, action);
 
            return handled;
        }
    };
 
    protected boolean performActionForVirtualViewId(int virtualViewId, int action)
    {
        //noinspection CommentedOutCode
        {
            // Log.i(TAG, "ACTION " + action + " on " + virtualViewId);
            // dumpNodes(virtualViewId);
        }
        boolean success = false;
        switch (action) {
        case AccessibilityNodeInfo.ACTION_CLICK:
            success = QtNativeAccessibility.clickAction(virtualViewId);
            if (success)
                sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_VIEW_CLICKED);
            break;
        case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD:
            success = QtNativeAccessibility.scrollForward(virtualViewId);
            if (success)
                sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_VIEW_SCROLLED);
            break;
        case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD:
            success = QtNativeAccessibility.scrollBackward(virtualViewId);
            if (success)
                sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_VIEW_SCROLLED);
            break;
        }
        return success;
    }
}