AccessibilityService + Flutter + Node.js的自动化控制方案小探

如果要控制手机打开App自动化操作现成的方案是目前已Appium之类的，
但是这些方案都有一个特点：得依赖pc执行具体的程序。倘若需要独立在手机自身上运行自动化控制怎么办？

经过一番调研，发现基于AccessibilityService可以做到很多，比如说拿到当前的UI树，对UI节点可以通过performAction广播click等各种事件，包括监听App的UI变化等。

其实包括UiAutomator等测试框架也都是基于AccessibilityService来做的。

这样一来便可以基于此提供一个nodejs runtime，宿主暴露一些控制API。接着就可以通过JS脚本来执行自动化控制了，只要保证稳定性，这个脚本就可以脱离PC在手机上自动运行了，控制端只需要对这些手机下发特定的自动化控制脚本即可。

宿主APP我打算基于Flutter，因为写起UI来真的是方便。

其次是因为有个第三方包flutter_liquidcore 封装了LiquidCorea提供的nodejs runtime.

宿主API

fun getSource(): String {
    var xmlView = AccessibilityNodeInfoDumper.dumpWindowXmlString(MyAccessibilityService.instance?.rootInActiveWindow, 0, 1080, 1920);
    return xmlView.toString();
}

接着只需要实现一些简单的api，比如说使用AccessibilityNodeInfoDumper获取当前的UI XML树

fun dumpNodeRec(node: AccessibilityNodeInfo, serializer: XmlSerializer, index: Int,
                            width: Int, height: Int, skipNext: Boolean) {
  val eid = node.hashCode().toString();
   serializer.attribute("", "element-id", eid);
   MainActivity.collectNodeToCache(node);
}

fun collectNodeToCache(node: AccessibilityNodeInfo){
  val eid = node.hashCode().toString();
  if(node.isScrollable || node.isClickable || node.isLongClickable || node.isCheckable || node.isEditable ||  node.isFocusable || node.isDismissable){
      if(MainActivity.cacheElements.get(eid) == null){
          MainActivity.cacheElements.put(eid, node);
      }else{
          MainActivity.cacheElements.remove(eid);
          MainActivity.cacheElements.put(eid, node);
      }
  }
}

给每个节点加上个guid并把可交互节点cache起来。

广播交互

接着提供一个通用API doActionToElement来广播下click之类的事件

fun doActionToElement(elementId: String, action: String, actionData: JSONObject): Boolean {

        Log.d("stdout", "handle doActionToElement ");

//        val element = MainActivity.knowElements.get(elementId);
        val element = MainActivity.getNodeFromCache(elementId);

        Log.d("stdout", elementId);
        Log.d("stdout", action);

        if(element == null){
            Log.d("MainActivity", "element not found");
            MainActivity.channel?.invokeMethod("onMicroServiceStatus", elementId+" element not found");
            return false;
        }

        Log.d("stdout", element.hashCode().toString());
        Log.d("stdout", MainActivity.accessibilityNodeToJson(element).toString());

        if(action.equals("click")){
            if(element.isClickable){
                element.performAction(AccessibilityNodeInfo.ACTION_CLICK);
            }else{
                return false;
            }
        }


        if(action.equals("long-click")){
            if(element.isLongClickable){
                element.performAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
            }else{
                return false;
            }
        }

        if(action.equals("scroll-backward")){
            if(element.isScrollable){
                if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {

                    Log.d("stdout", "scroll-backward not in actionList");
                    if(!element.actionList.contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_BACKWARD)){
                        return false;
                    };
                }
                element.performAction(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD);
            }else{
                return false;
            }
        }

        if(action.equals("scroll-forward")){
            if(element.isScrollable){
                if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
                    Log.d("stdout", "scroll-forward not in actionList");
                    if(!element.actionList.contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_FORWARD)){
                        return false;
                    };
                }
                element.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);
            }else{
                return false;
            }
        }


        if(action.equals("setText")){
            var text = actionData.optString("text");
            val arguments = Bundle()
            arguments.putCharSequence(AccessibilityNodeInfo
                    .ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
            element.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments);
        }


        return true;
}

还有启动App等API, 详见

NodeJs Runtime

接着还需要封装一层API给跑在虚拟机的js的用

function getGuid() {
    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
        var r = Math.random() * 16 | 0,
            v = c == 'x' ? r : (r & 0x3 | 0x8);
        return v.toString(16);
    });
}


var watchers = {};

LiquidCore.on('actionResponse', (reponse) => {
    var originalEvent = reponse.event;
    var eventId = originalEvent.eventId;
    var actionName = originalEvent.actionName;
    if (watchers[actionName]) {
        if (watchers[actionName]) {
            try {
                watchers[actionName][eventId](reponse.result);
                delete watchers[actionName][eventId];
            } catch (e) { }
        }
    }
})


// Wrapper
export function sendAction(actionName, data, timeout) {
    return new Promise((resolve, reject) => {
        // var respName = actionName + 'Response';
        var eventId = getGuid();
        data.eventId = eventId;
        data.actionName = actionName;

        timeout = timeout || 3000;

        var isCalled = false;

        watchers[actionName] = {};
        watchers[actionName][eventId] = (re) => {
            isCalled = true;
            resolve(re);
        }

        setTimeout(() => {
            if(isCalled) return;
            try {
                delete watchers[actionName][eventId];
            } catch (e) {
            }
            console.log('timeout', actionName, JSON.stringify(data));
            reject('timeout');
        }, timeout);

        console.log('send', actionName, JSON.stringify(data));
        LiquidCore.emit(actionName, data);
    });
}


export class Driver {

    static findByText(text) {
        return sendAction('findElement', {
            strategy: 'text',
            selector: text
        });
    }

    static getSource() {
        return sendAction('getSource', {});
    }

    static clickElement(elementId) {
        return sendAction('doActionToElement', {
            elementId,
            action: 'click'
        });
    }

    static triggerEventToElement(elementId, type) {
        return sendAction('doActionToElement', {
            elementId,
            action: type
        });
    }
    // static 
}

通过LiquidCore的事件API来和Native APP通讯，调用宿主提供的getSource，doActionToElement等

export function createDoc(xmlString){
    
    var doc = cheerio.load(xmlString, { ignoreWhitespace: true, xmlMode: true });

    doc.prototype.click = async function () {
        const element = this.eq(0);
        var status = false;
        try{
            status = await Driver.triggerEventToElement(element.attr(ATTR_ID), 'click');
        }catch(e){
        }
        return status;
    }

    doc.prototype.scroll = async function (type) {
        type = type || 'forward';
        const element = this.eq(0);
        var status = false;
        try{
            status = await Driver.triggerEventToElement(element.attr(ATTR_ID), 'scroll-' + type);
        }catch(e){
        }
        return status;
    }

    doc.prototype.text = function() {
        return getText(this);
    };


    function parseBounds(bounds){
        bounds = bounds.split('][');
        var boundsOne = bounds[0].replace('[', '').split(',')
        var boundsTwo = bounds[1].replace(']', '').split(',');
        return {
            x: parseInt(boundsOne[0]),
            y: parseInt(boundsOne[1]),
            width: boundsTwo[0] - boundsOne[0],
            height: boundsTwo[1] - boundsOne[1]
        }
    }
    
    function getElementRect(el){
        return parseBounds(el.attr('bounds'));
    }

    doc.prototype.getElementRect = function(){
        return getElementRect(this);
    }

    doc.prototype.getArea = function(){
        var rect = getElementRect(this);
        return rect.width * rect.height;
    }

    doc.prototype.isClickable = function(){
        return this.attr('clickable') == "true";
    }

    doc.prototype.isScrollable = function(){
        return this.attr('scollable') == "true";
    }

    doc.prototype.filterClickable = function(){
        var els = [];
        for (let index = 0; index < this.length; index++) {
            const element = this.eq(index);
            if(element.isClickable()){
                els.push(element);
            }
        }
        return this._make(els);
    }


    doc.prototype.toSelector = function(includeSelf){
        try{
            var parent = [];
            var startNode = this;
            for (let index = 0; index < 20; index++) {
                if(startNode.length == 0) break;
                var className = startNode.attr('class');
                if(className) {
                    var currentSelector = [startNode[0].tagName, "[class='"+className+"']"].join('')
                    parent.push(currentSelector);
                }else{
                    parent.push(startNode[0].tagName);
                };
                startNode = startNode.parent();
            }
            // console.log(parent.reverse().join(" > "), node.attr('class'), node.index());
            return parent.reverse().join(" > ");
        }catch(e){
            console.log(e);
        }
    }


    function getSelector(direc){
        var selector = "[scrollable='true']";
        if(direc){
            selector = "[scroll-"+direc+"='true']";
        }
        return selector;
    }



    _.merge(doc, {
        scrollForwardAble: function(){
            return this(getSelector('forward'));
        },

        scrollBackwardAble: function(){
            return this(getSelector('backward'));
        },

        scrollable: function(direc){
            return this(getSelector(direc));
        },

        clickable: function(){
            return this("[clickable='true']");
        },

        tabs: function(type, maxHeight = 1185){
            var clickElements = this.clickable();
            var els = [];
            for (let index = 0; index < clickElements.length; index++) {
                const clickElement = clickElements.eq(index);
                const clickAbleSiblings = clickElement.siblings().filterClickable();
                try{
                    if(clickAbleSiblings.length > 1){
                        els.push(clickElement);
                    }
                }catch(e){
                    console.log('findError', e.toString());
                }
            }

            var sameSkyLine = {};

            els.forEach((el) => {
                var rect = el.getElementRect();
                sameSkyLine[rect.y] = sameSkyLine[rect.y] || [];
                sameSkyLine[rect.y].push(el);
            });

            var SameLineUiSet = {};
            var bottomX = maxHeight;

            Object.keys(sameSkyLine).forEach((startX) => {
                var nodes = sameSkyLine[startX];
                if(nodes.length > 1){

                    var position = 'nav';
                    if(startX > bottomX){
                        position = 'bottom' 
                    }

                    if(startX > 100 && startX < bottomX){
                        position = 'top' 
                    }

                    SameLineUiSet[position] = {
                        startX: startX,
                        position: position,
                        nodes: nodes
                    }
                }
            });

            if(type){
                if(SameLineUiSet[type]){
                    return clickElements._make(SameLineUiSet[type].nodes);
                }
                return null;
            }else{
                var all = {};
                for(var atype in SameLineUiSet){
                    all[atype] = clickElements._make(SameLineUiSet[atype].nodes);
                }
            }
        },

        mainContentView: function(){
            var scrollableElements = this.scrollable();
            var contentViewByBigArea = null;

            for (let index = 0; index < scrollableElements.length; index++) {
                const element = scrollableElements.eq(index);
                // 排除滚屏组件
                if(element.attr('class') == "android.support.v4.view.ViewPager"){
                    continue;
                }

                // 排除banner
                if(element.attr('class') == "android.widget.HorizontalScrollView"){
                    continue;
                }

                var area = element.getArea();
                var comp = {
                    area: area,
                    node: element
                };

                // android.support.v4.view.ViewPager
                if(contentViewByBigArea == null){
                    contentViewByBigArea = comp
                }else{
                    if(comp.area > contentViewByBigArea.area){
                        contentViewByBigArea = comp;
                    }
                }
            }

            if(contentViewByBigArea){
                return contentViewByBigArea.node;
            }

            return contentViewByBigArea;
        }
    });

    return doc;
}

export async function getDoc() {
    var viewTree = await Driver.getSource();
    return createDoc(viewTree);
}

通过cheerio 对UI树做节点查找，那么就可以用jquery的方式再做操作比如说

$(“[text*=’知乎’]”).click()

来对属性包含知乎的节点进行点击，mainContentView获取当前可滚动的内容区域节点等，源码详见：https://github.com/lljxx1/App-Walker/blob/master/src/driver.js

一个简单的遍历

async function doTest() {
  await sendAction('launchPackage', {
    appName: '豆瓣'
  });

  var $ = await getDoc();
  var mainView = $.mainContentView();
  if(mainView){
    var childSelector = mainView.children("[clickable='true']").eq(0).toSelector();
    for (let index = 0; index < 10; index++) {
        var childs = $(childSelector);
        if(childs.length){
            await PlayDetail(childs.eq(0));
        }
    }
    console.log(childSelector);
    await $.mainContentView().scroll();
  }
}

(async function loop() {
  
    var $ = await getDoc();
    var chrome = $("[text*='今日头条']");
    if (chrome.length) {
        var icon = chrome.eq(0);
        icon.parent().click();
    }

    await wait(10 * 1000);

    var $ = await getDoc();
    var dialog = $("[text*='个人信息保护指引']");
    var knowButton = $("[text*='我知道了']");

    if(dialog.length && knowButton.length){
        console.log("try to click");
        knowButton.click();
      	doTest();
        return;
    }

    setTimeout(loop, 5000);
})();

获取列表页的可点击项，对内页进行逐个遍历，滚动列表页等

开发者工具

var originalLog = console.log;

console.log = function() {
    var arr = Array.prototype.slice.call(arguments);
    if(!remoteDebugger){
        return originalLog.apply(null, arr);
    }

    try{
        var consoleStr = arr.join("\t");
        originalLog(remoteId, consoleStr);
        remoteDebugger.sendText(JSON.stringify({
            method: 'sendMessage',
            did: remoteId,
            message: JSON.stringify({
                method: "logger",
                log: consoleStr
            })
        }));
    }catch(e){
        originalLog(e);
    }
}

var con = ws.connect(serverURL, () => {
        con.sendText(JSON.stringify({
            'method': 'registerDevice'
        }));
    })
    
    con.on("text", function (str) {
        originalLog(str);
        str = JSON.parse(str);

        try{
            var msg = JSON.parse(str.msg);
            if(msg.method == "eval"){
                eval(msg.code);
            }

            if(msg.method == 'inspect'){
                remoteDebugger = con;
                remoteId = str.from;
            }
        }catch(e){
        }
        
    })

为了便于开发测试脚本，我还做了个类似chrome devtool的开发者工具，因为nodejs runtime是具备网络通讯能力的，只需要websocket桥接下，即可实时获取手机的UI树，把runtime里的console收集显示，实时执行代码测试等。