如果要控制手机打开App自动化操作现成的方案是目前已Appium之类的,
但是这些方案都有一个特点:得依赖pc执行具体的程序。倘若需要独立在手机自身上运行自动化控制怎么办?

经过一番调研,发现基于AccessibilityService可以做到很多,比如说拿到当前的UI树,对UI节点可以通过performAction广播click等各种事件,包括监听App的UI变化等。

其实包括UiAutomator等测试框架也都是基于AccessibilityService来做的。

image-20210509074234680

这样一来便可以基于此提供一个nodejs runtime,宿主暴露一些控制API。接着就可以通过JS脚本来执行自动化控制了,只要保证稳定性,这个脚本就可以脱离PC在手机上自动运行了,控制端只需要对这些手机下发特定的自动化控制脚本即可。

宿主APP我打算基于Flutter,因为写起UI来真的是方便。

其次是因为有个第三方包flutter_liquidcore 封装了LiquidCorea提供的nodejs runtime.

宿主API

1
2
3
4
fun getSource(): String {
var xmlView = AccessibilityNodeInfoDumper.dumpWindowXmlString(MyAccessibilityService.instance?.rootInActiveWindow, 0, 1080, 1920);
return xmlView.toString();
}

接着只需要实现一些简单的api,比如说使用AccessibilityNodeInfoDumper获取当前的UI XML树

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
fun dumpNodeRec(node: AccessibilityNodeInfo, serializer: XmlSerializer, index: Int,
width: Int, height: Int, skipNext: Boolean) {
val eid = node.hashCode().toString();
serializer.attribute("", "element-id", eid);
MainActivity.collectNodeToCache(node);
}

fun collectNodeToCache(node: AccessibilityNodeInfo){
val eid = node.hashCode().toString();
if(node.isScrollable || node.isClickable || node.isLongClickable || node.isCheckable || node.isEditable || node.isFocusable || node.isDismissable){
if(MainActivity.cacheElements.get(eid) == null){
MainActivity.cacheElements.put(eid, node);
}else{
MainActivity.cacheElements.remove(eid);
MainActivity.cacheElements.put(eid, node);
}
}
}

给每个节点加上个guid并把可交互节点cache起来。

广播交互

接着提供一个通用API doActionToElement来广播下click之类的事件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
fun doActionToElement(elementId: String, action: String, actionData: JSONObject): Boolean {

Log.d("stdout", "handle doActionToElement ");

// val element = MainActivity.knowElements.get(elementId);
val element = MainActivity.getNodeFromCache(elementId);

Log.d("stdout", elementId);
Log.d("stdout", action);

if(element == null){
Log.d("MainActivity", "element not found");
MainActivity.channel?.invokeMethod("onMicroServiceStatus", elementId+" element not found");
return false;
}

Log.d("stdout", element.hashCode().toString());
Log.d("stdout", MainActivity.accessibilityNodeToJson(element).toString());

if(action.equals("click")){
if(element.isClickable){
element.performAction(AccessibilityNodeInfo.ACTION_CLICK);
}else{
return false;
}
}


if(action.equals("long-click")){
if(element.isLongClickable){
element.performAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
}else{
return false;
}
}

if(action.equals("scroll-backward")){
if(element.isScrollable){
if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {

Log.d("stdout", "scroll-backward not in actionList");
if(!element.actionList.contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_BACKWARD)){
return false;
};
}
element.performAction(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD);
}else{
return false;
}
}

if(action.equals("scroll-forward")){
if(element.isScrollable){
if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
Log.d("stdout", "scroll-forward not in actionList");
if(!element.actionList.contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_FORWARD)){
return false;
};
}
element.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);
}else{
return false;
}
}


if(action.equals("setText")){
var text = actionData.optString("text");
val arguments = Bundle()
arguments.putCharSequence(AccessibilityNodeInfo
.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
element.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments);
}


return true;
}

还有启动App等API, 详见

NodeJs Runtime

接着还需要封装一层API给跑在虚拟机的js的用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

function getGuid() {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
var r = Math.random() * 16 | 0,
v = c == 'x' ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
}


var watchers = {};

LiquidCore.on('actionResponse', (reponse) => {
var originalEvent = reponse.event;
var eventId = originalEvent.eventId;
var actionName = originalEvent.actionName;
if (watchers[actionName]) {
if (watchers[actionName]) {
try {
watchers[actionName][eventId](reponse.result);
delete watchers[actionName][eventId];
} catch (e) { }
}
}
})


// Wrapper
export function sendAction(actionName, data, timeout) {
return new Promise((resolve, reject) => {
// var respName = actionName + 'Response';
var eventId = getGuid();
data.eventId = eventId;
data.actionName = actionName;

timeout = timeout || 3000;

var isCalled = false;

watchers[actionName] = {};
watchers[actionName][eventId] = (re) => {
isCalled = true;
resolve(re);
}

setTimeout(() => {
if(isCalled) return;
try {
delete watchers[actionName][eventId];
} catch (e) {
}
console.log('timeout', actionName, JSON.stringify(data));
reject('timeout');
}, timeout);

console.log('send', actionName, JSON.stringify(data));
LiquidCore.emit(actionName, data);
});
}


export class Driver {

static findByText(text) {
return sendAction('findElement', {
strategy: 'text',
selector: text
});
}

static getSource() {
return sendAction('getSource', {});
}

static clickElement(elementId) {
return sendAction('doActionToElement', {
elementId,
action: 'click'
});
}

static triggerEventToElement(elementId, type) {
return sendAction('doActionToElement', {
elementId,
action: type
});
}
// static
}

通过LiquidCore的事件API来和Native APP通讯,调用宿主提供的getSource,doActionToElement等

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233

export function createDoc(xmlString){

var doc = cheerio.load(xmlString, { ignoreWhitespace: true, xmlMode: true });

doc.prototype.click = async function () {
const element = this.eq(0);
var status = false;
try{
status = await Driver.triggerEventToElement(element.attr(ATTR_ID), 'click');
}catch(e){
}
return status;
}

doc.prototype.scroll = async function (type) {
type = type || 'forward';
const element = this.eq(0);
var status = false;
try{
status = await Driver.triggerEventToElement(element.attr(ATTR_ID), 'scroll-' + type);
}catch(e){
}
return status;
}

doc.prototype.text = function() {
return getText(this);
};


function parseBounds(bounds){
bounds = bounds.split('][');
var boundsOne = bounds[0].replace('[', '').split(',')
var boundsTwo = bounds[1].replace(']', '').split(',');
return {
x: parseInt(boundsOne[0]),
y: parseInt(boundsOne[1]),
width: boundsTwo[0] - boundsOne[0],
height: boundsTwo[1] - boundsOne[1]
}
}

function getElementRect(el){
return parseBounds(el.attr('bounds'));
}

doc.prototype.getElementRect = function(){
return getElementRect(this);
}

doc.prototype.getArea = function(){
var rect = getElementRect(this);
return rect.width * rect.height;
}

doc.prototype.isClickable = function(){
return this.attr('clickable') == "true";
}

doc.prototype.isScrollable = function(){
return this.attr('scollable') == "true";
}

doc.prototype.filterClickable = function(){
var els = [];
for (let index = 0; index < this.length; index++) {
const element = this.eq(index);
if(element.isClickable()){
els.push(element);
}
}
return this._make(els);
}


doc.prototype.toSelector = function(includeSelf){
try{
var parent = [];
var startNode = this;
for (let index = 0; index < 20; index++) {
if(startNode.length == 0) break;
var className = startNode.attr('class');
if(className) {
var currentSelector = [startNode[0].tagName, "[class='"+className+"']"].join('')
parent.push(currentSelector);
}else{
parent.push(startNode[0].tagName);
};
startNode = startNode.parent();
}
// console.log(parent.reverse().join(" > "), node.attr('class'), node.index());
return parent.reverse().join(" > ");
}catch(e){
console.log(e);
}
}


function getSelector(direc){
var selector = "[scrollable='true']";
if(direc){
selector = "[scroll-"+direc+"='true']";
}
return selector;
}



_.merge(doc, {
scrollForwardAble: function(){
return this(getSelector('forward'));
},

scrollBackwardAble: function(){
return this(getSelector('backward'));
},

scrollable: function(direc){
return this(getSelector(direc));
},

clickable: function(){
return this("[clickable='true']");
},

tabs: function(type, maxHeight = 1185){
var clickElements = this.clickable();
var els = [];
for (let index = 0; index < clickElements.length; index++) {
const clickElement = clickElements.eq(index);
const clickAbleSiblings = clickElement.siblings().filterClickable();
try{
if(clickAbleSiblings.length > 1){
els.push(clickElement);
}
}catch(e){
console.log('findError', e.toString());
}
}

var sameSkyLine = {};

els.forEach((el) => {
var rect = el.getElementRect();
sameSkyLine[rect.y] = sameSkyLine[rect.y] || [];
sameSkyLine[rect.y].push(el);
});

var SameLineUiSet = {};
var bottomX = maxHeight;

Object.keys(sameSkyLine).forEach((startX) => {
var nodes = sameSkyLine[startX];
if(nodes.length > 1){

var position = 'nav';
if(startX > bottomX){
position = 'bottom'
}

if(startX > 100 && startX < bottomX){
position = 'top'
}

SameLineUiSet[position] = {
startX: startX,
position: position,
nodes: nodes
}
}
});

if(type){
if(SameLineUiSet[type]){
return clickElements._make(SameLineUiSet[type].nodes);
}
return null;
}else{
var all = {};
for(var atype in SameLineUiSet){
all[atype] = clickElements._make(SameLineUiSet[atype].nodes);
}
}
},

mainContentView: function(){
var scrollableElements = this.scrollable();
var contentViewByBigArea = null;

for (let index = 0; index < scrollableElements.length; index++) {
const element = scrollableElements.eq(index);
// 排除滚屏组件
if(element.attr('class') == "android.support.v4.view.ViewPager"){
continue;
}

// 排除banner
if(element.attr('class') == "android.widget.HorizontalScrollView"){
continue;
}

var area = element.getArea();
var comp = {
area: area,
node: element
};

// android.support.v4.view.ViewPager
if(contentViewByBigArea == null){
contentViewByBigArea = comp
}else{
if(comp.area > contentViewByBigArea.area){
contentViewByBigArea = comp;
}
}
}

if(contentViewByBigArea){
return contentViewByBigArea.node;
}

return contentViewByBigArea;
}
});

return doc;
}

export async function getDoc() {
var viewTree = await Driver.getSource();
return createDoc(viewTree);
}

通过cheerio 对UI树做节点查找,那么就可以用jquery的方式再做操作比如说

$(“[text*=’知乎’]”).click()

来对属性包含知乎的节点进行点击,mainContentView获取当前可滚动的内容区域节点等,源码详见:https://github.com/lljxx1/App-Walker/blob/master/src/driver.js

一个简单的遍历

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

async function doTest() {
await sendAction('launchPackage', {
appName: '豆瓣'
});

var $ = await getDoc();
var mainView = $.mainContentView();
if(mainView){
var childSelector = mainView.children("[clickable='true']").eq(0).toSelector();
for (let index = 0; index < 10; index++) {
var childs = $(childSelector);
if(childs.length){
await PlayDetail(childs.eq(0));
}
}
console.log(childSelector);
await $.mainContentView().scroll();
}
}

(async function loop() {

var $ = await getDoc();
var chrome = $("[text*='今日头条']");
if (chrome.length) {
var icon = chrome.eq(0);
icon.parent().click();
}

await wait(10 * 1000);

var $ = await getDoc();
var dialog = $("[text*='个人信息保护指引']");
var knowButton = $("[text*='我知道了']");

if(dialog.length && knowButton.length){
console.log("try to click");
knowButton.click();
doTest();
return;
}

setTimeout(loop, 5000);
})();

获取列表页的可点击项,对内页进行逐个遍历,滚动列表页等

开发者工具

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
var originalLog = console.log;

console.log = function() {
var arr = Array.prototype.slice.call(arguments);
if(!remoteDebugger){
return originalLog.apply(null, arr);
}

try{
var consoleStr = arr.join("\t");
originalLog(remoteId, consoleStr);
remoteDebugger.sendText(JSON.stringify({
method: 'sendMessage',
did: remoteId,
message: JSON.stringify({
method: "logger",
log: consoleStr
})
}));
}catch(e){
originalLog(e);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
var con = ws.connect(serverURL, () => {
con.sendText(JSON.stringify({
'method': 'registerDevice'
}));
})

con.on("text", function (str) {
originalLog(str);
str = JSON.parse(str);

try{
var msg = JSON.parse(str.msg);
if(msg.method == "eval"){
eval(msg.code);
}

if(msg.method == 'inspect'){
remoteDebugger = con;
remoteId = str.from;
}
}catch(e){
}

})

为了便于开发测试脚本,我还做了个类似chrome devtool的开发者工具,因为nodejs runtime是具备网络通讯能力的,只需要websocket桥接下,即可实时获取手机的UI树,把runtime里的console收集显示,实时执行代码测试等。